The main idea is to convert TFRecords into numpy arrays. Assume that the TFRecord stores images. Specifically:
Read a TFRecord File and convert each image into a numpy array.
Write the image into 1.jpg, 2.jpg, etc.
At the same time, write the file name and label to the text file like this:
1.jpg 2
2.jpg 4
3.jpg 5
I currently use the following code:
import tensorflow as tf
import os
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'image_raw': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64),
'height': tf.FixedLenFeature([], tf.int64),
'width': tf.FixedLenFeature([], tf.int64),
'depth': tf.FixedLenFeature([], tf.int64)
})
image = tf.decode_raw(features['image_raw'], tf.uint8)
label = tf.cast(features['label'], tf.int32)
height = tf.cast(features['height'], tf.int32)
width = tf.cast(features['width'], tf.int32)
depth = tf.cast(features['depth'], tf.int32)
return image, label, height, width, depth
with tf.Session() as sess:
filename_queue = tf.train.string_input_producer(["../data/svhn/svhn_train.tfrecords"])
image, label, height, width, depth = read_and_decode(filename_queue)
image = tf.reshape(image, tf.pack([height, width, 3]))
image.set_shape([32,32,3])
init_op = tf.initialize_all_variables()
sess.run(init_op)
print (image.eval())
I'm just reading trying to get at least one image for starters. The code just gets stuck when I run this.
Oops, it was a silly mistake on my part. I used a string_input_producer but forgot to run the queue_runners.
with tf.Session() as sess:
filename_queue = tf.train.string_input_producer(["../data/svhn/svhn_train.tfrecords"])
image, label, height, width, depth = read_and_decode(filename_queue)
image = tf.reshape(image, tf.pack([height, width, 3]))
image.set_shape([32,32,3])
init_op = tf.initialize_all_variables()
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(1000):
example, l = sess.run([image, label])
print (example,l)
coord.request_stop()
coord.join(threads)
Related
when i use tf.data.TFRecordDataset read many tfrecords. I read the label and image from tfrecord. I use tensorboard summary the image,and i write the label into the log file.But when i look at the log file and tensorboard. The label and image is not Corresponding. As follows，My code to read tfrecrods.
def parser(record):
features = tf.parse_single_example(record,
features={
'label': tf.FixedLenFeature([], tf.int64),
'image': tf.FixedLenFeature([], tf.string)
}) # 取出包含image和label的feature对象
recode_image = tf.decode_raw(features['image'], tf.uint8)
real_image = tf.reshape(recode_image, shape=[38, 38, 1])
lable = tf.cast(features['label'], tf.int64)
return real_image,lable
def read_data(file_path):
min_after_dequeue = 100
batch_size = 3
data=tf.data.TFRecordDataset(file_path)
dataset=data.map(parser).
shuffle(buffer_size=min_after_dequeue).
batch(batch_size=batch_size)
dataset=dataset.repeat()
dataset.prefetch(100)
iterator = dataset.make_one_shot_iterator()
image_batch, lable_batch = iterator.get_next()
image_batch=input_float(image_batch)
return image_batch,lable_batch
use read_data in main thread code is:
file_list=glob.glob("./tfcode/training_image/*.tfrecord")
file_list = list(
map(lambda image: image.replace('\\', '/'), file_list))
image_batch, lable_batch= read_data(file_list)
tf.summary.image(tensor=image_batch,name="image")
input_lable = sess.run(lable_batch)
logger.info(input_lable)
As follows is what i look in tensorboard:
enter image description here
As follows is what i look in log file:
enter image description here
the label of image which tensorboard summary is [1,3,3], but the log file is [3,3,3].
how can i deal with it.
Why don't you use https://www.tensorflow.org/api_docs/python/tf/image/decode_jpeg instead ot tf.decode_raw ?
def parser(record):
features = tf.parse_single_example(record,
features={
'label': tf.FixedLenFeature([], tf.int64),
'image': tf.FixedLenFeature([], tf.string)
})
recode_image = tf.image.decode_jpeg(features['image'], channels=1)
real_image = tf.reshape(recode_image, shape=[38, 38])
lable = tf.cast(features['label'], tf.int64)
return real_image,lable
def read_data(file_path):
min_after_dequeue = 100
batch_size = 3
data=tf.data.TFRecordDataset(file_path)
dataset=data.map(parser).
shuffle(buffer_size=min_after_dequeue).
batch(batch_size=batch_size)
dataset=dataset.repeat()
dataset.prefetch(100)
iterator = dataset.make_one_shot_iterator()
image_batch, label_batch = iterator.get_next()
image_batch=input_float(image_batch) # I'm assuming you are rescalling the image to [0,1]
return image_batch,label_batch
I am trying to make use of tensorflow protobuffer feeding pipeline. The easiest way seemed to use tf.estimator.Estimator with tf.contrib.data.TFRecordDataset. However, I came across the issue that it creates a new Graph in spite of being launched within with g.as_default(). In following code I see that both model tensors and tensors returned by the TFRecordDataset are the same before I feed them to Estimator, but become different within the Estimator. Any ideas how to put them on the same graph?
# coding: utf-8
import sys
import tensorflow as tf
from keras.applications.inception_v3 import InceptionV3
import numpy as np
final_activation='linear'
g = tf.Graph()
with g.as_default():
model = InceptionV3(weights='imagenet',
include_top=True,
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
def model_fn(mode, features, labels, params):
optimizer = params["optimizer"]
opt_params= params.get("opt_params", {})
predictions = model(features)
if (mode == tf.estimator.ModeKeys.TRAIN or
mode == tf.estimator.ModeKeys.EVAL):
loss = tf.contrib.keras.backend.categorical_crossentropy(predictions, labels)
#loss = tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logyhat)
else:
loss = None
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = getattr(tf.train, optimizer)
train_op = optimizer(opt_params).minimize(loss)
else:
train_op = None
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op)
def parser(record):
keys_to_features = {
'height': tf.FixedLenFeature([], tf.int64),
'width': tf.FixedLenFeature([], tf.int64),
'image_raw': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64)
}
features = tf.parse_single_example(
record,
features=keys_to_features)
# Convert from a scalar string tensor to a uint8 tensor
image = tf.decode_raw(features['image_raw'], tf.float32)
height = tf.cast(features['height'], tf.int32)
width = tf.cast(features['width'], tf.int32)
image_shape = tf.stack([height, width, 3])
image = tf.reshape(image, image_shape)
label = tf.cast(features["label"], tf.int32)
return image, label
def get_dataset_inp_fn(filenames, epochs=20):
def dataset_input_fn():
dataset = tf.contrib.data.TFRecordDataset(filenames)
# Use `Dataset.map()` to build a pair of a feature dictionary and a label
# tensor for each example.
dataset = dataset.map(parser)
dataset = dataset.shuffle(buffer_size=10000)
dataset = dataset.batch(32)
dataset = dataset.repeat(epochs)
iterator = dataset.make_one_shot_iterator()
features, labels = iterator.get_next()
return features, labels
return dataset_input_fn
inpfun = get_dataset_inp_fn(["mydataset.tfrecords"], epochs=20)
x,y = inpfun()
print("X", x.graph)
print("DEFAULT", g)
print("MODEL", model.input.graph)
# everything is on the same graph
if not x.graph is tf.get_default_graph():
raise ValueError()
with tf.Session(graph=g) as sess:
est = tf.estimator.Estimator(
model_fn,
model_dir=None,
config=None,
params={"optimizer": "AdamOptimizer",
"opt_params":{}}
)
est.train(inpfun)
I have a FIFO Queue reading from tfrecords file in tensorflow. Each record is consisted of an image and its annotation, that is, a set of features. I was trying to skip some images that is, not feeding them into the graph, or not viewing them, according to some features in mind. Therefore, I thought that the best case scenario was to use on a while loop. That loop is going to test the value of the specified feature and decide whether to proceed or not.
Kindly look at the following code:
import tensorflow as tf
import numpy as np
num_epoch = 100
tfrecords_filename_seq = ["C:/Users/user/PycharmProjects/AffectiveComputing/P16_db.tfrecords"]
filename_queue = tf.train.string_input_producer(tfrecords_filename_seq, num_epochs=num_epoch, shuffle=False, name='queue')
reader = tf.TFRecordReader()
current_image_confidence = tf.constant(0.0, dtype=tf.float32)
def body(i):
key, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'height': tf.FixedLenFeature([], tf.int64),
'width': tf.FixedLenFeature([], tf.int64),
'image_raw': tf.FixedLenFeature([], tf.string),
'annotation_raw': tf.FixedLenFeature([], tf.string)
})
# This is how we create one example, that is, extract one example from the database.
image = tf.decode_raw(features['image_raw'], tf.uint8)
# The height and the weights are used to
height = tf.cast(features['height'], tf.int32)
width = tf.cast(features['width'], tf.int32)
# The image is reshaped since when stored as a binary format, it is flattened. Therefore, we need the
# height and the weight to restore the original image back.
image = tf.reshape(image, [height, width, 3])
annotation = tf.cast(features['annotation_raw'], tf.string)
t1 = tf.string_split([annotation], delimiter=',')
t2 = tf.reshape(t1.values, [1, -1])
t3 = tf.string_to_number(t2, out_type=tf.float32)
t_ = tf.slice(t3, begin=[0, 3], size=[1, 1])
# Note that t_ is holding a value of 1.0 or 0.0. So its a particular feature I'm interested in.
t_ = tf.Print(t_, data=[tf.shape(t_)], message='....')
z = tf.cond(t_[0][0] < 1.0, lambda: tf.add(i, 0.0), lambda: tf.add(i, 1.0))
return z
cond = lambda i: tf.equal(i, tf.constant(0.0, dtype=tf.float32))
loop = tf.while_loop(cond, body, [current_image_confidence])
init_op = tf.group(tf.local_variables_initializer(),
tf.global_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
sess.run(loop)
Finally, when trying to run the following code, it seems that the body is not executing and hence stuck in an infinite loop. And the tf.Print(...) in the body was not executed.
Why this is the case?
Any help is much appreciated!!
Your program is getting stick because you're not starting queue runners. Run tf.start_queue_runners() before running your loop op.
I have converted a CSV file ("test03.txt") to a TFRecords-formated file ("test03.tfrecords"), but when I then read in the TFRecords file and try to use tf.train.shuffle_batch I get the error message
RandomShuffleQueue '_2_shuffle_batch_1/random_shuffle_queue' is closed and has insufficient elements (requested 10, current size 0)
The CSV file is
1,0
2,0
3,0
4,0
5,1
6,0
7,1
8,1
9,1
10,1
which I convert to a TFRecords file using
import pandas
import tensorflow as tf
csv = pandas.read_csv(r"test03.txt", header=None).values
with tf.python_io.TFRecordWriter("test03.tfrecords") as writer:
for row in csv:
features, label = row[:-1], row[-1]
example = tf.train.Example()
example.features.feature["features"].float_list.value.extend(features)
example.features.feature["label"].int64_list.value.append(label)
writer.write(example.SerializeToString())
But I get the above error message when I run the following code:
import tensorflow as tf
batch_size = 10
with tf.Session() as sess:
filename_queue = tf.train.string_input_producer(["test03.tfrecords"],num_epochs=1)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
feature_dict = {'features': tf.FixedLenFeature([], tf.int64),'label': tf.FixedLenFeature([], tf.int64)}
featuresLabel = tf.parse_single_example(serialized_example, features=feature_dict)
xdata = tf.cast(featuresLabel['features'], tf.int32)
label = tf.cast(featuresLabel['label'], tf.int32)
min_after_dequeue = 1
capacity = min_after_dequeue + 3 * batch_size
batch_of_xs, batch_of_labels = tf.train.shuffle_batch([xdata, label], batch_size=batch_size, capacity=capacity, num_threads=1, min_after_dequeue=min_after_dequeue)
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
single_batch_xs, single_batch_ys = sess.run([batch_of_xs, batch_of_labels])
Your issue is located in the feature_dict. In your initial example you perform the conversion to TFRecords as follows:
example.features.feature["features"].float_list.value.extend(features)
example.features.feature["label"].int64_list.value.append(label)
Hence your features are encoded as floats and your labels as int64. However when you read them back you turn them into int64:
feature_dict = {'features': tf.FixedLenFeature([], tf.int64),'label': tf.FixedLenFeature([], tf.int64)}
You issue is as simply as matching the feature_dict to your initial encoding, hence changing the line above into:
feature_dict = {'features': tf.FixedLenFeature([], tf.float32),'label': tf.FixedLenFeature([], tf.int64)}
Solves the issue for me (along with a print of the single_batch_xs and ys at the end).
I have a set of images which I'm going to feed into a graph in tensorflow. Fetching the data is done through a FIFOQueue. The problem is that in some images, the face is not detected, that is, the image does not contain a face. Therefore, I am going to ignore these images before feeding them into the graph. My code is as follows:
import tensorflow as tf
import numpy as np
num_epoch = 100
tfrecords_filename_seq = ["C:/Users/user/PycharmProjects/AffectiveComputing/P16_db.tfrecords"]
filename_queue = tf.train.string_input_producer(tfrecords_filename_seq, num_epochs=num_epoch, shuffle=False, name='queue')
reader = tf.TFRecordReader()
current_image_confidence = tf.Variable(tf.constant(0.0, dtype=tf.float32))
image = tf.Variable(tf.ones([112, 112, 3]), dtype=tf.float32)
annotation = tf.Variable('', dtype=tf.string)
def body():
key, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'height': tf.FixedLenFeature([], tf.int64),
'width': tf.FixedLenFeature([], tf.int64),
'image_raw': tf.FixedLenFeature([], tf.string),
'annotation_raw': tf.FixedLenFeature([], tf.string)
})
# This is how we create one example, that is, extract one example from the database.
image_ = tf.decode_raw(features['image_raw'], tf.uint8)
# The height and the weights are used to
height = tf.cast(features['height'], tf.int32)
width = tf.cast(features['width'], tf.int32)
# The image is reshaped since when stored as a binary format, it is flattened. Therefore, we need the
# height and the weight to restore the original image back.
image.assign(tf.reshape(image_, [height, width, 3]))
annotation.assign(tf.cast(features['annotation_raw'], tf.string))
current_image_confidence.assign(tf.slice(tf.string_to_number(tf.string_split(annotation, delimiter=','),
out_type=tf.float32),
begin=[0, 3],
size=[1, 1]))
def cond():
tf.equal(current_image_confidence, tf.constant(0.0, dtype=tf.float32))
loop = tf.while_loop(cond, body, [current_image_confidence, reader, image, annotation])
Therefore, I need a while loop that will run until I get an image with face. That is when I need to terminate the loop and send the image to the graph.
Please note that my data is stored in a tfrecord file. So each record contains one image and a set of features called annotation, saved as a tf.string. So the current_image_confidence Variable is used to hold a value of 1 or 0 based whether a face is present or not.
How to fix the code??
Any help is much appreciated!!