How to convert numpy.ndarray to tfrecord? - tensorflow

Question

I have a large dataset ,The dataset has two feature,first feature is data,the second feature is label,the dataset size is about 6GB,when I run the code as follows:
#data_from_dataset represent data from 4G dataset, data_from_dataset
#type is ndarray,The data_from_dataset shape is two dimension like (a
#very large num,15)
#label_from_dataset represent label from 4G dataset,,label_from_dataset type
#is ndarray also ndarray
#label_from_dataset #shape is two dimension like (a very large num,15)
data_from_dataset, label_from_dataset = load_train_data()
#calc total batch count
num_batch = len(data_from_dataset) // hp.batch_size
# Convert to tensor
X = tf.convert_to_tensor(data_from_dataset, tf.int32)
Y = tf.convert_to_tensor(label_from_dataset, tf.int32)
# Create Queues
input_queues = tf.train.slice_input_producer([X, Y])
# create batch queues
x, y = tf.train.shuffle_batch(input_queues,
num_threads=20,
batch_size=hp.batch_size,
capacity=hp.batch_size*64,
min_after_dequeue=hp.batch_size*32,
allow_smaller_final_batch=False)
it runs very slowly after wating a long time ,the console hints the error as follows:
Error:cannot create a tensor larger than 2GB
it seems problem in these code line:
# Convert to tensor
X = tf.convert_to_tensor(data_from_dataset, tf.int32)
Y = tf.convert_to_tensor(label_from_dataset, tf.int32)
I MODIFY THE CODE CONVERT NUMPY to TFRECORD AS FOLLOWS:
def _int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def save_tfrecords(data_from_dataset, label_from_dataset, desfile):
with tf.python_io.TFRecordWriter(desfile) as writer:
for i in range(len(data_from_dataset)):
features = tf.train.Features(
feature = {
"data": _int64_feature(data[i]),
"label": _int64_feature(label[i])
}
)
example = tf.train.Example(features = features)
serialized = example.SerializeToString()
writer.write(serialized)
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'data': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.string),
})
sent = features['data']
tag = features['label']
sent_decode=tf.decode_raw(sent,tf.int32)
sent_decode=tf.decode_raw(tag,tf.int32)
return sent, tag
fname_out="out.tfrecord"
save_tfrecords(data_from_dataset, label_from_dataset, fname_out)
filename_queue = tf.train.string_input_producer(fname_out, shuffle=True)
example, label = read_and_decode(filename_queue, 2)
x, y = tf.train.shuffle_batch([example, label],
num_threads=20,
batch_size=hp.batch_size,
capacity=hp.batch_size*64,
min_after_dequeue=hp.batch_size*32,
allow_smaller_final_batch=False)
it hints the error on code line as follows:
def _int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
Error:only length-1 arrays can be converted to python scalars
How to convert numpy to tfrecord ?is there any other method?

Answer 1

Function tf.train.Int64List is not for arrays.
You need to use tf.train.BytesList instead
data = np.random.rand(15,)
writer = tf.python_io.TFRecordWriter('file.tfrecords')
str = data.tostring()
example = tf.train.Example(features=tf.train.Features(feature={'1': _bytes_feature(str)}))
writer.write(example.SerializeToString())
writer.close()
You can then decode it with tf.decode_raw of you can inspect tfrecord file with
for str_rec in tf.python_io.tf_record_iterator('file.tfrecords'):
example = tf.train.Example()
example.ParseFromString(str_rec)
str = (example.features.feature['1'].bytes_list.value[0])
your_data = np.fromstring(str, dtype)