How to save large float into TFRecord format? float_list/float32 seems to truncate the values - tensorflow

Question

We write processed data into TFRecords and we are noticing data loss when read back from TFRecords. Reproducible example below. Strange thing is that it doesn't just drop the decimals but seem to randomly roundup/down values. Since it only allows float32, int64 and string, we are not sure what other options to try.
We are writing these values
[20191221.1, 20191222.1, 20191223.1, 20191224.1, 20191225.1, 20191226.1, 20191227.1, 20191228.1, 20191229.1, 20191230.1]
But reading from tfrecords returns these values
tf.Tensor(
[20191222. 20191222. 20191224. 20191224. 20191226. 20191226. 20191228.
20191228. 20191230. 20191230.], shape=(10,), dtype=float32)
Reproducible Code
import tensorflow as tf
def write_date_tfrecord():
#writes 10 dummy values to replicate the issue
data = [20191221.1 + x for x in range(0,10)]
print("Writing data - ", data)
example = tf.train.Example(
features = tf.train.Features(
feature = {
'data':tf.train.Feature(float_list=tf.train.FloatList(value=data))
}
))
writer = tf.io.TFRecordWriter("data.tf_record")
writer.write(example.SerializeToString())
def parse_function(serialized_example):
features = {
'data': tf.io.FixedLenSequenceFeature([], tf.float32,allow_missing=True)
}
features = tf.io.parse_single_example(serialized=serialized_example, features=features)
data = features['data']
return data
def dataset_generator():
trRecordDataset = tf.data.TFRecordDataset("data.tf_record")
trRecordDataset = trRecordDataset.map(parse_function, num_parallel_calls = tf.data.experimental.AUTOTUNE)
return trRecordDataset
if __name__ == '__main__':
write_date_tfrecord()
generator = dataset_generator()
for data in generator:
print(data)

Answer 1

This solved my issue. I had this issue when writing audio files as floating point matrix using FloatList.. but when i used BytesList and stored the data into tfrecords and then read the data by decoding it.. the issue resolved.. note that even decoding with tf.float32 will lead not solve the issue. we need to decode it with tf.float64..
def _bytes_feature2(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def serialize_example(sound):
feature = {
'snd': _bytes_feature2(sound.tobytes()),
}
example_proto = tf.train.Example(features=tf.train.Features(feature=feature))
return example_proto.SerializeToString()
def write_tfrecords(rf,snd):
nsamples = len(snd)
with tf.io.TFRecordWriter(rf) as writer:
for i in range(nsamples):
SND = snd[i]
tf_example = serialize_example(SND)
writer.write(tf_example)
# writing records
write_tfrecords(os.getcwd()+'\\tfrec\\'+'train.tfrecords',train)
# loading records
raw_dataset = tf.data.TFRecordDataset(os.getcwd()+'\\tfrec\\'+'train.tfrecords')
def parse_record(record):
name_to_features= {
'snd':tf.io.FixedLenFeature([],tf.string),
}
return tf.io.parse_single_example(record, name_to_features)
def decode_record(record):
aud = tf.io.decode_raw(
record['snd'], out_type=tf.float64
)
return aud
for record in raw_dataset:
parsed_record = parse_record(record)
decoded_record = decode_record(parsed_record)
aud = decoded_record
print(aud.numpy()[0:10])
print(train[0][0:10])
output:
[ 417.69951205 -231.58708746 -10.05624011 -146.10342256 -66.60317323
-159.91550792 -3.93602823 29.94517981 106.22196629 65.53008959]
[ 417.69951205 -231.58708746 -10.05624011 -146.10342256 -66.60317323
-159.91550792 -3.93602823 29.94517981 106.22196629 65.53008959]