Problem about training tensorflow estimator with tf.records files - iterator

Question

I'm using tensorflow estimator with tf.records files to training but I can't solve some errors.
My dataset size is big...So I have to change my dataset npz->tfrecords. TF estimator is running well with numpy file dataset.
First, My saved tfrecords file shape = (N, 437, 256). My batch_size is 30. Here's my code.
def parser(serialized_example):
"""Parses a single tf.Example into image and label tensors."""
features = tf.parse_single_example(
serialized_example,
features={
'train_input_enc/time': tf.FixedLenFeature([], tf.float32),
'train_input_enc/fft': tf.FixedLenFeature([], tf.float32),
'train_input_enc/data_raw': tf.FixedLenFeature([], tf.string),
'train_output_dec/time': tf.FixedLenFeature([], tf.float32),
'train_output_dec/fft': tf.FixedLenFeature([], tf.float32),
'train_output_dec/data_raw': tf.FixedLenFeature([], tf.string),
'train_target_dec/time': tf.FixedLenFeature([], tf.float32),
'train_target_dec/fft': tf.FixedLenFeature([], tf.float32),
'train_target_dec/data_raw': tf.FixedLenFeature([], tf.string)
})
inp_enc_spec = tf.decode_raw(features['train_input_enc/data_raw'], tf.float32)
oup_dec_spec = tf.decode_raw(features['train_output_dec/data_raw'], tf.float32)
trg_dec_spec = tf.decode_raw(features['train_target_dec/data_raw'], tf.float32)
inp_enc_spectrogram = tf.reshape(inp_enc_spec, [DEFINES.max_sequence_length, DEFINES.embedding_size])
oup_dec_spectrogram = tf.reshape(oup_dec_spec, [DEFINES.max_sequence_length, DEFINES.embedding_size])
trg_dec_spectrogram = tf.reshape(trg_dec_spec, [DEFINES.max_sequence_length, DEFINES.embedding_size])
return inp_enc_spectrogram, oup_dec_spectrogram, trg_dec_spectrogram
def rearrange(input, output, target):
features = {"input": input, "output": output}
return features, target
def train_input_fn(train_input_enc, train_output_dec, train_target_dec, batch_size):
dataset = tf.data.TFRecordDataset([train_input_enc, train_output_dec, train_target_dec])
print("input dataset", dataset)
dataset = dataset.map(map_func=parser)
print("dataset map with parser",dataset) # <TFRecordDataset shapes: (), types: tf.string>
dataset = dataset.shuffle(buffer_size=900)
print("dataset shuffle", dataset) # <MapDataset shapes: ((437, 256), (437, 256), (437, 256)), types: (tf.float32, tf.float32, tf.float32)>
dataset = dataset.batch(batch_size = batch_size, drop_remainder=True)
print("dataset batch", dataset) # <BatchDataset shapes: ((30, 437, 256), (30, 437, 256), (30, 437, 256)), types: (tf.float32, tf.float32, tf.float32)>
dataset = dataset.map(rearrange)
print("dataset after rearrange",dataset) # <MapDataset shapes: ({output: (30, 437, 256), input: (30, 437, 256)}, (30, 437, 256)), types: ({output: tf.float32, input: tf.float32}, tf.float32)>
dataset = dataset.repeat()
print("dataset repeat",dataset) # <RepeatDataset shapes: ({output: (30, 437, 256), input: (30, 437, 256)}, (30, 437, 256)), types: ({output: tf.float32, input: tf.float32}, tf.float32)>
iterator = dataset.make_initializable_iterator()
print("iterator init", iterator) # <tensorflow.python.data.ops.iterator_ops.Iterator object at 0x7f9ed1700128>
# iterator = dataset.make_one_shot_iterator()
# print("iterator oneshow",iterator)
iterator = iterator.get_next()
print("iterator",iterator) # ({'output': <tf.Tensor 'IteratorGetNext:1' shape=(30, 437, 256) dtype=float32>, 'input': <tf.Tensor 'IteratorGetNext:0' shape=(30, 437, 256) dtype=float32>}, <t
f.Tensor 'IteratorGetNext:2' shape=(30, 437, 256) dtype=float32>)
return iterator
def main()
filename_inp_enc = './train_input_enc.tfrecords'
filename_oup_dec = './train_output_dec.tfrecords'
filename_trg_dec = './train_target_dec.tfrecords'
check_point_path = os.path.join(os.getcwd(), DEFINES.check_point_path)
os.makedirs(check_point_path, exist_ok=True)
classifier = tf.estimator.Estimator(
model_fn=ml.Model,
model_dir=DEFINES.check_point_path,
config=estimator_config,
params={
...
})
classifier.train(input_fn=lambda: data.train_input_fn(filename_inp_enc, filename_oup_dec, filename_trg_dec, DEFINES.batch_size),steps=DEFINES.train_steps)
Error messages are:
FailedPreconditionError (see above for traceback): GetNext() failed because the iterator has not been initialized. Ensure that you have run the initializer operation for this iterator before getting the next element.
[[{{node IteratorGetNext}} = IteratorGetNext[output_shapes=[[30,437,256], [30,437,256], [30,437,256]], output_types=[DT_FLOAT, DT_FLOAT, DT_FLOAT], _device="/job:localhost/replica:0/task:0/device:CPU:0"](IteratorV2)]]
When I put
iterator = dataset.make_one_shot_iterator()
return iterator.get_next()
the error messages are follows:
tensorflow.python.framework.errors_impl.InvalidArgumentError: Feature: train_output_dec/data_raw (data type: string) is required but could not be found.
[[{{node ParseSingleExample/ParseSingleExample}} = ParseSingleExample[Tdense=[DT_STRING, DT_FLOAT, DT_FLOAT, DT_STRING, DT_FLOAT, DT_FLOAT, DT_STRING, DT_FLOAT, DT_FLOAT], dense_keys=["train_input_enc/data_raw", "train_input_enc/fft", "train_input_enc/time", "train_output_dec/data_raw", "train_output_dec/fft", "train_output_dec/time", "train_target_dec/data_raw", "train_target_dec/fft", "train_target_dec/time"], dense_shapes=[[], [], [], [], [], [], [], [], []], num_sparse=0, sparse_keys=[], sparse_types=[], _device="/device:CPU:0"](arg0, ParseSingleExample/Const, ParseSingleExample/Const_1, ParseSingleExample/Const_1, ParseSingleExample/Const, ParseSingleExample/Const_1, ParseSingleExample/Const_1, ParseSingleExample/Const, ParseSingleExample/Const_1, ParseSingleExample/Const_1)]]
[[{{node IteratorGetNext}} = IteratorGetNext[output_shapes=[[30,437,256], [30,437,256], [30,437,256]], output_types=[DT_FLOAT, DT_FLOAT, DT_FLOAT], _device="/job:localhost/replica:0/task:0/device:CPU:0"](OneShotIterator)]]
[[{{node IteratorGetNext/_4351}} = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device_incarnation=1, tensor_name="edge_192_IteratorGetNext", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:GPU:0"]()]]
How can I solve this problem?