Here is the piece of code I cloned off of Github and I am having a hard time getting it to work.
def lstm_doc_enc(input_cnn,
batch_size=20,
num_rnn_layers=2,
rnn_size=650,
max_doc_length=35,
dropout=0.0):
# lstm document encoder
with tf.variable_scope('LSTMenc') as scope:
def create_rnn_cell():
cell = tf.contrib.rnn.BasicLSTMCell(rnn_size, state_is_tuple=True, forget_bias=0.0, reuse=True)
if dropout > 0.0:
cell = tf.contrib.rnn.DropoutWrapper(cell, output_keep_prob=1.-dropout)
return cell
if num_rnn_layers > 1:
cell = tf.contrib.rnn.MultiRNNCell([create_rnn_cell() for _ in range(num_rnn_layers)], state_is_tuple=True)
else:
cell = create_rnn_cell()
initial_rnn_state = cell.zero_state(batch_size, dtype=tf.float32)
input_cnn = tf.reshape(input_cnn, [batch_size, max_doc_length, -1])
input_cnn2 = [tf.squeeze(x, [1]) for x in tf.split(input_cnn, max_doc_length, 1)]
outputs, final_rnn_state = tf.contrib.rnn.static_rnn(cell, input_cnn2,
initial_state=initial_rnn_state, dtype=tf.float32)
return adict(
initial_enc_state=initial_rnn_state,
final_enc_state=final_rnn_state,
enc_outputs=outputs
)
I cloned it from the repository of NeuralSum.
If I leave reuse=True while creating the BasicLSTMCell, it gives the following error
Traceback (most recent call last):
File "pretrain.py", line 358, in <module>
tf.app.run()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 48, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "pretrain.py", line 244, in main
train_model = build_model(word_vocab, train=True)
File "pretrain.py", line 146, in build_model
dropout=FLAGS.dropout))
File "/home/raghuram.vadapalli/styletransfer/NeuralSum/model.py", line 218, in lstm_doc_enc
initial_state=initial_rnn_state, dtype=tf.float32)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/rnn/python/ops/core_rnn.py", line 197, in static_rnn
(output, state) = call_cell()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/rnn/python/ops/core_rnn.py", line 184, in <lambda>
call_cell = lambda: cell(input_, state)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py", line 713, in __call__
output, new_state = self._cell(inputs, state, scope)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py", line 241, in __call__
concat = _linear([inputs, h], 4 * self._num_units, True)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py", line 1044, in _linear
_WEIGHTS_VARIABLE_NAME, [total_arg_size, output_size], dtype=dtype)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.py", line 1049, in get_variable
use_resource=use_resource, custom_getter=custom_getter)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.py", line 948, in get_variable
use_resource=use_resource, custom_getter=custom_getter)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.py", line 356, in get_variable
validate_shape=validate_shape, use_resource=use_resource)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.py", line 341, in _true_getter
use_resource=use_resource)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.py", line 671, in _get_single_variable
"VarScope?" % name)
ValueError: Variable Model/LSTMenc/rnn/basic_lstm_cell/weights does not exist, or was not created with tf.get_variable(). Did you mean to set reuse=None in VarScope?
If I don't put reuse=True, I get this other error:
Traceback (most recent call last):
File "pretrain.py", line 358, in <module>
tf.app.run()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 48, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "pretrain.py", line 251, in main
valid_model = build_model(word_vocab, train=False)
File "pretrain.py", line 200, in build_model
dropout=FLAGS.dropout))
File "/home/raghuram.vadapalli/styletransfer/NeuralSum/model.py", line 218, in lstm_doc_enc
initial_state=initial_rnn_state, dtype=tf.float32)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/rnn/python/ops/core_rnn.py", line 197, in static_rnn
(output, state) = call_cell()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/rnn/python/ops/core_rnn.py", line 184, in <lambda>
call_cell = lambda: cell(input_, state)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py", line 235, in __call__
with _checked_scope(self, scope or "basic_lstm_cell", reuse=self._reuse):
File "/usr/lib/python2.7/contextlib.py", line 17, in __enter__
return self.gen.next()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py", line 93, in _checked_scope
"the argument reuse=True." % (scope_name, type(cell).__name__))
ValueError: Attempt to have a second RNNCell use the weights of a variable scope that already has weights: 'Model/LSTMenc/rnn/basic_lstm_cell'; and the cell was not constructed as BasicLSTMCell(..., reuse=True). To share the weights of an RNNCell, simply reuse it in your second calculation, or create a new one with the argument reuse=True.
I don't have a lot of experience with variable scopes in tensorflow. I tried to google a lot and it came of no use. I hope someone understands what is wrong here and help me.
Related
Following this guide, I'm converting a tensor [batch_size, 16000, 1] to an MFCC using the method described in the link:
def gen_spectrogram(wav, sr=16000):
# A 1024-point STFT with frames of 64 ms and 75% overlap.
stfts = tf.contrib.signal.stft(wav, frame_length=1024, frame_step=256, fft_length=1024)
spectrograms = tf.abs(stfts)
# Warp the linear scale spectrograms into the mel-scale.
num_spectrogram_bins = stfts.shape[-1].value
lower_edge_hertz, upper_edge_hertz, num_mel_bins = 80.0, 7600.0, 80
linear_to_mel_weight_matrix = tf.contrib.signal.linear_to_mel_weight_matrix(
num_mel_bins, num_spectrogram_bins,
sample_rate, lower_edge_hertz, upper_edge_hertz)
mel_spectrograms = tf.tensordot(spectrograms, linear_to_mel_weight_matrix, 1)
mel_spectrograms.set_shape(
spectrograms.shape[:-1].concatenate(
linear_to_mel_weight_matrix.shape[-1:]
)
)
# Compute a stabilized log to get log-magnitude mel-scale spectrograms.
log_mel_spectrograms = tf.log(mel_spectrograms + 1e-6)
# Compute MFCCs from log_mel_spectrograms and take the first 13.
return tf.contrib.signal.mfccs_from_log_mel_spectrograms(log_mel_spectrograms)[..., :13]
I then reshape the output of that to [batch_size, 125, 128, 1]. If I send that to a tf.layers.conv2d, things seem to work fine. However, if I try to tf.summary.image, I get the following error:
print(spec)
// => Tensor("spectrogram/Reshape:0", shape=(?, 125, 128, 1), dtype=float32)
tf.summary.image('spec', spec)
Caused by op u'spectrogram/stft/rfft', defined at:
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py", line 162, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/Users/rsilveira/rnd/ml-engine/trainer/flatv1.py", line 103, in <module>
runner.run(model_fn)
File "trainer/runner.py", line 88, in run
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
File "/Library/Python/2.7/site-packages/tensorflow/python/estimator/training.py", line 432, in train_and_evaluate
executor.run_local()
File "/Library/Python/2.7/site-packages/tensorflow/python/estimator/training.py", line 611, in run_local
hooks=train_hooks)
File "/Library/Python/2.7/site-packages/tensorflow/python/estimator/estimator.py", line 302, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/Library/Python/2.7/site-packages/tensorflow/python/estimator/estimator.py", line 711, in _train_model
features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
File "/Library/Python/2.7/site-packages/tensorflow/python/estimator/estimator.py", line 694, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/Users/rsilveira/rnd/ml-engine/trainer/flatv1.py", line 53, in model_fn
spec = gen_spectrogram(x)
File "/Users/rsilveira/rnd/ml-engine/trainer/flatv1.py", line 22, in gen_spectrogram
step,
File "/Library/Python/2.7/site-packages/tensorflow/contrib/signal/python/ops/spectral_ops.py", line 91, in stft
return spectral_ops.rfft(framed_signals, [fft_length])
File "/Library/Python/2.7/site-packages/tensorflow/python/ops/spectral_ops.py", line 136, in _rfft
return fft_fn(input_tensor, fft_length, name)
File "/Library/Python/2.7/site-packages/tensorflow/python/ops/gen_spectral_ops.py", line 619, in rfft
"RFFT", input=input, fft_length=fft_length, name=name)
File "/Library/Python/2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/Library/Python/2.7/site-packages/tensorflow/python/framework/ops.py", line 2956, in create_op
op_def=op_def)
File "/Library/Python/2.7/site-packages/tensorflow/python/framework/ops.py", line 1470, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): Input dimension 4 must have length of at least 512 but got: 320
Not sure where to start troubleshooting this. What am I missing here?
Trying to incorporate tensorflow staging area along with dataset api.
compute_stage_put_op = compute_stage.put(iterator.get_next())
if compute_stage_put_op.type == 'Stage':
compute_stage_ops.append(compute_stage_put_op)
Getting the below error after completing a few 100 steps.
ValueError: Fetch argument <tf.Operation 'group_deps' type=NoOp>
cannot be interpreted as a Tensor. (Operation name:
"group_deps" op: "NoOp")
Stack Trace:
Traceback (most recent call last):
File "timit_trainer.py", line 5, in
timit_trainer.train()
File "/mnt/sdc/nlp/workspace/hci/nlp/mapc/core/model/model.py", line 43, in train
hparams=self.hyper_params # HParams
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/learn_runner.py", line 218, in run
return _execute_schedule(experiment, schedule)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/learn_runner.py", line 46, in _execute_schedule
return task()
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 625, in train_and_evaluate
self.train(delay_secs=0)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 367, in train
hooks=self._train_monitors + extra_hooks)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 807, in _call_train
hooks=hooks)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 302, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 783, in _train_model
_, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 521, in run
run_metadata=run_metadata)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 892, in run
run_metadata=run_metadata)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 967, in run
raise six.reraise(*original_exc_info)
File "/usr/local/lib/python3.5/dist-packages/six.py", line 693, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 952, in run
return self._sess.run(*args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1032, in run
run_metadata=run_metadata))
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 1196, in after_run
induce_stop = m.step_end(self._last_step, result)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 356, in step_end
return self.every_n_step_end(step, output)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 694, in every_n_step_end
validation_outputs = self._evaluate_estimator()
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 665, in _evaluate_estimator
name=self.name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 355, in evaluate
name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 839, in _evaluate_model
config=self._session_config)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/evaluation.py", line 206, in _evaluate_once
session.run(eval_ops, feed_dict)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 521, in run
run_metadata=run_metadata)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 892, in run
run_metadata=run_metadata)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 967, in run
raise six.reraise(*original_exc_info)
File "/usr/local/lib/python3.5/dist-packages/six.py", line 693, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 952, in run
return self._sess.run(*args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 1024, in run
run_metadata=run_metadata)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/monitored_session.py", line 827, in run
return self._sess.run(*args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 889, in run
run_metadata_ptr)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 1105, in _run
self._graph, fetches, feed_dict_tensor, feed_handles=feed_handles)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 414, in init
self._fetch_mapper = _FetchMapper.for_fetch(fetches)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 236, in for_fetch
return _DictFetchMapper(fetch)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 374, in init
for fetch in fetches.values()]
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 374, in
for fetch in fetches.values()]
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 234, in for_fetch
return _ListFetchMapper(fetch)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 341, in init
self._mappers = [_FetchMapper.for_fetch(fetch) for fetch in fetches]
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 341, in
self._mappers = [_FetchMapper.for_fetch(fetch) for fetch in fetches]
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 234, in for_fetch
return _ListFetchMapper(fetch)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 341, in init
self._mappers = [_FetchMapper.for_fetch(fetch) for fetch in fetches]
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 341, in
self._mappers = [_FetchMapper.for_fetch(fetch) for fetch in fetches]
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 242, in for_fetch
return _ElementFetchMapper(fetches, contraction_fn)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py", line 278, in init
'Tensor. (%s)' % (fetch, str(e)))
ValueError: Fetch argument cannot be interpreted as a Tensor. (Operation name: "StagingArea_put"
Code:
def read(self, category: DatasetCategory, devices: list, proc_device: str, shuffle=False):
batch_size = ds.BATCH_SIZE
record_store_exists, record_store = self.__get_store_info(store_path=fu.join_path(self.store_dir, self.store_name + '_' + category.name + '.tfrecord'), create_new=False)
logger.info('Reading records. category: {}, store_exists:{}, store;{}'.format(category.name, str(record_store_exists), record_store))
if not record_store_exists:
self.__process()
record_store_exists, record_store = self.__get_store_info(store_path=fu.join_path(self.store_dir, self.store_name + '_' + category.name + '.tfrecord'), create_new=False)
iterator_init_hook = SessionRunHook()
map_fn = self.__parse_function
gpu_copy_stage_ops = []
gpu_compute_stage_ops = []
def input_fn():
file_names = tf.placeholder(dtype=tf.string, shape=[None], name='data_store')
dataset = tf.data.TFRecordDataset(filenames=file_names, buffer_size=2000000000) # 2.0GB
dataset = dataset.map(map_func=map_fn, num_parallel_calls=tf.constant(value=20000, dtype=tf.int32))
if shuffle:
dataset = dataset.shuffle(buffer_size=tf.constant(value=1000 * batch_size, dtype=tf.int64))
dataset = dataset.repeat(None) # Infinite iterations
dataset = dataset.padded_batch(batch_size=tf.constant(value=batch_size, dtype=tf.int64), padded_shapes=([None, ds.NUM_INPUT_FEATURES], [], [None], []))
iterator = dataset.make_initializable_iterator()
iterator_init_hook.run_func = lambda session: session.run(iterator.initializer, feed_dict={file_names: [record_store]})
gpu_copy_stage = StagingArea(dtypes=[tf.float32, tf.int32, tf.int32, tf.int32],
shapes=[[batch_size, None, ds.NUM_INPUT_FEATURES], [batch_size], [batch_size, None], [batch_size]])
gpu_compute_stage = StagingArea(dtypes=[tf.float32, tf.int32, tf.int32, tf.int32],
shapes=[[batch_size, None, ds.NUM_INPUT_FEATURES], [batch_size], [batch_size, None], [batch_size]])
features_dict = {}
labels_dict = {}
for index, device in enumerate(devices):
with tf.device(proc_device):
gpu_copy_stage_ops.append(gpu_copy_stage.put(iterator.get_next()))
with tf.device(device):
gpu_compute_stage_ops.append(gpu_compute_stage.put(gpu_copy_stage.get()))
source, source_len, target, target_len = gpu_compute_stage.get()
if ds.USE_WARP_CTC:
targets = []
for bi in range(batch_size):
targets.append(target[bi])
target = tf.concat(targets, axis=0)
features_dict[device] = {'source': source, 'source_len': source_len}
labels_dict[device] = {'target': target, 'target_len': target_len}
return features_dict, labels_dict
copy_stage_hook = StepOpsRunHook(ops=[gpu_copy_stage_ops], every_n_secs=1)
compute_stage_hook = StepOpsRunHook(ops=[gpu_compute_stage_ops], every_n_steps=1)
return input_fn, [iterator_init_hook, copy_stage_hook, compute_stage_hook]
I got an error when I changed new images to train the im2txt model. Don't know why.
Build the model.
bazel build -c opt im2txt/...
bazel-bin/im2txt/train
--input_file_pattern="${MY_DATA_DIR}/train-?????-of-00256"
--inception_checkpoint_file="${INCEPTION_CHECKPOINT}"
--train_dir="${MODEL_DIR}/train"
--train_inception=false
--number_of_steps=10000
It went to error when running below sentence
sequence_length = tf.reduce_sum(self.input_mask, 1)
lstm_outputs, _ = tf.nn.dynamic_rnn(cell=lstm_cell,
inputs=self.seq_embeddings,
sequence_length=sequence_length,
initial_state=initial_state,
dtype=tf.float32,
scope=lstm_scope)
The detail info is below
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:global step 1: loss = 9.5415 (37.21 sec/step)
INFO:tensorflow:global step 2: loss = 6.6332 (12.90 sec/step)
INFO:tensorflow:global step 3: loss = 3.1327 (13.01 sec/step)
INFO:tensorflow:global step 4: loss = 6.2893 (12.04 sec/step)
INFO:tensorflow:Error reported to Coordinator: <class 'tensorflow.python.framework.errors_impl.UnimplementedError'>, TensorArray has size zero, but element shape is not fully defined. Currently only static shapes are supported when packing zero-size TensorArrays.
[[Node: OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGatherV3 = TensorArrayGatherV3[_class=["loc:#lstm/lstm/TensorArray_1"], dtype=DT_FLOAT, element_shape=, _device="/job:localhost/replica:0/task:0/cpu:0"](OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/TensorArrayGradV3, lstm/lstm/TensorArrayUnstack/range, OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/gradient_flow)]]
Caused by op u'OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGatherV3', defined at:
File "/data/projects/content_creator/image2text/im2txt/bazel-bin/im2txt/train.runfiles/im2txt/im2txt/train.py", line 155, in
tf.app.run()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 44, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "/data/projects/content_creator/image2text/im2txt/bazel-bin/im2txt/train.runfiles/im2txt/im2txt/train.py", line 135, in main
learning_rate_decay_fn=learning_rate_decay_fn)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/layers/python/layers/optimizers.py", line 226, in optimize_loss
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/optimizer.py", line 345, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gradients_impl.py", line 482, in gradients
in_grads = grad_fn(op, *out_grads)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/tensor_array_grad.py", line 186, in _TensorArrayScatterGrad
grad = g.gather(indices)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/tensor_array_ops.py", line 328, in gather
element_shape=element_shape)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_data_flow_ops.py", line 2226, in _tensor_array_gather_v3
element_shape=element_shape, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 2327, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1226, in init
self._traceback = _extract_stack()
...which was originally created as op u'lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3', defined at:
File "/data/projects/content_creator/image2text/im2txt/bazel-bin/im2txt/train.runfiles/im2txt/im2txt/train.py", line 155, in
tf.app.run()
[elided 0 identical lines from previous traceback]
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 44, in run
_sys.exit(main(sys.argv[:1] + flags_passthrough))
File "/data/projects/content_creator/image2text/im2txt/bazel-bin/im2txt/train.runfiles/im2txt/im2txt/train.py", line 89, in main
model.build()
File "/data/projects/content_creator/image2text/im2txt/im2txt/show_and_tell_model.py", line 437, in build
self.build_model()
File "/data/projects/content_creator/image2text/im2txt/im2txt/show_and_tell_model.py", line 356, in build_model
scope=lstm_scope)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 546, in dynamic_rnn
dtype=dtype)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 664, in dynamic_rnn_loop
for ta, input in zip(input_ta, flat_input))
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 664, in
for ta, input in zip(input_ta, flat_input))
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/tensor_array_ops.py", line 380, in unstack
indices=math_ops.range(0, num_elements), value=value, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/tensor_array_ops.py", line 408, in scatter
name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_data_flow_ops.py", line 2492, in _tensor_array_scatter_v3
name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
op_def=op_def)
UnimplementedError (see above for traceback): TensorArray has size zero, but element shape is not fully defined. Currently only static shapes are supported when packing zero-size TensorArrays.
[[Node: OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGatherV3 = TensorArrayGatherV3[_class=["loc:#lstm/lstm/TensorArray_1"], dtype=DT_FLOAT, element_shape=, _device="/job:localhost/replica:0/task:0/cpu:0"](OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/TensorArrayGradV3, lstm/lstm/TensorArrayUnstack/range, OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/gradient_flow)]]
Traceback (most recent call last):
File "/data/projects/content_creator/image2text/im2txt/bazel-bin/im2txt/train.runfiles/im2txt/im2txt/train.py", line 155, in
tf.app.run()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 44, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "/data/projects/content_creator/image2text/im2txt/bazel-bin/im2txt/train.runfiles/im2txt/im2txt/train.py", line 152, in main
saver=saver)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/slim/python/slim/learning.py", line 793, in train
train_step_kwargs)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/slim/python/slim/learning.py", line 530, in train_step
run_metadata=run_metadata)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 767, in run
run_metadata_ptr)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 965, in _run
feed_dict_string, options, run_metadata)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 1015, in _do_run
target_list, options, run_metadata)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 1035, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.UnimplementedError: TensorArray has size zero, but element shape is not fully defined. Currently only static shapes are supported when packing zero-size TensorArrays.
[[Node: OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGatherV3 = TensorArrayGatherV3[_class=["loc:#lstm/lstm/TensorArray_1"], dtype=DT_FLOAT, element_shape=, _device="/job:localhost/replica:0/task:0/cpu:0"](OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/TensorArrayGradV3, lstm/lstm/TensorArrayUnstack/range, OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/gradient_flow)]]
Caused by op u'OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGatherV3', defined at:
File "/data/projects/content_creator/image2text/im2txt/bazel-bin/im2txt/train.runfiles/im2txt/im2txt/train.py", line 155, in
tf.app.run()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 44, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "/data/projects/content_creator/image2text/im2txt/bazel-bin/im2txt/train.runfiles/im2txt/im2txt/train.py", line 135, in main
learning_rate_decay_fn=learning_rate_decay_fn)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/layers/python/layers/optimizers.py", line 226, in optimize_loss
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/optimizer.py", line 345, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gradients_impl.py", line 482, in gradients
in_grads = grad_fn(op, *out_grads)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/tensor_array_grad.py", line 186, in _TensorArrayScatterGrad
grad = g.gather(indices)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/tensor_array_ops.py", line 328, in gather
element_shape=element_shape)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_data_flow_ops.py", line 2226, in _tensor_array_gather_v3
element_shape=element_shape, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 2327, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1226, in init
self._traceback = _extract_stack()
...which was originally created as op u'lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3', defined at:
File "/data/projects/content_creator/image2text/im2txt/bazel-bin/im2txt/train.runfiles/im2txt/im2txt/train.py", line 155, in
tf.app.run()
[elided 0 identical lines from previous traceback]
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 44, in run
_sys.exit(main(sys.argv[:1] + flags_passthrough))
File "/data/projects/content_creator/image2text/im2txt/bazel-bin/im2txt/train.runfiles/im2txt/im2txt/train.py", line 89, in main
model.build()
File "/data/projects/content_creator/image2text/im2txt/im2txt/show_and_tell_model.py", line 437, in build
self.build_model()
File "/data/projects/content_creator/image2text/im2txt/im2txt/show_and_tell_model.py", line 356, in build_model
scope=lstm_scope)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 546, in dynamic_rnn
dtype=dtype)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 664, in dynamic_rnn_loop
for ta, input in zip(input_ta, flat_input))
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 664, in
for ta, input in zip(input_ta, flat_input))
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/tensor_array_ops.py", line 380, in unstack
indices=math_ops.range(0, num_elements), value=value, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/tensor_array_ops.py", line 408, in scatter
name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_data_flow_ops.py", line 2492, in _tensor_array_scatter_v3
name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
op_def=op_def)
UnimplementedError (see above for traceback): TensorArray has size zero, but element shape is not fully defined. Currently only static shapes are supported when packing zero-size TensorArrays.
[[Node: OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGatherV3 = TensorArrayGatherV3[_class=["loc:#lstm/lstm/TensorArray_1"], dtype=DT_FLOAT, element_shape=, _device="/job:localhost/replica:0/task:0/cpu:0"](OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/TensorArrayGradV3, lstm/lstm/TensorArrayUnstack/range, OptimizeLoss/gradients/lstm/lstm/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/gradient_flow)]]
I'm getting the above error when attempting to call tf.nn.bidirectional_dynamic_rnn(). I've called tf.global_variables_initializer(). At first I thought it's because I didn't pass in sequence_length to tf.nn.bidirectional_dynamic_rnn(). However, even after I did, it's still shows the same error.
Any idea?
Stacktrace:
Traceback (most recent call last):
File "/Applications/PyCharm CE.app/Contents/helpers/pydev/pydevd.py", line 1580, in <module>
globals = debugger.run(setup['file'], None, None, is_module)
File "/Applications/PyCharm CE.app/Contents/helpers/pydev/pydevd.py", line 964, in run
pydev_imports.execfile(file, globals, locals) # execute the script
File "/Users/Keven/Documents/stanford local/cs224n project/224n-project/bi_lstm_encoder.py", line 49, in <module>
test_bilstm()
File "/Users/Keven/Documents/stanford local/cs224n project/224n-project/bi_lstm_encoder.py", line 43, in test_bilstm
out = session.run(pred, feed_dict={input_placeholder: doc, sequence_placeholder: sequence_length})
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 766, in run
run_metadata_ptr)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 964, in _run
feed_dict_string, options, run_metadata)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1014, in _do_run
target_list, options, run_metadata)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1034, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value BiRNN/FW/LSTMCell/B
[[Node: BiRNN/FW/LSTMCell/B/read = Identity[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](BiRNN/FW/LSTMCell/B)]]
Caused by op u'BiRNN/FW/LSTMCell/B/read', defined at:
File "/Applications/PyCharm CE.app/Contents/helpers/pydev/pydevd.py", line 1580, in <module>
globals = debugger.run(setup['file'], None, None, is_module)
File "/Applications/PyCharm CE.app/Contents/helpers/pydev/pydevd.py", line 964, in run
pydev_imports.execfile(file, globals, locals) # execute the script
File "/Users/Keven/Documents/stanford local/cs224n project/224n-project/bi_lstm_encoder.py", line 49, in <module>
test_bilstm()
File "/Users/Keven/Documents/stanford local/cs224n project/224n-project/bi_lstm_encoder.py", line 42, in test_bilstm
pred = BidirectionalLSTMEncoder().add_prediction_op(input_placeholder, sequence_placeholder, 6)
File "/Users/Keven/Documents/stanford local/cs224n project/224n-project/bi_lstm_encoder.py", line 20, in add_prediction_op
preds, _ = tf.nn.bidirectional_dynamic_rnn(cell_forward, cell_backward, inputs, sequence_length=sequence_length, dtype=tf.float32)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py", line 652, in bidirectional_dynamic_rnn
time_major=time_major, scope=fw_scope)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py", line 845, in dynamic_rnn
dtype=dtype)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py", line 1012, in _dynamic_rnn_loop
swap_memory=swap_memory)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2636, in while_loop
result = context.BuildLoop(cond, body, loop_vars, shape_invariants)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2469, in BuildLoop
pred, body, original_loop_vars, loop_vars, shape_invariants)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2419, in _BuildLoop
body_result = body(*packed_vars_for_body)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py", line 995, in _time_step
skip_conditionals=True)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py", line 403, in _rnn_step
new_output, new_state = call_cell()
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/rnn.py", line 983, in <lambda>
call_cell = lambda: cell(input_t, state)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/rnn_cell.py", line 500, in __call__
initializer=init_ops.zeros_initializer, dtype=dtype)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 1024, in get_variable
custom_getter=custom_getter)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 850, in get_variable
custom_getter=custom_getter)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 346, in get_variable
validate_shape=validate_shape)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 331, in _true_getter
caching_device=caching_device, validate_shape=validate_shape)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 677, in _get_single_variable
expected_shape=shape)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 224, in __init__
expected_shape=expected_shape)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 367, in _init_from_args
self._snapshot = array_ops.identity(self._variable, name="read")
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py", line 1424, in identity
result = _op_def_lib.apply_op("Identity", input=input, name=name)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 759, in apply_op
op_def=op_def)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2240, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1128, in __init__
self._traceback = _extract_stack()
FailedPreconditionError (see above for traceback): Attempting to use uninitialized value BiRNN/FW/LSTMCell/B
[[Node: BiRNN/FW/LSTMCell/B/read = Identity[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](BiRNN/FW/LSTMCell/B)]]
Code:
import tensorflow as tf
import numpy as np
from SubModel import SubModel
# input:
# shape=(?, max_timestep_doc2, 3 * word_vector_size)
#
# output:
# shape=(?, max_timestep_doc2, 2 * word_vector_size)
class BidirectionalLSTMEncoder(SubModel):
def add_prediction_op(self, inputs, output_size=None):
sequence_length = [5, 5]
cell_forward = tf.nn.rnn_cell.LSTMCell(output_size, num_proj=output_size)
cell_backward = tf.nn.rnn_cell.LSTMCell(output_size, num_proj=output_size)
preds, _ = tf.nn.bidirectional_dynamic_rnn(cell_forward, cell_backward, inputs, sequence_length=sequence_length, dtype=tf.float32)
return preds
def __init__(self):
pass
def test_bilstm():
print('testing bidirectional lstm layer')
with tf.variable_scope("test_bilstm_layer"):
input_placeholder = tf.placeholder(tf.float32, shape=(None, 5, 9))
sequence_placeholder = tf.placeholder(tf.int32, shape=(None,))
init = tf.global_variables_initializer()
with tf.Session() as session:
session.run(init)
doc = np.ones(shape=(2, 5, 9), dtype=np.float32) * 0.5
pred = BidirectionalLSTMEncoder().add_prediction_op(input_placeholder, 6)
out = session.run(pred, feed_dict={input_placeholder: doc})
print("out = " + str(out))
# assert np.allclose(CD_correct, out, atol=1e-2), "new state vector does not seem to be correct."
if __name__ == "__main__":
test_bilstm()
Never mind. It turns out I didn't initialize the variables properly..
I moved this line:
pred = BidirectionalLSTMEncoder().add_prediction_op(input_placeholder, 6)
above this line:
with tf.Session() as session:
Then everything worked fine. The functions containing variable initialization need to be called before tf.global_variables_initializer()
I am trying to reverse my inputs with array_ops.reverse_sequence() before sending it to dynamic_rnn(), the inference graph can be build with no problem, but when building the training graph, I got the following error:
Traceback (most recent call last):
File "bin/trainer.py", line 158, in <module>
kmer_len=args.kmer_len)
File "/home/ubuntu/GIT/IvyMike/ivymike/base_model.py", line 193, in run_training
train_op = model.training(loss, learning_rate)
File "/home/ubuntu/GIT/IvyMike/ivymike/base_model.py", line 100, in training
train_op = optimizer.minimize(loss, global_step=global_step)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/optimizer.py", line 190, in minimize
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/optimizer.py", line 241, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gradients.py", line 481, in gradients
in_grads = _AsList(grad_fn(op, *out_grads))
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/array_grad.py", line 307, in _ReverseSequenceGrad
seq_lengths=seq_lengths),
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_array_ops.py", line 1143, in reverse_sequence
batch_dim=batch_dim, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/op_def_library.py", line 655, in apply_op
op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 2119, in create_op
set_shapes_for_outputs(ret)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1586, in set_shapes_for_outputs
shapes = shape_func(op)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/array_ops.py", line 1257, in _ReverseSequenceShape
(batch_dim, input_shape.ndims))
TypeError: %d format: a number is required, not NoneType
Any idea what went wrong?
This has been fixed at the mater in TensorFlow