I'm trying to use dynamic_decode in Tensorflow for an attention model. The original version is provided by https://github.com/tensorflow/nmt#decoder
learning_rate = 0.001
n_hidden = 128
total_epoch = 10000
num_units=128
n_class = n_input = 47
num_steps=8
embedding_size=30
mode = tf.placeholder(tf.bool)
embed_enc = tf.placeholder(tf.float32, shape=[None,num_steps,300])
embed_dec = tf.placeholder(tf.float32, shape=[None,num_steps,300])
targets=tf.placeholder(tf.int32, shape=[None,num_steps])
enc_seqlen = tf.placeholder(tf.int32, shape=[None])
dec_seqlen = tf.placeholder(tf.int32, shape=[None])
decoder_weights= tf.placeholder(tf.float32, shape=[None, num_steps])
with tf.variable_scope('encode'):
enc_cell = tf.contrib.rnn.BasicRNNCell(n_hidden)
enc_cell = tf.contrib.rnn.DropoutWrapper(enc_cell, output_keep_prob=0.5)
outputs, enc_states = tf.nn.dynamic_rnn(enc_cell, embed_enc,sequence_length=enc_seqlen,
dtype=tf.float32,time_major=True )
attention_states = tf.transpose(outputs, [1, 0, 2])
# Create an attention mechanism
attention_mechanism = tf.contrib.seq2seq.LuongAttention(
num_units, attention_states,
memory_sequence_length=enc_seqlen)
decoder_cell = tf.contrib.rnn.BasicLSTMCell(num_units)
decoder_cell = tf.contrib.seq2seq.AttentionWrapper(
decoder_cell, attention_mechanism,
attention_layer_size=num_units)
helper = tf.contrib.seq2seq.TrainingHelper(
embed_dec, dec_seqlen, time_major=True)
# Decoder
projection_layer = Dense(
47, use_bias=False)
decoder = tf.contrib.seq2seq.BasicDecoder(
decoder_cell, helper, enc_states,
output_layer=projection_layer)
# Dynamic decoding
outputs, _ = tf.contrib.seq2seq.dynamic_decode(decoder)
But i got an error when i ran
tf.contrib.seq2seq.dynamic_decode(decoder)
and error shows as below
Traceback (most recent call last):
File "<ipython-input-19-0708495dbbfb>", line 27, in <module>
outputs, _ = tf.contrib.seq2seq.dynamic_decode(decoder)
File "D:\Anaconda3\lib\site-packages\tensorflow\contrib\seq2seq\python\ops\decoder.py", line 286, in dynamic_decode
swap_memory=swap_memory)
File "D:\Anaconda3\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2775, in while_loop
result = context.BuildLoop(cond, body, loop_vars, shape_invariants)
File "D:\Anaconda3\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2604, in BuildLoop
pred, body, original_loop_vars, loop_vars, shape_invariants)
File "D:\Anaconda3\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2554, in _BuildLoop
body_result = body(*packed_vars_for_body)
File "D:\Anaconda3\lib\site-packages\tensorflow\contrib\seq2seq\python\ops\decoder.py", line 234, in body
decoder_finished) = decoder.step(time, inputs, state)
File "D:\Anaconda3\lib\site-packages\tensorflow\contrib\seq2seq\python\ops\basic_decoder.py", line 139, in step
cell_outputs, cell_state = self._cell(inputs, state)
File "D:\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 180, in __call__
return super(RNNCell, self).__call__(inputs, state)
File "D:\Anaconda3\lib\site-packages\tensorflow\python\layers\base.py", line 450, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "D:\Anaconda3\lib\site-packages\tensorflow\contrib\seq2seq\python\ops\attention_wrapper.py", line 1143, in call
cell_inputs = self._cell_input_fn(inputs, state.attention)
AttributeError: 'Tensor' object has no attribute 'attention'
I tried installed the latest tensorflow 1.2.1 but it didn't work.
Thank you for your help.
UPDATE:
The problem is if I change initial_states of BasicDecoder:
decoder = tf.contrib.seq2seq.BasicDecoder(
decoder_cell, helper, enc_states,
output_layer=projection_layer)
into:
decoder = tf.contrib.seq2seq.BasicDecoder(
decoder_cell, helper,
decoder_cell.zero_state(dtype=tf.float32,batch_size=batch_size),
output_layer=projection_layer)
Then it works. I have no idea if it is a correct solution because initial_states set to zero seems wired.
Thank you for your help.
Your approach is correct. i added better error messaging in tf master branch for future users. since you're using attention, you probably don't really need to pass anything through to the decoder initial state. however it's still common to feed the encoder final state in. you can do this by creating a decoder cell zero state the way you're doing, and calling its .clone method with arg cell_state=encoder_final_state. use the resulting object as the initial decoder state.
Can you write all your calls to use kwargs everywhere? i.e., tf.nn.dynamic_rnn(cell=..., inputs=...) etc. i think your args are misplaced somewhere and using kwargs should resolve it.
Related
So, I have done some modifications with the VGG16 neural network to make a linear-regression and classification model.
And at last I am writing the following code to compile and fit my data into the model to initiate training->
class Facetracker(Model):
# The initialization function-
def __init__(self,eyetracker,**kwargs):
super().__init__(**kwargs)
self.model = eyetracker #Instantiating the model
def compile(self,opt,classlosss,localization_loss,**kwargs):
super().compile(**kwargs)
self.classloss = class_loss
self.localization_loss = regress_loss
self.opt = optimizer
# Defining the training step
def train_step(self,batch,**kwargs):
X,y = batch #unpacking our data
with tf.GradientTape() as tape:
classes,coords = self.model(X,training=True)
batch_classloss = self.classloss(y[0],classes)
batch_localloss = self.localization_loss(tf.cast(y[1],tf.float32),coords)
# calculating total loss-
total_loss = batch_localloss+0.5*batch_classloss
grad = tape.gradient(total_loss,self.model.trainable_variables)
optimizer.apply_gradients(zip(grad,self.model.trainable_variables))
return{
"total_loss":total_loss,
"class_loss":batch_classloss,
"localilzation_loss":batch_localloss
}
def test_step(self,batch):
X,y = batch
classes,coords = self.model(X,training=False)
batch_classloss = self.classloss(y[0],classes)
batch_localloss = self.localization_loss(tf.cast(y[1],tf.float32),coords)
total_loss = batch_localloss+0.5*batch_classloss
return{
"total_loss": total_loss,
"class_loss": batch_classloss,
"localilzation_loss": batch_localloss
}
# def call(self, X, **kwargs):
# return self.model(X,**kwargs)
# Replacing the call function with a lambda function
lambda self,X,**kwargs: self.model(X,**kwargs)
# Subclassing our model-
print("Subclassing.....")
model = Facetracker(facetracker)
print("Compiling......")
model.compile(optimizer,classlosss=class_loss,localization_loss=localization_loss)
# Preparing the log directory
logdir="logdir"
tensorboard_callbacks = tf.keras.callbacks.TensorBoard(log_dir=logdir)
print("Fitting the model")
hist = model.fit(train.take(80),
epochs=16,
initial_epoch =8,
validation_data=val,
validation_steps =8,
validation_freq=2,
callbacks = [[tensorboard_callbacks]])
This includes the class prepared for training the model and the last few lines are subclassing the model and fitting the prepared data into the model.
The error I now get seems pretty hefty to me and it goes like this->
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\main.py", line 535, in <module>
hist = model.fit(train.take(80),
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\utils\traceback_utils.py", line 67, in error_handler
raise e.with_traceback(filtered_tb) from None
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\tensorflow\python\eager\execute.py", line 54, in quick_execute
tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
tensorflow.python.framework.errors_impl.InvalidArgumentError: Graph execution error:
Detected at node 'gradient_tape/sub_2/BroadcastGradientArgs' defined at (most recent call last):
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\main.py", line 535, in <module>
hist = model.fit(train.take(80),
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\engine\training.py", line 1409, in fit
tmp_logs = self.train_function(iterator)
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\engine\training.py", line 1051, in train_function
return step_function(self, iterator)
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\engine\training.py", line 1040, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\engine\training.py", line 1030, in run_step
outputs = model.train_step(data)
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\main.py", line 498, in train_step
grad = tape.gradient(total_loss,self.model.trainable_variables)
Node: 'gradient_tape/sub_2/BroadcastGradientArgs'
Incompatible shapes: [2,4] vs. [8]
[[{{node gradient_tape/sub_2/BroadcastGradientArgs}}]] [Op:__inference_train_function_22026]
It would be a great help if someone could examine this problem and help me get out of it. I have been quite struggling with it now.
Thanks in advance!!!
I has planned to initiate training with the code provided, but I firstly received an error for the call function and as I resolved it by converting it into lambda function- This issue of incompatible shapes came up. I tries to adjust the integers in the input data and batch related fields but nothing worked.
I want to build a predictor from a an tf.estimator.Estimator model. Therefore I need to specify a input_receiver_fn that specifies the preprocessing graph from the receiver tensors to the features that will be passed to the model_fn by the predictor.
Here is an example for an eval_input_fn for the Estimator:
def eval_input_fn(params):
ds = tf.data.Dataset.from_generator(
generator=Eval_Generator(params),
output_types=(tf.uint16,tf.uint16),
output_shapes = ([3]+params['crop_size'],[2]+params['crop_size']))
augmentations = [Convert,Downsample,Clip]
ds = ds.repeat()
for augmentation in augmentations:
ds = ds.map(augmentation, num_parallel_calls=params['threads'])
ds = ds.batch(1).prefetch(None)
return ds
I changed the augmentation functions from taking in two arguments (features: tf.Tensor, labels: tf.Tensor) to taking only one argument (features: tf.Tensor) and wrote the according input_receiver_fn that looks like this:
def serving_input_receiver_fn():
rec_raw = tf.placeholder(tf.float32, [3, 256, 256, 256],name='raw')
raw = Convert(rec_raw)
raw = Downsample(raw)
raw = Clip(raw)
raw = tf.expand_dims(raw,0)
return tf.estimator.export.TensorServingInputReceiver(features=raw,receiver_tensors=rec_raw)
The function returns the following object:
TensorServingInputReceiver(features=<tf.Tensor 'ExpandDims_1:0' shape=(1, 3, 128, 128, 128) dtype=float32>, receiver_tensors={'input': <tf.Tensor 'raw:0' shape=(3, 256, 256, 256) dtype=float32>}, receiver_tensors_alternatives=None)
which seems pretty right. But when it try to instantiate the predictor by:
config = tf.estimator.RunConfig(model_dir = params['model_dir'])
estimator = tf.estimator.Estimator(model_fn=model_fn, params=params,config=config)
predict_fn = tf.contrib.predictor.from_estimator(estimator, serving_input_receiver_fn)
I'll get the following error message:
INFO:tensorflow:Calling model_fn.
Traceback (most recent call last):
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 510, in _apply_op_helper
preferred_dtype=default_dtype)
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1146, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py", line 229, in _constant_tensor_conversion_function
return constant(v, dtype=dtype, name=name)
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py", line 208, in constant
value, dtype=dtype, shape=shape, verify_shape=verify_shape))
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/python/framework/tensor_util.py", line 430, in make_tensor_proto
raise ValueError("None values not supported.")
ValueError: None values not supported.
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/contrib/predictor/predictor_factories.py", line 105, in from_estimator
config=config)
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/contrib/predictor/core_estimator_predictor.py", line 72, in __init__
serving_input_receiver, estimator, output_key)
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/contrib/predictor/core_estimator_predictor.py", line 37, in _get_signature_def
estimator.config)
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 235, in public_model_fn
return self._call_model_fn(features, labels, mode, config)
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1195, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/fast/AG_Kainmueller/jrumber/flylight_01/train_tf.py", line 227, in model_fn
gt,fg = tf.unstack(labels,num=2,axis=1)
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 1027, in unstack
return gen_array_ops.unpack(value, num=num, axis=axis, name=name)
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py", line 9429, in unpack
"Unpack", value=value, num=num, axis=axis, name=name)
File "/home/jrumber/anaconda3/envs/tf1.12_gpuenv/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 528, in _apply_op_helper
(input_name, err))
ValueError: Tried to convert 'value' to a tensor and failed. Error: None values not supported.
Since it could be a problem with my model_fn, I'll post it too:
def model_fn(features,labels,mode,params):
gt,fg = tf.unstack(labels,num=2,axis=1)
gt.set_shape([1]+params['input_size'])
fg.set_shape([1]+params['input_size'])
features.set_shape([1,3]+params['input_size'])
# first layer to set input_shape
features = tf.keras.layers.Conv3D(
input_shape = tuple([3]+params['input_size']),
data_format = 'channels_first',
filters = params['chan'],
kernel_size = [3,3,3],
strides=(1, 1, 1),
padding='same',
activation='relu',
kernel_regularizer=tf.keras.regularizers.l2(l=0.01))(features)
# U-Net
out = unet(features, params['unet_initial_filters'], params['width_factor'], params['architecture'])
# Embedding conv pass
output_batched = conv_pass(
out,
kernel_size=1,
num_fmaps=params['chan'],
num_repetitions=1,
activation=None,
name='conv_embedding')
output = tf.squeeze(output_batched)
# Fg/Bg segmentation conv pass
mask_batched = conv_pass(
out,
kernel_size=1,
num_fmaps=1,
num_repetitions=1,
activation='sigmoid',
name='conv_mask')
prob_mask = tf.squeeze(mask_batched)
logits_mask = logit(prob_mask)
# store predictions in dict
predictions = {
'prob_mask': tf.expand_dims(prob_mask,0),
'embedding': output,
'gt': tf.squeeze(gt,0)}
# RAIN mode
if mode == tf.contrib.learn.ModeKeys.TRAIN:
loss , l_var, l_dist, l_reg = discriminative_loss_single(prediction=output,
correct_label=tf.squeeze(gt),
feature_dim=params['chan'],
delta_v= params['delta_v'],
delta_d= params['delta_d'],
param_var= params['param_var'],
param_dist= params['param_dist'],
param_reg= params['param_reg']
)
mask_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.squeeze(fg),
logits=logits_mask))
reg_loss = tf.losses.get_regularization_loss() * 1e-6
loss += mask_loss + reg_loss
opt = tf.train.AdamOptimizer(
learning_rate=0.5e-4,
beta1=0.95,
beta2=0.999,
epsilon=1e-8)
optimizer = opt.minimize(loss, global_step=tf.train.get_global_step())
global_step = tf.Variable(1, name='global_step', trainable=False, dtype=tf.int32)
increment_global_step_op = tf.assign(global_step, global_step+1)
logging_hook = tf.train.LoggingTensorHook({"loss" : loss,'global_step':increment_global_step_op}, every_n_iter=1)
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions, loss=loss, train_op=optimizer, training_hooks=[logging_hook])
# PREDICT mode
if mode == tf.estimator.ModeKeys.PREDICT:
export_outputs = {
'predict_output': tf.estimator.export.PredictOutput(predictions)
}
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions, export_outputs=export_outputs)
# EVAL mode
if mode == tf.estimator.ModeKeys.EVAL:
export_outputs = {
'eval_output': tf.estimator.export.EvalOutput(predictions)
}
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions, export_outputs=export_outputs)
Does anybody spot my mistake here?
Best :)
The error was in the model_fn. The following lines have to be moved down to the # TRAIN mode part of the function
gt,fg = tf.unstack(labels,num=2,axis=1)
gt.set_shape([1]+params['input_size'])
fg.set_shape([1]+params['input_size'])
Estimator.predict will feed only the features and None instead of labels, therefore tf.unstack will throw an exception, so all operations that work on the labels have to be moved to the # train mode part of the model_fn.
I try to export the model for text classification with tf hub modules, and then infer a prediction from it for a single string example using predictor.from_saved_model(). I saw some examples of similar ideas, but still couldn't make it work for the case when using tf hub modules to build features. Here is what I do:
train_input_fn = tf.estimator.inputs.pandas_input_fn(
train_df, train_df['label_ids'], num_epochs= None, shuffle=True)
# Prediction on the whole training set.
predict_train_input_fn = tf.estimator.inputs.pandas_input_fn(
train_df, train_df['label_ids'], shuffle=False)
embedded_text_feature_column = hub.text_embedding_column(
key='sentence',
module_spec='https://tfhub.dev/google/nnlm-de-dim128/1')
#Estimator
estimator = tf.estimator.DNNClassifier(
hidden_units=[500, 100],
feature_columns=[embedded_text_feature_column],
n_classes=num_of_class,
optimizer=tf.train.AdagradOptimizer(learning_rate=0.003) )
# Training
estimator.train(input_fn=train_input_fn, steps=1000)
#prediction on training set
train_eval_result = estimator.evaluate(input_fn=predict_train_input_fn)
print('Training set accuracy: {accuracy}'.format(**train_eval_result))
feature_spec = tf.feature_column.make_parse_example_spec([embedded_text_feature_column])
serving_input_receiver_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec)
export_dir_base = self.cfg['model_path']
servable_model_path = estimator.export_savedmodel(export_dir_base, serving_input_receiver_fn)
# Example message for inference
message = "Was ist denn los"
saved_model_predictor = predictor.from_saved_model(export_dir=servable_model_path)
content_tf_list = tf.train.BytesList(value=[str.encode(message)])
example = tf.train.Example(
features=tf.train.Features(
feature={
'sentence': tf.train.Feature(
bytes_list=content_tf_list
)
}
)
)
with tf.python_io.TFRecordWriter('the_message.tfrecords') as writer:
writer.write(example.SerializeToString())
reader = tf.TFRecordReader()
data_path = 'the_message.tfrecords'
filename_queue = tf.train.string_input_producer([data_path], num_epochs=1)
_, serialized_example = reader.read(filename_queue)
output_dict = saved_model_predictor({'inputs': [serialized_example]})
And the output:
Traceback (most recent call last):
File "/Users/dimitrs/component-pythia/src/pythia.py", line 321, in _train
model = algo.generate_model(samples, generation_id)
File "/Users/dimitrs/component-pythia/src/algorithm_layer/algorithm.py", line 56, in generate_model
model = self._process_training(samples, generation)
File "/Users/dimitrs/component-pythia/src/algorithm_layer/tf_hub_classifier.py", line 91, in _process_training
output_dict = saved_model_predictor({'inputs': [serialized_example]})
File "/Users/dimitrs/anaconda3/envs/pythia/lib/python3.6/site-packages/tensorflow/contrib/predictor/predictor.py", line 77, in __call__
return self._session.run(fetches=self.fetch_tensors, feed_dict=feed_dict)
File "/Users/dimitrs/anaconda3/envs/pythia/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 900, in run
run_metadata_ptr)
File "/Users/dimitrs/anaconda3/envs/pythia/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1135, in _run
feed_dict_tensor, options, run_metadata)
File "/Users/dimitrs/anaconda3/envs/pythia/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1316, in _do_run
run_metadata)
File "/Users/dimitrs/anaconda3/envs/pythia/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1335, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InternalError: Unable to get element as bytes.
Isn't serialized_example the right input that is suggested by serving_input_receiver_fn ?
So, all I need was serialized_example = example.SerializeToString()
Writing the example on a file requires to start a session before reading it back. Simply serialising is enough:
# Example message for inference
message = "Was ist denn los"
saved_model_predictor = predictor.from_saved_model(export_dir=servable_model_path)
content_tf_list = tf.train.BytesList(value=[message.encode('utf-8')])
sentence = tf.train.Feature(bytes_list=content_tf_list)
sentence_dict = {'sentence': sentence}
features = tf.train.Features(feature=sentence_dict)
example = tf.train.Example(features=features)
serialized_example = example.SerializeToString()
output_dict = saved_model_predictor({'inputs': [serialized_example]})
I'm trying to learn how to use the Estimator API, using input_fn to provide Dataset backed input to a feature_column generated input layer.
My code looks like
import tensorflow as tf import random
tf.logging.set_verbosity(tf.logging.DEBUG)
def input_fn():
def gen():
for i in range(100000):
for j in range(10):
yield {"in": str(j)}, [str(j+1)]
data = tf.data.Dataset.from_generator(gen, ({"in": tf.string}, tf.string))
data = data.batch(10)
iterator = data.make_one_shot_iterator()
return iterator.get_next()
vocabulary_feature_column = tf.feature_column.categorical_column_with_vocabulary_list(
key="in",
vocabulary_list=map(lambda i: str(i), range(11)))
embedding_column = tf.feature_column.embedding_column(
categorical_column=vocabulary_feature_column,
dimension=2)
with tf.Session() as sess:
print(sess.run(input_fn()))
classifier = tf.estimator.DNNClassifier(
feature_columns = [embedding_column],
hidden_units = [5,5],
n_classes = 11,
model_dir = '/tmp/predict/snap')
classifier.train(
input_fn=input_fn)
but running it I get
Traceback (most recent call last):
File "predict.py", line 33, in
input_fn=input_fn)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 302, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 711, in _train_model
features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 694, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/canned/dnn.py", line 334, in _model_fn
config=config)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/canned/dnn.py", line 190, in _dnn_model_fn
logits = logit_fn(features=features, mode=mode)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/canned/dnn.py", line 89, in dnn_logit_fn
features=features, feature_columns=feature_columns)
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 230, in input_layer
trainable=trainable)
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 1837, in _get_dense_tensor
inputs, weight_collections=weight_collections, trainable=trainable)
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 2123, in _get_sparse_tensors
return _CategoricalColumn.IdWeightPair(inputs.get(self), None)
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 1533, in get
transformed = column._transform_feature(self) # pylint: disable=protected-access
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 2091, in _transform_feature
input_tensor = _to_sparse_input(inputs.get(self.key))
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 1631, in _to_sparse_input
raise ValueError('Undefined input_tensor shape.')
ValueError: Undefined input_tensor shape.
Looking at the tf sources I get the impression I that the categorical_column_with_vocabulary_list expects a tensor as output instead of a string, but I have a hard time understanding how to make my input_fn provide that the right way.
Does anyone have any idea what I'm doing wrong here?
As a comparison, the following code works perfectly fine: https://pastebin.com/28QUNAjA
EDIT
I noticed that replacing tf.data.Dataset.from_generator with tf.data.Dataset.from_tensor_slices makes the code run.
I.e. the following actually works:
import tensorflow as tf
import random
tf.logging.set_verbosity(tf.logging.DEBUG)
def input_fn():
data = tf.data.Dataset.from_tensor_slices(({"in": map(lambda i: str(i), range(10))}, range(1,11)))
data = data.repeat(1000)
data = data.batch(10)
iterator = data.make_one_shot_iterator()
return iterator.get_next()
vocabulary_feature_column = tf.feature_column.categorical_column_with_vocabulary_list(
key="in",
vocabulary_list=map(lambda i: str(i), range(11)))
embedding_column = tf.feature_column.embedding_column(
categorical_column=vocabulary_feature_column,
dimension=2)
with tf.Session() as sess:
print(sess.run(input_fn()))
classifier = tf.estimator.DNNClassifier(
feature_columns = [embedding_column],
hidden_units = [5,5],
n_classes = 11,
model_dir = '/usr/local/google/home/zond/tmp/predict/snap')
classifier.train(
input_fn=input_fn)
This ought to be a bug, so I created https://github.com/tensorflow/tensorflow/issues/15178.
I have the following code:
def dense_layers(pool3):
with tf.variable_scope('local1') as scope:
# Move everything into depth so we can perform a single matrix multiply.
shape_d = pool3.get_shape()
shape = shape_d[1] * shape_d[2] * shape_d[3]
# tf_shape = tf.stack(shape)
tf_shape = 1024
print("shape:", shape, shape_d[1], shape_d[2], shape_d[3])
# So note that tf_shape = 1024, this means that we have 1024 features are fed into the network. And
# the batch size = 1024. Therefore, the aim is to divide the batch_size into num_steps so that
reshape = tf.reshape(pool3, [-1, tf_shape])
# Now we need to reshape/divide the batch_size into num_steps so that we would be feeding a sequence
# And note that most importantly is to have batch_partition_length followed by step_size in the parameter list.
lstm_inputs = tf.reshape(reshape, [batch_partition_length, step_size, tf_shape])
# print('RNN inputs shape: ', lstm_inputs.get_shape()) # -> (128, 8, 1024).
# Note that the state_size is the number of neurons.
lstm = tf.contrib.rnn.BasicLSTMCell(state_size)
lstm_outputs, final_state = tf.nn.dynamic_rnn(cell=lstm, inputs=lstm_inputs, initial_state=init_state)
tf.assign(init_state, final_state)
So, I am taking the output of the pool layer and try to feed it into the LSTM in the network.
Initially I have declared the following:
state_size = 16
step_size = 8
batch_partition_length = int(batch_size / step_size)
init_state = tf.Variable(tf.zeros([batch_partition_length, state_size])) # -> [128, 16].
Therefore, I am getting an error on:
lstm_outputs, final_state = tf.nn.dynamic_rnn(cell=lstm, inputs=lstm_inputs, initial_state=init_state)
As follows:
Traceback (most recent call last):
File "C:/Users/user/PycharmProjects/AffectiveComputing/Brady_with_LSTM.py", line 197, in <module>
predictions = dense_layers(conv_nets_output)
File "C:/Users/user/PycharmProjects/AffectiveComputing/Brady_with_LSTM.py", line 162, in dense_layers
lstm_outputs, final_state = tf.nn.dynamic_rnn(cell=lstm, inputs=lstm_inputs, initial_state=init_state)
File "C:\Users\user\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py", line 553, in dynamic_rnn
dtype=dtype)
File "C:\Users\user\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py", line 720, in _dynamic_rnn_loop
swap_memory=swap_memory)
File "C:\Users\user\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2623, in while_loop
result = context.BuildLoop(cond, body, loop_vars, shape_invariants)
File "C:\Users\user\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2456, in BuildLoop
pred, body, original_loop_vars, loop_vars, shape_invariants)
File "C:\Users\user\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2406, in _BuildLoop
body_result = body(*packed_vars_for_body)
File "C:\Users\user\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py", line 705, in _time_step
(output, new_state) = call_cell()
File "C:\Users\user\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py", line 691, in <lambda>
call_cell = lambda: cell(input_t, state)
File "C:\Users\user\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\contrib\rnn\python\ops\core_rnn_cell_impl.py", line 238, in __call__
c, h = state
File "C:\Users\user\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 504, in __iter__
raise TypeError("'Tensor' object is not iterable.")
TypeError: 'Tensor' object is not iterable.
Any help is much appreciated!!
The state for LSTMs really consists of two parts
State for the cell(s)
Previous outputs
This is alluded to in the docs for BasicLSTMCell. This paper has a good explanation of how LSTMs work which will help you understand why you need to keep two sets of states in an LSTM implementation. The reason an error is being thrown is because you need to supply a tuple of tensors for the initial state.
That said you have two options:
Supply an initial state that consists of two tensors.
Let the RNN cell generate its own initial state.
You would usually only do 1. if you wanted to override default behavior. In this case you are using the default (zero) initial state so you can do 2.
lstm_outputs, final_state = tf.nn.dynamic_rnn(cell=lstm, inputs=lstm_inputs, dtype=tf.float32)