I am trying to train on a data containing sequences of 43 records of 3-dimensional vectors. While trying to add this Conv1D layer here:
model = Sequential()
model.add(Conv1D(input_shape=(43, 3),
filters=16,
kernel_size=4,
padding='same')) # This is line 24 of bcl_model_builder.py
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dropout(0.5))
I am getting the following error. And I got no clue what went wrong here:
Traceback (most recent call last):
File "/home/shx/programs/pycharm-community-2017.1.3/helpers/pydev/pydevd.py", line 1585, in <module>
globals = debugger.run(setup['file'], None, None, is_module)
File "/home/shx/programs/pycharm-community-2017.1.3/helpers/pydev/pydevd.py", line 1015, in run
pydev_imports.execfile(file, globals, locals) # execute the script
File "/home/shx/programs/pycharm-community-2017.1.3/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "/home/shx/PycharmProjects/FBS/bcl/bcl_train_model.py", line 34, in <module>
model = mb.build_model()
File "/home/shx/PycharmProjects/FBS/bcl/bcl_model_builder.py", line 24, in build_model
padding='same'))
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/keras/models.py", line 442, in add
layer(x)
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/keras/engine/topology.py", line 602, in __call__
output = self.call(inputs, **kwargs)
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/keras/layers/convolutional.py", line 156, in call
dilation_rate=self.dilation_rate[0])
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 3124, in conv1d
data_format=tf_data_format)
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 672, in convolution
op=op)
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 338, in with_space_to_batch
return op(input, num_spatial_dims, padding)
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 664, in op
name=name)
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 116, in _non_atrous_convolution
name=scope)
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 2013, in conv1d
data_format=data_format)
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 397, in conv2d
data_format=data_format, name=name)
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 589, in apply_op
param_name=input_name)
File "/home/shx/pyenvs/finainpy_env1/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 60, in _SatisfiesTypeConstraint
", ".join(dtypes.as_dtype(x).name for x in allowed_list)))
TypeError: Value passed to parameter 'input' has DataType float64 not in list of allowed values: float16, float32
After giving so much thought on float64 in the error log, I realized that keras itself was configured that way in my machine
{
"floatx": "float64",
"epsilon": 1e-07,
"backend": "tensorflow",
"image_data_format": "channels_last"
}
I just had to update the property floatx to float32.
Related
After training a CNN model in TensorFlow-Keras I got this error mentioned in the title while opening the model file with the following function:
from tensorflow.keras import load_model
# I also tried:
# from tensorflow.compat.v1.keras.models import load_model
model = load_model('weights-improvement-70-0.57.hdf5', compile = False)
weights-improvement-70-0.57.hdf5 is my model file. Following is the detailed form of the error which I am quoting from the terminal:
Traceback (most recent call last):
File "testing_with_landmark.py", line 59, in <module>
PREDICTOR = load_model('weights-improvement-70-0.57.hdf5', compile = True)
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/saving/save.py", line 143, in load_model
return hdf5_format.load_model_from_hdf5(filepath, custom_objects, compile)
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/saving/hdf5_format.py", line 162, in load_model_from_hdf5
custom_objects=custom_objects)
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/saving/model_config.py", line 55, in model_from_config
return deserialize(config, custom_objects=custom_objects)
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/layers/serialization.py", line 105, in deserialize
printable_module_name='layer')
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/utils/generic_utils.py", line 191, in deserialize_keras_object
list(custom_objects.items())))
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/sequential.py", line 368, in from_config
custom_objects=custom_objects)
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/layers/serialization.py", line 105, in deserialize
printable_module_name='layer')
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/utils/generic_utils.py", line 193, in deserialize_keras_object
return cls.from_config(cls_config)
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py", line 601, in from_config
return cls(**config)
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/layers/convolutional.py", line 498, in __init__
**kwargs)
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/layers/convolutional.py", line 122, in __init__
**kwargs)
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/training/tracking/base.py", line 457, in _method_wrapper
result = method(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py", line 300, in __init__
generic_utils.validate_kwargs(kwargs, allowed_kwargs)
File "/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/utils/generic_utils.py", line 599, in validate_kwargs
raise TypeError(error_message, kwarg)
TypeError: ('Keyword argument not understood:', 'groups')
Why this is happening?
The problem is because you are tying to load model weights as a model.
To load a model you should fist create the model or load from the saved model (a json file or model checkpoint) and then use: load_weights(<model_weight_path>)
I've been trying to visualise a simple model in Tensorboard. The model is build in Tensorflow Eager Execution mode, using keras.
When fitting the model I get the following error:
>>> history = model.fit(x_train, y_train, epochs=1, batch_size=BatchSize, callbacks=[tensorbrd])
Traceback (most recent call last):
File "C:\...\tensorflow\python\ops\gen_logging_ops.py", line 322, in histogram_summary
"HistogramSummary", name, _ctx._post_execution_callbacks, tag, values)
tensorflow.python.eager.core._FallbackException: This function does not handle the case of the path where all inputs are not already EagerTensors.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\...\tensorflow\python\keras\engine\training.py", line 1348, in fit
validation_steps=validation_steps)
File "C:\...\tensorflow\python\keras\engine\training_eager.py", line 990, in fit_loop
callbacks.set_model(callback_model)
File "C:\...\tensorflow\python\keras\callbacks.py", line 71, in set_model
callback.set_model(model)
File "C:\...\tensorflow\python\keras\callbacks.py", line 781, in set_model
tf_summary.histogram('{}_out'.format(layer.name), layer.output)
File "C:\...\tensorflow\python\summary\summary.py", line 187, in histogram
tag=tag, values=values, name=scope)
File "C:\...\tensorflow\python\ops\gen_logging_ops.py", line 326, in histogram_summary
tag, values, name=name, ctx=_ctx)
File "C:\...\tensorflow\python\ops\gen_logging_ops.py", line 340, in histogram_summary_eager_fallback
_attr_T, (values,) = _execute.args_to_matching_eager([values], _ctx, _dtypes.float32)
File "C:\...\tensorflow\python\eager\execute.py", line 191, in args_to_matching_eager
t, dtype, preferred_dtype=default_dtype, ctx=ctx))
File "C:\...\tensorflow\python\framework\ops.py", line 1094, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "C:\...\tensorflow\python\framework\constant_op.py", line 217, in _constant_tensor_conversion_function
return constant(v, dtype=dtype, name=name)
File "C:\...\tensorflow\python\framework\constant_op.py", line 167, in constant
t = convert_to_eager_tensor(value, ctx, dtype)
File "C:\...\tensorflow\python\framework\constant_op.py", line 113, in convert_to_eager_tensor
return ops.EagerTensor(value, context=handle, device=device, dtype=dtype)
ValueError: Attempt to convert a value (<DeferredTensor 'None' shape=(?, 5000) dtype=float32>) with an unsupported type (<class 'tensorflow.python.keras.engine.base_layer.DeferredTensor'>) to a Tensor.
This is based on Tensorflow 1.10 code:
import tensorflow as tf
from tensorflow import keras
import matplotlib.pyplot as plt
import numpy as np
import os
import h5py
tf.enable_eager_execution()
BatchSize = 256
model = keras.Sequential([
keras.layers.Flatten(input_shape=(1000,5)),
keras.layers.Dense(1, activation=tf.nn.relu)
])
model.compile(optimizer=tf.train.AdamOptimizer(learning_rate=0.01, beta1=0.9, beta2=0.999),
loss='mean_squared_error',
metrics=[ 'mean_squared_error'])
tensorbrd = keras.callbacks.TensorBoard(log_dir=SUMMARIES_FOLDER, histogram_freq=1, write_graph=False, write_images=False)
history = model.fit(x_train, y_train, epochs=1, batch_size=BatchSize, callbacks=[tensorbrd])
Am I making a mistake or is Tensorboard not compatible in this way?
Any help is greatly appreciated!
Following this guide, I'm converting a tensor [batch_size, 16000, 1] to an MFCC using the method described in the link:
def gen_spectrogram(wav, sr=16000):
# A 1024-point STFT with frames of 64 ms and 75% overlap.
stfts = tf.contrib.signal.stft(wav, frame_length=1024, frame_step=256, fft_length=1024)
spectrograms = tf.abs(stfts)
# Warp the linear scale spectrograms into the mel-scale.
num_spectrogram_bins = stfts.shape[-1].value
lower_edge_hertz, upper_edge_hertz, num_mel_bins = 80.0, 7600.0, 80
linear_to_mel_weight_matrix = tf.contrib.signal.linear_to_mel_weight_matrix(
num_mel_bins, num_spectrogram_bins,
sample_rate, lower_edge_hertz, upper_edge_hertz)
mel_spectrograms = tf.tensordot(spectrograms, linear_to_mel_weight_matrix, 1)
mel_spectrograms.set_shape(
spectrograms.shape[:-1].concatenate(
linear_to_mel_weight_matrix.shape[-1:]
)
)
# Compute a stabilized log to get log-magnitude mel-scale spectrograms.
log_mel_spectrograms = tf.log(mel_spectrograms + 1e-6)
# Compute MFCCs from log_mel_spectrograms and take the first 13.
return tf.contrib.signal.mfccs_from_log_mel_spectrograms(log_mel_spectrograms)[..., :13]
I then reshape the output of that to [batch_size, 125, 128, 1]. If I send that to a tf.layers.conv2d, things seem to work fine. However, if I try to tf.summary.image, I get the following error:
print(spec)
// => Tensor("spectrogram/Reshape:0", shape=(?, 125, 128, 1), dtype=float32)
tf.summary.image('spec', spec)
Caused by op u'spectrogram/stft/rfft', defined at:
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py", line 162, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/Users/rsilveira/rnd/ml-engine/trainer/flatv1.py", line 103, in <module>
runner.run(model_fn)
File "trainer/runner.py", line 88, in run
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
File "/Library/Python/2.7/site-packages/tensorflow/python/estimator/training.py", line 432, in train_and_evaluate
executor.run_local()
File "/Library/Python/2.7/site-packages/tensorflow/python/estimator/training.py", line 611, in run_local
hooks=train_hooks)
File "/Library/Python/2.7/site-packages/tensorflow/python/estimator/estimator.py", line 302, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/Library/Python/2.7/site-packages/tensorflow/python/estimator/estimator.py", line 711, in _train_model
features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
File "/Library/Python/2.7/site-packages/tensorflow/python/estimator/estimator.py", line 694, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/Users/rsilveira/rnd/ml-engine/trainer/flatv1.py", line 53, in model_fn
spec = gen_spectrogram(x)
File "/Users/rsilveira/rnd/ml-engine/trainer/flatv1.py", line 22, in gen_spectrogram
step,
File "/Library/Python/2.7/site-packages/tensorflow/contrib/signal/python/ops/spectral_ops.py", line 91, in stft
return spectral_ops.rfft(framed_signals, [fft_length])
File "/Library/Python/2.7/site-packages/tensorflow/python/ops/spectral_ops.py", line 136, in _rfft
return fft_fn(input_tensor, fft_length, name)
File "/Library/Python/2.7/site-packages/tensorflow/python/ops/gen_spectral_ops.py", line 619, in rfft
"RFFT", input=input, fft_length=fft_length, name=name)
File "/Library/Python/2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/Library/Python/2.7/site-packages/tensorflow/python/framework/ops.py", line 2956, in create_op
op_def=op_def)
File "/Library/Python/2.7/site-packages/tensorflow/python/framework/ops.py", line 1470, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): Input dimension 4 must have length of at least 512 but got: 320
Not sure where to start troubleshooting this. What am I missing here?
There is an in-detailed post on keras blog.
But when compiling the code I get the error as follows:
Using TensorFlow backend.
Traceback (most recent call last):
File "visulaize_cifar.py", line 24, in <module>
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/keras/models.py", line 332, in add
output_tensor = layer(self.outputs[0])
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/keras/engine/topology.py", line 572, in __call__
self.add_inbound_node(inbound_layers, node_indices, tensor_indices)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/keras/engine/topology.py", line 635, in add_inbound_node
Node.create_node(self, inbound_layers, node_indices, tensor_indices)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/keras/engine/topology.py", line 166, in create_node
output_tensors = to_list(outbound_layer.call(input_tensors[0], mask=input_masks[0]))
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/keras/layers/pooling.py", line 160, in call
dim_ordering=self.dim_ordering)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/keras/layers/pooling.py", line 210, in _pooling_function
pool_mode='max')
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py", line 2866, in pool2d
x = tf.nn.max_pool(x, pool_size, strides, padding=padding)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/tensorflow/python/ops/nn_ops.py", line 1617, in max_pool
name=name)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 1598, in _max_pool
data_format=data_format, name=name)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 759, in apply_op
op_def=op_def)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2242, in create_op
set_shapes_for_outputs(ret)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1617, in set_shapes_for_outputs
shapes = shape_func(op)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1568, in call_with_requiring
return call_cpp_shape_fn(op, require_shape_fn=True)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/tensorflow/python/framework/common_shapes.py", line 610, in call_cpp_shape_fn
debug_python_shape_fn, require_shape_fn)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/tensorflow/python/framework/common_shapes.py", line 675, in _call_cpp_shape_fn_impl
raise ValueError(err.message)
ValueError: Negative dimension size caused by subtracting 2 from 1 for 'MaxPool_1' (op: 'MaxPool') with input shapes: [1,1,64,128].
This error goes when I set dim_ordering='th'.
But as I am using tensorflow backend so dimension ordering should be dim_ordering='tf'.
Even after setting dim_ordering as 'th', I get error while loading weights from vgg16_weights.h5 as follows :
Traceback (most recent call last):
File "visulaize_cifar.py", line 67, in <module>
model.layers[k].set_weights(weights)
File "/home/dude_perf3ct/.local/lib/python2.7/site-packages/keras/engine/topology.py", line 985, in set_weights
'provided weight shape ' + str(w.shape))
ValueError: Layer weight shape (3, 3, 128, 64) not compatible with provided weight shape (64, 3, 3, 3).
As detailed in this post about 'th' and 'tf'. The above error implies layer weights are in 'tf' (but I set it to 'th' to avoid first error) and provided weight shape in 'th' ordering.
What seems to be the error?
Answer to this question was pretty simple. As, I was using tensorflow as backend. So, to convert I inserted the line
if K.backend()=='tensorflow':
K.set_image_dim_ordering("th")
after from keras import backend as K.
This is because the vgg16_weights.h5 has th format and also cifar10.load_data().
I am using the Tensorflow seq2seq tutorial to play with machine translation. Say I have trained the model for some time and determine that I want to supplement the original vocab with new words to enhance the quality of the model. Is there a way to pause training, add words to the vocabulary, and then resume training from the most recent checkpoint? I attempted to do so but when I began training again I got this error:
Traceback (most recent call last):
File "execute.py", line 405, in <module>
train()
File "execute.py", line 127, in train
model = create_model(sess, False)
File "execute.py", line 108, in create_model
model.saver.restore(session, ckpt.model_checkpoint_path)
File "/home/jrthom18/.local/lib/python2.7/site- packages/tensorflow/python/training/saver.py", line 1388, in restore
{self.saver_def.filename_tensor_name: save_path})
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 766, in run
run_metadata_ptr)
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 964, in _run
feed_dict_string, options, run_metadata)
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1014, in _do_run
target_list, options, run_metadata)
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1034, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Assign requires shapes of both tensors to match. lhs shape= [384633] rhs shape= [384617]
[[Node: save/Assign_82 = Assign[T=DT_FLOAT, _class=["loc:#proj_b"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/cpu:0"](proj_b, save/RestoreV2_82)]]
Caused by op u'save/Assign_82', defined at:
File "execute.py", line 405, in <module>
train()
File "execute.py", line 127, in train
model = create_model(sess, False)
File "execute.py", line 99, in create_model
model = seq2seq_model.Seq2SeqModel( gConfig['enc_vocab_size'], gConfig['dec_vocab_size'], _buckets, gConfig['layer_size'], gConfig['num_layers'], gConfig['max_gradient_norm'], gConfig['batch_size'], gConfig['learning_rate'], gConfig['learning_rate_decay_factor'], forward_only=forward_only)
File "/home/jrthom18/data/3x256_bs32/easy_seq2seq/seq2seq_model.py", line 166, in __init__
self.saver = tf.train.Saver(tf.global_variables(), keep_checkpoint_every_n_hours=2.0)
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/training/saver.py", line 1000, in __init__
self.build()
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/training/saver.py", line 1030, in build
restore_sequentially=self._restore_sequentially)
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/training/saver.py", line 624, in build
restore_sequentially, reshape)
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/training/saver.py", line 373, in _AddRestoreOps
assign_ops.append(saveable.restore(tensors, shapes))
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/training/saver.py", line 130, in restore
self.op.get_shape().is_fully_defined())
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 47, in assign
use_locking=use_locking, name=name)
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 759, in apply_op
op_def=op_def)
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2240, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/jrthom18/.local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1128, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): Assign requires shapes of both tensors to match. lhs shape= [384633] rhs shape= [384617]
[[Node: save/Assign_82 = Assign[T=DT_FLOAT, _class=["loc:#proj_b"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/cpu:0"](proj_b, save/RestoreV2_82)]]
Obviously the new vocab is larger and so the tensor sizes do not match. Is there some way around this?
You can not update your vocab once its set but you can always use shared wordpiece model. It will help you to directly copy out of vocab words form source to target output.