AttributeError: 'numpy.ndarray' object has no attribute 'op' - numpy

I am have a time series data and I am trying to build and train an LSTM model over it. I have 1 input and 1 Output corresponding to my model. I am trying to build a Many to Many model where Input length is exactly equal to output length.
The shape of my inputs are
print(np.shape(X))
(1700,70,401)
#(examples, Timestep, Features)
Shape of my output is
print(np.shape(Y_1))
(1700,70,3)
#(examples, Timestep, Features)
Now When I am trying to approach this problem via sequential API everything is running fine.
model = Sequential()
model.add(LSTM(32, input_shape=(70,401), return_sequences=True))
model.add(Dense(3,activation='softmax'))
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01),loss=tf.keras.losses.CategoricalCrossentropy())
model.fit(X, Y_1, epochs=2,verbose=1)
But When I am approaching it from the functional API approach then it is showing the error
AttributeError: 'numpy.ndarray' object has no attribute 'op'
input_layer = Input(shape=(70,401))
hidden = LSTM(32,return_sequences=True)(input_layer)
output_1 = Dense(3, activation='softmax')(hidden)
# output_2 = Dense(np.shape(Y_2)[2], activation='softmax')(hidden)
model_lstm = Model(inputs=X, outputs = Y_1)
My question is How do I resolve the error?
I can not use the sequential API to solve the problem because I want to use Multiple Outputs to train i.e. I have 2 different outputs on which I want to train(But for the scope of this question let's just assume I have one set of input and one set of output)!!
The Entire error that I am getting is
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-66-df3a5a1656f0> in <module>
----> 1 model_lstm = Model(X, Y_1)
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py in __init__(self, *args, **kwargs)
144
145 def __init__(self, *args, **kwargs):
--> 146 super(Model, self).__init__(*args, **kwargs)
147 _keras_api_gauge.get_cell('model').set(True)
148 # initializing _distribution_strategy here since it is possible to call
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/network.py in __init__(self, *args, **kwargs)
165 'inputs' in kwargs and 'outputs' in kwargs):
166 # Graph network
--> 167 self._init_graph_network(*args, **kwargs)
168 else:
169 # Subclassed network
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
455 self._self_setattr_tracking = False # pylint: disable=protected-access
456 try:
--> 457 result = method(self, *args, **kwargs)
458 finally:
459 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/network.py in _init_graph_network(self, inputs, outputs, name, **kwargs)
268
269 if any(not hasattr(tensor, '_keras_history') for tensor in self.outputs):
--> 270 base_layer_utils.create_keras_history(self._nested_outputs)
271
272 self._base_init(name=name, **kwargs)
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py in create_keras_history(tensors)
182 keras_tensors: The Tensors found that came from a Keras Layer.
183 """
--> 184 _, created_layers = _create_keras_history_helper(tensors, set(), [])
185 return created_layers
186
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py in _create_keras_history_helper(tensors, processed_ops, created_layers)
208 if getattr(tensor, '_keras_history', None) is not None:
209 continue
--> 210 op = tensor.op # The Op that created this Tensor.
211 if op not in processed_ops:
212 # Recursively set `_keras_history`.
AttributeError: 'numpy.ndarray' object has no attribute 'op'
Update
I tried type cast X and Y_1 to the tensor objects as suggested in the comments. It is perfectly working in the case of Sequential API but failing for Fnctional API.
X_tensor = tf.convert_to_tensor(X, dtype=tf.float32)
y_tensor=tf.convert_to_tensor(Y_1, dtype=tf.int32)
model_lstm = Model(X_tensor, y_tensor)
Error
AttributeError: Tensor.op is meaningless when eager execution is enabled.
AttributeError Traceback (most recent call last)
<ipython-input-100-d090ea2b5a90> in <module>
----> 1 model_lstm = Model(X_tensor, y_tensor)
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py in __init__(self, *args, **kwargs)
144
145 def __init__(self, *args, **kwargs):
--> 146 super(Model, self).__init__(*args, **kwargs)
147 _keras_api_gauge.get_cell('model').set(True)
148 # initializing _distribution_strategy here since it is possible to call
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/network.py in __init__(self, *args, **kwargs)
165 'inputs' in kwargs and 'outputs' in kwargs):
166 # Graph network
--> 167 self._init_graph_network(*args, **kwargs)
168 else:
169 # Subclassed network
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
455 self._self_setattr_tracking = False # pylint: disable=protected-access
456 try:
--> 457 result = method(self, *args, **kwargs)
458 finally:
459 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/network.py in _init_graph_network(self, inputs, outputs, name, **kwargs)
268
269 if any(not hasattr(tensor, '_keras_history') for tensor in self.outputs):
--> 270 base_layer_utils.create_keras_history(self._nested_outputs)
271
272 self._base_init(name=name, **kwargs)
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py in create_keras_history(tensors)
182 keras_tensors: The Tensors found that came from a Keras Layer.
183 """
--> 184 _, created_layers = _create_keras_history_helper(tensors, set(), [])
185 return created_layers
186
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py in _create_keras_history_helper(tensors, processed_ops, created_layers)
208 if getattr(tensor, '_keras_history', None) is not None:
209 continue
--> 210 op = tensor.op # The Op that created this Tensor.
211 if op not in processed_ops:
212 # Recursively set `_keras_history`.
/root/anaconda3/envs/TensorPy36/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in op(self)
1078 def op(self):
1079 raise AttributeError(
-> 1080 "Tensor.op is meaningless when eager execution is enabled.")
1081
1082 #property
AttributeError: Tensor.op is meaningless when eager execution is enabled.

I made a mistake in the code itself while executing the Model part of in the functional API version.
model_lstm = Model(inputs=X, outputs = Y_1)
I have given the variables inside the above part of the code only!! While in this part we just define what our model is going to be. Hence Here we will just write what is our input layer that needs to be considered and what will be my output layer here!! Input layer while constructing my model will be input_layer in the code and output layer will be output_1. Hence code should be
model_lstm = Model(inputs=input_layer, outputs = output_1)
and after that we can do
model_lstm.fit(X,Y_1)
This will work perfectly fine now!!

Related

Using tf.data.Dataset with tf Hub Modules

How do I feed a tf.keras model, that includes a 1D input TF Hub module, with a tf.data.Dataset?
(Ultimately, the aim is to use a single tf.data.Dataset with a multi-input, multi-output keras funtional api model.)
Tried this:
import tensorflow as tf
import tensorflow_hub as hub
embed = "https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1"
hub_layer = hub.KerasLayer(embed, output_shape=[20], input_shape=[],
dtype=tf.string, trainable=True, name='hub_layer')
# From tf hub webpage: "The module takes a batch of sentences in a 1-D tensor of strings as input."
input_tensor = tf.keras.Input(shape=(), dtype=tf.string)
hub_tensor = hub_layer(input_tensor)
x = tf.keras.layers.Dense(16, activation='relu')(hub_tensor)#(x)
main_output = tf.keras.layers.Dense(units=4, activation='softmax', name='main_output')(x)
model = tf.keras.models.Model(inputs=[input_tensor], outputs=[main_output])
# This works as expected.
X_tensor = tf.constant(['Hello World', 'The Quick Brown Fox'])
model(X_tensor)
# This fails
X_ds = tf.data.Dataset.from_tensors(X_tensor)
X_ds.element_spec
model(X_ds)
Expectation was that the 1D tensor in the dataset would be automatically extracted and consumed by the model.
Error message:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
in
21 X_ds = tf.data.Dataset.from_tensors(X_tensor)
22 X_ds.element_spec
---> 23 model(X_ds)
24
25
~/projects/email_analysis/email_venv/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
966 with base_layer_utils.autocast_context_manager(
967 self._compute_dtype):
--> 968 outputs = self.call(cast_inputs, *args, **kwargs)
969 self._handle_activity_regularization(inputs, outputs)
970 self._set_mask_metadata(inputs, outputs, input_masks)
~/projects/email_analysis/email_venv/lib/python3.6/site-packages/tensorflow/python/keras/engine/network.py in call(self, inputs, training, mask)
717 return self._run_internal_graph(
718 inputs, training=training, mask=mask,
--> 719 convert_kwargs_to_constants=base_layer_utils.call_context().saving)
720
721 def compute_output_shape(self, input_shape):
~/projects/email_analysis/email_venv/lib/python3.6/site-packages/tensorflow/python/keras/engine/network.py in _run_internal_graph(self, inputs, training, mask, convert_kwargs_to_constants)
835 tensor_dict = {}
836 for x, y in zip(self.inputs, inputs):
--> 837 y = self._conform_to_reference_input(y, ref_input=x)
838 x_id = str(id(x))
839 tensor_dict[x_id] = [y] * self._tensor_usage_count[x_id]
~/projects/email_analysis/email_venv/lib/python3.6/site-packages/tensorflow/python/keras/engine/network.py in _conform_to_reference_input(self, tensor, ref_input)
959 # Dtype handling.
960 if isinstance(ref_input, (ops.Tensor, composite_tensor.CompositeTensor)):
--> 961 tensor = math_ops.cast(tensor, dtype=ref_input.dtype)
962
963 return tensor
~/projects/email_analysis/email_venv/lib/python3.6/site-packages/tensorflow/python/util/dispatch.py in wrapper(*args, **kwargs)
178 """Call target, and fall back on dispatchers if there is a TypeError."""
179 try:
--> 180 return target(*args, **kwargs)
181 except (TypeError, ValueError):
182 # Note: convert_to_eager_tensor currently raises a ValueError, not a
~/projects/email_analysis/email_venv/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py in cast(x, dtype, name)
785 # allows some conversions that cast() can't do, e.g. casting numbers to
786 # strings.
--> 787 x = ops.convert_to_tensor(x, name="x")
788 if x.dtype.base_dtype != base_type:
789 x = gen_math_ops.cast(x, base_type, name=name)
~/projects/email_analysis/email_venv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, dtype_hint, ctx, accepted_result_types)
1339
1340 if ret is None:
-> 1341 ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
1342
1343 if ret is NotImplemented:
~/projects/email_analysis/email_venv/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py in _constant_tensor_conversion_function(v, dtype, name, as_ref)
319 as_ref=False):
320 _ = as_ref
--> 321 return constant(v, dtype=dtype, name=name)
322
323
~/projects/email_analysis/email_venv/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py in constant(value, dtype, shape, name)
260 """
261 return _constant_impl(value, dtype, shape, name, verify_shape=False,
--> 262 allow_broadcast=True)
263
264
~/projects/email_analysis/email_venv/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py in _constant_impl(value, dtype, shape, name, verify_shape, allow_broadcast)
268 ctx = context.context()
269 if ctx.executing_eagerly():
--> 270 t = convert_to_eager_tensor(value, ctx, dtype)
271 if shape is None:
272 return t
~/projects/email_analysis/email_venv/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
94 dtype = dtypes.as_dtype(dtype).as_datatype_enum
95 ctx.ensure_initialized()
---> 96 return ops.EagerTensor(value, ctx.device_name, dtype)
97
98
ValueError: Attempt to convert a value () with an unsupported type () to a Tensor.
The point of a dataset is to provide a sequence of tensors, like here:
all_data = tf.constant([['Hello', 'World'], ['Brown Fox', 'lazy dog']])
ds = tf.data.Dataset.from_tensor_slices(all_data)
for tensor in ds:
print(tensor)
which outputs
tf.Tensor([b'Hello' b'World'], shape=(2,), dtype=string)
tf.Tensor([b'Brown Fox' b'lazy dog'], shape=(2,), dtype=string)
Instead of just printing tensor, you can compute with it:
for tensor in ds:
print(hub_layer(tensor))
which outputs 2 tensors of shape (2,20) each.
For more, see https://www.tensorflow.org/guide/data.

Saving A Model That Contains Universal Sentence Encoder as its Embedding

I am trying to save a model which uses USE from tf-hub as its embedding layer and has a few FFN stacked upon it. The model seems to work fine, but I am facing a problem in saving and loading the model.
disable_eager_execution()
embed = hub.Module(module_url)
def UniversalEmbedding(x):
return embed(tf.squeeze(tf.cast(x, tf.string)))
input_text = Input(shape=[], dtype=tf.string)
response_text = Input(shape=[], dtype=tf.string)
text_embedding = Lambda(UniversalEmbedding, output_shape=(512, ))(input_text)
response_embedding = Lambda(UniversalEmbedding, output_shape=(512, ))(response_text)
response_embedding = Dense(512, activation='relu')(response_embedding)
response_embedding = Dense(512, activation='relu')(response_embedding)
score = Dot(axes=1, normalize=True)([text_embedding, response_embedding])
pred = Dense(2, activation='softmax')(score)
text_encoder = Model(inputs=[input_text], outputs=text_embedding)
response_encoder = Model(inputs=[response_text], outputs=response_embedding)
model = Model(inputs=[input_text, response_text], outputs=pred)
The code above is how I built my model (its a dual encoder model with USE as its encoder).
I had to disable eager execution because USE seems to be not working in eager execution environment yet. If not, and if there is a workaround for that, I'd really appreciate any help for this too :)
The model is trained and saved via the following code :
with tf.compat.v1.Session() as session:
K.set_session(session)
session.run(tf.compat.v1.global_variables_initializer())
session.run(tf.compat.v1.tables_initializer())
history = model.fit_generator(generator=train_neg_sample_generator,
validation_data=val_neg_sample_generator, epochs=20,
callbacks=[checkpointer, earlystopper], verbose=0)
and the model is loaded with no error when the weights in the checkpoints (saved in hdf5 files) are loaded to the model defined in the code above. So the code below works fine, only because the architecture 'model' is already defined above.
with tf.compat.v1.Session() as session:
K.set_session(session)
session.run(tf.compat.v1.global_variables_initializer())
session.run(tf.compat.v1.tables_initializer())
model.load_weights('./saved_models/weights.03-0.29.hdf5')
tf.keras.models.save_model(model, 'test_model2.hdf5')
predicts = model.predict([["how are you?", "how are you?", 'hi', 'my two favorites in one pic!'], ["i'm fine", "what the heck", 'hi', 'same!']])
print(predicts)
print(np.argmax(predicts, axis=1))
Then I tried 2 things. First, I tried to save the architecture in json format, load the model architecture and then load the weights, but it did not work. Then I tried to save the whole model via keras.models.save_model, but it did not work either.
In both cases, they returned
AttributeError: module 'tensorflow' has no attribute 'placeholder'
How can I save/load the whole model (if not at once, loading architecture / weight separately is fine too) ?
Here is the whole error log
AttributeError Traceback (most recent call last)
<ipython-input-31-47468f2533ad> in <module>()
1 from keras.models import load_model
2
----> 3 model2 = load_model('testest.h5')
13 frames
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in load_wrapper(*args, **kwargs)
456 os.remove(tmp_filepath)
457 return res
--> 458 return load_function(*args, **kwargs)
459
460 return load_wrapper
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in load_model(filepath, custom_objects, compile)
548 if H5Dict.is_supported_type(filepath):
549 with H5Dict(filepath, mode='r') as h5dict:
--> 550 model = _deserialize_model(h5dict, custom_objects, compile)
551 elif hasattr(filepath, 'write') and callable(filepath.write):
552 def load_function(h5file):
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in _deserialize_model(h5dict, custom_objects, compile)
241 raise ValueError('No model found in config.')
242 model_config = json.loads(model_config.decode('utf-8'))
--> 243 model = model_from_config(model_config, custom_objects=custom_objects)
244 model_weights_group = h5dict['model_weights']
245
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in model_from_config(config, custom_objects)
591 '`Sequential.from_config(config)`?')
592 from ..layers import deserialize
--> 593 return deserialize(config, custom_objects=custom_objects)
594
595
/usr/local/lib/python3.6/dist-packages/keras/layers/__init__.py in deserialize(config, custom_objects)
166 module_objects=globs,
167 custom_objects=custom_objects,
--> 168 printable_module_name='layer')
/usr/local/lib/python3.6/dist-packages/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
145 config['config'],
146 custom_objects=dict(list(_GLOBAL_CUSTOM_OBJECTS.items()) +
--> 147 list(custom_objects.items())))
148 with CustomObjectScope(custom_objects):
149 return cls.from_config(config['config'])
/usr/local/lib/python3.6/dist-packages/keras/engine/network.py in from_config(cls, config, custom_objects)
1041 # First, we create all layers and enqueue nodes to be processed
1042 for layer_data in config['layers']:
-> 1043 process_layer(layer_data)
1044
1045 # Then we process nodes in order of layer depth.
/usr/local/lib/python3.6/dist-packages/keras/engine/network.py in process_layer(layer_data)
1027
1028 layer = deserialize_layer(layer_data,
-> 1029 custom_objects=custom_objects)
1030 created_layers[layer_name] = layer
1031
/usr/local/lib/python3.6/dist-packages/keras/layers/__init__.py in deserialize(config, custom_objects)
166 module_objects=globs,
167 custom_objects=custom_objects,
--> 168 printable_module_name='layer')
/usr/local/lib/python3.6/dist-packages/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
147 list(custom_objects.items())))
148 with CustomObjectScope(custom_objects):
--> 149 return cls.from_config(config['config'])
150 else:
151 # Then `cls` may be a function returning a class.
/usr/local/lib/python3.6/dist-packages/keras/engine/base_layer.py in from_config(cls, config)
1101 A layer instance.
1102 """
-> 1103 return cls(**config)
1104
1105 def count_params(self):
/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name + '` call to the ' +
90 'Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
/usr/local/lib/python3.6/dist-packages/keras/engine/input_layer.py in __init__(self, input_shape, batch_size, batch_input_shape, dtype, input_tensor, sparse, name)
85 dtype=dtype,
86 sparse=self.sparse,
---> 87 name=self.name)
88 else:
89 self.is_placeholder = False
/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py in placeholder(shape, ndim, dtype, sparse, name)
539 x = tf.sparse_placeholder(dtype, shape=shape, name=name)
540 else:
--> 541 x = tf.placeholder(dtype, shape=shape, name=name)
542 x._keras_shape = shape
543 x._uses_learning_phase = False
AttributeError: module 'tensorflow' has no attribute 'placeholder'
Provide the whole error log, not just a part of it.
If it's really error because of saving then what about model.save('model.h5')?
Not using the module from tf.keras.models but call a methomd from the Model class itself.
But why these code?
with tf.compat.v1.Session() as session:
K.set_session(session)
session.run(tf.compat.v1.global_variables_initializer())
session.run(tf.compat.v1.tables_initializer())
I believe you could juat call model.fit right away, and your tf is version2 right? Why call compat.v1?
Tensorflow 2 doesn't have Placeholder so I think it's maybe about this.
Worked fine with tensorflow version 1.15
Looking forward for tf-hub to become completely compatible with tensorflow 2.0 and keras...

How do you write a custom activation function in python for Keras?

I'm trying to write a custom activation function for use with Keras. I can not write it with tensorflow primitives as it does properly compute the derivative. I followed How to make a custom activation function with only Python in Tensorflow? and it works very we in creating a tensorflow function. However, when I tried putting it into Keras as an activation function for the classic MNIST demo. I got errors. I also tried the tf_spiky function from the above reference.
Here is the sample code
tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(512, activation=tf_spiky),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)])
Here's my entire error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-48-73a57f81db19> in <module>
3 tf.keras.layers.Dense(512, activation=tf_spiky),
4 tf.keras.layers.Dropout(0.2),
----> 5 tf.keras.layers.Dense(10, activation=tf.nn.softmax)])
6 x=tf.keras.layers.Activation(tf_spiky)
7 y=tf.keras.layers.Flatten(input_shape=(28, 28))
/opt/conda/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py in _method_wrapper(self, *args, **kwargs)
472 self._setattr_tracking = False # pylint: disable=protected-access
473 try:
--> 474 method(self, *args, **kwargs)
475 finally:
476 self._setattr_tracking = previous_value # pylint: disable=protected-access
/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/sequential.py in __init__(self, layers, name)
106 if layers:
107 for layer in layers:
--> 108 self.add(layer)
109
110 #property
/opt/conda/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py in _method_wrapper(self, *args, **kwargs)
472 self._setattr_tracking = False # pylint: disable=protected-access
473 try:
--> 474 method(self, *args, **kwargs)
475 finally:
476 self._setattr_tracking = previous_value # pylint: disable=protected-access
/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/sequential.py in add(self, layer)
173 # If the model is being built continuously on top of an input layer:
174 # refresh its output.
--> 175 output_tensor = layer(self.outputs[0])
176 if isinstance(output_tensor, list):
177 raise TypeError('All layers in a Sequential model '
/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs)
728
729 # Check input assumptions set before layer building, e.g. input rank.
--> 730 self._assert_input_compatibility(inputs)
731 if input_list and self._dtype is None:
732 try:
/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py in _assert_input_compatibility(self, inputs)
1463 if x.shape.ndims is None:
1464 raise ValueError('Input ' + str(input_index) + ' of layer ' +
-> 1465 self.name + ' is incompatible with the layer: '
1466 'its rank is undefined, but the layer requires a '
1467 'defined rank.')
ValueError: Input 0 of layer dense_1 is incompatible with the layer: its rank is undefined, but the layer requires a defined rank.
From this I gather the last Dense layer is unable to get the dimensions of the output after the activation function or something to that. I did see in the tensorflow code that many activation functions register a shape. But either I'm not doing that correctly or I'm going in the wrong direction. But I'm guessing something needs to be done to the tensorflow function to make it an activation function that Keras can use.
I would appreciate any help you can give.
As requested here is the sample codes for tf_spiky, it works as described in the above reference. However, once put into Keras I get the errors shown. This is pretty much as shown in the *How to make a custom activation function with only Python in Tensorflow?" stackoverflow article.
def spiky(x):
print(x)
r = x % 1
if r <= 0.5:
return r
else:
return 0
def d_spiky(x):
r = x % 1
if r <= 0.5:
return 1
else:
return 0
np_spiky = np.vectorize(spiky)
np_d_spiky = np.vectorize(d_spiky)
np_d_spiky_32 = lambda x: np_d_spiky(x).astype(np.float32)
import tensorflow as tf
from tensorflow.python.framework import ops
def tf_d_spiky(x,name=None):
with tf.name_scope(name, "d_spiky", [x]) as name:
y = tf.py_func(np_d_spiky_32,
[x],
[tf.float32],
name=name,
stateful=False)
return y[0]
def py_func(func, inp, Tout, stateful=True, name=None, grad=None):
# Need to generate a unique name to avoid duplicates:
rnd_name = 'PyFuncGrad' + str(np.random.randint(0, 1E+8))
tf.RegisterGradient(rnd_name)(grad) # see _MySquareGrad for grad example
g = tf.get_default_graph()
with g.gradient_override_map({"PyFunc": rnd_name}):
return tf.py_func(func, inp, Tout, stateful=stateful, name=name)
def spikygrad(op, grad):
x = op.inputs[0]
n_gr = tf_d_spiky(x)
return grad * n_gr
np_spiky_32 = lambda x: np_spiky(x).astype(np.float32)
def tf_spiky(x, name=None):
with tf.name_scope(name, "spiky", [x]) as name:
y = py_func(np_spiky_32,
[x],
[tf.float32],
name=name,
grad=spikygrad) # <-- here's the call to the gradient
return y[0]
The solution is in this post Output from TensorFlow `py_func` has unknown rank/shape
The easiest fix is to add y[0].set_shape(x.get_shape()) before the return statement in the definition of tf_spiky.
Perhaps someone out there knows how to properly work with tensorflow shape functions. Digging around I found a unchanged_shape shape function in tensorflow.python.framework.common_shapes, which be appropriate here, but I don't know how to attach it to the tf_spiky function. Seems a python decorator is in order here. It would probably be a service to others to explain customizing tensorflow functions with shape functions.

Initializing RNNCell

I'm having issues with using any child class of tensorflows RNNCell. According to the tensorflow source, the state for any RNNCell should be a tuple, however when I give it a tuple, it throws an error saying that it's trying to ask for the ndims attribute of the state tuple, which doesn't exist.
I am trying to create an LSTM where I can control each individual input.
This is the simplest code I can make, and even that has the same issue, so I hope that I am doing something wrong that can easily be fixed. This is the simple code:
lstm_layer = tf.contrib.rnn.LSTMCell(num_units = 64)
initial_state = lstm_layer.zero_state(batch_size=1,dtype=tf.float32)
initial_input = np.expand_dims(np.array([1,2,3,4,5,6,7,8]),0)
output_single, state_single = lstm_layer(inputs=initial_input,state=initial_state)
Here is the error I get:
AttributeError Traceback (most recent call last)
<ipython-input-22-1dcce10906e5> in <module>
2 initial_state = lstm_layer.zero_state(batch_size=1,dtype=tf.float32)
3 initial_input = np.expand_dims(np.array([1,2,3,4,5,6,7,8]),0)
----> 4 output_single, state_single = lstm_layer(inputs=initial_input,state=initial_state)
~/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/rnn_cell_impl.py in __call__(self, inputs, state, scope, *args, **kwargs)
369 # method. See the class docstring for more details.
370 return base_layer.Layer.__call__(self, inputs, state, scope=scope,
--> 371 *args, **kwargs)
372
373
~/anaconda3/lib/python3.7/site-packages/tensorflow/python/layers/base.py in __call__(self, inputs, *args, **kwargs)
528
529 # Actually call layer
--> 530 outputs = super(Layer, self).__call__(inputs, *args, **kwargs)
531
532 if not context.executing_eagerly():
~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs)
536 if not self.built:
537 # Build layer if applicable (if the `build` method has been overridden).
--> 538 self._maybe_build(inputs)
539 # We must set self.built since user defined build functions are not
540 # constrained to set self.built.
~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in _maybe_build(self, inputs)
1589 # Check input assumptions set before layer building, e.g. input rank.
1590 input_spec.assert_input_compatibility(
-> 1591 self.input_spec, inputs, self.name)
1592 input_list = nest.flatten(inputs)
1593 if input_list and self._dtype is None:
~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/input_spec.py in assert_input_compatibility(input_spec, inputs, layer_name)
107 spec.min_ndim is not None or
108 spec.max_ndim is not None):
--> 109 if x.shape.ndims is None:
110 raise ValueError('Input ' + str(input_index) + ' of layer ' +
111 layer_name + ' is incompatible with the layer: '
AttributeError: 'tuple' object has no attribute 'ndims'
It looks as though there is some base method that gets called for all layers in the keras API, but it doesn't work with tuples. However, I find it strange that this would be an issue that nobody has ever come against before. So I hope its just me making a mistake
I figured out the issue. Tensorflow wasn't playing nicely with numpy in this case. Instead of
initial_input = np.expand_dims(np.array([1,2,3,4,5,6,7,8]),0)
I needed to give it
initial_input = tf.expand_dims(np.array([1,2,3,4,5,6,7,8],dtype=np.float32),0)

Input tensors to a Model must come from `tf.layers.Input` when I concatenate two models with Keras API on Tensorflow

I'm creating a wide and deep model using Keras functional API on tensorflow.
When I try to merge the two models, the below error occurred.
--------------------------------------------------------------------------- ValueError Traceback (most recent call
last) in ()
1 merged_out = tf.keras.layers.concatenate([wide_model.output, deep_model.output])
2 merged_out = tf.keras.layers.Dense(1)(merged_out)
----> 3 combined_model = tf.keras.Model(inputs=wide_model.input + [deep_model.input], outputs=merged_out)
4 print(combined_model.summary())
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py
in init(self, *args, **kwargs)
111
112 def init(self, *args, **kwargs):
--> 113 super(Model, self).init(*args, **kwargs)
114 # Create a cache for iterator get_next op.
115 self._iterator_get_next = weakref.WeakKeyDictionary()
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py
in init(self, *args, **kwargs)
77 'inputs' in kwargs and 'outputs' in kwargs):
78 # Graph network
---> 79 self._init_graph_network(*args, **kwargs)
80 else:
81 # Subclassed network
/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/checkpointable/base.py
in _method_wrapper(self, *args, **kwargs)
362 self._setattr_tracking = False # pylint: disable=protected-access
363 try:
--> 364 method(self, *args, **kwargs)
365 finally:
366 self._setattr_tracking = previous_value # pylint: disable=protected-access
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py
in _init_graph_network(self, inputs, outputs, name)
193 'must come from tf.layers.Input. '
194 'Received: ' + str(x) +
--> 195 ' (missing previous layer metadata).')
196 # Check that x is an input tensor.
197 # pylint: disable=protected-access
ValueError: Input tensors to a Model must come from tf.layers.Input.
Received: Tensor("add_1:0", shape=(1, ?, 163), dtype=float32) (missing
previous layer metadata).
Here is the code for concatenating the two.
merged_out = tf.keras.layers.concatenate([wide_model.output, deep_model.output])
merged_out = tf.keras.layers.Dense(1)(merged_out)
combined_model = tf.keras.Model(inputs=wide_model.input + [deep_model.input], outputs=merged_out)
print(combined_model.summary())
For each model's inputs, I tried using tf.layers.Inputwith
inputs = tf.placeholder(tf.float32, shape=(None,X_resampled.shape[1]))
deep_inputs = tf.keras.Input(tensor=(inputs))
to make them tf.layers.Input as this page mentions.
But I'm still facing the same issue.
I'm using tensorflow==1.10.0
Could someone help me solving this issue?
Thanks!
In inputs=wide_model.input + [deep_model.input], wide.model.input is probably not a list, so that you are passing a new Add tensor instead of a list of inputs. Try passing inputs=[wide_model.input] + [deep_model.input] instead