Define custom LSTM with multiple inputs - tensorflow

Following the tutorial writing custom layer, I am trying to implement a custom LSTM layer with multiple input tensors. I am providing two vectors input_1 and input_2 as a list [input_1, input_2] as suggested in the tutorial. The single input code is working but when I change the code for multiple inputs, its throwing the error,
self.kernel = self.add_weight(shape=(input_shape[0][-1], self.units),
TypeError: 'NoneType' object is not subscriptable.
What change I have to do to get rid of the error? Here is the modified code.
class MinimalRNNCell(keras.layers.Layer):
def __init__(self, units, **kwargs):
self.units = units
self.state_size = units
super(MinimalRNNCell, self).__init__(**kwargs)
def build(self, input_shape):
print(type(input_shape))
self.kernel = self.add_weight(shape=(input_shape[0][-1], self.units),
initializer='uniform',
name='kernel')
self.recurrent_kernel = self.add_weight(
shape=(self.units, self.units),
initializer='uniform',
name='recurrent_kernel')
self.built = True
def call(self, inputs, states):
prev_output = states[0]
h = K.dot(inputs[0], self.kernel)
output = h + K.dot(prev_output, self.recurrent_kernel)
return output, [output]
# Let's use this cell in a RNN layer:
cell = MinimalRNNCell(32)
input_1 = keras.Input((None, 5))
input_2 = keras.Input((None, 5))
layer = RNN(cell)
y = layer([input_1, input_2])

Error is because of the line, y = layer([input_1, input_2]).
Replacing that line with y = layer((input_1, input_2)) (passing as Tuple of Inputs rather than List of Inputs), will resolve the error.
Complete working code using tf.keras is shown below:
from tensorflow import keras
from tensorflow.keras import backend as K
from tensorflow.keras.layers import RNN
import tensorflow as tf
class MinimalRNNCell(tf.keras.layers.Layer):
def __init__(self, units, **kwargs):
self.units = units
self.state_size = units
#self.state_size = [tf.TensorShape([units])]
super(MinimalRNNCell, self).__init__(**kwargs)
def build(self, input_shape):
print(type(input_shape))
self.kernel = self.add_weight(shape=(input_shape[0][-1], self.units),
initializer='uniform',
name='kernel')
self.recurrent_kernel = self.add_weight(
shape=(self.units, self.units),
initializer='uniform',
name='recurrent_kernel')
self.built = True
def call(self, inputs, states):
prev_output = states[0]
h = K.dot(inputs[0], self.kernel)
output = h + K.dot(prev_output, self.recurrent_kernel)
return output, [output]
# Let's use this cell in a RNN layer:
cell = MinimalRNNCell(32)
input_1 = tf.keras.Input((None, 5))
input_2 = tf.keras.Input((None, 5))
layer = RNN(cell)
y = layer((input_1, input_2))
Output of the above code is:
<class 'tuple'>
Hope this helps. Happy Learning!

Related

TypeError: can't pickle _thread.RLock objects - saving model in keras

I have model with custom layer. Custom layers is as below
class CustomLayer(tf.keras.layers.Layer):
def __init__(self, loaded_model):
super(CustomLayer, self).__init__()
self.loaded_model = loaded_model
def build(self, input_shape):
pass
def get_config(self):
config = super().get_config().copy()
config.update({
'loaded_model': self.loaded_model
})
return config
def call(self, x):
stacked_output = []
for t in tf.unstack(x, axis=-1):
single_image = tf.expand_dims(t, axis=-1)
for i in range(10):
single_image = self.loaded_model.get_layer(index=i)(single_image)
stacked_output.append(single_image)
output = tf.stack(stacked_output, axis=-1)
# print(output.shape)
return output
Main model with custom layer:
def create_model():
trajectory_input = Input(shape=(n_steps, feature_count), name='trajectory_input')
image_input = Input(shape=(128, 128, n_steps), name='image_input')
x_aware = CustomLayer(loaded_model)(image_input)
x_aware = Reshape((1, 501760))(x_aware)
x = (Dense(32, activation='relu', kernel_regularizer=regularizers.l2(0.001)))(trajectory_input)
x = Reshape((1, 32 * n_steps))(x)
x = concatenate([x, x_aware])
output_regression = (Dense(2, name='main_output'))(x_reg)
adam = Adam(lr=learning_rate)
model = Model(inputs=[trajectory_input, image_input], outputs=output_regression)
model.compile(optimizer=adam, loss='mse', metrics=[euc_dist_1, euc_dist_2])
model.summary()
return model
when I try to save model using model.save("model.h5") initially I ended up with
NotImplementedError: Layers with arguments in __init__ must override `get_config`
but once I include get_config in CustomLayer solved the error.
But now I get
TypeError: can't pickle _thread.RLock objects
I am using pretrained model as a custom layer.

Is there a tensorflow keras that is a wrapper for a stack of Dense layers?

For example, this is trivial but is there a layer for this? Is not really a convolution ... there is one "Dense layer" (weights) per data point.
In [266]: X = np.random.randn(10, 3); W = np.random.randn(10, 3, 4); (X[:, :, None] * W).sum(axis=1).shape
Out[266]: (10, 4)
Create your own layer:
Warning: works only with fixed batch size, you need to define batch_shape or batch_input_shape in your models!!!!
class SampleDense(Layer):
def __init__(self, units, **kwargs):
self.units = units
super(SampleDense, self).__init__(**kwargs)
def build(self, input_shape):
weight_shape = input_shape + (self.units,)
self.kernel = self.add_weight(name='kernel',
shape=weight_shape,
initializer='uniform',
trainable=True)
self.built = True
def call(self, inputs):
inputs = K.expand_dims(inputs, axis=-1)
outputs = inputs * self.kernel
outputs = K.sum(outputs, axis=-2)
return outputs
def compute_output_shape(self, input_shape):
return input_shape[:-1] + (self.units,)

Tensorflow 2.0 How to handle tf.Variable as Weight in __call__ function?

I made 'Decoder' layer to make Product Item Matrix in Tensorflow.
I've tried to initialize W, bias and make a calculation to get loss using some value.
class Decoder(tf.keras.layers.Layer):
def __init__(self, units=len(counter), input_dim=300):
super(Decoder, self).__init__()
self.trainer = Trainer(vec_dict)
w_init = tf.random_normal_initializer()
self.w = tf.Variable(initial_value=w_init(shape=(units, input_dim), dtype='float32'), trainable=True)
b_init = tf.zeros_initializer()
self.b = tf.Variable(initial_value=b_init(shape=(units,), dtype='float32'), trainable=True)
def call(self, x_play, y_pos, y_neg):
x = self.trainer(x_play)
user_vector = x[..., tf.newaxis, :]
wp = self.w.numpy()[y_pos]
bp = self.b.numpy()[y_pos]
pos_mul = tf.multiply(user_vector, wp)
pos = tf.reduce_sum(pos_mul, axis=-1) + bp
wn = self.w.numpy()[y_neg]
bn = self.b.numpy()[y_neg]
neg_mul = tf.multiply(user_vector, wn)
neg = tf.reduce_sum(neg_mul, axis=-1) + bn
tot = tf.sigmoid(tf.concat([pos, neg], axis=1))
return tot
As you can see this, If I used '.numpy()', it can make me possible to get matrix which has the shape what I want to.
For instance, self.w has (10000, 300) shapes, and y_pos has (256, 100)
and self.w.numpy()[y_pos] has (256, 100, 300)
The problem is that if I ran this code for training, I got these kind of errors:
NotImplementedError: in converted code:
<ipython-input-835-8d5f0b593979>:4 train_step *
predictions = model(x, y_pos, y_neg)
/usr/local/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py:847 __call__
outputs = call_fn(cast_inputs, *args, **kwargs)
<ipython-input-838-b9b896ea4fff>:64 call
wp = self.w.numpy()[y_pos]
/usr/local/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:579 numpy
"numpy() is only available when eager execution is enabled.")
NotImplementedError: numpy() is only available when eager execution is enabled.
Somebody else who let me know what's going on.

Propagating through a custom layer in tensorflow just once

Given a custom layer in tensorflow, is it possible to let the model use it just during one epoch? The layer may just be ignored for all other epochs or simple be an identity.
For example: Given data I would like the layer to simply double the given data. The other layers should may work normally. How would one do that?
def do_stuff(data):
return 2*data
def run_once(data):
return tf.py_func(do_stuff,
[data],
'float32',
stateful=False,
name='I run once')
class CustomLayer(Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
self.trainable = False
super(CustomLayer, self).__init__(**kwargs)
def call(self, x):
res = tf.map_fn(run_once, x)
res.set_shape([x.shape[0],
self.output_dim[1],
self.output_dim[0],
x.shape[-1]])
return res
inputs = Input(shape=(224, 224, 1))
x = Lambda(preprocess_input(x), input_shape=(224, 224, 1), output_shape=(224, 224, 3))
outputs = Dense(1)(x)
model = Model(input=inputs, output=outputs)
output = model(x)
Interesting question. To execute a TF operation just in the first epoch, one could use tf.cond and tf.control_dependencies to check/update the value of a boolean tensor. For example, your custom layer could be implemented as follows:
class CustomLayer(Layer):
def __init__(self, **kwargs):
super(CustomLayer, self).__init__(**kwargs)
def build(self, input_shape):
self.first_epoch = tf.Variable(True)
def call(self, x):
res = tf.cond(self.first_epoch,
true_fn=lambda: run_once(x),
false_fn=lambda: x)
with tf.control_dependencies([res]):
assign_op = self.first_epoch.assign(False)
with tf.control_dependencies([assign_op]):
res = tf.identity(res)
return res
To validate that this layer works as expected, define run_once as:
def run_once(data):
print_op = tf.print('First epoch')
with tf.control_dependencies([print_op]):
out = tf.identity(data)
return out

Keras unable to calculate number of parameters in a Keras Custom Layer

I am building a Keras Custom layer with some Tensorflow support. Before that I wanted to test whether a Convolution2D layer works properly if I write a Keras layer with Tensorflow's conv2d in the call function.
class Convolutional2D(Layer):
def __init__(self, filters=None, kernel_size=None, padding='same', activation='linear', strides=(1,1), name ='Conv2D', **kwargs):
self.filters = filters
self.kernel_size = kernel_size
self.padding = padding
self.activation = activation
self.strides = strides
self.name = name
self.input_spec = [InputSpec(ndim=4)]
super(Convolutional2D, self).__init__(**kwargs)
def call(self, input):
out = tf.layers.conv2d(inputs=input, filters=self.filters, kernel_size=self.kernel_size, strides=self.strides, padding=self.padding,
data_format='channels_last')
return(out)
def compute_output_shape(self, input_shape):
batch_size = input_shape[0]
width = input_shape[1]/self.strides[0]
height = input_shape[2]/self.strides[1]
channels = self.filters
return(batch_size, width, height, channels)
def get_config(self):
config = {'filters': self.filters, 'kernel_size': self.kernel_size, 'padding': self.padding, 'activation':self.activation, 'strides':self.strides,
'name':self.name}
base_config = super(Convolutional2D, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
This compiles properly but when the I use model.summary() it does not calculate the number of parameters for this layer.
What do I have to do so that when I check the total number of parameters of the model the number includes the trainable number of parameters of this layer?
I have found the answer to this problem.
def build(self, input_shape):
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape[channel_axis] is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
input_dim = input_shape[channel_axis]
kernel_shape = self.kernel_size + (input_dim, self.filters)
self.kernel = self.add_weight(shape=kernel_shape,
initializer=self.kernel_initializer,
name='kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
if self.use_bias:
self.bias = self.add_weight(shape=(self.filters,),
initializer=self.bias_initializer,
name='bias',
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
else:
self.bias = None
# Set input spec.
self.input_spec = InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_dim})
self.built = True
The add weights defines the number of parameters which I have not done in my code. But that does not hamper the performance of the model. It works fine except for the fact one cannot get the number of parameters specification.