I'm learning tensorflow 2.0 from its older versions.
I found tensorflow model is changed Sequential-base from Class-base.
But I want to use Class-base model because it is easy to read for me.
I want to try translate : https://www.tensorflow.org/beta/tutorials/keras/basic_text_classification_with_tfhub
embedding = 'https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1'
hub_layer = hub.KerasLayer(embedding,
input_shape=[],
dtype=tf.string,
trainable=True)
# hub_layer(train_example_batch[:3])
# model = tf.keras.Sequential()
# model.add(hub_layer)
# model.add(tf.keras.layers.Dense(16, activation='relu'))
# model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
class MyModel(keras.Model):
def __init__(self, embedding):
super(MyModel, self).__init__()
self.embedding = embedding
self.d1 = keras.layers.Dense(16, activation='relu')
self.d2 = keras.layers.Dense(1, activation='sigmoid')
def call(self, x):
print(x.shape)
return reduce(lambda x, f: f(x), [x, self.embedding, self.d1, self.d2])
model = MyModel(hub_layer)
I got below error message.
InvalidArgumentError: 2 root error(s) found.
(0) Invalid argument: input must be a vector, got shape: [512,1]
[[{{node my_model_48/keras_layer_7/StatefulPartitionedCall/StatefulPartitionedCall/StatefulPartitionedCall/tokenize/StringSplit}}]]
(1) Invalid argument: input must be a vector, got shape: [512,1]
[[{{node my_model_48/keras_layer_7/StatefulPartitionedCall/StatefulPartitionedCall/StatefulPartitionedCall/tokenize/StringSplit}}]]
[[my_model_48/keras_layer_7/StatefulPartitionedCall/StatefulPartitionedCall/StatefulPartitionedCall/SparseFillEmptyRows/SparseFillEmptyRows/_24]]
0 successful operations.
0 derived errors ignored. [Op:__inference_keras_scratch_graph_303077]
Function call stack:
keras_scratch_graph -> keras_scratch_graph
Why I got this error? And also, please answer whether WE NEED THROWING AWAY Class-base Model?
Here is the right code.
# model = tf.keras.Sequential()
# model.add(hub_layer)
# model.add(tf.keras.layers.Dense(16, activation='relu'))
# model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
class MyModel(keras.Model):
def __init__(self, embedding):
super(MyModel, self).__init__()
self.embedding = embedding
self.d1 = keras.layers.Dense(16, activation='relu')
self.d2 = keras.layers.Dense(1, activation='sigmoid')
def call(self, x):
# tf.sequeeze is needed! because x's dimention is [None, 1]. (1 was inserted without permission ...)
return reduce(lambda x, f: f(x), [x, tf.squeeze, self.embedding, self.d1, self.d2])
model = MyModel(hub_layer)
# model.summary()
model.layers
It is because the officious framework has added extra features on their own...
I don't like this attribute, but I think someone who like tensorflow eager to do it...
Related
I have defined a custom LSTM Layer as follows:
class LSTMModel(tf.keras.Model):
def __init__(self, CNN_model, num_classes):
super().__init__()
self.cnn_model = CNN_model
self.lstm = tf.keras.layers.LSTM(units=64, return_state=True, dropout=0.3)
self.dense = tf.keras.layers.Dense(num_classes, activation="softmax")
def call(self, input):
pass
However, I am unclear what needs too occur in the call function here. I also wrote a generic CNN class like below:
class generic_vns_function(tf.keras.Model):
# Where would we use layer_units here?
def __init__(self, input_shape, layers, layer_units):
super().__init__()
self.convolutions = []
# Dynamically create Convolutional layers and MaxPools
for layer in range(len(layers)):
self.convolutions.append(tf.keras.layers.Conv2D(layer, 3, padding="same",
input_shape=input_shape, activation="relu"))
# Add MaxPooling layer
self.convolutions.append(tf.keras.layers.MaxPooling2D((2,2)))
# Flatten
self.flatten = tf.keras.layers.Flatten()
# Dense layer
self.dense1 = tf.keras.layers.Dense(1024, activation="relu")
def call(self, input):
x = input
for layer in self.convolutions:
x = layer(x)
x = self.flatten(x)
x = self.dense1(x)
return x
but here the required structure makes a lot more sense to me. I am just initializing all of the layers. What do I need to do to initialize my LSTM layers?
You could write it like this:
import tensorflow as tf
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import LSTM
from tensorflow.keras import Model
class LSTMModel(Model):
def __init__(self, num_classes, num_units=64, drop_prob=0.3):
super().__init__()
self.num_classes = num_classes
self.num_units = num_units
self.drop_prob = drop_prob
self.lstm = LSTM(
units=self.num_units,
return_state=True,
dropout=self.drop_prob)
self.dense = Dense(
num_classes,
activation="softmax")
def call(self, x, training=True):
x, *state = self.lstm(x, training=training)
x = self.dense(x)
return x
And then you would use it like:
model = LSTMModel(num_classes=2)
time_series = tf.random.normal((32, 64, 128))
x_pred = model(time_series)
# loss and gradients calculations ...
It is a common tensorflow idom to instantiate layers when initializing a custom layer/model, and then execute their call() methods by passing data through them in your custom call implementation.
I am trying to create a custom layer in tensorflow to output the running maximum of its inputs. The layer has a memory variable and comparison function. I wrote the following
class ComputeMax(tf.keras.layers.Layer):
def __init__(self):
super(ComputeMax, self).__init__()
def build(self, input_shape):
self.maxval = tf.Variable(initial_value=tf.zeros((input_shape)),
trainable=False)
def call(self, inputs):
self.maxval.assign(tf.maximum(inputs, self.maxval))
return self.maxval
my_sum = ComputeMax()
x = tf.ones((1,2))
y = my_sum(x)
print(y.numpy()) # [1, 1]
y = my_sum(x)
print(y.numpy()) # [1, 1]
It works as above. When I try it in a test model:
model = Sequential()
model.add(tf.keras.Input(shape=(2)))
model.add(Dense(1, activation='relu'))
model.add(ComputeMax())
model.compile(optimizer='adam', loss='mse')
I get the error on compile:
ValueError: Cannot convert a partially known TensorShape to a Tensor: (None, 1)
What am I missing?
Actually, the layer needs to know the input neurons from the previous layer, which is the last value in input_shape. You are using input_shape as it is which is actually batch shape, leading to a layer of the shape of batch.
This implementation might help.
class ComputeMax(tf.keras.layers.Layer):
def __init__(self):
super(ComputeMax, self).__init__()
def build(self, input_shape):
self.maxval = tf.Variable(initial_value=tf.zeros((input_shape[-1])),
trainable=False)
def call(self, inputs):
self.maxval.assign(tf.maximum(inputs, self.maxval))
return self.maxval
But probably it won't give you answers with numpy 1d array.
I have model with custom layer. Custom layers is as below
class CustomLayer(tf.keras.layers.Layer):
def __init__(self, loaded_model):
super(CustomLayer, self).__init__()
self.loaded_model = loaded_model
def build(self, input_shape):
pass
def get_config(self):
config = super().get_config().copy()
config.update({
'loaded_model': self.loaded_model
})
return config
def call(self, x):
stacked_output = []
for t in tf.unstack(x, axis=-1):
single_image = tf.expand_dims(t, axis=-1)
for i in range(10):
single_image = self.loaded_model.get_layer(index=i)(single_image)
stacked_output.append(single_image)
output = tf.stack(stacked_output, axis=-1)
# print(output.shape)
return output
Main model with custom layer:
def create_model():
trajectory_input = Input(shape=(n_steps, feature_count), name='trajectory_input')
image_input = Input(shape=(128, 128, n_steps), name='image_input')
x_aware = CustomLayer(loaded_model)(image_input)
x_aware = Reshape((1, 501760))(x_aware)
x = (Dense(32, activation='relu', kernel_regularizer=regularizers.l2(0.001)))(trajectory_input)
x = Reshape((1, 32 * n_steps))(x)
x = concatenate([x, x_aware])
output_regression = (Dense(2, name='main_output'))(x_reg)
adam = Adam(lr=learning_rate)
model = Model(inputs=[trajectory_input, image_input], outputs=output_regression)
model.compile(optimizer=adam, loss='mse', metrics=[euc_dist_1, euc_dist_2])
model.summary()
return model
when I try to save model using model.save("model.h5") initially I ended up with
NotImplementedError: Layers with arguments in __init__ must override `get_config`
but once I include get_config in CustomLayer solved the error.
But now I get
TypeError: can't pickle _thread.RLock objects
I am using pretrained model as a custom layer.
Layers in tensorflow keras have a method build that is used to defer the weights creation to a time when you have seen what the input is going to be. a layer's build method
I have a few questions i have not been able to find the answer of:
here it is said that
If you assign a Layer instance as attribute of another Layer, the outer layer will start tracking the weights of the inner layer.
What does it mean to track the weights of a layer?
The same link also mentions that
We recommend creating such sublayers in the init method (since the sublayers will typically have a build method, they will be built when the outer layer gets built).
Does it mean that while running the build method of child class (self), there will an iteration through all the attributes of self and whichever are found to be subclassed from (instances of) tf.keras.layer.Layer will have their build methods run automatically?
I can run this code:
class Net(tf.keras.Model):
"""A simple linear model."""
def __init__(self):
super(Net, self).__init__()
self.l1 = tf.keras.layers.Dense(5)
def call(self, x):
return self.l1(x)
net = Net()
print(net.variables)
But not this:
class Net(tf.keras.Model):
"""A simple linear model."""
def __init__(self):
super(Net, self).__init__()
self.l1 = tf.keras.layers.Dense(5)
def build(self,input_shape):
super().build()
def call(self, x):
return self.l1(x)
net = Net()
print(net.variables)
why?
I would say the build mentioned means, when you build a self-defined tf.keras.Model for example
net = Net()
then you will get all the tf.keras.layers.Layer objects create in __init__, and being stored in net which is a callable object. In this case, it will become a completed object for TF to train later, this is what it said to track. The next time you call net(inputs) you'll can get your outputs.
Here is a example of Tensorflow self-defined decoder with attention
class BahdanauAttention(tf.keras.layers.Layer):
def __init__(self, units):
super(BahdanauAttention, self).__init__()
self.W1 = tf.keras.layers.Dense(units)
self.W2 = tf.keras.layers.Dense(units)
self.V = tf.keras.layers.Dense(1)
def call(self, query, values):
# query hidden state shape == (batch_size, hidden size)
# query_with_time_axis shape == (batch_size, 1, hidden size)
# values shape == (batch_size, max_len, hidden size)
# we are doing this to broadcast addition along the time axis to calculate the score
query_with_time_axis = tf.expand_dims(query, 1)
# score shape == (batch_size, max_length, 1)
# we get 1 at the last axis because we are applying score to self.V
# the shape of the tensor before applying self.V is (batch_size, max_length, units)
score = self.V(tf.nn.tanh(
self.W1(query_with_time_axis) + self.W2(values)))
# attention_weights shape == (batch_size, max_length, 1)
attention_weights = tf.nn.softmax(score, axis=1)
# context_vector shape after sum == (batch_size, hidden_size)
context_vector = attention_weights * values
context_vector = tf.reduce_sum(context_vector, axis=1)
return context_vector, attention_weights
class Decoder(tf.keras.Model):
def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):
super(Decoder, self).__init__()
self.batch_sz = batch_sz
self.dec_units = dec_units
self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)
self.gru = tf.keras.layers.GRU(self.dec_units,
return_sequences=True,
return_state=True,
recurrent_initializer='glorot_uniform')
self.fc = tf.keras.layers.Dense(vocab_size)
# used for attention
self.attention = BahdanauAttention(self.dec_units)
def call(self, x, hidden, enc_output):
# enc_output shape == (batch_size, max_length, hidden_size)
context_vector, attention_weights = self.attention(hidden, enc_output)
# x shape after passing through embedding == (batch_size, 1, embedding_dim)
x = self.embedding(x)
# x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)
x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)
# passing the concatenated vector to the GRU
output, state = self.gru(x)
# output shape == (batch_size * 1, hidden_size)
output = tf.reshape(output, (-1, output.shape[2]))
# output shape == (batch_size, vocab)
x = self.fc(output)
return x, state, attention_weights
I have tried to put tf.keras.layers.Layer object in call and got really poor outcome, guess that was because if you put it in call then it will be call multiple times while each time a forward-backward propagation happends.
I have a problem with re-writing sequential type of describing to functional.
Using keras==2.1. I have an old code(written 2-3 years ago) of sequential describing:
def Word2VecModel(num_words, embedding_dim, seq_length, dropout_rate):
model = Sequential()
model.add(Embedding(num_words, embedding_dim,
input_length=seq_length, trainable=False))
model.add(LSTM(units=512, return_sequences=True, input_shape=(seq_length, embedding_dim)))
model.add(Dropout(dropout_rate))
model.add(LSTM(units=512, return_sequences=False))
model.add(Dropout(dropout_rate))
model.add(Dense(1024, activation='tanh'))
return model
def img_model(dropout_rate):
model = Sequential()
model.add(Dense(1024, input_shape=(4096,), activation='tanh'))
return model
def vqa_model(num_words, embedding_dim, seq_length, dropout_rate, num_classes):
vgg_model = img_model(dropout_rate)
lstm_model = Word2VecModel(num_words, embedding_dim, seq_length, dropout_rate)
fc_model = Sequential()
fc_model.add(Concatenate([vgg_model, lstm_model]))
fc_model.add(Dropout(dropout_rate))
fc_model.add(Dense(1000, activation='tanh'))
fc_model.add(Dropout(dropout_rate))
fc_model.add(Dense(num_classes, activation='softmax'))
fc_model.compile(optimizer='rmsprop', loss='categorical_crossentropy',
metrics=['accuracy'])
return fc_model
my version of functional describing:
def Word2VecModel(num_words, embedding_dim, seq_length, dropout_rate):
w2v_input = Input((seq_length,))
w2v_embed = Embedding(input_dim=num_words, output_dim=embedding_dim, input_length=seq_length,
trainable=False)(w2v_input)
w2v_lstm1 = LSTM(512, input_shape=(seq_length, embedding_dim),return_sequences=True)(w2v_embed)
w2v_drop1 = Dropout(dropout_rate)(w2v_lstm1)
w2v_lstm2 = LSTM(512, return_sequences=False)(w2v_drop1)
w2v_drop2 = Dropout(dropout_rate)(w2v_lstm2)
w2v_dense = Dense(1024, activation='tanh')(w2v_drop2)
model = Model(w2v_input, w2v_dense)
return model
def img_model(dropout_rate):
img_input = Input((4096,))
img_dense = Dense(1024, activation='tanh')(img_input)
model = Model(img_input, img_dense)
return model
def vqa_model(num_words, embedding_dim, seq_length, dropout_rate, num_classes):
vgg_model = img_model(dropout_rate)
lstm_model = Word2VecModel(num_words, embedding_dim, seq_length, dropout_rate)
fc_concat = Concatenate()([vgg_model.output, lstm_model.output])
fc_drop1 = Dropout(dropout_rate)(fc_concat)
fc_dense1 = Dense(1000, activation='tanh')(fc_drop1)
fc_drop2 = Dropout(dropout_rate)(fc_dense1)
fc_dense2 = Dense(num_classes, activation='softmax')(fc_drop2)
fc_model = Model([vgg_model.input, lstm_model.input], fc_dense2)
fc_model.compile(optimizer='rmsprop', loss='categorical_crossentropy',
metrics=['accuracy'])
return fc_model
First of all, I can't edit the old one to make it executable.
Secondly, when I trying to load weights of trained sequential type to functional I've got the error:
ValueError: Layer #0 (named "embedding_1" in the current model) was
found to correspond to layer dense_1 in the save file. However the
new layer embedding_1 expects 1 weights, but the saved weights have 2
elements.
So, I just want to have a possibility to load old weights to a new functional nn. Is it possible?
it's impossible because the old code was written on keras==1.x