I have two functional Keras models in the same level (same input and output shape), one of them is pre-trained, I would like to combine them horizontally and then retrain the whole model. I mean I want to initialize the pretrained with its weights and the other one randomly. How can I horizontally combine them by adding them in branches (not concatenate)?
def define_model_a(input_shape, initializer, outputs = 1):
input_layer = Input(shape=(input_shape))
# first path
path10 = input_layer
path11 = Conv1D(filters=1, kernel_size=3, strides=1, padding="same", use_bias = True, kernel_initializer=initializer)(path10)
path12 = Lambda(lambda x: abs(x))(path11)
output = Add()([path10, path12])
define_model_a = Model(inputs=input_layer, outputs=output)
define_model_a._name = 'model_a'
return define_model_a
def define_model_b(input_shape, initializer, outputs = 1):
input_layer = Input(shape=(input_shape))
# first path
path10 = input_layer
path11 = Conv1D(filters=1, kernel_size=3, strides=1, padding="same", use_bias = True, kernel_initializer=initializer)(path10)
path12 = ReLU()(path11)
path13 = Dense(1, use_bias = True)(path12)
output = path13
define_model_b = Model(inputs=input_layer, outputs=output)
define_model_b._name = 'model_b'
return define_model_b
def define_merge_interpretation()
????
????
output = Add()(model_a, model_b)
model = Model(inputs=input_layer, outputs=output)
return model
initializer = tf.keras.initializers.HeNormal()
model_a = define_model_a(input_shape, initializer, outputs = 1)
model_b = define_model_b(input_shape, initializer, outputs = 1)
model_a.load_weights(load_path)
merge_interpretation = def merge_interprettation( )
history = merge_interpretation.fit(......
As reference, I am looking for a final structure like this in the image, but with some pretrained branches.
Related
I am trying to create a design that would look something like this:
Right now, I am trying using the following code but what I need is the output of the second hidden layer in the 15th position of the next layer. In my case, it is being added to the 31st position using this code.
inputs = Input(shape=(30,), name='first_input')
hn = Dense(4, activation='relu')(inputs)
output = Dense(1, activation='linear')(hn)
first_model = Model(inputs=inputs, outputs=output)
second_input = Input(shape=(30,), name='second_input')
from_first_model = first_model.output
merge_layer = concatenate([second_input, from_first_model ])
hn = Dense(4, activation="relu")(merge_layer)
dnn_op_layer = Dense(1, activation='linear')(hn)
model_b = Model(inputs=[second_input, first_model.input], outputs=dnn_op_layer)
This should work - slicing the tensor into the two parts and then concatenating the two parts with the output from the second hidden layer.
test = tf.convert_to_tensor(np.random.rand(30), dtype=tf.float32)
test2 = tf.convert_to_tensor([100], dtype=tf.float32)
# create layer 3 as desired
temp, temp2 = test[:15], test[15:]
layer3 = tf.concat([temp, test2, temp2], axis=0)
edit - The error you were getting might have been from using the functional api. Additionally the previous example assumes the input has a shape (30,) when really the input should have a shape like (1, 30) to be consistent with tensorflow.
class model(tf.keras.Model):
def __init__(self):
super().__init__()
self.hidden1 = tf.keras.layers.Dense(4, activation='relu')
self.hidden2 = tf.keras.layers.Dense(1)
self.hidden4 = tf.keras.layers.Dense(4, activation='relu')
self.hidden5 = tf.keras.layers.Dense(1)
def call(self, inputs):
x = self.hidden1(inputs)
x = self.hidden2(x)
temp, temp2 = inputs[:,:15], inputs[:,15:]
layer3 = tf.concat([temp, x, temp2], axis=1)
x = self.hidden4(layer3)
x = self.hidden5(x)
return x
# test
mymodel = model()
inputs = tf.convert_to_tensor(np.random.rand(1,30), dtype=tf.float32)
mymodel(inputs)
I was able to create a siamese network similar to :
https://github.com/aspamers/siamese/
The problem happens if I try to add a third model as an input to the head of my network.
I will get the following error :
ValueError: Shape must be rank 2 but is rank 3 for '{{node head/concatenate/concat}} = ConcatV2[N=3, T=DT_FLOAT, Tidx=DT_INT32](simple/Identity, simple_1/Identity, different/Identity, head/concatenate/concat/axis)' with input shapes: [?,?], [?,?], [?,?,1], [].
Here is the code below, one thing I am not comfortable with is the line processed_a = base_model1(input_a) and what it does even after checking the Keras model doc. I understand that if I don't do it I cannot have the desired shape and provide the necessary inputs to the final network.
Note that if I replace the code with what is comment and just use a pure siamese network it will work ok.
Any idea what needs to be changed to resolve the above error and what base_model1(input_a) does.
import numpy as np
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.python.keras.layers import *
def getModel1(input_shape):
model_input = Input(shape=input_shape)
layer = layers.Dense(32, activation='relu')(model_input)
layer = layers.Flatten()(layer)
return tf.keras.Model(inputs=model_input, outputs= layer, name="simple")
def getModel3(input_shape):
model_input = Input(shape=input_shape)
layer = layers.Dense(1, activation='relu')(model_input)
return tf.keras.Model(inputs=model_input, outputs=layer, name="different")
def outputModel(models):
inputs = []
for model in models:
inputs.append(Input(shape=model.output_shape))
layer = layers.Concatenate()(inputs)
layer = layers.Dense(1)(layer)
return tf.keras.Model(inputs=inputs, outputs=layer, name="head")
dataset = []
inputs1 = []
for i in range(0, 128):
dataset.append([0.0, 1.0, 2.0])
train_dataset1 = np.asarray(dataset)
base_model1 = getModel1(train_dataset1.shape)
dataset3 = [0.0, 1.0, 2.0]
train_dataset3 = np.asarray(dataset3)
base_model3 = getModel3(train_dataset3.shape)
input_a = Input(shape=base_model1.input_shape)
input_b = Input(shape=base_model1.input_shape)
input_c = Input(shape=base_model3.input_shape)
oModel = outputModel([base_model1, base_model1, base_model3])
#oModel = outputModel([base_model1, base_model1])
processed_a = base_model1(input_a)
processed_b = base_model1(input_b)
processed_c = base_model3(input_c)
head = oModel([processed_a, processed_b, processed_c])
model = tf.keras.Model(inputs=[input_a, input_b, input_c], outputs=head, name="model")
#head = oModel([processed_a, processed_b])
#model = tf.keras.Model(inputs=[input_a, input_b], outputs=head, name="model")
optimizer = tf.keras.optimizers.RMSprop(0.001)
model.compile(loss='mse',
optimizer=optimizer,
metrics=['mae', 'mse'])
model.predict([np.asarray([train_dataset1]), np.asarray([train_dataset1]), np.asarray([train_dataset3])])
#model.predict([np.asarray([train_dataset1]), np.asarray([train_dataset1])])
model.fit([np.asarray([train_dataset1]), np.asarray([train_dataset1]), np.asarray([train_dataset3])], np.asarray([1.0]), epochs=1000, validation_split=0, verbose=0, callbacks=[])
#model.fit([np.asarray([train_dataset1]), np.asarray([train_dataset1])], np.asarray([1.0]), epochs=1000, validation_split=0, verbose=0, callbacks=[])
pay attention to the dimensionality that you define when u initialize the model input and output. the first dimension is always the batch size (None) and this can cause u some problem. here the correct example:
def getModel1(input_shape):
model_input = Input(shape=input_shape)
layer = Dense(32, activation='relu')(model_input)
layer = Flatten()(layer)
return Model(inputs=model_input, outputs= layer, name="simple")
def getModel3(input_shape):
model_input = Input(shape=input_shape)
layer = Dense(1, activation='relu')(model_input)
return Model(inputs=model_input, outputs=layer, name="different")
def outputModel(models):
inputs = []
for model in models:
inputs.append(Input(shape=model.output_shape[1:]))
layer = Concatenate()(inputs)
layer = Dense(1)(layer)
return Model(inputs=inputs, outputs=layer, name="head")
base_model1 = getModel1((128,3))
base_model3 = getModel3((3))
input_a = Input(shape=base_model1.input_shape[1:])
input_b = Input(shape=base_model1.input_shape[1:])
input_c = Input(shape=base_model3.input_shape[1:])
oModel = outputModel([base_model1, base_model1, base_model3])
processed_a = base_model1(input_a)
processed_b = base_model1(input_b)
processed_c = base_model3(input_c)
head = oModel([processed_a, processed_b, processed_c])
model = Model(inputs=[input_a, input_b, input_c], outputs=head, name="model")
optimizer = tf.keras.optimizers.RMSprop(0.001)
model.compile(loss='mse',
optimizer=optimizer,
metrics=['mae', 'mse'])
# create dummy data
n_sample = 5
train_dataset1 = np.random.uniform(0,1, (n_sample,128,3))
train_dataset3 = np.random.uniform(0,1, (n_sample,3))
y = np.random.uniform(0,1, n_sample)
model.fit([train_dataset1, train_dataset1, train_dataset3], y, epochs=3)
model.predict([train_dataset1, train_dataset1, train_dataset3]).shape
I have a problem with re-writing sequential type of describing to functional.
Using keras==2.1. I have an old code(written 2-3 years ago) of sequential describing:
def Word2VecModel(num_words, embedding_dim, seq_length, dropout_rate):
model = Sequential()
model.add(Embedding(num_words, embedding_dim,
input_length=seq_length, trainable=False))
model.add(LSTM(units=512, return_sequences=True, input_shape=(seq_length, embedding_dim)))
model.add(Dropout(dropout_rate))
model.add(LSTM(units=512, return_sequences=False))
model.add(Dropout(dropout_rate))
model.add(Dense(1024, activation='tanh'))
return model
def img_model(dropout_rate):
model = Sequential()
model.add(Dense(1024, input_shape=(4096,), activation='tanh'))
return model
def vqa_model(num_words, embedding_dim, seq_length, dropout_rate, num_classes):
vgg_model = img_model(dropout_rate)
lstm_model = Word2VecModel(num_words, embedding_dim, seq_length, dropout_rate)
fc_model = Sequential()
fc_model.add(Concatenate([vgg_model, lstm_model]))
fc_model.add(Dropout(dropout_rate))
fc_model.add(Dense(1000, activation='tanh'))
fc_model.add(Dropout(dropout_rate))
fc_model.add(Dense(num_classes, activation='softmax'))
fc_model.compile(optimizer='rmsprop', loss='categorical_crossentropy',
metrics=['accuracy'])
return fc_model
my version of functional describing:
def Word2VecModel(num_words, embedding_dim, seq_length, dropout_rate):
w2v_input = Input((seq_length,))
w2v_embed = Embedding(input_dim=num_words, output_dim=embedding_dim, input_length=seq_length,
trainable=False)(w2v_input)
w2v_lstm1 = LSTM(512, input_shape=(seq_length, embedding_dim),return_sequences=True)(w2v_embed)
w2v_drop1 = Dropout(dropout_rate)(w2v_lstm1)
w2v_lstm2 = LSTM(512, return_sequences=False)(w2v_drop1)
w2v_drop2 = Dropout(dropout_rate)(w2v_lstm2)
w2v_dense = Dense(1024, activation='tanh')(w2v_drop2)
model = Model(w2v_input, w2v_dense)
return model
def img_model(dropout_rate):
img_input = Input((4096,))
img_dense = Dense(1024, activation='tanh')(img_input)
model = Model(img_input, img_dense)
return model
def vqa_model(num_words, embedding_dim, seq_length, dropout_rate, num_classes):
vgg_model = img_model(dropout_rate)
lstm_model = Word2VecModel(num_words, embedding_dim, seq_length, dropout_rate)
fc_concat = Concatenate()([vgg_model.output, lstm_model.output])
fc_drop1 = Dropout(dropout_rate)(fc_concat)
fc_dense1 = Dense(1000, activation='tanh')(fc_drop1)
fc_drop2 = Dropout(dropout_rate)(fc_dense1)
fc_dense2 = Dense(num_classes, activation='softmax')(fc_drop2)
fc_model = Model([vgg_model.input, lstm_model.input], fc_dense2)
fc_model.compile(optimizer='rmsprop', loss='categorical_crossentropy',
metrics=['accuracy'])
return fc_model
First of all, I can't edit the old one to make it executable.
Secondly, when I trying to load weights of trained sequential type to functional I've got the error:
ValueError: Layer #0 (named "embedding_1" in the current model) was
found to correspond to layer dense_1 in the save file. However the
new layer embedding_1 expects 1 weights, but the saved weights have 2
elements.
So, I just want to have a possibility to load old weights to a new functional nn. Is it possible?
it's impossible because the old code was written on keras==1.x
I have built CNN model by using the principle of "Model Sublclassing" in Keras. Here is the class which represents my model:
class ConvNet(tf.keras.Model):
def __init__(self, data_format, classes):
super(ConvNet, self).__init__()
if data_format == "channels_first":
axis = 1
elif data_format == "channels_last":
axis = -1
self.conv_layer1 = tf.keras.layers.Conv2D(filters = 32, kernel_size = 3,strides = (1,1),
padding = "same",activation = "relu")
self.pool_layer1 = tf.keras.layers.MaxPooling2D(pool_size = (2,2), strides = (2,2))
self.conv_layer2 = tf.keras.layers.Conv2D(filters = 64, kernel_size = 3,strides = (1,1),
padding = "same",activation = "relu")
self.pool_layer2 = tf.keras.layers.MaxPooling2D(pool_size = (2,2), strides = (2,2))
self.conv_layer3 = tf.keras.layers.Conv2D(filters = 128, kernel_size = 5,strides = (1,1),
padding = "same",activation = "relu")
self.pool_layer3 = tf.keras.layers.MaxPooling2D(pool_size = (2,2), strides = (1,1),
padding = "same")
self.flatten = tf.keras.layers.Flatten()
self.dense_layer1 = tf.keras.layers.Dense(units = 512, activation = "relu")
self.dense_layer2 = tf.keras.layers.Dense(units = classes, activation = "softmax")
def call(self, inputs, training = True):
output_tensor = self.conv_layer1(inputs)
output_tensor = self.pool_layer1(output_tensor)
output_tensor = self.conv_layer2(output_tensor)
output_tensor = self.pool_layer2(output_tensor)
output_tensor = self.conv_layer3(output_tensor)
output_tensor = self.pool_layer3(output_tensor)
output_tensor = self.flatten(output_tensor)
output_tensor = self.dense_layer1(output_tensor)
return self.dense_layer2(output_tensor)
I would like to know how to train it "eagerly", and by that I mean avoiding the use of compile and fit methods.
I am not sure how to exactly construct the training loop. I understand that I must perform tf.GradientTape.gradient() function in order to calculate the gradients and then use optimizers.apply_gradients() in order to update my model parameters.
What I do not understand is how can I make predictions with my model in order to get logits and then use them to calculate the loss. If someone could help me with the idea of how to construct the training loop I would really appreciate it.
Eager execution is the imperative programming mode to let developers follow Python's natural control flow. Essentially, you wouldn't need to first create placeholders, computational graphs and then execute them in TensorFlow sessions. You can use automatic differentiation to compute gradients in your training loop:
for i in range(iterations):
with tf.GradientTape() as tape:
logits = model(batch_examples, training = True)
loss = tf.losses.sparse_softmax_cross_entropy(batch_labels, logits)
grads = tape.gradient(loss, model.trainable_variables)
opt.apply_gradients([grads, model.trainable_variables])
This is assuming that the model is of the class Model from Keras. I hope this solves your problem! You should also check out the TensorFlow Guide on Eager Execution.
In CNTK - how can I use several filter sizes on the same layer (e.g. filter sizes 2,3,4,5)?
Following the work done here (link to code in github below(1)), I want to take text, use an embedding layer, apply four different sizes of filters (2,3,4,5), concatenate the results and feed it to a fully connected layer.
Network architecture figure
Keras sample code:
main_input = Input(shape=(100,)
embedding = Embedding(output_dim=32, input_dim=100, input_length=100, dropout=0)(main_input)
conv1 = getconvmodel(2,256)(embedding)
conv2 = getconvmodel(3,256)(embedding)
conv3 = getconvmodel(4,256)(embedding)
conv4 = getconvmodel(5,256)(embedding)
merged = merge([conv1,conv2,conv3,conv4],mode="concat")
def getconvmodel(filter_length,nb_filter):
model = Sequential()
model.add(Convolution1D(nb_filter=nb_filter,
`enter code here`input_shape=(100,32),
filter_length=filter_length,
border_mode='same',
activation='relu',
subsample_length=1))
model.add(Lambda(sum_1d, output_shape=(nb_filter,)))
#model.add(BatchNormalization(mode=0))
model.add(Dropout(0.5))
return model
(1): /joshsaxe/eXposeDeepNeuralNetwork/blob/master/src/modeling/models.py
You can do something like this:
import cntk as C
import cntk.layers as cl
def getconvmodel(filter_length,nb_filter):
#Function
def model(x):
f = cl.Convolution(filter_length, nb_filter, activation=C.relu))(x)
f = C.reduce_sum(f, axis=0)
f = cl.Dropout(0.5) (f)
return model
main_input = C.input_variable(100)
embedding = cl.Embedding(32)(main_input)
conv1 = getconvmodel(2,256)(embedding)
conv2 = getconvmodel(3,256)(embedding)
conv3 = getconvmodel(4,256)(embedding)
conv4 = getconvmodel(5,256)(embedding)
merged = C.splice([conv1,conv2,conv3,conv4])
Or with Sequential() and a lambda:
def getconvmodel(filter_length,nb_filter):
return Sequential([
cl.Convolution(filter_length, nb_filter, activation=C.relu)),
lambda f: C.reduce_sum(f, axis=0),
cl.Dropout()
])