combined multiple tensorflow models - tensorflow

Question

i am new to deep learning, and tensorflow. and it seems that all example online are very simple sequential model. but i have a lit bit complex model that i am trying to implement using tensorflow 2.1. so in nutshell, i have two models A, B that i am trying to combined its output and used as input for model C. please refer to diagram i attached to have more clear understanding of model architecture that i am trying to achieve
here is un-completed code so far i am still getting a lot of errors. any suggestions of how i can implement below network. thanks in advance.
def model_a():
model_small = Sequential()
model_small.add(Conv1D(filters=64, kernel_size=50, activation=None, input_shape=(3000, 1)))
model_small.add(MaxPool1D(pool_size=8, strides=8))
model_small.add(Dropout(0.5))
model_small.add(Conv1D(filters=128, kernel_size=8, strides=1, activation=None))
model_small.add(Conv1D(filters=128, kernel_size=8, strides=1, activation=None))
model_small.add(Conv1D(filters=128, kernel_size=8, strides=1, activation=None))
model_small.add(MaxPool1D(pool_size=4, strides=4))
model_small.add(Flatten())
return model_small
#return model_small.add(Flatten())
def model_c():
model = Sequential()
model.add(Bidirectional(LSTM(512)))
model.add(Dropout(0.5))
model.add(Dense(4, activation='sigmoid'))
def model_b():
model_large = Sequential()
# the number of strides in this layer is too large at 50?
model_large.add(Conv1D(filters=64, kernel_size=400, activation=None, input_shape=(3000, 1)))
model_large.add(MaxPool1D(pool_size=4))
model_large.add(Dropout(0.5))
model_large.add(Conv1D(filters=128, kernel_size=6, activation=None))
model_large.add(Conv1D(filters=128, kernel_size=6, activation=None))
model_large.add(Conv1D(filters=128, kernel_size=6, strides=1, activation=None))
model_large.add(MaxPool1D(pool_size=2))
model_large.add(Flatten())
return model_large
#return model_large.add(Flatten())
def final_model():
input1 = model_a()
input2 = model_b()
model_concat = concatenate([input1.output, input2.output], axis=1)
model_concat = Dropout(0.5)(model_concat)
# try to fix model_c here but i don't how
model= Model(inputs=[input1.input, input2.input], outputs=model_concat)
return model
model_2 = final_model()
model_2.compile(
loss=tf.keras.losses.sparse_categorical_crossentropy,
optimizer=tf.keras.optimizers.SGD(learning_rate=0.01),
metrics=['accuracy'] # can add more metrics
)
model_2.fit(x=INPUT, epochs=10)

Answer 1

You're on the right track. You can do it two ways: build one model split into functional parts, or build multiple models and link them together. Functionally they're the same thing, since a layer is a model; a model can be a layer.
I'm going to use some simple Dense layers to represent model_a through model_c. As a caveat, there's a bug that you might run into doing this my way if you use the sequential api, so I'll demonstrate with a functional api, but I assure you it's just as easy to use (the model is just defined after the layers, rather than before).
As one model split into functions:
import tensorflow as tf
def model_a(x):
return tf.keras.layers.Dense(56,name='model_a_whatever')(x) # returning output of layer
def model_b(x):
x = tf.keras.layers.Dense(56,name='model_b_whatever')(x)
return tf.keras.layers.Dense(56,name='model_b_more_whatever')(x)
def model_c(x):
x = tf.keras.layers.Dense(56,name='model_c_whatever')(x)
x = tf.keras.layers.Dense(56,name='model_c_more_whatever')(x)
return tf.keras.layers.Dense(56,name='model_c_even_more_whatever')(x)
# in through the input layer
main_input = tf.keras.layers.Input(shape=(12,34,56),name='INPUT')
# now through functions containing different models' layers
left = model_a(main_input)
right = model_b(main_input)
# concatenate their outputs
concatenated = tf.keras.layers.Concatenate(axis=-1)([left,right])
# now through function containing layers of model c
left = model_c(concatenated)
# and the juke right to a fully connected layer
right = tf.keras.layers.Dense(56,name='FC')(concatenated)
# then add the outputs and apply softmax activation
added = tf.keras.layers.Add(name='add')([left,right])
outputs = tf.keras.layers.Activation('softmax',name='Softmax')(added)
# now define the model
model = tf.keras.models.Model(main_input,outputs) # Model(input layer, final output))
print(model.summary())
tf.keras.utils.plot_model(model, to_file='just_a_model.png')
The diagram will look more cluttered than yours, since all layers will be visible:
As many models joined together:
# as separate models linked together
def build_model_a():
a = tf.keras.layers.Input(shape=(12,34,56),name='model_a_input')
b = tf.keras.layers.Dense(56,name='model_a_whatever')(a) # whatever layers
return tf.keras.models.Model(a,b,name='MODEL_A') # returning model, not just layer output
def build_model_b():
a = tf.keras.layers.Input(shape=(12,34,56),name='model_b_input')
b = tf.keras.layers.Dense(56,name='model_b_whatever')(a)
b = tf.keras.layers.Dense(56,name='model_b_more_whatever')(b)
return tf.keras.models.Model(a,b,name='MODEL_B')
def build_model_c():
a = tf.keras.layers.Input(shape=(12,34,112),name='model_c_input') # axis 2 is doubled because concatenation.
b = tf.keras.layers.Dense(56,name='model_c_whatever')(a)
b = tf.keras.layers.Dense(56,name='model_c_more_whatever')(b)
b = tf.keras.layers.Dense(56,name='model_c_even_more_whatever')(b)
return tf.keras.models.Model(a,b,name='MODEL_C')
# define the main input
main_input = tf.keras.layers.Input(shape=(12,34,56),name='INPUT')
# build the models
model_a = build_model_a()
model_b = build_model_b()
model_c = build_model_c()
# pass input through models a and b
a = model_a(main_input)
b = model_b(main_input)
# concatenate their outputs
ab = tf.keras.layers.Concatenate(axis=-1,name='Concatenate')([a,b])
# pass through model c and fully-connected layer
c = model_c(ab)
d = tf.keras.layers.Dense(56,name='FC')(ab)
# add their outputs and apply softmax activation
add = tf.keras.layers.Add(name="add")([c,d])
outputs = tf.keras.layers.Activation('softmax',name='Softmax')(add)
model = tf.keras.models.Model(main_input,outputs)
print(model.summary())
tf.keras.utils.plot_model(model, to_file='multi_model.png')
Although this is functionally the same network as in the first case, the diagram now matches yours:
Either method will work. As you can see, the first method is just a code-cleanup, really; putting separate data-pipelines into functions for clarity. If you want to get more complicated, like having different loss functions for sub-models and such, then the second method might simplify the process. The bug I mentioned only happens if you use the sequential api with the second method.