How to customise a CNN layers with TensorFlow 2, Feed new inputs at Dense Layers of CNN [duplicate] - tensorflow

I have 1D sequences which I want to use as input to a Keras VGG classification model, split in x_train and x_test. For each sequence, I also have custom features stored in feats_train and feats_test which I do not want to input to the convolutional layers, but to the first fully connected layer.
A complete sample of train or test would thus consist of a 1D sequence plus n floating point features.
What is the best way to feed the custom features first to the fully connected layer? I thought about concatenating the input sequence and the custom features, but I do not know how to make them separate inside the model. Are there any other options?
The code without the custom features:
x_train, x_test, y_train, y_test, feats_train, feats_test = load_balanced_datasets()
model = Sequential()
model.add(Conv1D(10, 5, activation='relu', input_shape=(timesteps, 1)))
model.add(Conv1D(10, 5, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.5, seed=789))
model.add(Conv1D(5, 6, activation='relu'))
model.add(Conv1D(5, 6, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.5, seed=789))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5, seed=789))
model.add(Dense(2, activation='softmax'))
model.compile(loss='logcosh', optimizer='adam', metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=batch_size, epochs=20, shuffle=False, verbose=1)
y_pred = model.predict(x_test)

Sequential model is not very flexible. You should look into the functional API.
I would try something like this:
from keras.layers import (Conv1D, MaxPool1D, Dropout, Flatten, Dense,
Input, concatenate)
from keras.models import Model, Sequential
timesteps = 50
n = 5
def network():
sequence = Input(shape=(timesteps, 1), name='Sequence')
features = Input(shape=(n,), name='Features')
conv = Sequential()
conv.add(Conv1D(10, 5, activation='relu', input_shape=(timesteps, 1)))
conv.add(Conv1D(10, 5, activation='relu'))
conv.add(MaxPool1D(2))
conv.add(Dropout(0.5, seed=789))
conv.add(Conv1D(5, 6, activation='relu'))
conv.add(Conv1D(5, 6, activation='relu'))
conv.add(MaxPool1D(2))
conv.add(Dropout(0.5, seed=789))
conv.add(Flatten())
part1 = conv(sequence)
merged = concatenate([part1, features])
final = Dense(512, activation='relu')(merged)
final = Dropout(0.5, seed=789)(final)
final = Dense(2, activation='softmax')(final)
model = Model(inputs=[sequence, features], outputs=[final])
model.compile(loss='logcosh', optimizer='adam', metrics=['accuracy'])
return model
m = network()

Related

Merge two sequential models on Keras for hybrid model

I want to combine two sequential models for a hybrid model (with Keras 2.6.0). The first model is a succession of dense layer of a set of 4 parameters, and the second is a succession of 2D convolution of an image ((32,32)). The goal is to predict a curve of 128 points.
My actual model:
def get_model_v2(params_shape, img_shape):
params_model = models.Sequential()
params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1'))
params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n2'))
params_model.add(layers.Dense(256, name='Output'))
img_model = models.Sequential()
img_model.add(layers.Input(img_shape, name='InputLayer2'))
img_model.add(layers.Conv2D(64, kernel_size=4, strides=2, padding="same"))
img_model.add(layers.LeakyReLU(alpha=0.2))
img_model.add(layers.Conv2D(16, kernel_size=4, strides=2, padding="same"))
img_model.add(layers.LeakyReLU(alpha=0.2))
img_model.add(layers.Flatten())
concat = tf.keras.layers.concatenate([params_model, img_model])
model = models.Sequential()
model.add(layers.Input(concat, name='InputLayer3'))
model.add(layers.Dense(256, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1'))
model.add(layers.Dense(128, name='Output'))
model.compile(optimizer = 'adam',
loss = 'mse',
metrics = ['mae', 'mse'])
return model
model = get_model_v2 ( (4,), (32, 32, 1) )
My problem is when I have to combine the two models, I don't know what to use, with this "concatenate" example I have an error like: TypeError: 'NoneType' object is not subscriptable. I understand the problem, but I can't find an other solution...
Few issues here,
You are not using params_shape for your params_model (which comes out with an undefined shape).
As you understood, you can't concatenate models with a concatenation layer
The final model needs to through the Functional API
You got a bunch of layers with same name - you cannot have the same name for two layers in the same model
import tensorflow.keras.layers as layers
import tensorflow.keras.models as models
import tensorflow.keras.regularizers as regularizers
import tensorflow as tf
def get_model_v2(params_shape, img_shape):
params_model = models.Sequential()
params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1', input_shape=params_shape))
params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n2'))
params_model.add(layers.Dense(256, name='Output'))
img_model = models.Sequential()
img_model.add(layers.Input(img_shape, name='InputLayer2'))
img_model.add(layers.Conv2D(64, kernel_size=4, strides=2, padding="same"))
img_model.add(layers.LeakyReLU(alpha=0.2))
img_model.add(layers.Conv2D(16, kernel_size=4, strides=2, padding="same"))
img_model.add(layers.LeakyReLU(alpha=0.2))
img_model.add(layers.Flatten())
param_out = params_model.outputs[0]
img_out = img_model.outputs[0]
concat_out = tf.keras.layers.concatenate([param_out, img_out])
full_dense_out = layers.Dense(256, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n3')(concat_out)
final_out = layers.Dense(128, name='Output_final')(full_dense_out)
model = models.Model(inputs=[params_model.inputs, img_model.inputs], outputs=final_out)
model.summary()
model.compile(optimizer = 'adam',
loss = 'mse',
metrics = ['mae', 'mse'])
return model
model = get_model_v2 ( (4,), (32, 32, 1) )

Merging tensors based on a key

I am dealing with a problem in which network design is such that it requires merging output of one part of the network with a tabular input(other input) data based on a key and training the network further with the merged data. It appeared that there is no way two tensors can be merged based on a key. Hence though of converting tensor to numpy to pandas data and them merging. The merged data would be converted back to tensor and used further in the network. Below is the code for it:
def build_convnet(shape=(112, 112, 1)):
from keras.layers import Conv2D, BatchNormalization, MaxPool2D, GlobalMaxPool2D
momentum = .9
model = keras.Sequential()
model.add(Conv2D(64, (3,3), input_shape=shape,
padding='same', activation='relu'))
model.add(Conv2D(64, (3,3), padding='same', activation='relu'))
model.add(BatchNormalization(momentum=momentum))
model.add(MaxPool2D())
model.add(Conv2D(128, (3,3), padding='same', activation='relu'))
model.add(Conv2D(128, (3,3), padding='same', activation='relu'))
model.add(BatchNormalization(momentum=momentum))
model.add(MaxPool2D())
model.add(Conv2D(256, (3,3), padding='same', activation='relu'))
model.add(Conv2D(256, (3,3), padding='same', activation='relu'))
model.add(BatchNormalization(momentum=momentum))
model.add(MaxPool2D())
model.add(Conv2D(512, (3,3), padding='same', activation='relu'))
model.add(Conv2D(512, (3,3), padding='same', activation='relu'))
model.add(BatchNormalization(momentum=momentum))
# flatten...
model.add(GlobalMaxPool2D())
return model
def action_model(shape=(3, 112, 112, 1)):
from keras.layers import TimeDistributed, GRU, Dense, Dropout, Concatenate
# Create our convnet with (224, 224, 3) input shape
convnet = build_convnet(shape[1:])
# then create our final model
model = keras.Sequential()
# add the convnet with (5, 224, 224, 3) shape
model.add(TimeDistributed(convnet, input_shape=shape))
# here, you can also use GRU or LSTM
model.add(GRU(64))
# and finally, we make a decision network
model.add(Dense(1024, activation='relu'))
model.add(Dropout(.5))
model.add(Dense(512, activation='relu'))
model.add(Dropout(.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(.5))
model.add(Dense(64, activation='relu'))
model.add(Dense(4, activation='relu'))
return model
# create the tab_data and cnn_gru models
tab_dt = keras.Input(shape=(trainX.shape[1],))
cnn_gru = action_model(X_train.shape[1:])
# converting tensor to numpy array and merging with a tabular data on a key(Patient)
cnn_gru_np = cnn_gru.output.eval()
cnn_gru_pd = pd.Dataframe(cnn_gru_np, names = ["V1", "V2", "V3", "V4"])
cnn_gru_pd["Patient"] = train_p
tab_dt_np = tab_dt.eval()
tab_dt_pd = pd.Dataframe(tab_dt_np, names = ["Weeks", "Percent", "Age", "Sex_Male", "SmokingStatus_Ex-smoker", "SmokingStatus_Never smoked"])
tab_dt_pd["Patient"] = train_p.numpy()
combinedInput_pd = pd.merge(tab_dt_pd, cnn_gru_pd, on = ["Patient"], how = "left")
combinedInput_pd.drop(["Patient"], axis = 1, inplace = True)
combinedInput_np = np.array(combinedInput_pd)
combinedInput = tf.convert_to_tensor(combinedInput_np)
# being our regression head
x = Dense(8, activation="relu")(combinedInput)
x = Dense(1, activation="relu")(x)
model = Model(inputs=[tab_dt, cnn_gru.input], outputs=x)
I am getting the below error for eval function in the line "cnn_gru_np = cnn_gru.output.eval()"
ValueError: Cannot evaluate tensor u`enter code here`sing `eval()`: No default session is registered. Use `with sess.as_default()` or pass an explicit session to `eval(session=sess)`
Please help with suggesting what is going wrong here.
The reason you're getting a ValueError is that the output of a keras model isn't an eager tensor, and thus does not support eval like that.
Just try
some_model = keras.Sequential([keras.layers.Dense(10, input_shape=(5,))])
print(type(some_model.output))
print(type(tf.zeros((2,))))
some_model.output.eval()
# <class 'tensorflow.python.framework.ops.Tensor'>
# <class 'tensorflow.python.framework.ops.EagerTensor'>
# ValueError
However, there is a bigger problem with your approach: there is no connected computation graph from your models inputs to your models outputs because none of the pandas stuff are tensorflow ops. I.E. even if you were able to use eager tensors, you still wouldn't be able to train your model with automatic differentiation.
You're going to have to specify your entire model in tf I'm afraid.
Maybe you could do the data processing before giving it as input to the model? Then you only need split concat ops to put everything together?

Error when checking target: expected dense_18 to have shape (1,) but got array with shape (10,)

Y_train = to_categorical(Y_train, num_classes = 10)#
random_seed = 2
X_train,X_val,Y_train,Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed)
Y_train.shape
model = Sequential()
model.add(Flatten())
model.add(Dense(64, activation='relu'))
model.add(Dense(128, activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy',metrics=['accuracy'])
model.fit(X_train, Y_train, batch_size = 86, epochs = 3,validation_data = (X_val, Y_val), verbose =2)
I have to classify the MNIST data into 10 classes. I am converting the Y_train into one hot encoded array. I have gone through a number of answers but none have helped. Kindly guide me in this regard as I am a novice in ML and neural network.
It seems there is no need to use model.add(Flatten()) in your first layer. Instead of doing so, you can use a dense layer with a specific input size like: model.add(Dense(64, input_shape=your_input_shape, activation="relu").
To ensure this issue happens because of the layers, you can check whether to_categorical() function works alone with jupyter notebook.
Updated Answer
Before the model, you should reshape your model. In that case 28*28 to 784.
train_images = train_images.reshape((-1, 784))
test_images = test_images.reshape((-1, 784))
I also suggest to normalize the data that could be done by simply dividing the images to 255
After that step you should create your model.
model = Sequential([
Dense(64, activation='relu', input_shape=(784,)),
Dense(64, activation='relu'),
Dense(10, activation='softmax'),
])
Have you noticed input_shape=(784,) That is the shape of your flattened input.
Last step, compiling and fitting.
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'],
)
model.fit(
train_images,
train_labels,
epochs=10,
batch_size=16,
)
What you do is you have just flattened the input layer without feeding the network with an input. That's why you experience an issue. The point is you should manually reshape your inputs and feed forward to the Dense() layers with parameter input_shape

How to do early stopping with tensorflow.models.Sequential()?

Using a sequential model generated like this:
def generate_model():
model = Sequential()
model.add(Conv1D(64, kernel_size=10, strides=1,
activation='relu', padding='same',
input_shape=(MAXLENGTH, NAMESPACELENGTH)))
model.add(MaxPooling1D(pool_size=4, strides=2))
model.add(Conv1D(32, 3, activation='relu', padding='same'))
model.add(MaxPooling1D(pool_size=4))
model.add(Flatten())
model.add(Dense(10, activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(loss='mean_squared_error',
optimizer='adam', metrics=['mean_squared_error'])
return model
I want to do Kfold cross-validated modeling. So, I train K models in a loop:
models = []
for ndx_train, ndx_val in kfold.split(X, y):
model = generate_model()
N_train = len(ndx_train)
X_batch = X[ndx_train]
y_batch = y[ndx_train]
model.fit(X_batch, y_batch, epochs=100, verbose=1, steps_per_epoch=10,
validation_data=(X[ndx_val], y[ndx_val]), validation_steps=100)
models.append(model)
Now, I can see when I want each model to stop by looking at the output. I.e. when the validation error increases again. Is it possible to do that easily with pure tf and with this higher level api setup? There is some suggestions using along the lines using tflearn here.
By using EarlyStopping callback:
from tensorflow.keras.callbacks import EarlyStopping
callbacks = [
EarlyStopping(monitor='val_mean_squared_error', patience=2, verbose=1),
]
model.fit(..., callbacks=callbacks)

How to connect RNN at the end of a CNN to use to train video frames? [closed]

Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 4 years ago.
Improve this question
I'm trying to classify a video as image classification thus to use the frames as the classified method. But i have no idea how to code it out.
I'm using Inception ResNet as my CNN but don't know any RNN or how to use them.
this is ML_machine, here is what I wanted to show you,
this is an implementation of a CNN to classify the mnist data, it is not mine and come from here
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras import backend as K
batch_size = 128
num_classes = 10
epochs = 12
# input image dimensions
img_rows, img_cols = 28, 28
# the data, split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
if K.image_data_format() == 'channels_first':
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
else:
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),
activation='relu',
input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
to convert this CNN followed by a fully connected layer into a CNN into RNN, simply change the line
model.add(Dense(num_classes, activation='softmax'))
into
model.add(SimpleRNN(num_classes, activation='softmax'))
(of course you need to import it)
you might have to change the input dimension of your network and/or TimeDistribute the whole CNN part, I had trouble in some version of tensorflow and others not
EDIT:
I encountered some problem on my own with the code I gave you, it is harder than I thought because of the dimensions to end a CNN network with a recurrent one, here is how I managed to do it:
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=in_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
# NO MORE POOLING
model.add(Dropout(0.25))
# Reshape with the first argument being the number of filter in your last conv layer
model.add(Reshape((64, -1)))
# Just write this Permute after, its complicated why
model.add(Permute((2, 1)))
# it can also be an LSTM
model.add(SimpleRNN(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
EDIT2, dummy example of a simple fully connected NN in keras:
trng_input = np.random.uniform(size=(1000, 4))
trng_output = np.column_stack([np.sin(trng_input).sum(axis=1), np.cos(trng_input).sum(axis=1)])
model = Sequential()
model.add(Dense(6, input_shape=trng_input.shape, activation='relu'))
model.add(Dense(2, activation='sigmoid'))
model.compile(loss='MSE', optimizer=keras.optimizer.Adam(), metrics=['accuracy'])