Using a sequential model generated like this:
def generate_model():
model = Sequential()
model.add(Conv1D(64, kernel_size=10, strides=1,
activation='relu', padding='same',
input_shape=(MAXLENGTH, NAMESPACELENGTH)))
model.add(MaxPooling1D(pool_size=4, strides=2))
model.add(Conv1D(32, 3, activation='relu', padding='same'))
model.add(MaxPooling1D(pool_size=4))
model.add(Flatten())
model.add(Dense(10, activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(loss='mean_squared_error',
optimizer='adam', metrics=['mean_squared_error'])
return model
I want to do Kfold cross-validated modeling. So, I train K models in a loop:
models = []
for ndx_train, ndx_val in kfold.split(X, y):
model = generate_model()
N_train = len(ndx_train)
X_batch = X[ndx_train]
y_batch = y[ndx_train]
model.fit(X_batch, y_batch, epochs=100, verbose=1, steps_per_epoch=10,
validation_data=(X[ndx_val], y[ndx_val]), validation_steps=100)
models.append(model)
Now, I can see when I want each model to stop by looking at the output. I.e. when the validation error increases again. Is it possible to do that easily with pure tf and with this higher level api setup? There is some suggestions using along the lines using tflearn here.
By using EarlyStopping callback:
from tensorflow.keras.callbacks import EarlyStopping
callbacks = [
EarlyStopping(monitor='val_mean_squared_error', patience=2, verbose=1),
]
model.fit(..., callbacks=callbacks)
Related
I want to combine two sequential models for a hybrid model (with Keras 2.6.0). The first model is a succession of dense layer of a set of 4 parameters, and the second is a succession of 2D convolution of an image ((32,32)). The goal is to predict a curve of 128 points.
My actual model:
def get_model_v2(params_shape, img_shape):
params_model = models.Sequential()
params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1'))
params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n2'))
params_model.add(layers.Dense(256, name='Output'))
img_model = models.Sequential()
img_model.add(layers.Input(img_shape, name='InputLayer2'))
img_model.add(layers.Conv2D(64, kernel_size=4, strides=2, padding="same"))
img_model.add(layers.LeakyReLU(alpha=0.2))
img_model.add(layers.Conv2D(16, kernel_size=4, strides=2, padding="same"))
img_model.add(layers.LeakyReLU(alpha=0.2))
img_model.add(layers.Flatten())
concat = tf.keras.layers.concatenate([params_model, img_model])
model = models.Sequential()
model.add(layers.Input(concat, name='InputLayer3'))
model.add(layers.Dense(256, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1'))
model.add(layers.Dense(128, name='Output'))
model.compile(optimizer = 'adam',
loss = 'mse',
metrics = ['mae', 'mse'])
return model
model = get_model_v2 ( (4,), (32, 32, 1) )
My problem is when I have to combine the two models, I don't know what to use, with this "concatenate" example I have an error like: TypeError: 'NoneType' object is not subscriptable. I understand the problem, but I can't find an other solution...
Few issues here,
You are not using params_shape for your params_model (which comes out with an undefined shape).
As you understood, you can't concatenate models with a concatenation layer
The final model needs to through the Functional API
You got a bunch of layers with same name - you cannot have the same name for two layers in the same model
import tensorflow.keras.layers as layers
import tensorflow.keras.models as models
import tensorflow.keras.regularizers as regularizers
import tensorflow as tf
def get_model_v2(params_shape, img_shape):
params_model = models.Sequential()
params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1', input_shape=params_shape))
params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n2'))
params_model.add(layers.Dense(256, name='Output'))
img_model = models.Sequential()
img_model.add(layers.Input(img_shape, name='InputLayer2'))
img_model.add(layers.Conv2D(64, kernel_size=4, strides=2, padding="same"))
img_model.add(layers.LeakyReLU(alpha=0.2))
img_model.add(layers.Conv2D(16, kernel_size=4, strides=2, padding="same"))
img_model.add(layers.LeakyReLU(alpha=0.2))
img_model.add(layers.Flatten())
param_out = params_model.outputs[0]
img_out = img_model.outputs[0]
concat_out = tf.keras.layers.concatenate([param_out, img_out])
full_dense_out = layers.Dense(256, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n3')(concat_out)
final_out = layers.Dense(128, name='Output_final')(full_dense_out)
model = models.Model(inputs=[params_model.inputs, img_model.inputs], outputs=final_out)
model.summary()
model.compile(optimizer = 'adam',
loss = 'mse',
metrics = ['mae', 'mse'])
return model
model = get_model_v2 ( (4,), (32, 32, 1) )
I am dealing with a problem in which network design is such that it requires merging output of one part of the network with a tabular input(other input) data based on a key and training the network further with the merged data. It appeared that there is no way two tensors can be merged based on a key. Hence though of converting tensor to numpy to pandas data and them merging. The merged data would be converted back to tensor and used further in the network. Below is the code for it:
def build_convnet(shape=(112, 112, 1)):
from keras.layers import Conv2D, BatchNormalization, MaxPool2D, GlobalMaxPool2D
momentum = .9
model = keras.Sequential()
model.add(Conv2D(64, (3,3), input_shape=shape,
padding='same', activation='relu'))
model.add(Conv2D(64, (3,3), padding='same', activation='relu'))
model.add(BatchNormalization(momentum=momentum))
model.add(MaxPool2D())
model.add(Conv2D(128, (3,3), padding='same', activation='relu'))
model.add(Conv2D(128, (3,3), padding='same', activation='relu'))
model.add(BatchNormalization(momentum=momentum))
model.add(MaxPool2D())
model.add(Conv2D(256, (3,3), padding='same', activation='relu'))
model.add(Conv2D(256, (3,3), padding='same', activation='relu'))
model.add(BatchNormalization(momentum=momentum))
model.add(MaxPool2D())
model.add(Conv2D(512, (3,3), padding='same', activation='relu'))
model.add(Conv2D(512, (3,3), padding='same', activation='relu'))
model.add(BatchNormalization(momentum=momentum))
# flatten...
model.add(GlobalMaxPool2D())
return model
def action_model(shape=(3, 112, 112, 1)):
from keras.layers import TimeDistributed, GRU, Dense, Dropout, Concatenate
# Create our convnet with (224, 224, 3) input shape
convnet = build_convnet(shape[1:])
# then create our final model
model = keras.Sequential()
# add the convnet with (5, 224, 224, 3) shape
model.add(TimeDistributed(convnet, input_shape=shape))
# here, you can also use GRU or LSTM
model.add(GRU(64))
# and finally, we make a decision network
model.add(Dense(1024, activation='relu'))
model.add(Dropout(.5))
model.add(Dense(512, activation='relu'))
model.add(Dropout(.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(.5))
model.add(Dense(64, activation='relu'))
model.add(Dense(4, activation='relu'))
return model
# create the tab_data and cnn_gru models
tab_dt = keras.Input(shape=(trainX.shape[1],))
cnn_gru = action_model(X_train.shape[1:])
# converting tensor to numpy array and merging with a tabular data on a key(Patient)
cnn_gru_np = cnn_gru.output.eval()
cnn_gru_pd = pd.Dataframe(cnn_gru_np, names = ["V1", "V2", "V3", "V4"])
cnn_gru_pd["Patient"] = train_p
tab_dt_np = tab_dt.eval()
tab_dt_pd = pd.Dataframe(tab_dt_np, names = ["Weeks", "Percent", "Age", "Sex_Male", "SmokingStatus_Ex-smoker", "SmokingStatus_Never smoked"])
tab_dt_pd["Patient"] = train_p.numpy()
combinedInput_pd = pd.merge(tab_dt_pd, cnn_gru_pd, on = ["Patient"], how = "left")
combinedInput_pd.drop(["Patient"], axis = 1, inplace = True)
combinedInput_np = np.array(combinedInput_pd)
combinedInput = tf.convert_to_tensor(combinedInput_np)
# being our regression head
x = Dense(8, activation="relu")(combinedInput)
x = Dense(1, activation="relu")(x)
model = Model(inputs=[tab_dt, cnn_gru.input], outputs=x)
I am getting the below error for eval function in the line "cnn_gru_np = cnn_gru.output.eval()"
ValueError: Cannot evaluate tensor u`enter code here`sing `eval()`: No default session is registered. Use `with sess.as_default()` or pass an explicit session to `eval(session=sess)`
Please help with suggesting what is going wrong here.
The reason you're getting a ValueError is that the output of a keras model isn't an eager tensor, and thus does not support eval like that.
Just try
some_model = keras.Sequential([keras.layers.Dense(10, input_shape=(5,))])
print(type(some_model.output))
print(type(tf.zeros((2,))))
some_model.output.eval()
# <class 'tensorflow.python.framework.ops.Tensor'>
# <class 'tensorflow.python.framework.ops.EagerTensor'>
# ValueError
However, there is a bigger problem with your approach: there is no connected computation graph from your models inputs to your models outputs because none of the pandas stuff are tensorflow ops. I.E. even if you were able to use eager tensors, you still wouldn't be able to train your model with automatic differentiation.
You're going to have to specify your entire model in tf I'm afraid.
Maybe you could do the data processing before giving it as input to the model? Then you only need split concat ops to put everything together?
I need to run the following method after every 5K iterations.
def evaluation_matrix(path_true,path_pred):
print(path_true,"\n",path_pred)
true_list_new, pred_list_new = read_from_folder(path_true = path_true , path_pred = path_pred)
try:
scikit_metrix(true_list_new = true_list_new,pred_list_new = pred_list_new)
except:
print("An exception occurred")
I'm hoping to use it as a callback in model.fit_generator function. How to achive this? That is parameter passing + 5K interval?
history = model.fit_generator(generator = myGene, steps_per_epoch=steps_per_epoch, epochs=epoch, verbose = 1, callbacks=[],shuffle=True)
A custom callback is a powerful tool to customize the behavior of a Keras model during training, evaluation, or inference.
Below is an example where we are calculating gradient after every epochs. Similarly you can do more customize with many inbuilt methods. You can find more about it here - https://www.tensorflow.org/guide/keras/custom_callback
Note: I was using tensorflow 1.15.0
# (1) Importing dependency
import tensorflow as tf
import keras
from keras import backend as K
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D
from keras.layers.normalization import BatchNormalization
import numpy as np
np.random.seed(1000)
# (2) Get Data
import tflearn.datasets.oxflower17 as oxflower17
x, y = oxflower17.load_data(one_hot=True)
# (3) Create a sequential model
model = Sequential()
# 1st Convolutional Layer
model.add(Conv2D(filters=96, input_shape=(224,224,3), kernel_size=(11,11), strides=(4,4), padding='valid'))
model.add(Activation('relu'))
# Pooling
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='valid'))
# Batch Normalisation before passing it to the next layer
model.add(BatchNormalization())
# 2nd Convolutional Layer
model.add(Conv2D(filters=256, kernel_size=(11,11), strides=(1,1), padding='valid'))
model.add(Activation('relu'))
# Pooling
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='valid'))
# Batch Normalisation
model.add(BatchNormalization())
# 3rd Convolutional Layer
model.add(Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), padding='valid'))
model.add(Activation('relu'))
# Batch Normalisation
model.add(BatchNormalization())
# 4th Convolutional Layer
model.add(Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), padding='valid'))
model.add(Activation('relu'))
# Batch Normalisation
model.add(BatchNormalization())
# 5th Convolutional Layer
model.add(Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), padding='valid'))
model.add(Activation('relu'))
# Pooling
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='valid'))
# Batch Normalisation
model.add(BatchNormalization())
# Passing it to a dense layer
model.add(Flatten())
# 1st Dense Layer
model.add(Dense(4096, input_shape=(224*224*3,)))
model.add(Activation('relu'))
# Add Dropout to prevent overfitting
model.add(Dropout(0.4))
# Batch Normalisation
model.add(BatchNormalization())
# 2nd Dense Layer
model.add(Dense(4096))
model.add(Activation('relu'))
# Add Dropout
model.add(Dropout(0.4))
# Batch Normalisation
model.add(BatchNormalization())
# 3rd Dense Layer
model.add(Dense(1000))
model.add(Activation('relu'))
# Add Dropout
model.add(Dropout(0.4))
# Batch Normalisation
model.add(BatchNormalization())
# Output Layer
model.add(Dense(17))
model.add(Activation('softmax'))
model.summary()
# (4) Compile
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
epoch_gradient = []
# Define the Required Callback Function
class GradientCalcCallback(tf.keras.callbacks.Callback):
def get_gradient_func(model):
grads = K.gradients(model.total_loss, model.trainable_weights)
inputs = model.model._feed_inputs + model.model._feed_targets + model.model._feed_sample_weights
func = K.function(inputs, grads)
return func
def on_epoch_end(self, epoch, logs=None):
get_gradient = get_gradient_func(model)
grads = get_gradient([x, y, np.ones(len(y))])
epoch_gradient.append(grads)
model.fit(x, y, batch_size=64, epochs= 4, verbose=1, validation_split=0.2, shuffle=True, callbacks=[GradientCalcCallback()])
# (7) Convert to a 2 dimensiaonal array of (epoch, gradients) type
gradient = np.asarray(epoch_gradient)
print("Total number of epochs run:", epoch)
print("Gradient Array has the shape:",gradient.shape)
I have 1D sequences which I want to use as input to a Keras VGG classification model, split in x_train and x_test. For each sequence, I also have custom features stored in feats_train and feats_test which I do not want to input to the convolutional layers, but to the first fully connected layer.
A complete sample of train or test would thus consist of a 1D sequence plus n floating point features.
What is the best way to feed the custom features first to the fully connected layer? I thought about concatenating the input sequence and the custom features, but I do not know how to make them separate inside the model. Are there any other options?
The code without the custom features:
x_train, x_test, y_train, y_test, feats_train, feats_test = load_balanced_datasets()
model = Sequential()
model.add(Conv1D(10, 5, activation='relu', input_shape=(timesteps, 1)))
model.add(Conv1D(10, 5, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.5, seed=789))
model.add(Conv1D(5, 6, activation='relu'))
model.add(Conv1D(5, 6, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.5, seed=789))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5, seed=789))
model.add(Dense(2, activation='softmax'))
model.compile(loss='logcosh', optimizer='adam', metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=batch_size, epochs=20, shuffle=False, verbose=1)
y_pred = model.predict(x_test)
Sequential model is not very flexible. You should look into the functional API.
I would try something like this:
from keras.layers import (Conv1D, MaxPool1D, Dropout, Flatten, Dense,
Input, concatenate)
from keras.models import Model, Sequential
timesteps = 50
n = 5
def network():
sequence = Input(shape=(timesteps, 1), name='Sequence')
features = Input(shape=(n,), name='Features')
conv = Sequential()
conv.add(Conv1D(10, 5, activation='relu', input_shape=(timesteps, 1)))
conv.add(Conv1D(10, 5, activation='relu'))
conv.add(MaxPool1D(2))
conv.add(Dropout(0.5, seed=789))
conv.add(Conv1D(5, 6, activation='relu'))
conv.add(Conv1D(5, 6, activation='relu'))
conv.add(MaxPool1D(2))
conv.add(Dropout(0.5, seed=789))
conv.add(Flatten())
part1 = conv(sequence)
merged = concatenate([part1, features])
final = Dense(512, activation='relu')(merged)
final = Dropout(0.5, seed=789)(final)
final = Dense(2, activation='softmax')(final)
model = Model(inputs=[sequence, features], outputs=[final])
model.compile(loss='logcosh', optimizer='adam', metrics=['accuracy'])
return model
m = network()
I'm running a model using Keras and TensorFlow backend. Everything works perfect:
model = Sequential()
model.add(Dense(dim, input_dim=dim, activation='relu'))
model.add(Dense(200, activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(loss='mse', optimizer='Adam', metrics=['mae'])
history = model.fit(X, Y, epochs=12,
batch_size=100,
validation_split=0.2,
shuffle=True,
verbose=2)
But as soon as I include logger and callbacks so I can log for tensorboard, I get
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'input_layer_input_2' with dtype float and shape [?,1329]...
Here's my code: (and actually, it worked 1 time, the very first time, then ecer since been getting that error)
model = Sequential()
model.add(Dense(dim, input_dim=dim, activation='relu'))
model.add(Dense(200, activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(loss='mse', optimizer='Adam', metrics=['mae'])
logger = keras.callbacks.TensorBoard(log_dir='/tf_logs',
write_graph=True,
histogram_freq=1)
history = model.fit(X, Y,
epochs=12,
batch_size=100,
validation_split=0.2,
shuffle=True,
verbose=2,
callbacks=[logger])
A tensorboard callback uses tf.summary.merge_all function in order to collect all tensors for histogram computations. Because of that - your summary is collecting tensors from previous models not cleared from previous model runs. In order to clear these previous models try:
from keras import backend as K
K.clear_session()
model = Sequential()
model.add(Dense(dim, input_dim=dim, activation='relu'))
model.add(Dense(200, activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(loss='mse', optimizer='Adam', metrics=['mae'])
logger = keras.callbacks.TensorBoard(log_dir='/tf_logs',
write_graph=True,
histogram_freq=1)
history = model.fit(X, Y,
epochs=12,
batch_size=100,
validation_split=0.2,
shuffle=True,
verbose=2,
callbacks=[logger])