TensorFlow get stuck after use concatenate layer - tensorflow

I have the next model:
import tensorflow as tf
input1 = tf.keras.layers.Input(shape = (10, 300, 1))
input2 = tf.keras.layers.Input(shape = (24, ))
x = tf.keras.layers.Conv2D(64, (3,3), activation='relu')(input1)
x = tf.keras.layers.MaxPooling2D(2,2)(x)
x = tf.keras.layers.Dropout(0.25)(x)
x = tf.keras.layers.Conv2D(128, (2,2), activation='relu')(x)
x = tf.keras.layers.MaxPooling2D(2,2)(x)
x = tf.keras.layers.Dropout(0.25)(x)
x = tf.keras.layers.Flatten()(x)
x = tf.keras.layers.Dense(512, activation = 'relu')(x)
x = tf.keras.layers.Dropout(0.25)(x)
x = tf.keras.layers.Concatenate()([x, input2])
x = tf.keras.layers.Dense(128, activation = 'relu')(x)
x = tf.keras.layers.Dropout(0.25)(x)
output = tf.keras.layers.Dense(1, activation='sigmoid')(x)
model = tf.keras.models.Model(inputs = [input1,input2], outputs = output)
model.summary()
model.compile(optimizer = 'rmsprop',
loss ='binary_crossentropy',
metrics = ['acc'])
history = model.fit([X_train, X_features], y_train,
batch_size=64,
epochs=100)
But when I try to fit it, get stuck and only appears Epoch 1/100 and nothing more happens even if i let it run for hours. But when I remove the concatenate layer, everything go well. I'm using Google colab. Why is this happening?

Related

Problem with classification of flat-color image and image with object (Keras CNN)

I'm using CNN described below from the Keras tutorial to classify two sets of images.
The first set is just a flat color image, and the second set is pictures with an objects.
But can't make CNN to distiguish classes.
How to tune the CNN to distinguish flat-color images and images with objects on it?
(Originally this CNN was designed to distinguish cats and dogs.
https://keras.io/examples/vision/image_classification_from_scratch/#train-the-model)
Flat-color image:
Images with object in it:
def make_model(input_shape, num_classes):
inputs = keras.Input(shape=input_shape)
# Image augmentation block
x = data_augmentation(inputs)
# Entry block
x = layers.Rescaling(1.0 / 255)(x)
x = layers.Conv2D(32, 3, strides=2, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.Activation("relu")(x)
x = layers.Conv2D(64, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.Activation("relu")(x)
previous_block_activation = x # Set aside residual
for size in [128, 256, 512, 728]:
x = layers.Activation("relu")(x)
x = layers.SeparableConv2D(size, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.Activation("relu")(x)
x = layers.SeparableConv2D(size, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.MaxPooling2D(3, strides=2, padding="same")(x)
# Project residual
residual = layers.Conv2D(size, 1, strides=2, padding="same")(
previous_block_activation
)
x = layers.add([x, residual]) # Add back residual
previous_block_activation = x # Set aside next residual
x = layers.SeparableConv2D(1024, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.Activation("relu")(x)
x = layers.GlobalAveragePooling2D()(x)
if num_classes == 2:
activation = "sigmoid"
units = 1
else:
activation = "softmax"
units = num_classes
x = layers.Dropout(0.5)(x)
outputs = layers.Dense(units, activation=activation)(x)
return keras.Model(inputs, outputs)
model = make_model(input_shape=image_size + (3,), num_classes=2)

('Input has undefined rank:', TensorShape(None)) <- Error in Building ResNet

I am getting an undefined rank error, while building ResNet; I have mentioned a reproducible code below:
Here is my Identity Block:
def IdentityBlock(X, f, filters):
F1, F2, F3 = filters
X_shortcut = X
X = Conv2D(filters = F1, kernel_size = (3, 3), padding = 'valid')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F2, kernel_size = (f, f), padding = 'same')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')(X)
X = BatchNormalization()(X)
X = Add()([X, X_shortcut])
X = Activation('relu')(X)
return X
Here is my Convolution Block
def ConvBlock(X, f, filters):
F1, F2, F3 = filters
X_shortcut = X
X = Conv2D(filters = F1, kernel_size = (3, 3), padding = 'valid')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F2, kernel_size = (f, f), padding = 'same')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')(X)
X = BatchNormalization()(X)
X_shortcut = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')
X_shortcut = BatchNormalization()(X_shortcut)
X = Add()([X, X_shortcut])
X = Activation('relu')(X)
return X
And my resnet model:
def ResNet(input_shape = (224, 224, 3)):
X_input = Input(input_shape)
X = Conv2D(64, (7, 7))(X_input)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = MaxPooling2D((3, 3))(X)
X = ConvBlock(X, f = 3, filters = [64, 64, 128])
X = IdentityBlock(X, 3, filters = [64, 64, 128])
X = IdentityBlock(X, 3, filters = [64, 64, 128])
X = ConvBlock(X, f = 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
X = MaxPooling2D((2, 2))(X)
model = Model(input = X_input, output = X)
return model
When I call RenNet Like this:
base_model = ResNet50(input_shape=(224, 224, 3))
I get the following error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-22-f81766b0bb4e> in <module>
----> 1 base_model = ResNet50(input_shape=(224, 224, 3))
<ipython-input-21-309ae6f634f4> in ResNet50(input_shape)
7 X = MaxPooling2D((3, 3))(X)
8
----> 9 X = ConvBlock(X, f = 3, filters = [64, 64, 128])
10 X = IdentityBlock(X, 3, filters = [64, 64, 128])
11 X = IdentityBlock(X, 3, filters = [64, 64, 128])
<ipython-input-20-aeab857c5df6> in ConvBlock(X, f, filters)
16
17 X_shortcut = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')
---> 18 X_shortcut = BatchNormalization()(X_shortcut)
19
20 X = Add()([X, X_shortcut])
.
.
.
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/normalization.py in build(self, input_shape)
296 input_shape = tensor_shape.TensorShape(input_shape)
297 if not input_shape.ndims:
--> 298 raise ValueError('Input has undefined rank:', input_shape)
299 ndims = len(input_shape)
300
ValueError: ('Input has undefined rank:', TensorShape(None))

You are passing the Conv2D operation to the BatchNormalization Layer instead of a tensor. Try changing
X_shortcut = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')
to
X_shortcut = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')(X_shortcut)

These were the steps I followed to reach the final solution:
1 - the add function works like add([x,x_shortcut]) , im not too sure about the other way.
2 - Made a change in the variable, instead of using X in the IdentityBlock Function I changed it to input_
3 - The shape of the inputs in the add layer we not in the desired way, so I
played around with the kernel size and stride values (not too good at these)
to make that layer work. Did this for the ConvBlock and IdentityBlock.
4 - Final small error was when calling the keras model its Model(inputs = , outputs = ), input and output don't work.
def IdentityBlock(input_, f, filters):
F1, F2, F3 = filters
X = Conv2D(filters = F1, kernel_size = (1, 1), padding = 'valid') (input_)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F2, kernel_size = (f, f), padding = 'same')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F3, kernel_size = (1, 1), padding = 'same')(X)
X = BatchNormalization()(X)
X = add([X, input_])
X = Activation('relu')(X)
return X
def ConvBlock(X, f, filters):
F1, F2, F3 = filters
X_shortcut = X
X = Conv2D(filters = F1, kernel_size = (1, 1),strides = (1,1), padding
= 'valid')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F2, kernel_size = (f, f), padding = 'same')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F3, kernel_size = (1, 1), padding = 'same')(X)
X = BatchNormalization()(X)
X_shortcut = Conv2D(filters = F3,padding = 'same' , kernel_size = (1,1)
, strides = (1,1))(X_shortcut)
X_shortcut = BatchNormalization()(X_shortcut)
X = add([X, X_shortcut])
X = Activation('relu')(X)
return X
def ResNet(input_shape = (224, 224, 3)):
X_input = Input(input_shape)
X = ZeroPadding2D((3,3))(X_input)
X = Conv2D(64, (7, 7) , strides = (2,2))(X_input)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = ZeroPadding2D((1,1))(X)
X = MaxPooling2D((3, 3) , strides = (2,2))(X)
X = ConvBlock(X, f = 3, filters = [64, 64, 128])
X = IdentityBlock(X, 3, filters = [64, 64, 128])
X = IdentityBlock(X, 3, filters = [64, 64, 128])
X = ConvBlock(X, f = 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
model = Model(inputs = X_input, outputs = X)
I hope it helped!

How to add a convolutional layer in the shortcut connection using functional API in Keras?

I am trying to implement a custom CNN model like the one shown in the attached figure using the functional API in Keras. I have written the code to implement the main branch but having issues in adding the 1x1 convolutions as a shortcut connection. The shortcut convolution is added for each pair of convolution blocks, just before the max-pooling layer. The code is given below:
input_shape = (256,256,3)
model_input = Input(shape=input_shape)
print(model_input)
def custom_cnn(model_input):
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(model_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(32, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(32, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(48, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(48, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(64, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(80, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(80, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = GlobalAveragePooling2D()(x)
x = Dense(num_classes, activation='softmax')(x)
model = Model(inputs=model_input, outputs=x, name='custom_cnn')
return model
#instantiate the model
custom_model = custom_cnn(model_input)
#display model summary
custom_model.summary()

here the implementation of the residual blocks inside your network following the schema:
num_classes = 3
input_shape = (256,256,3)
model_input = Input(shape=input_shape)
def custom_cnn(model_input):
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(model_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
xx = Conv2D(16, (1,1), strides= (4,4), padding = 'same')(model_input)
x = Add()([x,xx])
xx = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(32, (3, 3), strides = (1,1), padding = 'same')(xx)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(32, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
xx = Conv2D(32, (1,1), strides= (1,1), padding = 'same')(xx)
x = Add()([x,xx])
xx = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(48, (3, 3), strides = (1,1), padding = 'same')(xx)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(48, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
xx = Conv2D(48, (1,1), strides= (1,1), padding = 'same')(xx)
x = Add()([x,xx])
xx = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(64, (3, 3), strides = (1,1), padding = 'same')(xx)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
xx = Conv2D(64, (1,1), strides= (1,1), padding = 'same')(xx)
x = Add()([x,xx])
xx = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(80, (3, 3), strides = (1,1), padding = 'same')(xx)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(80, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
xx = Conv2D(80, (1,1), strides= (1,1), padding = 'same')(xx)
x = Add()([x,xx])
xx = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = GlobalAveragePooling2D()(xx)
x = Dense(num_classes, activation='softmax')(x)
model = Model(inputs=model_input, outputs=x, name='custom_cnn')
return model
#instantiate the model
custom_model = custom_cnn(model_input)
#display model summary
custom_model.summary()

You can merge multiple layers with keras.layers.concatenate function so if you have 2 "parallel" layers x and y, you can do merge them into layer z like this:
z = layers.concatenate([x, y])
In your code, it would look something like this:
First, create the one "parallel" layer (note that both layers x and later y will be applied on the same layer starting with the model_input layer)
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(model_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
Then you would create the second "parallel" layer:
y = Conv2D(...)(model_input)
Merge them
z = layers.concatenate([x, y])
and apply max pooling on the z (on the layer that is the result of the concatenation)
z = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(z)
And again, create two branches, both taking z as an input (where z is equivalent to the model_input from the previous iteration) and repeat for each of these blocks where you need to apply concatenation.

Add extra kernel to a CNN layer while maintaining the weights learned for the other kernels

I'm training a simple feed forward conv neural network on the cifar10 dataset. After running a few epochs, I want to increase the kernel count in the 2nd conv layer from 16 to some number k.
How do I do this while keeping the trained weights for the other kernels and layers in the model intact?
def conv_layer(inp, fltrs):
inp = Conv2D(filters = fltrs, kernel_size = 3, strides = 1, padding = 'valid')(inp)
inp = BatchNormalization()(inp)
inp = Dropout(0.25)(inp)
inp = Activation('relu')(inp)
return inp
inp = Input(shape = (32, 32, 3))
x0 = conv_layer(inp, 8)
x1 = conv_layer(x0, 16)
x2 = MaxPooling2D(pool_size= 2, strides=None, padding='valid', data_format=None)(x1)
x3 = conv_layer(x2, 32)
x4 = conv_layer(x3, 48)
x5 = conv_layer(x4, 64)
x6 = MaxPooling2D(pool_size= 2, strides=None, padding='valid', data_format=None)(x5)
x7 = Flatten()(x6)
x8 = Dense(512)(x7)
x9 = BatchNormalization()(x8)
x10 = Dropout(0.25)(x9)
x11 = Activation('relu')(x10)
x12 = Dense(num_classes, activation='softmax')(x11)
model = Model(inputs = [inp], outputs = [x12])

Tensorflow - ValueError: Cannot feed value of shape

I have 19 input integer features. Output and labels is 1 or 0. I examine MNIST example from tensorflow website.
My code is here :
validation_images, validation_labels, train_images, train_labels = ld.read_data_set()
print "\n"
print len(train_images[0])
print len(train_labels)
import tensorflow as tf
sess = tf.InteractiveSession()
x = tf.placeholder(tf.float32, shape=[None, 19])
y_ = tf.placeholder(tf.float32, shape=[None, 2])
W = tf.Variable(tf.zeros([19,2]))
b = tf.Variable(tf.zeros([2]))
sess.run(tf.initialize_all_variables())
y = tf.nn.softmax(tf.matmul(x,W) + b)
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
start = 0
batch_1 = 50
end = 100
for i in range(1000):
#batch = mnist.train.next_batch(50)
x1 = train_images[start:end]
y1 = train_labels[start:end]
start = start + batch_1
end = end + batch_1
x1 = np.reshape(x1, (-1, 19))
y1 = np.reshape(y1, (-1, 2))
train_step.run(feed_dict={x: x1[0], y_: y1[0]})
I run upper code, I get an error. The compiler says that
% (np_val.shape, subfeed_t.name, str(subfeed_t.get_shape())))
ValueError: Cannot feed value of shape (19,) for Tensor u'Placeholder:0', which has shape '(?, 19)'
How can I handle this error?

Try
train_step.run(feed_dict={x: x1, y_: y1})

You can reshape your feed's value by the following code:
x1 = np.column_stack((x1))
x1 = np.transpose(x1) # if necessary
Thus, the shape of the input value will be (1, 19) instead of (19,)