How to give input to the model if not a numpy array?
def createmodel():
myInput = Input(shape=(96, 96, 3))
x = ZeroPadding2D(padding=(3, 3), input_shape=(96, 96, 3))(myInput)
x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn1')(x)
x = Activation('relu')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = MaxPooling2D(pool_size=3, strides=2)(x)
x = Lambda(LRN2D, name='lrn_1')(x)
x = Conv2D(64, (1, 1), name='conv2')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn2')(x)
x = Activation('relu')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = Conv2D(192, (3, 3), name='conv3')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn3')(x)
x = Activation('relu')(x)
x = Lambda(LRN2D, name='lrn_2')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = MaxPooling2D(pool_size=3, strides=2)(x)
# Inception3a
inception_3a_3x3 = Conv2D(96, (1, 1), name='inception_3a_3x3_conv1')(x)
inception_3a_3x3 = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3a_3x3_bn1')(inception_3a_3x3)
inception_3a_3x3 = Activation('relu')(inception_3a_3x3)
inception_3a_3x3 = ZeroPadding2D(padding=(1, 1))(inception_3a_3x3)
inception_3a_3x3 = Conv2D(128, (3, 3), name='inception_3a_3x3_conv2')(inception_3a_3x3)
inception_3a_3x3 = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3a_3x3_bn2')(inception_3a_3x3)
inception_3a_3x3 = Activation('relu')(inception_3a_3x3)
inception_3a_5x5 = Conv2D(16, (1, 1), name='inception_3a_5x5_conv1')(x)
inception_3a_5x5 = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3a_5x5_bn1')(inception_3a_5x5)
inception_3a_5x5 = Activation('relu')(inception_3a_5x5)
inception_3a_5x5 = ZeroPadding2D(padding=(2, 2))(inception_3a_5x5)
inception_3a_5x5 = Conv2D(32, (5, 5), name='inception_3a_5x5_conv2')(inception_3a_5x5)
inception_3a_5x5 = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3a_5x5_bn2')(inception_3a_5x5)
inception_3a_5x5 = Activation('relu')(inception_3a_5x5)
inception_3a_pool = MaxPooling2D(pool_size=3, strides=2)(x)
inception_3a_pool = Conv2D(32, (1, 1), name='inception_3a_pool_conv')(inception_3a_pool)
inception_3a_pool = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3a_pool_bn')(inception_3a_pool)
inception_3a_pool = Activation('relu')(inception_3a_pool)
inception_3a_pool = ZeroPadding2D(padding=((3, 4), (3, 4)))(inception_3a_pool)
inception_3a_1x1 = Conv2D(64, (1, 1), name='inception_3a_1x1_conv')(x)
inception_3a_1x1 = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3a_1x1_bn')(inception_3a_1x1)
inception_3a_1x1 = Activation('relu')(inception_3a_1x1)
inception_3a = concatenate([inception_3a_3x3, inception_3a_5x5, inception_3a_pool, inception_3a_1x1], axis=3)
# Inception3b
inception_3b_3x3 = Conv2D(96, (1, 1), name='inception_3b_3x3_conv1')(inception_3a)
inception_3b_3x3 = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3b_3x3_bn1')(inception_3b_3x3)
inception_3b_3x3 = Activation('relu')(inception_3b_3x3)
inception_3b_3x3 = ZeroPadding2D(padding=(1, 1))(inception_3b_3x3)
inception_3b_3x3 = Conv2D(128, (3, 3), name='inception_3b_3x3_conv2')(inception_3b_3x3)
inception_3b_3x3 = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3b_3x3_bn2')(inception_3b_3x3)
inception_3b_3x3 = Activation('relu')(inception_3b_3x3)
inception_3b_5x5 = Conv2D(32, (1, 1), name='inception_3b_5x5_conv1')(inception_3a)
inception_3b_5x5 = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3b_5x5_bn1')(inception_3b_5x5)
inception_3b_5x5 = Activation('relu')(inception_3b_5x5)
inception_3b_5x5 = ZeroPadding2D(padding=(2, 2))(inception_3b_5x5)
inception_3b_5x5 = Conv2D(64, (5, 5), name='inception_3b_5x5_conv2')(inception_3b_5x5)
inception_3b_5x5 = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3b_5x5_bn2')(inception_3b_5x5)
inception_3b_5x5 = Activation('relu')(inception_3b_5x5)
inception_3b_pool = Lambda(lambda x: x**2, name='power2_3b')(inception_3a)
inception_3b_pool = AveragePooling2D(pool_size=(3, 3), strides=(3, 3))(inception_3b_pool)
inception_3b_pool = Lambda(lambda x: x*9, name='mult9_3b')(inception_3b_pool)
inception_3b_pool = Lambda(lambda x: K.sqrt(x), name='sqrt_3b')(inception_3b_pool)
inception_3b_pool = Conv2D(64, (1, 1), name='inception_3b_pool_conv')(inception_3b_pool)
inception_3b_pool = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3b_pool_bn')(inception_3b_pool)
inception_3b_pool = Activation('relu')(inception_3b_pool)
inception_3b_pool = ZeroPadding2D(padding=(4, 4))(inception_3b_pool)
inception_3b_1x1 = Conv2D(64, (1, 1), name='inception_3b_1x1_conv')(inception_3a)
inception_3b_1x1 = BatchNormalization(axis=3, epsilon=0.00001, name='inception_3b_1x1_bn')(inception_3b_1x1)
inception_3b_1x1 = Activation('relu')(inception_3b_1x1)
inception_3b = concatenate([inception_3b_3x3, inception_3b_5x5, inception_3b_pool, inception_3b_1x1], axis=3)
# Inception3c
inception_3c_3x3 = utils.conv2d_bn(inception_3b,
layer='inception_3c_3x3',
cv1_out=128,
cv1_filter=(1, 1),
cv2_out=256,
cv2_filter=(3, 3),
cv2_strides=(2, 2),
padding=(1, 1))
inception_3c_5x5 = utils.conv2d_bn(inception_3b,
layer='inception_3c_5x5',
cv1_out=32,
cv1_filter=(1, 1),
cv2_out=64,
cv2_filter=(5, 5),
cv2_strides=(2, 2),
padding=(2, 2))
inception_3c_pool = MaxPooling2D(pool_size=3, strides=2)(inception_3b)
inception_3c_pool = ZeroPadding2D(padding=((0, 1), (0, 1)))(inception_3c_pool)
inception_3c = concatenate([inception_3c_3x3, inception_3c_5x5, inception_3c_pool], axis=3)
#inception 4a
inception_4a_3x3 = utils.conv2d_bn(inception_3c,
layer='inception_4a_3x3',
cv1_out=96,
cv1_filter=(1, 1),
cv2_out=192,
cv2_filter=(3, 3),
cv2_strides=(1, 1),
padding=(1, 1))
inception_4a_5x5 = utils.conv2d_bn(inception_3c,
layer='inception_4a_5x5',
cv1_out=32,
cv1_filter=(1, 1),
cv2_out=64,
cv2_filter=(5, 5),
cv2_strides=(1, 1),
padding=(2, 2))
inception_4a_pool = Lambda(lambda x: x**2, name='power2_4a')(inception_3c)
inception_4a_pool = AveragePooling2D(pool_size=(3, 3), strides=(3, 3))(inception_4a_pool)
inception_4a_pool = Lambda(lambda x: x*9, name='mult9_4a')(inception_4a_pool)
inception_4a_pool = Lambda(lambda x: K.sqrt(x), name='sqrt_4a')(inception_4a_pool)
inception_4a_pool = utils.conv2d_bn(inception_4a_pool,
layer='inception_4a_pool',
cv1_out=128,
cv1_filter=(1, 1),
padding=(2, 2))
inception_4a_1x1 = utils.conv2d_bn(inception_3c,
layer='inception_4a_1x1',
cv1_out=256,
cv1_filter=(1, 1))
inception_4a = concatenate([inception_4a_3x3, inception_4a_5x5, inception_4a_pool, inception_4a_1x1], axis=3)
#inception4e
inception_4e_3x3 = utils.conv2d_bn(inception_4a,
layer='inception_4e_3x3',
cv1_out=160,
cv1_filter=(1, 1),
cv2_out=256,
cv2_filter=(3, 3),
cv2_strides=(2, 2),
padding=(1, 1))
inception_4e_5x5 = utils.conv2d_bn(inception_4a,
layer='inception_4e_5x5',
cv1_out=64,
cv1_filter=(1, 1),
cv2_out=128,
cv2_filter=(5, 5),
cv2_strides=(2, 2),
padding=(2, 2))
inception_4e_pool = MaxPooling2D(pool_size=3, strides=2)(inception_4a)
inception_4e_pool = ZeroPadding2D(padding=((0, 1), (0, 1)))(inception_4e_pool)
inception_4e = concatenate([inception_4e_3x3, inception_4e_5x5, inception_4e_pool], axis=3)
#inception5a
inception_5a_3x3 = utils.conv2d_bn(inception_4e,
layer='inception_5a_3x3',
cv1_out=96,
cv1_filter=(1, 1),
cv2_out=384,
cv2_filter=(3, 3),
cv2_strides=(1, 1),
padding=(1, 1))
inception_5a_pool = Lambda(lambda x: x**2, name='power2_5a')(inception_4e)
inception_5a_pool = AveragePooling2D(pool_size=(3, 3), strides=(3, 3))(inception_5a_pool)
inception_5a_pool = Lambda(lambda x: x*9, name='mult9_5a')(inception_5a_pool)
inception_5a_pool = Lambda(lambda x: K.sqrt(x), name='sqrt_5a')(inception_5a_pool)
inception_5a_pool = utils.conv2d_bn(inception_5a_pool,
layer='inception_5a_pool',
cv1_out=96,
cv1_filter=(1, 1),
padding=(1, 1))
inception_5a_1x1 = utils.conv2d_bn(inception_4e,
layer='inception_5a_1x1',
cv1_out=256,
cv1_filter=(1, 1))
inception_5a = concatenate([inception_5a_3x3, inception_5a_pool, inception_5a_1x1], axis=3)
#inception_5b
inception_5b_3x3 = utils.conv2d_bn(inception_5a,
layer='inception_5b_3x3',
cv1_out=96,
cv1_filter=(1, 1),
cv2_out=384,
cv2_filter=(3, 3),
cv2_strides=(1, 1),
padding=(1, 1))
inception_5b_pool = MaxPooling2D(pool_size=3, strides=2)(inception_5a)
inception_5b_pool = utils.conv2d_bn(inception_5b_pool,
layer='inception_5b_pool',
cv1_out=96,
cv1_filter=(1, 1))
inception_5b_pool = ZeroPadding2D(padding=(1, 1))(inception_5b_pool)
inception_5b_1x1 = utils.conv2d_bn(inception_5a,
layer='inception_5b_1x1',
cv1_out=256,
cv1_filter=(1, 1))
inception_5b = concatenate([inception_5b_3x3, inception_5b_pool, inception_5b_1x1], axis=3)
av_pool = AveragePooling2D(pool_size=(3, 3), strides=(1, 1))(inception_5b)
reshape_layer = Flatten()(av_pool)
dense_layer = Dense(128, name='dense_layer')(reshape_layer)
norm_layer = Lambda(lambda x: K.l2_normalize(x, axis=1), name='norm_layer')(dense_layer)
# Final Model
return Model(inputs=[myInput], outputs=norm_layer)
Calling the above model as:
from keras import backend as K
from keras.models import Model
from keras.layers import Input, Layer
# Input for anchor, positive and negative images
in_a =np.random.randint(10,100,(1,96,96,3)).astype(float)
in_p =np.random.randint(10,100,(1,96,96,3)).astype(float)
in_n =np.random.randint(10,100,(1,96,96,3)).astype(float)
in_a_a =K.variable(value=in_a)
in_p_p =K.variable(value=in_p)
in_n_n =K.variable(value=in_n)
# # Output for anchor, positive and negative embedding vectors
# # The nn4_small model instance is shared (Siamese network)
emb_a = nn4_small2(in_a_a)
emb_p = nn4_small2(in_p_p)
emb_n = nn4_small2(in_n_n)
class TripletLossLayer(Layer):
def __init__(self, alpha, **kwargs):
self.alpha = alpha
super(TripletLossLayer, self).__init__(**kwargs)
def triplet_loss(self, inputs):
a, p, n = inputs
p_dist = K.sum(K.square(a-p), axis=-1)
n_dist = K.sum(K.square(a-n), axis=-1)
return K.sum(K.maximum(p_dist - n_dist + self.alpha, 0), axis=0)
def call(self, inputs):
loss = self.triplet_loss(inputs)
self.add_loss(loss)
return loss
# # # Layer that computes the triplet loss from anchor, positive and negative embedding vectors
triplet_loss_layer = TripletLossLayer(alpha=0.2, name='triplet_loss_layer')([emb_a, emb_p, emb_n])
# # # Model that can be trained with anchor, positive negative images
nn4_small2_train = Model([in_a, in_p, in_n], triplet_loss_layer)
Gives a type error on this line:
nn4_small2_train = Model([in_a, in_p, in_n], triplet_loss_layer)
c:\users\amark\anaconda3\envs\python3.5\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your ' + object_name +
90 ' call to the Keras 2 API: ' +signature,stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
c:\users\amark\anaconda3\envs\python3.5\lib\site-packages\keras\engine\topology.py in init(self, inputs, outputs, name)
1526
1527 # Check for redundancy in inputs.
-> 1528 if len(set(self.inputs)) != len(self.inputs):
1529 raise ValueError('The list of inputs passed to the model '
1530 'is redundant. '
TypeError: unhashable type: 'numpy.ndarray'
If I try to use the following:
nn4_small2_train = Model([in_a_a, in_p_p, in_n_n], triplet_loss_layer)
then the error raised is:
TypeError: Input tensors to a Model must be Keras tensors. Found: (missing Keras metadata)
You are passing numpy arrays as inputs to build a Model, and that is not right, you should pass instances of Input.
In your specific case, you are passing in_a, in_p, in_n but instead to build a Model you should be giving instances of Input, not K.variables (your in_a_a, in_p_p, in_n_n) or numpy arrays. Also it makes no sense to give values to the varibles. First you build the model symbolically, without any specific input values, and then you can train it or predict on it with real input values.
Related
Hi I'm working on a CT scan image segmentation task. I've extracted jpeg image from DICOM from two different CT window (WW and WL). But the masks are the same.
Now, I want to add an additional path to U-net with a second set of the input images (see image). I think it might give additional context to the network. But, I'm not sure how to implement this.
This is my U-net model.
import tensorflow as tf
IMG_WIDTH = 128
IMG_HEIGHT = 128
IMG_CHANNELS = 3
#Build the model
inputs = tf.keras.layers.Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))
s = tf.keras.layers.Lambda(lambda x: x / 255)(inputs)
#Contraction path
c1 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(s)
c1 = tf.keras.layers.Dropout(0.1)(c1)
c1 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c1)
p1 = tf.keras.layers.MaxPooling2D((2, 2))(c1)
c2 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p1)
c2 = tf.keras.layers.Dropout(0.1)(c2)
c2 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c2)
p2 = tf.keras.layers.MaxPooling2D((2, 2))(c2)
c3 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p2)
c3 = tf.keras.layers.Dropout(0.2)(c3)
c3 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c3)
p3 = tf.keras.layers.MaxPooling2D((2, 2))(c3)
c4 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p3)
c4 = tf.keras.layers.Dropout(0.2)(c4)
c4 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c4)
p4 = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))(c4)
c5 = tf.keras.layers.Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p4)
c5 = tf.keras.layers.Dropout(0.3)(c5)
c5 = tf.keras.layers.Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c5)
#Expansive path
u6 = tf.keras.layers.Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)
u6 = tf.keras.layers.concatenate([u6, c4])
c6 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u6)
c6 = tf.keras.layers.Dropout(0.2)(c6)
c6 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c6)
u7 = tf.keras.layers.Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)
u7 = tf.keras.layers.concatenate([u7, c3])
c7 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u7)
c7 = tf.keras.layers.Dropout(0.2)(c7)
c7 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c7)
u8 = tf.keras.layers.Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)
u8 = tf.keras.layers.concatenate([u8, c2])
c8 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u8)
c8 = tf.keras.layers.Dropout(0.1)(c8)
c8 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c8)
u9 = tf.keras.layers.Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)
u9 = tf.keras.layers.concatenate([u9, c1], axis=3)
c9 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u9)
c9 = tf.keras.layers.Dropout(0.1)(c9)
c9 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c9)
outputs = tf.keras.layers.Conv2D(1, (1, 1), activation='sigmoid')(c9)
model = tf.keras.Model(inputs=[inputs], outputs=[outputs])
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
model.summary()
you could use a labda layer to split the channels like this
x0 = Lambda(lambda x : x[:,:,:,0])(x)
x1 = Lambda(lambda x : x[:,:,:,1])(x)
in this way you get the channel with the new input and then you concatenate it at the last layer like this
u9 = tf.keras.layers.concatenate([u9, c1, x1], axis=3)
so you elaborate on x0 and on the last block you concatenate again with x1
I am trying to pick up an output from a specific layer in CNN and visualize it with matplotlib.pyplot.contour. This is my CNN architecture
input_img = Input(shape=(384, 192, 2))
## Encoder
x = Conv2D(16, (3, 3), activation='tanh', padding='same')(input_img)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(8, (3, 3), activation='tanh', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(8, (3, 3), activation='tanh', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(8, (3, 3), activation='tanh', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(4, (3, 3), activation='tanh', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(4, (3, 3), activation='tanh', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Reshape([6*3*4])(x)
encoded = Dense(2,activation='tanh')(x)
## Two variables
val1= Lambda(lambda x: x[:,0:1])(encoded)
val2= Lambda(lambda x: x[:,1:2])(encoded)
## Decoder 1
x1 = Dense(6*3*4,activation='tanh')(val1)
x1 = Reshape([6,3,4])(x1)
x1 = UpSampling2D((2,2))(x1)
x1 = Conv2D(4,(3,3),activation='tanh',padding='same')(x1)
x1 = UpSampling2D((2,2))(x1)
x1 = Conv2D(8,(3,3),activation='tanh',padding='same')(x1)
x1 = UpSampling2D((2,2))(x1)
x1 = Conv2D(8,(3,3),activation='tanh',padding='same')(x1)
x1 = UpSampling2D((2,2))(x1)
x1 = Conv2D(8,(3,3),activation='tanh',padding='same')(x1)
x1 = UpSampling2D((2,2))(x1)
x1 = Conv2D(16,(3,3),activation='tanh',padding='same')(x1)
x1 = UpSampling2D((2,2))(x1)
x1d = Conv2D(2,(3,3),activation='linear',padding='same')(x1)
## Decoder 2
x2 = Dense(6*3*4,activation='tanh')(val2)
x2 = Reshape([6,3,4])(x2)
x2 = UpSampling2D((2,2))(x2)
x2 = Conv2D(4,(3,3),activation='tanh',padding='same')(x2)
x2 = UpSampling2D((2,2))(x2)
x2 = Conv2D(8,(3,3),activation='tanh',padding='same')(x2)
x2 = UpSampling2D((2,2))(x2)
x2 = Conv2D(8,(3,3),activation='tanh',padding='same')(x2)
x2 = UpSampling2D((2,2))(x2)
x2 = Conv2D(8,(3,3),activation='tanh',padding='same')(x2)
x2 = UpSampling2D((2,2))(x2)
x2 = Conv2D(16,(3,3),activation='tanh',padding='same')(x2)
x2 = UpSampling2D((2,2))(x2)
x2d = Conv2D(2,(3,3),activation='linear',padding='same')(x2)
decoded = Add()([x1d,x2d])
autoencoder = Model(input_img, decoded)
autoencoder.compile(optimizer='adam', loss='mse')
# Check the network structure
autoencoder.summary()
I got the output of x1d layer as the code below with the output shape TensorShape([None, 72])
r1 = autoencoder.layers[13].output
r1.shape
How do I visualize this r1 output with matplotlib contour, the output should be like this photo.
I appriciate any help or suggestion. Thank you very much!
This pcolormesh works for me.
plt.pcolormesh(X_train[0][:,:,0].T, cmap = cm.jet)
I am getting an undefined rank error, while building ResNet; I have mentioned a reproducible code below:
Here is my Identity Block:
def IdentityBlock(X, f, filters):
F1, F2, F3 = filters
X_shortcut = X
X = Conv2D(filters = F1, kernel_size = (3, 3), padding = 'valid')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F2, kernel_size = (f, f), padding = 'same')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')(X)
X = BatchNormalization()(X)
X = Add()([X, X_shortcut])
X = Activation('relu')(X)
return X
Here is my Convolution Block
def ConvBlock(X, f, filters):
F1, F2, F3 = filters
X_shortcut = X
X = Conv2D(filters = F1, kernel_size = (3, 3), padding = 'valid')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F2, kernel_size = (f, f), padding = 'same')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')(X)
X = BatchNormalization()(X)
X_shortcut = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')
X_shortcut = BatchNormalization()(X_shortcut)
X = Add()([X, X_shortcut])
X = Activation('relu')(X)
return X
And my resnet model:
def ResNet(input_shape = (224, 224, 3)):
X_input = Input(input_shape)
X = Conv2D(64, (7, 7))(X_input)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = MaxPooling2D((3, 3))(X)
X = ConvBlock(X, f = 3, filters = [64, 64, 128])
X = IdentityBlock(X, 3, filters = [64, 64, 128])
X = IdentityBlock(X, 3, filters = [64, 64, 128])
X = ConvBlock(X, f = 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
X = MaxPooling2D((2, 2))(X)
model = Model(input = X_input, output = X)
return model
When I call RenNet Like this:
base_model = ResNet50(input_shape=(224, 224, 3))
I get the following error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-22-f81766b0bb4e> in <module>
----> 1 base_model = ResNet50(input_shape=(224, 224, 3))
<ipython-input-21-309ae6f634f4> in ResNet50(input_shape)
7 X = MaxPooling2D((3, 3))(X)
8
----> 9 X = ConvBlock(X, f = 3, filters = [64, 64, 128])
10 X = IdentityBlock(X, 3, filters = [64, 64, 128])
11 X = IdentityBlock(X, 3, filters = [64, 64, 128])
<ipython-input-20-aeab857c5df6> in ConvBlock(X, f, filters)
16
17 X_shortcut = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')
---> 18 X_shortcut = BatchNormalization()(X_shortcut)
19
20 X = Add()([X, X_shortcut])
.
.
.
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/normalization.py in build(self, input_shape)
296 input_shape = tensor_shape.TensorShape(input_shape)
297 if not input_shape.ndims:
--> 298 raise ValueError('Input has undefined rank:', input_shape)
299 ndims = len(input_shape)
300
ValueError: ('Input has undefined rank:', TensorShape(None))
You are passing the Conv2D operation to the BatchNormalization Layer instead of a tensor. Try changing
X_shortcut = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')
to
X_shortcut = Conv2D(filters = F3, kernel_size = (3, 3), padding = 'same')(X_shortcut)
These were the steps I followed to reach the final solution:
1 - the add function works like add([x,x_shortcut]) , im not too sure about the other way.
2 - Made a change in the variable, instead of using X in the IdentityBlock Function I changed it to input_
3 - The shape of the inputs in the add layer we not in the desired way, so I
played around with the kernel size and stride values (not too good at these)
to make that layer work. Did this for the ConvBlock and IdentityBlock.
4 - Final small error was when calling the keras model its Model(inputs = , outputs = ), input and output don't work.
def IdentityBlock(input_, f, filters):
F1, F2, F3 = filters
X = Conv2D(filters = F1, kernel_size = (1, 1), padding = 'valid') (input_)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F2, kernel_size = (f, f), padding = 'same')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F3, kernel_size = (1, 1), padding = 'same')(X)
X = BatchNormalization()(X)
X = add([X, input_])
X = Activation('relu')(X)
return X
def ConvBlock(X, f, filters):
F1, F2, F3 = filters
X_shortcut = X
X = Conv2D(filters = F1, kernel_size = (1, 1),strides = (1,1), padding
= 'valid')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F2, kernel_size = (f, f), padding = 'same')(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = Conv2D(filters = F3, kernel_size = (1, 1), padding = 'same')(X)
X = BatchNormalization()(X)
X_shortcut = Conv2D(filters = F3,padding = 'same' , kernel_size = (1,1)
, strides = (1,1))(X_shortcut)
X_shortcut = BatchNormalization()(X_shortcut)
X = add([X, X_shortcut])
X = Activation('relu')(X)
return X
def ResNet(input_shape = (224, 224, 3)):
X_input = Input(input_shape)
X = ZeroPadding2D((3,3))(X_input)
X = Conv2D(64, (7, 7) , strides = (2,2))(X_input)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = ZeroPadding2D((1,1))(X)
X = MaxPooling2D((3, 3) , strides = (2,2))(X)
X = ConvBlock(X, f = 3, filters = [64, 64, 128])
X = IdentityBlock(X, 3, filters = [64, 64, 128])
X = IdentityBlock(X, 3, filters = [64, 64, 128])
X = ConvBlock(X, f = 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
X = IdentityBlock(X, 3, filters = [128, 128, 512])
model = Model(inputs = X_input, outputs = X)
I hope it helped!
I am trying to implement a custom CNN model like the one shown in the attached figure using the functional API in Keras. I have written the code to implement the main branch but having issues in adding the 1x1 convolutions as a shortcut connection. The shortcut convolution is added for each pair of convolution blocks, just before the max-pooling layer. The code is given below:
input_shape = (256,256,3)
model_input = Input(shape=input_shape)
print(model_input)
def custom_cnn(model_input):
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(model_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(32, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(32, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(48, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(48, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(64, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(80, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(80, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = GlobalAveragePooling2D()(x)
x = Dense(num_classes, activation='softmax')(x)
model = Model(inputs=model_input, outputs=x, name='custom_cnn')
return model
#instantiate the model
custom_model = custom_cnn(model_input)
#display model summary
custom_model.summary()
here the implementation of the residual blocks inside your network following the schema:
num_classes = 3
input_shape = (256,256,3)
model_input = Input(shape=input_shape)
def custom_cnn(model_input):
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(model_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
xx = Conv2D(16, (1,1), strides= (4,4), padding = 'same')(model_input)
x = Add()([x,xx])
xx = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(32, (3, 3), strides = (1,1), padding = 'same')(xx)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(32, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
xx = Conv2D(32, (1,1), strides= (1,1), padding = 'same')(xx)
x = Add()([x,xx])
xx = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(48, (3, 3), strides = (1,1), padding = 'same')(xx)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(48, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
xx = Conv2D(48, (1,1), strides= (1,1), padding = 'same')(xx)
x = Add()([x,xx])
xx = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(64, (3, 3), strides = (1,1), padding = 'same')(xx)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
xx = Conv2D(64, (1,1), strides= (1,1), padding = 'same')(xx)
x = Add()([x,xx])
xx = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = Conv2D(80, (3, 3), strides = (1,1), padding = 'same')(xx)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(80, (3, 3), strides = (1,1), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
xx = Conv2D(80, (1,1), strides= (1,1), padding = 'same')(xx)
x = Add()([x,xx])
xx = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(x)
x = GlobalAveragePooling2D()(xx)
x = Dense(num_classes, activation='softmax')(x)
model = Model(inputs=model_input, outputs=x, name='custom_cnn')
return model
#instantiate the model
custom_model = custom_cnn(model_input)
#display model summary
custom_model.summary()
You can merge multiple layers with keras.layers.concatenate function so if you have 2 "parallel" layers x and y, you can do merge them into layer z like this:
z = layers.concatenate([x, y])
In your code, it would look something like this:
First, create the one "parallel" layer (note that both layers x and later y will be applied on the same layer starting with the model_input layer)
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(model_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(16, (3, 3), strides = (2,2), padding = 'same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
Then you would create the second "parallel" layer:
y = Conv2D(...)(model_input)
Merge them
z = layers.concatenate([x, y])
and apply max pooling on the z (on the layer that is the result of the concatenation)
z = MaxPooling2D(pool_size=(3, 3), strides=(2,2))(z)
And again, create two branches, both taking z as an input (where z is equivalent to the model_input from the previous iteration) and repeat for each of these blocks where you need to apply concatenation.
I'm running an extremely large model with IMAGE_SIZE = [2, 5928, 4728, 1]
The model structure can be found below. We are using Tesla K80 to train that so we have 12GB of VRAM. We're using CUDA 9.0, Tensorflow 1.6.0 as backend for Keras 2.1.5.
There's always some memory problems. I settled with batch_size=1 because it failed otherwise. At first I couldn't even execute fit() for more then couple steps. It crashed with memory error. Then I restricted memory usage to 0.9 and it trained fine for some time. Right now I'm trying to load weights from previous training and I get memory error again.
Is there a way to debug this somehow? Why does it fail when I load weights? I would like to get any insight about what to do in this situation.
Thanks in advance.
def make_model():
img_1_inp = Input(shape=IMAGE_SIZE)
img_2_inp = Input(shape=IMAGE_SIZE)
conv_1 = Conv2D(32, kernel_size=(3, 3),
strides=[2, 2],
activation=None)
img_1 = conv_1(img_1_inp)
img_2 = conv_1(img_2_inp)
batch_norm_1 = BatchNormalization()
img_1 = batch_norm_1(img_1)
img_2 = batch_norm_1(img_2)
activation_1 = Activation('relu')
img_1 = activation_1(img_1)
img_2 = activation_1(img_2)
pooling_1 = MaxPooling2D(pool_size=(3, 3))
img_1 = pooling_1(img_1)
img_2 = pooling_1(img_2)
conv_2_1 = Conv2D(64, kernel_size=(3, 3),
strides=[2, 2],
activation=None)
img_1 = conv_2_1(img_1)
img_2 = conv_2_1(img_2)
batch_norm_2_1 = BatchNormalization()
img_1 = batch_norm_2_1(img_1)
img_2 = batch_norm_2_1(img_2)
activation_2_1 = Activation('relu')
img_1 = activation_2_1(img_1)
img_2 = activation_2_1(img_2)
conv_2_2 = Conv2D(64, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_2_2(img_1)
img_2 = conv_2_2(img_2)
batch_norm_2_2 = BatchNormalization()
img_1 = batch_norm_2_2(img_1)
img_2 = batch_norm_2_2(img_2)
activation_2_2 = Activation('relu')
img_1 = activation_2_2(img_1)
img_2 = activation_2_2(img_2)
conv_2_3 = Conv2D(64, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_2_3(img_1)
img_2 = conv_2_3(img_2)
batch_norm_2_3 = BatchNormalization()
img_1 = batch_norm_2_3(img_1)
img_2 = batch_norm_2_3(img_2)
activation_2_3 = Activation('relu')
img_1 = activation_2_3(img_1)
img_2 = activation_2_3(img_2)
pooling_2 = MaxPooling2D(pool_size=(3, 3))
img_1 = pooling_2(img_1)
img_2 = pooling_2(img_2)
conv_3_1 = Conv2D(128, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_3_1(img_1)
img_2 = conv_3_1(img_2)
batch_norm_3_1 = BatchNormalization()
img_1 = batch_norm_3_1(img_1)
img_2 = batch_norm_3_1(img_2)
activation_3_1 = Activation('relu')
img_1 = activation_3_1(img_1)
img_2 = activation_3_1(img_2)
conv_3_2 = Conv2D(128, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_3_2(img_1)
img_2 = conv_3_2(img_2)
batch_norm_3_2 = BatchNormalization()
img_1 = batch_norm_3_2(img_1)
img_2 = batch_norm_3_2(img_2)
activation_3_2 = Activation('relu')
img_1 = activation_3_2(img_1)
img_2 = activation_3_2(img_2)
conv_3_3 = Conv2D(128, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_3_3(img_1)
img_2 = conv_3_3(img_2)
batch_norm_3_3 = BatchNormalization()
img_1 = batch_norm_3_3(img_1)
img_2 = batch_norm_3_3(img_2)
activation_3_3 = Activation('relu')
img_1 = activation_3_3(img_1)
img_2 = activation_3_3(img_2)
pooling_3 = MaxPooling2D(pool_size=(3, 3))
img_1 = pooling_3(img_1)
img_2 = pooling_3(img_2)
conv_4_1 = Conv2D(256, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_4_1(img_1)
img_2 = conv_4_1(img_2)
batch_norm_4_1 = BatchNormalization()
img_1 = batch_norm_4_1(img_1)
img_2 = batch_norm_4_1(img_2)
activation_4_1 = Activation('relu')
img_1 = activation_4_1(img_1)
img_2 = activation_4_1(img_2)
conv_4_2 = Conv2D(256, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_4_2(img_1)
img_2 = conv_4_2(img_2)
batch_norm_4_2 = BatchNormalization()
img_1 = batch_norm_4_2(img_1)
img_2 = batch_norm_4_2(img_2)
activation_4_2 = Activation('relu')
img_1 = activation_4_2(img_1)
img_2 = activation_4_2(img_2)
conv_4_3 = Conv2D(256, kernel_size=(3, 3),
strides=[10, 10],
activation=None)
img_1 = conv_4_3(img_1)
img_2 = conv_4_3(img_2)
batch_norm_4_3 = BatchNormalization()
img_1 = batch_norm_4_3(img_1)
img_2 = batch_norm_4_3(img_2)
activation_4_3 = Activation('relu')
img_1 = activation_4_3(img_1)
img_2 = activation_4_3(img_2)
g_pooling = GlobalAveragePooling2D(data_format='channels_last')
img_1 = g_pooling(img_1)
img_2 = g_pooling(img_2)
# flat_1 = Flatten()
# flatten_1 = flat_1(img_1)
# flatten_2 = flat_1(img_2)
#
# merged = keras.layers.concatenate([flatten_1, flatten_2])
merged = keras.layers.concatenate([img_1, img_2])
dropout_1 = Dropout(0.2)
merged = dropout_1(merged)
dense_1 = (Dense(1024, activation='relu'))(merged)
dropout_2 = Dropout(0.2)
dense_1 = dropout_2(dense_1)
dense_2 = (Dense(num_classes, activation='softmax'))(dense_1)
model = Model(inputs=[img_1_inp, img_2_inp], output=dense_2)
return model