Related
I am trying to develop a GAN, I have created the generator and the discriminator and now I am trying to train it. I am using the Mnist dataset but I plan to use some more. The problem is that when I train it I get this error: Input 0 of layer "conv2d_transpose_4" is incompatible with the layer: expected ndim=4, found ndim=2. Full shape received: (None, 100)
I don't really know if the problem is in the networks or in the data used to train the GAN, can someone tell me how should I train it or where the problem is?
imports:
import tensorflow
import keras
from keras.models import Sequential, Model
from keras.layers import Dense, Dropout, Flatten, Input, BatchNormalization,
LeakyReLU, Reshape
from keras.layers import Conv2D, Conv2DTranspose, MaxPooling2D
from tensorflow.keras.optimizers import Adam
from keras import backend as K
from keras.utils import np_utils
from keras.datasets import mnist
import numpy as np
import matplotlib.pyplot as plt
import os
import cv2
generator:
def generator():
model = Sequential()
model.add(Conv2DTranspose(32, (3,3), strides=(2,
2), activation='relu', use_bias=False,
input_shape=img_shape))
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2DTranspose(128,(3,3),strides=
(2,2), activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
model.add(Conv2DTranspose(128,(3,3),strides=
(2,2), activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(BatchNormalization(momentum=0.3))
model.add(LeakyReLU(alpha=0.2))
model.add(Conv2DTranspose(128,(3,3),strides=
(2,2), activation='relu', padding='same',
use_bias=False))
model.add(BatchNormalization())
model.add(Dense(512,
activation=LeakyReLU(alpha=0.2)))
model.add(BatchNormalization(momentum=0.7))
model.build()
model.summary()
return model
discriminator:
def discriminator():
model = Sequential()
model.add(Conv2D(32, (5,5), strides=(2, 2),
activation='relu', use_bias=False,
input_shape=img_shape))
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2D(64,(5,5),strides=(2,2),
activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
model.add(Conv2D(64,(5,5),strides=(2,2),
activation='relu', padding='same',
use_bias=False))
model.add(Dropout(0.5))
model.add(BatchNormalization(momentum=0.3))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(512,
activation=LeakyReLU(alpha=0.2)))
model.add(Flatten())
model.add(BatchNormalization(momentum=0.7))
model.add(Dense(1, activation='sigmoid'))
model.build()
model.summary()
return model
train function:
def train(epochs, batch_size, save_interval):
(x_train, _), (_, _) = mnist.load_data()
x_train = (x_train.astype(np.float32) - 127.5) / 127.5
x_train = np.expand_dims(x_train, axis=3)
half_batch = int(batch_size / 2)
for epoch in range(epochs):
idx = np.random.randint(0, x_train.shape[0], half_batch)
imgs = x_train[idx]
noise = np.random.normal(0, 1, (half_batch, 100))
gen_imgs = generator.predict(noise)
d_loss_real = discriminator.train_on_batch(imgs, np.ones((half_batch, 1)))
d_loss_fake = discriminator.train_on_batch(gen_imgs, np.zeros((half_batch, 1)))
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
noise = np.random.normal(0, 1, (batch_size, 100))
valid_y = np.array([1] * batch_size)
g_loss = combined.train_on_batch(noise, valid_y)
print ("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss))
if epoch % save_interval == 0:
save_imgs(epoch)
Data used:
img_rows = 28
img_cols = 28
channels = 1
img_shape = (img_rows, img_cols, channels)
optimizer = Adam(0.0002, 0.5)
discriminator = discriminator()
discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
generator = generator()
generator.compile(loss='binary_crossentropy',
optimizer=optimizer)
z = Input(shape=(100,))
img = generator(z) #error
discriminator.trainable = False
valid = discriminator(img)
combined = Model(z, valid)
combined.compile(loss='binary_crossentropy',
optimizer=optimizer)
train(epochs=100000, batch_size=32,
save_interval=10)
generator.save('generator_model.h5')
The problem is coming from the first Flatten layer in the Discriminator model, which is converting your n-dimensional tensor to a 1D tensor. Since a MaxPooling2D layer cannot work with a 1D tensor, you are seeing that error. If you remove it, it should work:
def discriminator():
model = Sequential()
model.add(Conv2D(32, (5,5), strides=(2, 2),
activation='relu', use_bias=False,
input_shape=img_shape))
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2D(64,(5,5),strides=(2,2),
activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
model.add(Conv2D(64,(5,5),strides=(2,2),
activation='relu', padding='same',
use_bias=False))
model.add(Dropout(0.5))
model.add(BatchNormalization(momentum=0.3))
model.add(LeakyReLU(alpha=0.2))
model.add(Flatten())
model.add(Dense(512,
activation=LeakyReLU(alpha=0.2)))
model.add(BatchNormalization(momentum=0.7))
model.add(Dense(1, activation='sigmoid'))
model.build()
model.summary()
return model
Update 1:
Try rewriting your Generator model like this:
def generator():
model = Sequential()
model = tf.keras.Sequential()
model.add(Dense(7*7*256, use_bias=False, input_shape=(100,)))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Reshape((7, 7, 256)))
assert model.output_shape == (None, 7, 7, 256) # Note: None is the batch size
model.add(Conv2DTranspose(128, (5, 5), strides=(1, 1), padding='same', use_bias=False))
assert model.output_shape == (None, 7, 7, 128)
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Conv2DTranspose(64, (5, 5), strides=(2, 2), padding='same', use_bias=False))
assert model.output_shape == (None, 14, 14, 64)
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Conv2DTranspose(1, (5, 5), strides=(2, 2), padding='same', use_bias=False, activation='tanh'))
assert model.output_shape == (None, 28, 28, 1)
model.summary()
return model
It should then work, but you should definitely go through this tutorial to understand everything.
First the discriminator is wrong you expecting the input noise to generator with same diemnsions where it is similaities for crossentrory, noises can generate into multiple levels where the batch is create !
WHen you looking at each layer you will see how much the layer identification need !
Simply doing by measure the input / output you don't need to removed or edit meaning of the model.
It is not the generator flase when you input is wrong try see the discriminator layers where they are training on the batch sizes and the genrated input image.
( It is noises similarlities cross-entrophy )
Model layers and shapes is not chagne when you use the input correct way
### name='conv2d_transpose_input'), name='conv2d_transpose_input', description="created by layer 'conv2d_transpose_input'"),
### but it was called on an input with incompatible shape (None, 100).
model = Sequential()
model.add(Conv2DTranspose(32, (3,3), strides=(2,
2), activation='relu', use_bias=False,
input_shape=(28, 28, 1)))
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2DTranspose(128,(3,3),strides=
(2,2), activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
########################
def discriminator():
model = Sequential()
### Input 0 of layer "conv2d" is incompatible with the layer: expected axis -1 of input shape to have value 1, but received input with shape (None, 114, 114, 512)
### model.add(tf.keras.layers.Reshape((1, 100), input_shape=img_shape))
model.add(Conv2D(32, (5,5), strides=(2, 2),
activation='relu', use_bias=False,
input_shape=( 28, 28, 1))) ### img_shape
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2D(64,(5,5),strides=(2,2),
activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
...
I am developing a GAN using the Mnist dataset. I have developed the Generator and Discriminator. However, when I combine them together I get this error: Input 0 of layer "conv2d" is incompatible with the layer: expected axis -1 of input shape to have value 1, but received input with shape (None, 57, 57, 1024). Does anyone know why this happens? Do I have to add something else?
The preprocessing:
(x_train, _), (x_test, _) = mnist.load_data()
x_train = x_train.reshape(60000, 28, 28, 1)
x_test = x_test.reshape(10000, 28, 28, 1)
x_train = x_train.astype('float32')/255
x_test = x_test.astype('float32')/255
img_rows, img_cols = 28, 28
channels = 1
img_shape = (img_rows, img_cols, channels)
The Generator:
def generator():
model = Sequential()
model.add(Conv2DTranspose(32, (3,3), strides=(2, 2), activation='relu', use_bias=False,
input_shape=img_shape))
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2DTranspose(64,(3,3),strides=(2,2), activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
model.add(Conv2DTranspose(64,(3,3),strides=(2,2), activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(BatchNormalization(momentum=0.3))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(512, activation=LeakyReLU(alpha=0.2)))
model.add(BatchNormalization(momentum=0.7))
model.add(Dense(1024, activation='tanh'))
model.summary()
model.compile(loss=keras.losses.binary_crossentropy, optimizer=Adam(learning_rate=0.02))
return model
generator = generator()
The Discriminator:
def discriminator():
model = Sequential()
model.add(Conv2D(32, (5,5), strides=(2, 2), activation='relu', use_bias=False,
input_shape=img_shape))
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2D(64,(5,5),strides=(2,2), activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
model.add(Conv2D(64,(5,5),strides=(2,2), activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(BatchNormalization(momentum=0.3))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(512, activation=LeakyReLU(alpha=0.2)))
model.add(BatchNormalization(momentum=0.7))
model.add(Dense(1024, activation='tanh'))
model.summary()
model.compile(loss=keras.losses.binary_crossentropy, optimizer=Adam(learning_rate=0.02))
return model
discriminator = discriminator()
Both models combined (Where I get the error):
def GAN(generator, discriminator):
model = Sequential()
model.add(generator)
discriminator.trainable = False
model.add(discriminator)
model.summary()
model.compile()
return model
gan = GAN(generator, discriminator)
Your generator needs to produce images, thus the output shape of the generator must be the same shape as the images. The activation also must be compatible with the range in the images. I don't think your images go from -1 to +1, so you should not use "tanh". You must choose an activation compatible with the images.
Last generator layer:
Dense(img_shape[-1], ...)
Your discriminator needs to say whether the images are true or false, thus its output must have one value only, 0 or 1.
Last discriminator layer:
Dense(1, activation="sigmoid")
I am a newbie in machine learning.
Actually, I used my unet code for image segmentation using one input image slice (192x912) and one output mask image (192x192)
My Unet code is contained several CNN layer and I usually used one input image (192x912) and one its corresponding mask binary image for training.
Code related with above explanation is as below.
def get_unet():
inputs = Input((img_rows, img_cols, 1))
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(inputs)
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
drop1 = Dropout(0.2)(pool1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(drop1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
drop2 = Dropout(0.2)(pool2)
'''''''
return model
model.fit(imgs_train, imgs_mask_train, batch_size=32, epochs=100, verbose=2, shuffle=True, validation_split=0.1, callbacks=[model_checkpoint])
it works well. But, now, I would like to use multi input image when I train my network. So, I add another train data and edit my code like below.
def get_unet():
inputs = Input((img_rows, img_cols, 1))
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(inputs)
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
drop1 = Dropout(0.2)(pool1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(drop1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
drop2 = Dropout(0.2)(pool2)
'''''''
return model
model.fit([imgs_train, imgs_other_train], imgs_mask_train, batch_size=32, epochs=100, verbose=2, shuffle=True, validation_split=0.1, callbacks=[model_checkpoint])
but when I run the train code, I got error message as below.
"ValueError: Error when checking model input: the list of Numpy arrays that you are passing to your model is not the size the model expected. Expected to see 1 array(s), but instead got the following list of 2 arrays: "
I think my U net needs to be changed for multi input, but I don't know where I have to change.
Please help and give me any comments.
Thanks.
This is actually rather easy. All you need to do would adjust your input size I believe.
inputs = Input((img_rows, img_cols, size)) would have worked. Or you could have used concat like:
inputs = []
for _ in range(num_inputs):
inputs.append(Input((self.input_height, self.input_width, self.input_features)))
x = concatenate(inputs)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
You can check something similar I implemented here
I want to build an end-to-end trainable model with the following proprieties:
CNN to extract features from image
The features is reshaped to a matrix
Each row of this matrix is then fed to LSTM1
Each column of this matrix is then fed to LSTM2
The output of LSTM1 and LSTM2 are concatenated for the final output
(it's more or less similar to Figure 2 in this paper: https://arxiv.org/pdf/1611.07890.pdf)
My problem now is after the reshape, how can I feed the values of feature matrix to LSTM with Keras or Tensorflow?
This is my code so far with VGG16 net (also a link to Keras issues):
# VGG16
model = Sequential()
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', input_shape=(224, 224, 3)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
# block 2
model.add(Conv2D(128, (3, 3), activation='relu', padding='same'))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
# block 3
model.add(Conv2D(256, (3, 3), activation='relu', padding='same'))
model.add(Conv2D(256, (3, 3), activation='relu'))
model.add(Conv2D(256, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
# block 4
model.add(Conv2D(512, (3, 3), activation='relu', padding='same'))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
# block 5
model.add(Conv2D(512, (3, 3), activation='relu', padding='same'))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
# block 6
model.add(Flatten())
model.add(Dense(4096, activation='relu'))
model.add(Dense(4096, activation='relu'))
# reshape the feature 4096 = 64 * 64
model.add(Reshape((64, 64)))
# How to feed each row of this to LSTM?
# This is my first solution but it doesn’t look correct:
# model.add(LSTM(256, input_shape=(64, 1))) # 256 hidden units, sequence length = 64, feature dim = 1
Consider building your CNN model with Conv2D and MaxPool2D layers, until you reach your Flatten layer, because the vectorized output from the Flatten layer will be you input data to the LSTM part of your structure.
So, build your CNN model like this:
model_cnn = Sequential()
model_cnn.add(Conv2D...)
model_cnn.add(MaxPooling2D...)
...
model_cnn.add(Flatten())
Now, this is an interesting point, the current version of Keras has some incompatibility with some TensorFlow structures that will not let you stack your entire layers in just one Sequential object.
So it's time to use the Keras Model Object to complete you neural network with a trick:
input_lay = Input(shape=(None, ?, ?, ?)) #dimensions of your data
time_distribute = TimeDistributed(Lambda(lambda x: model_cnn(x)))(input_lay) # keras.layers.Lambda is essential to make our trick work :)
lstm_lay = LSTM(?)(time_distribute)
output_lay = Dense(?, activation='?')(lstm_lay)
And finally, now it's time to put together our 2 separated models:
model = Model(inputs=[input_lay], outputs=[output_lay])
model.compile(...)
OBS: Note that you can substitute my model_cnn example by your VGG without including the top layers, once the vectorized output from the VGG Flatten layer will be the input of the LSTM model.
I'm trying to obtain output of an intermediate layer in Keras, Following is my code:
XX = model.input # Keras Sequential() model object
YY = model.layers[0].output
F = K.function([XX], [YY]) # K refers to keras.backend
Xaug = X_train[:9]
Xresult = F([Xaug.astype('float32')])
Running this, I got an Error :
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'dropout_1/keras_learning_phase' with dtype bool
i came to know that because I'm using dropout layer in my model, I have to specify a learning_phase() flag to my function as per keras documentation.
I changed my code to the following:
XX = model.input
YY = model.layers[0].output
F = K.function([XX, K.learning_phase()], [YY])
Xaug = X_train[:9]
Xresult = F([Xaug.astype('float32'), 0])
Now I'm getting a new Error that I'm unable to figure out:
TypeError: Cannot interpret feed_dict key as Tensor: Can not convert a int into a Tensor.
Any help would be appreciated.
PS : I'm new to TensorFlow and Keras.
Edit 1 :
Following is the complete code that I'm using. I'm using Spatial Transformer Network as discussed in this NIPS paper and it's Kera's implementation here
input_shape = X_train.shape[1:]
# initial weights
b = np.zeros((2, 3), dtype='float32')
b[0, 0] = 1
b[1, 1] = 1
W = np.zeros((100, 6), dtype='float32')
weights = [W, b.flatten()]
locnet = Sequential()
locnet.add(Convolution2D(64, (3, 3), input_shape=input_shape, padding='same'))
locnet.add(Activation('relu'))
locnet.add(Convolution2D(64, (3, 3), padding='same'))
locnet.add(Activation('relu'))
locnet.add(MaxPooling2D(pool_size=(2, 2)))
locnet.add(Convolution2D(128, (3, 3), padding='same'))
locnet.add(Activation('relu'))
locnet.add(Convolution2D(128, (3, 3), padding='same'))
locnet.add(Activation('relu'))
locnet.add(MaxPooling2D(pool_size=(2, 2)))
locnet.add(Convolution2D(256, (3, 3), padding='same'))
locnet.add(Activation('relu'))
locnet.add(Convolution2D(256, (3, 3), padding='same'))
locnet.add(Activation('relu'))
locnet.add(MaxPooling2D(pool_size=(2, 2)))
locnet.add(Dropout(0.5))
locnet.add(Flatten())
locnet.add(Dense(100))
locnet.add(Activation('relu'))
locnet.add(Dense(6, weights=weights))
model = Sequential()
model.add(SpatialTransformer(localization_net=locnet,
output_size=(128, 128), input_shape=input_shape))
model.add(Convolution2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(128, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Convolution2D(128, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(256, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Convolution2D(256, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(256, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Convolution2D(256, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
#==============================================================================
# Start Training
#==============================================================================
#define training results logger callback
csv_logger = keras.callbacks.CSVLogger(training_logs_path+'.csv')
model.fit(X_train, y_train,
batch_size=batch_size,
epochs=20,
validation_data=(X_valid, y_valid),
shuffle=True,
callbacks=[SaveModelCallback(), csv_logger])
#==============================================================================
# Visualize what Transformer layer has learned
#==============================================================================
XX = model.input
YY = model.layers[0].output
F = K.function([XX, K.learning_phase()], [YY])
Xaug = X_train[:9]
Xresult = F([Xaug.astype('float32'), 0])
# input
for i in range(9):
plt.subplot(3, 3, i+1)
plt.imshow(np.squeeze(Xaug[i]))
plt.axis('off')
for i in range(9):
plt.subplot(3, 3, i + 1)
plt.imshow(np.squeeze(Xresult[0][i]))
plt.axis('off')
The easiest way is to create a new model in Keras, without calling the backend. You'll need the functional model API for this:
from keras.models import Model
XX = model.input
YY = model.layers[0].output
new_model = Model(XX, YY)
Xaug = X_train[:9]
Xresult = new_model.predict(Xaug)
You could try:
model1 = tf.keras.models.Sequential(base_model.layers[:1])
model2 = tf.keras.models.Sequential(base_model.layers[1:])
Xaug = X_train[:9]
out = model1(Xaug)