Related
I am trying to develop a GAN, I have created the generator and the discriminator and now I am trying to train it. I am using the Mnist dataset but I plan to use some more. The problem is that when I train it I get this error: Input 0 of layer "conv2d_transpose_4" is incompatible with the layer: expected ndim=4, found ndim=2. Full shape received: (None, 100)
I don't really know if the problem is in the networks or in the data used to train the GAN, can someone tell me how should I train it or where the problem is?
imports:
import tensorflow
import keras
from keras.models import Sequential, Model
from keras.layers import Dense, Dropout, Flatten, Input, BatchNormalization,
LeakyReLU, Reshape
from keras.layers import Conv2D, Conv2DTranspose, MaxPooling2D
from tensorflow.keras.optimizers import Adam
from keras import backend as K
from keras.utils import np_utils
from keras.datasets import mnist
import numpy as np
import matplotlib.pyplot as plt
import os
import cv2
generator:
def generator():
model = Sequential()
model.add(Conv2DTranspose(32, (3,3), strides=(2,
2), activation='relu', use_bias=False,
input_shape=img_shape))
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2DTranspose(128,(3,3),strides=
(2,2), activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
model.add(Conv2DTranspose(128,(3,3),strides=
(2,2), activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(BatchNormalization(momentum=0.3))
model.add(LeakyReLU(alpha=0.2))
model.add(Conv2DTranspose(128,(3,3),strides=
(2,2), activation='relu', padding='same',
use_bias=False))
model.add(BatchNormalization())
model.add(Dense(512,
activation=LeakyReLU(alpha=0.2)))
model.add(BatchNormalization(momentum=0.7))
model.build()
model.summary()
return model
discriminator:
def discriminator():
model = Sequential()
model.add(Conv2D(32, (5,5), strides=(2, 2),
activation='relu', use_bias=False,
input_shape=img_shape))
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2D(64,(5,5),strides=(2,2),
activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
model.add(Conv2D(64,(5,5),strides=(2,2),
activation='relu', padding='same',
use_bias=False))
model.add(Dropout(0.5))
model.add(BatchNormalization(momentum=0.3))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(512,
activation=LeakyReLU(alpha=0.2)))
model.add(Flatten())
model.add(BatchNormalization(momentum=0.7))
model.add(Dense(1, activation='sigmoid'))
model.build()
model.summary()
return model
train function:
def train(epochs, batch_size, save_interval):
(x_train, _), (_, _) = mnist.load_data()
x_train = (x_train.astype(np.float32) - 127.5) / 127.5
x_train = np.expand_dims(x_train, axis=3)
half_batch = int(batch_size / 2)
for epoch in range(epochs):
idx = np.random.randint(0, x_train.shape[0], half_batch)
imgs = x_train[idx]
noise = np.random.normal(0, 1, (half_batch, 100))
gen_imgs = generator.predict(noise)
d_loss_real = discriminator.train_on_batch(imgs, np.ones((half_batch, 1)))
d_loss_fake = discriminator.train_on_batch(gen_imgs, np.zeros((half_batch, 1)))
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
noise = np.random.normal(0, 1, (batch_size, 100))
valid_y = np.array([1] * batch_size)
g_loss = combined.train_on_batch(noise, valid_y)
print ("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss))
if epoch % save_interval == 0:
save_imgs(epoch)
Data used:
img_rows = 28
img_cols = 28
channels = 1
img_shape = (img_rows, img_cols, channels)
optimizer = Adam(0.0002, 0.5)
discriminator = discriminator()
discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
generator = generator()
generator.compile(loss='binary_crossentropy',
optimizer=optimizer)
z = Input(shape=(100,))
img = generator(z) #error
discriminator.trainable = False
valid = discriminator(img)
combined = Model(z, valid)
combined.compile(loss='binary_crossentropy',
optimizer=optimizer)
train(epochs=100000, batch_size=32,
save_interval=10)
generator.save('generator_model.h5')
The problem is coming from the first Flatten layer in the Discriminator model, which is converting your n-dimensional tensor to a 1D tensor. Since a MaxPooling2D layer cannot work with a 1D tensor, you are seeing that error. If you remove it, it should work:
def discriminator():
model = Sequential()
model.add(Conv2D(32, (5,5), strides=(2, 2),
activation='relu', use_bias=False,
input_shape=img_shape))
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2D(64,(5,5),strides=(2,2),
activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
model.add(Conv2D(64,(5,5),strides=(2,2),
activation='relu', padding='same',
use_bias=False))
model.add(Dropout(0.5))
model.add(BatchNormalization(momentum=0.3))
model.add(LeakyReLU(alpha=0.2))
model.add(Flatten())
model.add(Dense(512,
activation=LeakyReLU(alpha=0.2)))
model.add(BatchNormalization(momentum=0.7))
model.add(Dense(1, activation='sigmoid'))
model.build()
model.summary()
return model
Update 1:
Try rewriting your Generator model like this:
def generator():
model = Sequential()
model = tf.keras.Sequential()
model.add(Dense(7*7*256, use_bias=False, input_shape=(100,)))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Reshape((7, 7, 256)))
assert model.output_shape == (None, 7, 7, 256) # Note: None is the batch size
model.add(Conv2DTranspose(128, (5, 5), strides=(1, 1), padding='same', use_bias=False))
assert model.output_shape == (None, 7, 7, 128)
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Conv2DTranspose(64, (5, 5), strides=(2, 2), padding='same', use_bias=False))
assert model.output_shape == (None, 14, 14, 64)
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Conv2DTranspose(1, (5, 5), strides=(2, 2), padding='same', use_bias=False, activation='tanh'))
assert model.output_shape == (None, 28, 28, 1)
model.summary()
return model
It should then work, but you should definitely go through this tutorial to understand everything.
First the discriminator is wrong you expecting the input noise to generator with same diemnsions where it is similaities for crossentrory, noises can generate into multiple levels where the batch is create !
WHen you looking at each layer you will see how much the layer identification need !
Simply doing by measure the input / output you don't need to removed or edit meaning of the model.
It is not the generator flase when you input is wrong try see the discriminator layers where they are training on the batch sizes and the genrated input image.
( It is noises similarlities cross-entrophy )
Model layers and shapes is not chagne when you use the input correct way
### name='conv2d_transpose_input'), name='conv2d_transpose_input', description="created by layer 'conv2d_transpose_input'"),
### but it was called on an input with incompatible shape (None, 100).
model = Sequential()
model.add(Conv2DTranspose(32, (3,3), strides=(2,
2), activation='relu', use_bias=False,
input_shape=(28, 28, 1)))
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2DTranspose(128,(3,3),strides=
(2,2), activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
########################
def discriminator():
model = Sequential()
### Input 0 of layer "conv2d" is incompatible with the layer: expected axis -1 of input shape to have value 1, but received input with shape (None, 114, 114, 512)
### model.add(tf.keras.layers.Reshape((1, 100), input_shape=img_shape))
model.add(Conv2D(32, (5,5), strides=(2, 2),
activation='relu', use_bias=False,
input_shape=( 28, 28, 1))) ### img_shape
model.add(BatchNormalization(momentum=0.3))
model.add(Conv2D(64,(5,5),strides=(2,2),
activation='relu', padding='same',
use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(alpha=0.2))
...
from keras import layers
from keras import models
model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.Flatten())
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10, activation='softmax'))
from keras.datasets import mnist
from keras.utils import to_categorical
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()
train_images = train_images.reshape((60000, 28, 28, 1))
train_images = train_images.astype('float32') / 255
test_images = test_images.reshape((10000, 28, 28, 1))
test_images = test_images.astype('float32') / 255
train_labels = to_categorical(train_labels)
test_labels = to_categorical(test_labels)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(train_images, train_labels, epochs=5, batch_size=64)
I am getting an error:
ValueError: Input 0 of layer sequential_22 is incompatible with the layer: expected ndim=4, found ndim=3. Full shape received: [28, 28, 1]
I tried using the Keras 2.8, fashion_mnist dataset and it is working.
import keras
print(keras.__version__)
from keras import layers
from keras import models
model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.Flatten())
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10, activation='softmax'))
from keras.datasets import fashion_mnist
from tensorflow.keras.utils import to_categorical
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()
train_images = train_images.reshape((60000, 28, 28, 1))
train_images = train_images.astype('float32') / 255
test_images = test_images.reshape((10000, 28, 28, 1))
test_images = test_images.astype('float32') / 255
train_labels = to_categorical(train_labels)
test_labels = to_categorical(test_labels)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(train_images, train_labels, epochs=5, batch_size=64)
Output:
2.8.0
Epoch 1/5
938/938 [==============================] - 53s 55ms/step - loss: 0.5523 - accuracy: 0.7965
Epoch 2/5
938/938 [==============================] - 51s 55ms/step - loss: 0.3333 - accuracy: 0.8780
Epoch 3/5
938/938 [==============================] - 54s 58ms/step - loss: 0.2832 - accuracy: 0.8968
Epoch 4/5
938/938 [==============================] - 52s 55ms/step - loss: 0.2508 - accuracy: 0.9079
Epoch 5/5
938/938 [==============================] - 51s 55ms/step - loss: 0.2264 - accuracy: 0.9168
<keras.callbacks.History at 0x7f1c1b0a5390>
So I have a sort of AlexNet CNN for regression. When I train it, the training loss is great, but then the loss for my test images is bad. I thought it was simple overfitting, but then when I test the trained model on the train images, I get an equally bad loss even though it is the images it trained on. Maybe I'm mistaken, but shouldn't the loss when predicting on train images be the same as the loss when the model finished? One curious bit of information is that the outputs are all very close to each other.
Here are two possible culprits:
I added a ModelCheckpoint callback to save the best epoch weights as a .ckpt file. and call it back at the end of training. I was having some issues with loading it before, but I am not getting any actual errors for it at the moment.
I have imbalanced data so I recently added class_weights. Although it is regression, all the outputs for training are 0-10 in increments of 0.5, so I can create 20 class weights.
Here is my code for training and testing, and then the code for testing the ckpt model again is below.
from __future__ import absolute_import, division, print_function
import os
os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin'
from tensorflow import keras
import keras.backend as K
from keras.utils import plot_model
import numpy as np
import matplotlib.pyplot as plt
import datetime
(train_images, train_labels), (test_images, test_labels) = np.load("dataset.npy", allow_pickle=True)
train_images = train_images / 255
test_images = test_images / 255
train_labels = list(map(float, train_labels))
test_labels = list(map(float, test_labels))
train_labels = [i/10 for i in train_labels]
test_labels = [i/10 for i in test_labels]
start_time = datetime.datetime.now()
model = keras.Sequential([
keras.layers.Conv2D(filters=96, kernel_size=(11, 11), strides=(4, 4), activation='relu', input_shape=(128, 128, 3), padding='same'),
keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
keras.layers.Conv2D(filters=256, kernel_size=(11, 11), strides=(1, 1), activation='relu', padding='same'),
keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
keras.layers.Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), activation='relu', padding='same'),
keras.layers.Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), activation='relu', padding='same'),
keras.layers.Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), activation='relu', padding='same'),
keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
keras.layers.Flatten(),
keras.layers.Dense(4096, activation='relu'),
keras.layers.Dropout(0.4),
keras.layers.Dense(4096, activation='relu'),
keras.layers.Dropout(0.4),
keras.layers.Dense(1000, activation='relu'),
keras.layers.Dropout(0.4),
keras.layers.Dense(1)
])
plot_model(model, to_file='model.png')
class_weights = {.05: 93.0,
.10: 31.0,
.15: 93.0,
.20: 18.6,
.25: 46.5,
.30: 4.894736842105263,
.35: 7.75,
.40: 4.043478260869565,
.45: 2.90625,
.50: 2.2142857142857144,
.55: 2.066666666666667,
.60: 1.5,
.65: 1.453125,
.70: 1.0,
.75: 1.273972602739726,
.80: 1.6607142857142858,
.85: 3.72,
.90: 6.642857142857143,
.95: 15.5,
1.0: 1}
model.compile(loss='mean_squared_error',
optimizer=keras.optimizers.SGD(0.01),
metrics=['mean_squared_error'],
weighted_metrics=['mean_squared_error'])
train_images = train_images.reshape(462, 128, 128, 3)
test_images = test_images.reshape(116, 128, 128, 3)
history = model.fit(train_images, train_labels, epochs=200, callbacks=[keras.callbacks.ModelCheckpoint("./model.ckpt", monitor='mean_squared_error', save_best_only=True, verbose=1)], class_weight=class_weights)
# Plot training & validation loss values
plt.plot(history.history['loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper left')
plt.ylim(0, 0.5)
plt.show()
model.load_weights("./model.ckpt")
predictions = model.predict(test_images)
totalDifference = 0
for i in range(116):
print("%s: %s" % (test_labels[i] * 10, predictions[i] * 10))
totalDifference += abs(test_labels[i] - predictions[i])
avgDifference = totalDifference / 11.6
print("\n%s\n" % avgDifference)
print("Time Elapsed:")
print(datetime.datetime.now() - start_time)
Testing by loading the weights through a .ckpt file.
from __future__ import absolute_import, division, print_function
import os
os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin'
from tensorflow import keras
import numpy as np
(train_images, train_labels), (test_images, test_labels) = np.load("dataset.npy", allow_pickle=True)
train_images = train_images / 255
test_images = test_images / 255
train_labels = list(map(float, train_labels))
test_labels = list(map(float, test_labels))
train_labels = [i/10 for i in train_labels]
test_labels = [i/10 for i in test_labels]
model = keras.Sequential([
keras.layers.Conv2D(filters=96, kernel_size=(11, 11), strides=(4, 4), activation='relu', input_shape=(128, 128, 3), padding='same'),
keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
keras.layers.Conv2D(filters=256, kernel_size=(11, 11), strides=(1, 1), activation='relu', padding='same'),
keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
keras.layers.Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), activation='relu', padding='same'),
keras.layers.Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), activation='relu', padding='same'),
keras.layers.Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), activation='relu', padding='same'),
keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'),
keras.layers.Flatten(),
keras.layers.Dense(4096, activation='relu'),
keras.layers.Dropout(0.4),
keras.layers.Dense(4096, activation='relu'),
keras.layers.Dropout(0.4),
keras.layers.Dense(1000, activation='relu'),
keras.layers.Dropout(0.4),
keras.layers.Dense(1)
])
model.compile(loss='mean_squared_error',
optimizer=keras.optimizers.SGD(0.01),
metrics=['mean_squared_error'],
weighted_metrics=['mean_squared_error'])
train_images = train_images.reshape(462, 128, 128, 3)
test_images = test_images.reshape(116, 128, 128, 3)
model.load_weights("./model.ckpt")
train_pred = model.predict(train_images)
totalDifference = 0
for i in range(116):
print("%s: %s" % (train_labels[i] * 10, train_pred[i] * 10))
totalDifference += abs(train_labels[i] - train_pred[i])
avgDifference = totalDifference / 11.6
print("\n%s\n" % avgDifference)
predictions = model.predict(test_images)
totalDifference = 0
for i in range(116):
print("%s: %s" % (test_labels[i] * 10, predictions[i] * 10))
totalDifference += abs(test_labels[i] - predictions[i])
avgDifference = totalDifference / 11.6
print("\n%s\n" % avgDifference)
Any ideas are helpful, thanks.
I'm not sure about this but on the line
history = model.fit(train_images,
train_labels,
epochs=200,
callbacks=[keras.callbacks.ModelCheckpoint("./model.ckpt", monitor='mean_squared_error', save_best_only=True, verbose=1)],
class_weight=class_weights)
You might be saving the model with Maximum mean squared error, so pass mode=min as an argument to make sure that the model with minimum mean_squared_error is being saved at the end.
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.layers import MaxPooling2D
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Dropout
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
import numpy as np
import cv2
import os
from imutils import paths
import imutils
def image_to_feature_vector(image, size=(32, 32)):
# resize the image to a fixed size, then flatten the image into
# a list of raw pixel intensities
return np.reshape(cv2.resize(image, size).flatten(), (32, 32, 3))
def extract_color_histogram(image, bins=(8, 8, 8)):
# extract a 3D color histogram from the HSV color space using
# the supplied number of `bins` per channel
hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
hist = cv2.calcHist([hsv], [0, 1, 2], None, bins,
[0, 180, 0, 256, 0, 256])
# handle normalizing the histogram if we are using OpenCV 2.4.X
if imutils.is_cv2():
hist = cv2.normalize(hist)
# otherwise, perform "in place" normalization in OpenCV 3
else:
cv2.normalize(hist, hist)
# return the flattened histogram as the feature vector
return hist.flatten()
DATA_SET_PATH = '../training_images'
# grab the list of images that we'll be describing
print("[INFO] describing images...")
imagePaths = list(paths.list_images(DATA_SET_PATH))
# initialize the raw pixel intensities matrix, the features matrix,
# and labels list
rawImages = []
features = []
labels = []
# loop over the input images
for (i, imagePath) in enumerate(imagePaths):
# load the image and extract the class label (assuming that our
# path as the format: /path/to/dataset/{class}.{image_num}.jpg
image = cv2.imread(imagePath)
label = imagePath.split(os.path.sep)[-2].split(".")[1]
# extract raw pixel intensity "features", followed by a color
# histogram to characterize the color distribution of the pixels
# in the image
pixels = image_to_feature_vector(image)
hist = extract_color_histogram(image)
# update the raw images, features, and labels matricies,
# respectively
rawImages.append(pixels)
labels.append(label)
features.append(hist)
# show an update every 1,000 images
if i > 0 and i % 1000 == 0:
print("[INFO] processed {}/{}".format(i, len(imagePaths)))
# show some information on the memory consumed by the raw images
# matrix and features matrix
rawImages = np.array(rawImages)
features = np.array(features)
labels = np.array(labels)
print("[INFO] pixels matrix: {:.2f}MB".format(
rawImages.nbytes / (1024 * 1000.0)))
encoder = LabelEncoder()
encoder.fit(labels)
labels = to_categorical(encoder.transform(labels))
# partition the data into training and testing splits, using 75%
# of the data for training and the remaining 25% for testing
(trainRI, testRI, trainRL, testRL) = train_test_split(
rawImages, labels, test_size=0.15, random_state=42)
model = Sequential()
model.add(Conv2D(32, (3, 3), padding='same', activation='relu', input_shape=(32, 32, 3)))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(257, activation='softmax'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
aug = ImageDataGenerator(rotation_range=30, width_shift_range=0.1,
height_shift_range=0.1, shear_range=0.2, zoom_range=0.2,
horizontal_flip=True, fill_mode="nearest")
model.fit_generator(aug.flow(trainRI, trainRL, batch_size=256), epochs=10, validation_data=(testRI, testRL))
model.evaluate(testRI, testRL)
I'm a beginner in cnn and keras and I'm trying to create a cnn to classify 257 different objects using keras. Above is my code. When I run the code, both training accuracy and validation are incredibly high.
100/102 [============================>.] - ETA: 1s - loss: 0.0265 - acc: 0.9960
101/102 [============================>.] - ETA: 0s - loss: 0.0264 - acc: 0.9960
102/102 [==============================] - 56s 553ms/step - loss: 0.0264 - acc: 0.9960 - val_loss: 0.0250 - val_acc: 0.9961
Could anyone tell me what's wrong with my code? Thanks.
I built a model to colorize a grayscale image, during the training phase i feed the network 100 RGB images of a forest, and then i convert the images to the LAB color space to split the training set to L and AB,
Based on the trained AB data, the model will predict these two channels for the grayscale input image during the testing phase.
Now i have a problem, i trained the model with a different architecture than this one with 10 images, the loss decreased to 0.0035 and it worked good, for that, i wanted to increase the size of the dataset to acquire a better result, but in exchange, the loss and the accuracy kept being constant and the model output is a mess,
My code is the following, i wish anyone can direct me of what i am doing wrong, is it because of the optimizer? the loss function? the batch size? or anything else i'm not aware of,
Thank you in advance.
# Import images
MODEL_NAME = 'forest'
X = []
Y = []
for filename in os.listdir('forest/'):
if (filename != '.DS_Store'):
image = img_to_array(load_img("/Users/moos/Desktop/Project-Master/forest/" + filename))
image = np.array(image, dtype=float)
imL = rgb2lab(1.0 / 255 * image)[:, :,0]
X.append(imL)
imAB = rgb2lab(1.0 / 255 * image)[:, :,1:]
imAB = imAB/128
Y.append(imAB)
X = np.array(X)
Y = np.array(Y)
X = X.reshape(1, 256 , np.size(X)/256, 1)
Y = Y.reshape(1, 256, np.size(Y)/256/2, 2)
# Building the neural network
model = Sequential()
model.add(InputLayer(input_shape=(256, np.size(X)/256, 1)))
model.add(Conv2D(8, (3, 3), activation='relu', padding='same', strides=2))
model.add(Conv2D(32, (3, 3), activation='relu', padding='same'))
model.add(Conv2D(32, (3, 3), activation='relu', padding='same', strides=2))
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', strides=1))
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', strides=2))
model.add(Conv2D(128, (3, 3), activation='relu', padding='same', strides=1))
model.add(Conv2D(128, (3, 3), activation='relu', padding='same', strides=1))
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', strides=1))
model.add(Conv2D(32, (3, 3), activation='relu', padding='same', strides=1))
model.add(UpSampling2D((2, 2)))
model.add(Conv2D(8, (3, 3), activation='relu', padding='same'))
model.add(Conv2D(8, (3, 3), activation='relu', padding='same'))
model.add(UpSampling2D((2, 2)))
model.add(Conv2D(2, (3, 3), activation='relu', padding='same'))
model.add(Conv2D(2, (3, 3), activation='tanh', padding='same'))
model.add(UpSampling2D((2, 2)))
# Finish model
model.compile(optimizer='rmsprop',loss='mse', metrics=['acc'])
#Train the neural network
model.fit(x=X, y=Y, batch_size=100, epochs=1000)
print(model.evaluate(X, Y, batch_size=100))
output
Epoch 1/1000 1/1 [==============================] - 7s 7s/step - loss: 0.0214 - acc: 0.4987
Epoch 2/1000 1/1 [==============================] - 7s 7s/step - loss: 0.0214 - acc:
0.4987
Epoch 3/1000 1/1 [==============================] - 9s 9s/step - loss: 0.0214 - acc: 0.4987
Epoch 4/1000 1/1 [==============================] - 8s 8s/step - loss: 0.0214 - acc:
0.4987 . . . .
First of all, I have simplified the image loading code, and also normalized (subtract mean, divide by standard deviation) all channels (L, A, B) separately, also renamed the variables, that usually helps a lot. (5 minute free Coursera video about normalizing inputs (will bug you to subscribe but just click that away.).) So the loading part now looks like this:
# Import images
MODEL_NAME = 'forest'
imgLABs = []
for filename in os.listdir('./forest/'):
if (filename != '.DS_Store'):
image = img_to_array( load_img("./forest/" + filename) )
imgLABs.append( rgb2lab( image / 255.0 ) )
imgLABs_arr = np.array( imgLABs )
L, A, B = imgLABs_arr[ :, :, :, 0 : 1 ], imgLABs_arr[ :, :, :, 1 : 2 ], imgLABs_arr[ :, :, :, 2 : 3 ]
L_mean, L_std = np.mean( L ), np.std( L )
A_mean, A_std = np.mean( A ), np.std( A )
B_mean, B_std = np.mean( B ), np.std( B )
L, A, B = ( L - L_mean ) / L_std, ( A - A_mean ) / A_std, ( B - B_mean ) / B_std
AB = np.concatenate( ( A, B ), axis = 3)
Also changed the model around, added more feature depth, and a few max pool layers (don't forget to include them in the imports, not shown). Note that the activation function at the final few layers are set to None to allow for negative values, since we're expecting normalized results:
# Building the neural network
model = Sequential()
model.add(InputLayer( input_shape = L.shape[ 1: ] ) )
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', strides=2,
kernel_initializer='truncated_normal'))
model.add(MaxPooling2D( (3, 3), strides = 1, padding='same' ) )
model.add(Conv2D(64, (3, 3), activation='relu', padding='same',
kernel_initializer='truncated_normal'))
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', strides=2,
kernel_initializer='truncated_normal'))
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', strides=1,
kernel_initializer='truncated_normal'))
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', strides=2,
kernel_initializer='truncated_normal'))
model.add(MaxPooling2D( (3, 3), strides = 1, padding='same' ) )
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', strides=1,
kernel_initializer='truncated_normal'))
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', strides=1,
kernel_initializer='truncated_normal'))
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', strides=1,
kernel_initializer='truncated_normal'))
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', strides=1,
kernel_initializer='truncated_normal'))
model.add(UpSampling2D((2, 2)))
model.add(Conv2D(32, (3, 3), activation='relu', padding='same',
kernel_initializer='truncated_normal'))
model.add(Conv2D(32, (3, 3), activation='relu', padding='same',
kernel_initializer='truncated_normal'))
model.add(UpSampling2D((2, 2)))
model.add(Conv2D(32, (3, 3), activation=None, padding='same',
kernel_initializer='truncated_normal'))
model.add(Conv2D(2, (3, 3), activation=None, padding='same',
kernel_initializer='truncated_normal'))
model.add(UpSampling2D((2, 2)))
# Finish model
optimizer = optimizers.RMSprop( lr = 0.0005, decay = 1e-5 )
model.compile( optimizer=optimizer, loss='mse', metrics=['acc'] )
#Train the neural network
model.fit( x=L, y=AB, batch_size=1, epochs=1800 )
model.save("forest-model-v2.h5")
Note the learning rate of 0.0005, I've experimented with some values, this looked best. Then the learning rate decay can help later in the training, reducing the learning rate as we go along. Also, I've changed the batch_size to 1 - this is very specific to this network and is not generally recommended. But here you mostly have straight convolutions, so it makes sense to update the kernels after each exemplar, as every exemplar itself is affecting the weights from each pixel. But if you change the architecture, then this might not make sense any more, and you should change the batch size back. I've also increased the epochs to 1,800, because it runs fairly quickly on my machine and I had the time to run it. It reaches its maximum around 1,000 though.
With all that, here's the output from the training (first and last few lines only):
Epoch 1/1800
100/100 [==============================] - 6s 63ms/step - loss: 1.0554 - acc: 0.5217
Epoch 2/1800
100/100 [==============================] - 1s 13ms/step - loss: 1.1097 - acc: 0.5703
...
Epoch 1000/1800
100/100 [==============================] - 1s 13ms/step - loss: 0.0533 - acc: 0.9338
...
Epoch 1800/1800
100/100 [==============================] - 1s 13ms/step - loss: 0.0404 - acc: 0.9422
To print the re-colored image I used the following code, please note 5 is just an arbitrary index of the image I picked from the 100; also we need to add back the means and standard deviations for L, A and B (you have to treat these six numbers as part of your network, when you want to use it for actual recoloring - you need to preprocess the input with L_std, L_mean, and then postprocess the output with A, B means and std-s):
predicted = model.predict( x = L[ 5 : 6 ], batch_size = 1, verbose = 1 )
plt.imshow( lab2rgb( np.concatenate(
( ( L[ 5 ] * L_std ) + L_mean,
( predicted[ 0, :, :, 0 : 1 ] * A_std ) + A_mean,
( predicted[ 0, :, :, 1 : 2 ] * B_std ) + B_mean),
axis = 2 ) ) )
img_pred = lab2rgb( np.concatenate(
( ( L[ 5 ] * L_std ) + L_mean,
( predicted[ 0, :, :, 0 : 1 ] * A_std ) + A_mean,
( predicted[ 0, :, :, 1 : 2 ] * B_std ) + B_mean),
axis = 2 ) )
img_orig = lab2rgb( np.concatenate(
( ( L[ 5 ] * L_std ) + L_mean,
( A[ 5 ] * A_std ) + A_mean,
( B[ 5 ] * B_std ) + B_mean ),
axis = 2 ) )
diff = img_orig - img_pred
plt.imshow( diff * 10 )
And with all that the images are (original; greyscale network input; network output (colors restored); difference between original and restored):
Pretty neat! :) Mainly some of the detail on the mountains what's lost only. Since it's only 100 training images, it might be seriously overfitted, though. Still, I hope this gives you a good start!