I have a dataset of about 500 .mat files 300 train and 200 test and these are really small sized cropped images that are at most 3kb each. when I try training on the below architecture with the following parameters, I get a test accuracy and loss of 69% and the validation accuracy over 25 epochs remains around 51%. I want to know how to improve my test accuracy and fix the constant validation accuracy problem.
note: The problem is a binary classification problem and the class split is in the 60:40 ratio
weight_decay = 1e-3
model = models.Sequential()
model.add(layers.Conv2D(16, (3, 3), kernel_regularizer=regularizers.l2(weight_decay),padding='same',input_shape=X_train.shape[1:]))
model.add(layers.Activation('relu'))
model.add(layers.Dropout(0.2))
model.add(layers.Conv2D(32, (3, 3),kernel_regularizer=regularizers.l2(weight_decay), padding='same'))
model.add(layers.Activation('relu'))
#model.add(layers.Dropout(0.2))
model.add(layers.Flatten())
#model.add(layers.Dropout(0.4))
model.add(layers.Dense(20, activation='relu'))
model.add(layers.Dropout(0.50))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer=optimizers.adam(lr=0.001), metrics=['acc'])
es_callback = callbacks.EarlyStopping(monitor='val_loss', patience=5)
history= model.fit(#train_generator,
X_train,Y_train,
batch_size= batch_size,
#steps_per_epoch=trainSize,
epochs=25,
validation_data=(X_val,Y_val),#val_generator,
#validation_steps=valSize,
#callbacks=[LearningRateScheduler(lr_schedule)]
callbacks=[es_callback]
)
Related
I am training my first ML model. I am working on a 10-class classification problem. From what I can see, the model is overfitting since there is a significant difference between the training and validation accuracy.
This is the relevant code for the model
model = keras.Sequential()
model.add(keras.Input(shape=(x_train[0].shape)))
model.add(tf.keras.layers.Conv2D(filters=32,kernel_size=3, strides = (3, 3), padding = "same", activation = "relu", kernel_regularizer=tf.keras.regularizers.l1_l2(0.01)))
model.add(tf.keras.layers.MaxPool2D(strides=2))
model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=(3,3), padding='valid', activation='relu', kernel_regularizer=tf.keras.regularizers.l1_l2(0.01)))
model.add(tf.keras.layers.MaxPool2D(strides=2))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(128, activation='relu'))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.BatchNormalization())
model.add(tf.keras.layers.Dense(10))
optimizer = tf.keras.optimizers.Adam(learning_rate=0.001/2)
model.summary()
model.compile(optimizer=optimizer,
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs = 30, validation_data = (x_val, y_val), callbacks=tf.keras.callbacks.EarlyStopping(verbose=1, patience=4))
There are large fluctuations in the validation accuracy and I am not sure why.
I have tried augmenting the data and have also injected noise into the training data. (This is an audio classification problem with 10 different classes)
https://i.stack.imgur.com/TXe50.png
I am (very) new to deep learning and I am trying to train a dog breed classifier using Tensorflow/Keras. I have selected a subset of 10 breeds to speed up calculations, and I am using all the images available in the Stanford dataset for those breeds, which I have placed in train/test/val directories. I have 1338 images for training, 379 images for validation and 200 images for test.
I have first tried building a simple CNN from scratch without data augmentation, and I quickly reached 99% accuracy for the training set and got stuck at 30% for the val set (which I assume is quite normal without augmentation ?)
Then I applied data augmentation and tried two approaches, building a CNN from scratch and using transfer learning. With the "home-made" CNN I can't reach more than around 30 % accuracy even for the training set, and I can't figure out what the problem is. And I am stuck around 80 % with transfer learning, which I guess is not that good either ?
Here is the code for data augmentation:
`
# Creating image generator steps
train_datagen = ImageDataGenerator(rescale=1.0/255.0,
rotation_range=60,
width_shift_range=0.3,
height_shift_range=0.3,
shear_range=0.2,
zoom_range=[0.5, 1.5],
brightness_range=[0.5, 1.5],
horizontal_flip=True
)
val_datagen = ImageDataGenerator(rescale=1.0/255.0)
test_datagen = ImageDataGenerator(rescale=1.0/255.0)
train_generator = train_datagen.flow_from_directory(
directory="split_output/train",
target_size=(224,224),
color_mode="rgb",
batch_size=8,
class_mode='sparse',
shuffle='True',
seed=42
)
val_generator = val_datagen.flow_from_directory(
directory="split_output/val",
target_size=(224,224),
color_mode="rgb",
batch_size=8,
class_mode='sparse',
shuffle='True',
seed=42
)
test_generator = test_datagen.flow_from_directory(
directory="split_output/test",
target_size=(224,224),
color_mode="rgb",
batch_size=8,
class_mode='sparse',
shuffle='False',
seed=42
)
`
Here is the first CNN I tried (for which accuracies are both stuck around 25 %):
`
# The CNN architecture
model = Sequential()
model.add(Conv2D(32,(3,3), padding="same", activation='relu',input_shape = (224,224,3)))
model.add(MaxPooling2D((2,2)))
# 32 = number of filters
# (3, 3) = kernel size
model.add(Conv2D(64,(3,3), padding="same", activation='relu'))
model.add(MaxPooling2D((2,2)))
model.add(Conv2D(64,(3,3), padding="same", activation='relu'))
model.add(MaxPooling2D((2,2)))
model.add(Flatten())
model.add(Dense(64,activation='relu'))
model.add(Dense(10,activation='softmax'))
# Fitting the model
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy']
)
history = model.fit_generator(train_generator,
# steps_per_epoch=1000,
epochs=50,
validation_data=val_generator,
# validation_steps=250,
verbose=1
)
`
And the second one, a bit deeper and including BatchNorm and Dropout (accuracies are stuck around 35%):
`
# The CNN architecture
model = Sequential()
model.add(Conv2D(32,(3,3), padding="same", activation='relu',input_shape = (224,224,3)))
model.add(MaxPooling2D((2,2)))
# 32 = number of filters
# (3, 3) = kernel size
model.add(Conv2D(32,(3,3),activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2,2)))
model.add(Conv2D(64,(3,3), padding="same", activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2,2)))
model.add(Conv2D(128,(3,3), padding="same", activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2,2)))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(512,activation='relu'))
model.add(Dense(10,activation='softmax'))
model.summary()
opt = Adam(lr=0.0001)
# Fitting the model
model.compile(optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy']
)
history = model.fit(train_generator,
# steps_per_epoch=1000,
epochs=50,
validation_data=val_generator,
# validation_steps=250,
verbose=1
)
`
Here is the history for that second CNN:
accuracies for 2nd CNN
And finally I tried with a resnet, which gets stuck around 90% for train and 80% for val:
`
model = Sequential()
model.add(ResNet50(include_top=False, pooling='avg', weights="imagenet"))
model.add(Flatten())
model.add(BatchNormalization())
model.add(Dense(2048, activation='relu'))
model.add(BatchNormalization())
model.add(Dense(1024, activation='relu'))
model.add(BatchNormalization())
model.add(Dense(10, activation='softmax'))
opt = Adam(lr=0.0001)
model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=['accuracy'])
history = model.fit(train_generator,
# steps_per_epoch=1000,
epochs=150,
validation_data=val_generator,
# validation_steps=250,
verbose=1
)
`
And the history for this last one:
resnet history
I'm a bit surprised at how the accuracies (especially val) get stuck so fast at a nearly constant value...
Again I'm very new at this so there could be very basic mistakes!
I have a model for identification object by spectrum
model = Sequential()
model.add(Conv1D(filters = 64, input_shape=(train_generator.get_half_spec_size(160000), 1), kernel_size = 20, activation='relu'))
model.add(BatchNormalization())
model.add(Conv1D(filters = 64, kernel_size = 16, activation='relu', kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4),
bias_regularizer=regularizers.l2(1e-4),
activity_regularizer=regularizers.l2(1e-5)))
model.add(BatchNormalization())
model.add(MaxPool1D(strides=5))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(256, activation='relu'))
#model.add(BatchNormalization())
model.add(Dense(2, activation='softmax'))
model.compile(optimizer=Adam(), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
It seems that I have strange thing with 'accuracy' metric. First of all, almost in all epoches I have 0.66 accuracy for train set and 0.99 for validation set.
Okay. Than when model has learned - I just took one sample and check if validation is correct on sample. The validation was not correct (there were 1 but predicted 0), but answer of evaluate was
[2.060739278793335, 1.0]
so big loss ( expectingly ) and wrong accuracy (1.0)
Then I took another sample and predicted again. answer of 'evaluate' was
[0.18439120054244995, 1.0]
so correct loss and accuracy now (and in fact the prediction of model was correct).
My suggestion is that 'accuracy' metrics works wrong there. Or where there is a mistake?
For my first Machine Learning experience i have a basic classification to do.
I have 3 different folders :
train_path = './dataset/pneumonia/train/'
test_path = './dataset/pneumonia/test/'
val_path = './dataset/pneumonia/val/
each folders :
os.listdir(train_path)
returns
['NORMAL', 'PNEUMONIA']
In each sets :
Training set:
Normal: 949
Pneumonia: 949
Test set:
Normal: 317
Pneumonia: 317
Validation set:
Normal: 317
Pneumonia: 317
I use tensorflow :
from tensorflow.keras.preprocessing.image import ImageDataGenerator
image_gen = ImageDataGenerator(
rotation_range=10, # rotate the image 10 degrees
width_shift_range=0.10, # Shift the pic width by a max of 5%
height_shift_range=0.10, # Shift the pic height by a max of 5%
rescale=1/255, # Rescale the image by normalzing it.
shear_range=0.1, # Shear means cutting away part of the image (max 10%)
zoom_range=0.1, # Zoom in by 10% max
horizontal_flip=True, # Allow horizontal flipping
fill_mode='nearest' # Fill in missing pixels with the nearest filled value
)
image_gen.flow_from_directory(train_path)
image_gen.flow_from_directory(test_path)
I create a model (basic model) :
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=(image_width, image_height, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), input_shape=(image_width, image_height, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(128, (3, 3), input_shape=(image_width, image_height, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(256, (3, 3), input_shape=(image_width, image_height, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(512, (3, 3), input_shape=(image_width, image_height, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
# Dropouts help reduce overfitting by randomly turning neurons off during training.
# Here we say randomly turn off 50% of neurons.
model.add(Dropout(0.5))
# Last layer, remember its binary so we use sigmoid
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
Then I train it :
train_image_gen = image_gen.flow_from_directory(
train_path,
target_size=image_shape[:2],
color_mode='rgb',
batch_size=batch_size,
class_mode='binary'
)
results = model.fit_generator(train_image_gen,epochs=20,
validation_data=test_image_gen,
callbacks=[early_stop, board])
So far so good results are correct :
pred_probabilities = model.predict_generator(test_image_gen)
predictions = pred_probabilities > 0.5
confusion_matrix(test_image_gen.classes,predictions)
I obtain rather good results :
My issue is when I want to predict images it returns results which are far from being correct:
val_image_gen = image_gen.flow_from_directory(
val_path,
target_size=image_shape[:2],
color_mode='rgb',
class_mode='binary',
)
pred_probabilities = model.predict_generator(val_image_gen)
predictions = pred_probabilities > 0.5
Here are some output I obtain :
precision recall f1-score support
0 0.51 0.57 0.53 317
1 0.51 0.44 0.47 317
accuracy 0.51 634
macro avg 0.51 0.51 0.50 634
weighted avg 0.51 0.51 0.50 634
THe confusion matrix on this data set is the following :
[[180 137]
[176 141]]
A few issues with your code:
You are using test set for validation and validation set for testing.
This may be a problem or not, depending on your data and how it was
split.
Augmentation should be applied only to training set. Use separate
instance of ImageDataGenerator(rescale=1/255) for testing and
validation.
Your test results look like they were got from untrained model. Check if the model object you are running test on is the same one you were training. You may want to use model.save() and load_model() functions to preserve model weights after training.
I replaced :
val_image_gen = image_gen.flow_from_directory(
val_path,
target_size=image_shape[:2],
color_mode='rgb',
class_mode='binary',
)
by:
val_image_gen = image_gen.flow_from_directory(
val_path,
target_size=image_shape[:2],
color_mode='rgb',
batch_size=batch_size,
class_mode='binary',
shuffle=False
)
I obtain nice results :
[[269 48]
[ 3 314]]
I am trying to build a image classification model, using ImageDataGenerator().
It seems that the model trains and performs poorly. The training loss stays at around 15 and the accuracy is barely 10%, the validation is about the same.
Just to see what would happen, I tried training without using the ImageDataGenerator() and set up the data in a similar way. It performed much better in training, validation and testing. With training loss of 0.71 and accuracy of 75% and validation loss of 0.8 and accuracy of 72%.
I need to figure out this model with the data generator because I will be moving on to a larger dataset, where it will not fit into memory.
So, I guess my question is what am I doing wrong with the ImageDataGenerator() that it is performing so badly and how can I improve the outcome?
When setting up the files (in all Train, Test, Validation folders), there are the classes with its own folder and in those folders is where the images are.
Here is the code:
import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import pickle
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation, Flatten, Conv2D, MaxPooling2D, Dropout
data_gen = ImageDataGenerator()
IMG_SIZE = 100
train_it = data_gen.flow_from_directory('D:/.../Train/', class_mode='sparse',
target_size=(IMG_SIZE, IMG_SIZE),color_mode='grayscale', shuffle=True,batch_size=32)
val_it = data_gen.flow_from_directory('D:/.../Validation/', class_mode='sparse',
target_size=(IMG_SIZE, IMG_SIZE),color_mode='grayscale', shuffle=True,batch_size=32)
IMAGE_SIZE = [100, 100]
model=Sequential()
model.add(Conv2D(32,(3,3), input_shape=[*IMAGE_SIZE, 1]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.5))
model.add(Conv2D(32,(3,3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.5))
model.add(Conv2D(32,(3,3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(len(train_it.class_indices), activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit_generator(train_it, epochs=20, validation_data=val_it )
Here is my code without ImageDataGenerator():
SETUP the data, using OpenCV
DATADIR='D:\...\Train'
CATEGORIES = pickle.load(open("CATEGORIES.p" , "rb"))
print(len(CATEGORIES))
IMG_SIZE = 100
training_data=[]
def create_training_data():
for category in CATEGORIES:
path = os.path.join(DATADIR,category)
class_num = CATEGORIES.index(category)
for img in os.listdir(path):
try:
img_array = cv2.imread(os.path.join(path,img),cv2.IMREAD_GRAYSCALE)
new_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE))
training_data.append([new_array, class_num])
except:
print(category)
print(img)
create_training_data()
random.shuffle(training_data)
X=[]
y=[]
for features, label in training_data:
X.append(features)
y.append(label)
X=np.array(X).reshape(-1,IMG_SIZE, IMG_SIZE, 1)
X=X/255.0
MODEL SETUP:
model=Sequential()
model.add(Conv2D(32,(3,3), input_shape=[*IMAGE_SIZE, 1]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.5))
model.add(Conv2D(32,(3,3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.5))
model.add(Conv2D(32,(3,3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(len(CATEGORIES), activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(X,y, epochs=20, batch_size=32, validation_split=0.1)
#acho,
Mentioning the solution to this issue cited by you in the comments, for the benefit of the community.
Reason for the issue is that Input Data is not Normalized by dividing each Pixel Value by 255. It has an impact on Training because of the reasons mentioned below:
It converts Pixel Values from Integers to Float, in a range of 0.0-1.0 where 0.0 means 0 (0x00) and 1.0 means 255 (0xFF). Conv Nets work better on Float Values compared to Integer Values, and by normalizing it in a range of 0-1, computations will be reduced.
Normalization will help you to remove distortions caused by lights and shadows in an image.