The following model is defined in TF1, I am trying to migrate it to TF2 without using compat API.
# Define the tensorflow neural network
# 1. Input:
self.input_states = tf.placeholder(
tf.float32, shape=[None, 4, board_height, board_width])
self.input_state = tf.transpose(self.input_states, [0, 2, 3, 1])
# 2. Common Networks Layers
self.conv1 = tf.layers.conv2d(inputs=self.input_state,
filters=32, kernel_size=[3, 3],
padding="same", data_format="channels_last",
activation=tf.nn.relu)
self.conv2 = tf.layers.conv2d(inputs=self.conv1, filters=64,
kernel_size=[3, 3], padding="same",
data_format="channels_last",
activation=tf.nn.relu)
self.conv3 = tf.layers.conv2d(inputs=self.conv2, filters=128,
kernel_size=[3, 3], padding="same",
data_format="channels_last",
activation=tf.nn.relu)
# 3-1 Action Networks
self.action_conv = tf.layers.conv2d(inputs=self.conv3, filters=4,
kernel_size=[1, 1], padding="same",
data_format="channels_last",
activation=tf.nn.relu)
# Flatten the tensor
self.action_conv_flat = tf.reshape(
self.action_conv, [-1, 4 * board_height * board_width])
# 3-2 Full connected layer, the output is the log probability of moves
# on each slot on the board
self.action_fc = tf.layers.dense(inputs=self.action_conv_flat,
units=board_height * board_width,
activation=tf.nn.log_softmax)
# 4 Evaluation Networks
self.evaluation_conv = tf.layers.conv2d(inputs=self.conv3, filters=2,
kernel_size=[1, 1],
padding="same",
data_format="channels_last",
activation=tf.nn.relu)
self.evaluation_conv_flat = tf.reshape(
self.evaluation_conv, [-1, 2 * board_height * board_width])
self.evaluation_fc1 = tf.layers.dense(inputs=self.evaluation_conv_flat,
units=64, activation=tf.nn.relu)
# output the score of evaluation on current state
self.evaluation_fc2 = tf.layers.dense(inputs=self.evaluation_fc1,
units=1, activation=tf.nn.tanh)
# Define the Loss function
# 1. Label: the array containing if the game wins or not for each state
self.labels = tf.placeholder(tf.float32, shape=[None, 1])
# 2. Predictions: the array containing the evaluation score of each state
# which is self.evaluation_fc2
# 3-1. Value Loss function
self.value_loss = tf.losses.mean_squared_error(self.labels,
self.evaluation_fc2)
# 3-2. Policy Loss function
self.mcts_probs = tf.placeholder(
tf.float32, shape=[None, board_height * board_width])
self.policy_loss = tf.negative(tf.reduce_mean(
tf.reduce_sum(tf.multiply(self.mcts_probs, self.action_fc), 1)))
# 3-3. L2 penalty (regularization)
l2_penalty_beta = 1e-4
vars = tf.trainable_variables()
l2_penalty = l2_penalty_beta * tf.add_n(
[tf.nn.l2_loss(v) for v in vars if 'bias' not in v.name.lower()])
# 3-4 Add up to be the Loss function
self.loss = self.value_loss + self.policy_loss + l2_penalty
# Define the optimizer we use for training
self.learning_rate = tf.placeholder(tf.float32)
self.optimizer = tf.train.AdamOptimizer(
learning_rate=self.learning_rate).minimize(self.loss)
And here is my TF2 code
l2_penalty_beta = 1e-4
# Define the tensorflow neural network
# 1. Input:
self.inputs = tf.keras.Input( shape=(4, board_height, board_width), dtype=tf.dtypes.float32)
self.transposed_inputs = tf.keras.layers.Lambda( lambda x: tf.transpose(x, [0, 2, 3, 1]) )(self.inputs)
# 2. Common Networks Layers
self.conv1 = tf.keras.layers.Conv2D( name="conv1",
filters=32,
kernel_size=(3, 3),
padding="same",
data_format="channels_last",
activation=tf.keras.activations.relu,
kernel_regularizer=tf.keras.regularizers.L2(l2_penalty_beta))(self.transposed_inputs)
self.conv2 = tf.keras.layers.Conv2D( name="conv2",
filters=64,
kernel_size=(3, 3),
padding="same",
data_format="channels_last",
activation=tf.keras.activations.relu,
kernel_regularizer=tf.keras.regularizers.L2(l2_penalty_beta))(self.conv1)
self.conv3 = tf.keras.layers.Conv2D( name="conv3",
filters=128,
kernel_size=(3, 3),
padding="same",
data_format="channels_last",
activation=tf.keras.activations.relu,
kernel_regularizer=tf.keras.regularizers.L2(l2_penalty_beta))(self.conv2)
# 3-1 Action Networks
self.action_conv = tf.keras.layers.Conv2D( name="action_conv",
filters=4,
kernel_size=(1, 1),
padding="same",
data_format="channels_last",
activation=tf.keras.activations.relu,
kernel_regularizer=tf.keras.regularizers.L2(l2_penalty_beta))(self.conv3)
# flatten tensor
self.action_conv_flat = tf.keras.layers.Reshape( (-1, 4 * board_height * board_width), name="action_conv_flat" )(self.action_conv)
# 3-2 Full connected layer, the output is the log probability of moves
# on each slot on the board
self.action_fc = tf.keras.layers.Dense( board_height * board_width,
activation=tf.nn.log_softmax,
name="action_fc",
kernel_regularizer=tf.keras.regularizers.L2(l2_penalty_beta))(self.action_conv_flat)
# 4 Evaluation Networks
self.evaluation_conv = tf.keras.layers.Conv2D( name="evaluation_conv",
filters=2,
kernel_size=(1, 1),
padding="same",
data_format="channels_last",
activation=tf.keras.activations.relu,
kernel_regularizer=tf.keras.regularizers.L2(l2_penalty_beta))(self.conv3)
self.evaluation_conv_flat = tf.keras.layers.Reshape( (-1, 2 * board_height * board_width),
name="evaluation_conv_flat" )(self.evaluation_conv)
self.evaluation_fc1 = tf.keras.layers.Dense( 64,
activation=tf.keras.activations.relu,
name="evaluation_fc1",
kernel_regularizer=tf.keras.regularizers.L2(l2_penalty_beta))(self.evaluation_conv_flat)
self.evaluation_fc2 = tf.keras.layers.Dense( 1,
activation=tf.keras.activations.tanh,
name="evaluation_fc2",
kernel_regularizer=tf.keras.regularizers.L2(l2_penalty_beta))(self.evaluation_fc1)
self.outputs = tf.keras.layers.Concatenate()([self.action_fc, self.evaluation_fc2])
self.model = tf.keras.Model(inputs=self.inputs, outputs=self.outputs, name="policy_value_model")
self.model.summary()
def custom_loss(labels, predictions):
expected_act_probs, expected_value = tf.split(labels, [self.board_height*self.board_width, -1], axis=2)
pred_act_probs, pred_value = tf.split(predictions, [self.board_height*self.board_width, -1], axis=2)
value_loss = tf.losses.mean_squared_error(expected_value[0], pred_value[0])
policy_loss = tf.negative(tf.reduce_mean(
tf.reduce_sum(tf.multiply(expected_act_probs, pred_act_probs), 1)))
return value_loss + policy_loss
#print(tf.autograph.to_code(custom_loss))
self.model.compile(optimizer=tf.keras.optimizers.Adam(),
loss=tf.function(custom_loss),
metrics=['accuracy'])
Here is summary of this model.
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_1 (InputLayer) [(None, 4, 15, 15)] 0 []
lambda (Lambda) (None, 15, 15, 4) 0 ['input_1[0][0]']
conv1 (Conv2D) (None, 15, 15, 32) 1184 ['lambda[0][0]']
conv2 (Conv2D) (None, 15, 15, 64) 18496 ['conv1[0][0]']
conv3 (Conv2D) (None, 15, 15, 128) 73856 ['conv2[0][0]']
evaluation_conv (Conv2D) (None, 15, 15, 2) 258 ['conv3[0][0]']
action_conv (Conv2D) (None, 15, 15, 4) 516 ['conv3[0][0]']
evaluation_conv_flat (Reshape) (None, 1, 450) 0 ['evaluation_conv[0][0]']
action_conv_flat (Reshape) (None, 1, 900) 0 ['action_conv[0][0]']
evaluation_fc1 (Dense) (None, 1, 64) 28864 ['evaluation_conv_flat[0][0]']
action_fc (Dense) (None, 1, 225) 202725 ['action_conv_flat[0][0]']
evaluation_fc2 (Dense) (None, 1, 1) 65 ['evaluation_fc1[0][0]']
concatenate (Concatenate) (None, 1, 226) 0 ['action_fc[0][0]',
'evaluation_fc2[0][0]']
==================================================================================================
As you can notice, the TF1 model outputs two tensors (action_fc and evaluation_fc2). And in TF2 model I added a concatenate layer in the last to combine them into a single tensor so that I can have a single loss function on them.
The loss function in TF1 model sum up three parts. The l2_penalty is the L2 lost of all weights in the model.
l2_penalty_beta = 1e-4
vars = tf.trainable_variables()
l2_penalty = l2_penalty_beta * tf.add_n(
[tf.nn.l2_loss(v) for v in vars if 'bias' not in v.name.lower()])
self.loss = self.value_loss + self.policy_loss + l2_penalty
In TF2 model, all trainable layers are added kernel_regularizer=tf.keras.regularizers.L2(l2_penalty_beta). Is that the same as TF1 model? Or do I make some mistake?
The TF2 model will be executed in an environment without Python interpreter, that is to say the model will be compiled into graph. I guess my loss function has to be stateless -- it must not rely on any varialble out of the function scope.
UPDATE:
Here is the training logs from the fress TF2 model. The loss looks abnormal. It is too small at the start. The loss is greater than 4.0 in TF1 model. Will the L2 regularizer be part of loss in TF2?
batch i:2, episode_len:113
1/1 - 0s - loss: 0.7342 - accuracy: 0.0000e+00 - 18ms/epoch - 18ms/step
1/1 - 0s - loss: 0.4714 - accuracy: 0.0000e+00 - 12ms/epoch - 12ms/step
1/1 - 0s - loss: 0.1785 - accuracy: 0.0000e+00 - 14ms/epoch - 14ms/step
1/1 - 0s - loss: 0.0829 - accuracy: 0.0000e+00 - 30ms/epoch - 30ms/step
1/1 - 0s - loss: 0.0743 - accuracy: 0.0000e+00 - 13ms/epoch - 13ms/step
kl:0.00178,lr_multiplier:2.250,loss:[0.07430928945541382]
batch i:3, episode_len:92
1/1 - 0s - loss: 0.0764 - accuracy: 1.0000 - 19ms/epoch - 19ms/step
1/1 - 0s - loss: 0.0718 - accuracy: 1.0000 - 28ms/epoch - 28ms/step
1/1 - 0s - loss: 0.0705 - accuracy: 1.0000 - 12ms/epoch - 12ms/step
1/1 - 0s - loss: 0.0693 - accuracy: 1.0000 - 12ms/epoch - 12ms/step
1/1 - 0s - loss: 0.0681 - accuracy: 1.0000 - 34ms/epoch - 34ms/step
kl:0.00068,lr_multiplier:3.375,loss:[0.06813239306211472]
batch i:4, episode_len:118
1/1 - 0s - loss: 0.0676 - accuracy: 1.0000 - 16ms/epoch - 16ms/step
1/1 - 0s - loss: 0.0665 - accuracy: 1.0000 - 12ms/epoch - 12ms/step
1/1 - 0s - loss: 0.0654 - accuracy: 1.0000 - 12ms/epoch - 12ms/step
1/1 - 0s - loss: 0.0643 - accuracy: 1.0000 - 15ms/epoch - 15ms/step
1/1 - 0s - loss: 0.0631 - accuracy: 1.0000 - 11ms/epoch - 11ms/step
kl:0.00425,lr_multiplier:5.062,loss:[0.06307009607553482]
batch i:5, episode_len:84
1/1 - 0s - loss: 4.0628 - accuracy: 0.0000e+00 - 17ms/epoch - 17ms/step
1/1 - 0s - loss: 4.0618 - accuracy: 0.0000e+00 - 12ms/epoch - 12ms/step
1/1 - 0s - loss: 4.0606 - accuracy: 0.0000e+00 - 16ms/epoch - 16ms/step
1/1 - 0s - loss: 4.0592 - accuracy: 0.0000e+00 - 11ms/epoch - 11ms/step
1/1 - 0s - loss: 4.0577 - accuracy: 0.0000e+00 - 13ms/epoch - 13ms/step
kl:0.07804,lr_multiplier:3.375,loss:[4.057666778564453]
batch i:6, episode_len:96
1/1 - 0s - loss: 0.0599 - accuracy: 1.0000 - 17ms/epoch - 17ms/step
1/1 - 0s - loss: 0.0589 - accuracy: 1.0000 - 11ms/epoch - 11ms/step
1/1 - 0s - loss: 0.0579 - accuracy: 1.0000 - 12ms/epoch - 12ms/step
1/1 - 0s - loss: 0.0568 - accuracy: 1.0000 - 31ms/epoch - 31ms/step
1/1 - 0s - loss: 0.0556 - accuracy: 1.0000 - 11ms/epoch - 11ms/step
kl:0.06898,lr_multiplier:2.250,loss:[0.055556993931531906]
batch i:7, episode_len:62
1/1 - 0s - loss: 0.0577 - accuracy: 1.0000 - 28ms/epoch - 28ms/step
1/1 - 0s - loss: 0.0569 - accuracy: 1.0000 - 11ms/epoch - 11ms/step
1/1 - 0s - loss: 0.0559 - accuracy: 1.0000 - 14ms/epoch - 14ms/step
1/1 - 0s - loss: 0.0549 - accuracy: 1.0000 - 16ms/epoch - 16ms/step
1/1 - 0s - loss: 0.0538 - accuracy: 1.0000 - 19ms/epoch - 19ms/step
kl:0.03346,lr_multiplier:2.250,loss:[0.05379907414317131]
batch i:8, episode_len:118
1/1 - 0s - loss: 0.0552 - accuracy: 1.0000 - 28ms/epoch - 28ms/step
1/1 - 0s - loss: 0.0543 - accuracy: 1.0000 - 48ms/epoch - 48ms/step
1/1 - 0s - loss: 0.0532 - accuracy: 1.0000 - 12ms/epoch - 12ms/step
1/1 - 0s - loss: 0.0521 - accuracy: 1.0000 - 11ms/epoch - 11ms/step
1/1 - 0s - loss: 0.0510 - accuracy: 1.0000 - 11ms/epoch - 11ms/step
kl:0.04336,lr_multiplier:1.500,loss:[0.051018256694078445]
Yes, I think that's the right way to add L2 regularization to the weights (but not the biases) in TF2
My only thought is that you don't have to concatenate the outputs tensors in TF2, you can define
self.outputs = [self.action_fc, self.evaluation_fc2]
if that's easier. (Then also specify your losses as a length 2 list etc.).
Related
I’m trying to do a binary classification on a very unbalanced dataset.
The model is doing great, but after some random epochs the loss becomes nan, also precision, recall, TP, and FP, all become ZERO.
Sometimes it happens after the 3rd epoch, sometimes after the 20th epoch.
The code:
import numpy as np
import os
from keras import regularizers
from tensorflow.keras import layers
from tensorflow.keras.utils import Sequence
from tensorflow.keras.models import Sequential
import pandas
nodules_csv = pandas.read_csv("/cropped_nodules.csv")
base_dir = "/cropped_nodules/"
all_image_paths = os.listdir(base_dir)
all_image_paths = sorted(all_image_paths,key=lambda x: int(os.path.splitext(x)[0]))
nodules = nodules_csv.rename(columns = {'SN':'ID'})
labels= nodules.iloc[:,1]
labels = labels.to_numpy()
class DataGenerator(Sequence):
# Learned from https://mahmoudyusof.github.io/facial-keypoint-detection/data-generator/
def __init__(self, all_image_paths, labels, base_dir, output_size, shuffle=False, batch_size=10):
"""
Initializes a data generator object
:param csv_file: file in which image names and numeric labels are stored
:param base_dir: the directory in which all images are stored
:param output_size: image output size after preprocessing
:param shuffle: shuffle the data after each epoch
:param batch_size: The size of each batch returned by __getitem__
"""
self.imgs = all_image_paths
self.base_dir = base_dir
self.output_size = output_size
self.shuffle = shuffle
self.batch_size = batch_size
self.labels = labels
self.on_epoch_end()
def on_epoch_end(self):
self.indices = np.arange(len(self.imgs))
if self.shuffle:
np.random.shuffle(self.indices)
def __len__(self):
return int(len(self.imgs) / self.batch_size)
def __getitem__(self, idx):
## Initializing Batch
# that one in the shape is just for a one channel images
# if you want to use colored images you might want to set that to 3
X = np.empty((self.batch_size, *self.output_size,1))
# (x, y, h, w)
y = np.empty((self.batch_size, 1))
# get the indices of the requested batch
indices = self.indices[idx*self.batch_size:(idx+1)*self.batch_size]
for i, data_index in enumerate(indices):
img_path = os.path.join(self.base_dir,
self.imgs[data_index])
img = np.load(img_path)
while img.shape == (31,31,31):
img = np.expand_dims(img, axis=3)
## this is where you preprocess the image
## make sure to resize it to be self.output_size
label = self.labels[data_index]
## if you have any preprocessing for
## the labels too do it here
X[i,] = img
y[i] = label
return X, y
## Defining and training the model
model = Sequential([
## define the model's architecture
layers.Conv3D(filters=32, kernel_size=3, activation="relu",padding='same'),
layers.BatchNormalization(),
layers.Conv3D(filters=32, kernel_size=3, activation="relu",padding='same'),
layers.BatchNormalization(),
layers.MaxPool3D(pool_size=2),
layers.BatchNormalization(),
layers.Conv3D(filters=64, kernel_size=3, activation="relu",padding='same'),
layers.BatchNormalization(),
layers.Conv3D(filters=64, kernel_size=3, activation="relu",padding='same'),
layers.BatchNormalization(),
layers.MaxPool3D(pool_size=2),
layers.BatchNormalization(),
layers.Conv3D(filters=128, kernel_size=3, activation="relu",padding='same'),
layers.BatchNormalization(),
layers.Conv3D(filters=128, kernel_size=3, activation="relu",padding='same'),
layers.BatchNormalization(),
layers.MaxPool3D(pool_size=2),
layers.BatchNormalization(),
layers.Conv3D(filters=256, kernel_size=3, activation="relu", padding='same'),
layers.BatchNormalization(),
layers.Conv3D(filters=256, kernel_size=3, activation="relu", padding='same'),
layers.BatchNormalization(),
layers.MaxPool3D(pool_size=2),
layers.BatchNormalization(),
layers.GlobalAveragePooling3D(),
layers.Dense(units=512, activation="relu"),
layers.BatchNormalization(),
layers.Dropout(0.3),
layers.Dense(units=1, activation="sigmoid"),
])
train_gen = DataGenerator(all_image_paths, labels, base_dir, (31, 31, 31), batch_size=128, shuffle=False)
## compile the model first of course
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy', 'Precision', 'Recall', 'FalseNegatives', 'FalsePositives', 'TrueNegatives', 'TruePositives'])
model.build(input_shape= (128,31,31,31,1))
model.summary()
# now let's train the model
history = model.fit(train_gen, epochs=25)
and the results below:
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv3d (Conv3D) (128, 31, 31, 31, 32) 896
batch_normalization (BatchN (128, 31, 31, 31, 32) 128
ormalization)
conv3d_1 (Conv3D) (128, 31, 31, 31, 32) 27680
batch_normalization_1 (Batc (128, 31, 31, 31, 32) 128
hNormalization)
max_pooling3d (MaxPooling3D (128, 15, 15, 15, 32) 0
)
batch_normalization_2 (Batc (128, 15, 15, 15, 32) 128
hNormalization)
conv3d_2 (Conv3D) (128, 15, 15, 15, 64) 55360
batch_normalization_3 (Batc (128, 15, 15, 15, 64) 256
hNormalization)
conv3d_3 (Conv3D) (128, 15, 15, 15, 64) 110656
batch_normalization_4 (Batc (128, 15, 15, 15, 64) 256
hNormalization)
max_pooling3d_1 (MaxPooling (128, 7, 7, 7, 64) 0
3D)
batch_normalization_5 (Batc (128, 7, 7, 7, 64) 256
hNormalization)
conv3d_4 (Conv3D) (128, 7, 7, 7, 128) 221312
batch_normalization_6 (Batc (128, 7, 7, 7, 128) 512
hNormalization)
conv3d_5 (Conv3D) (128, 7, 7, 7, 128) 442496
batch_normalization_7 (Batc (128, 7, 7, 7, 128) 512
hNormalization)
max_pooling3d_2 (MaxPooling (128, 3, 3, 3, 128) 0
3D)
batch_normalization_8 (Batc (128, 3, 3, 3, 128) 512
hNormalization)
conv3d_6 (Conv3D) (128, 3, 3, 3, 256) 884992
batch_normalization_9 (Batc (128, 3, 3, 3, 256) 1024
hNormalization)
conv3d_7 (Conv3D) (128, 3, 3, 3, 256) 1769728
batch_normalization_10 (Bat (128, 3, 3, 3, 256) 1024
chNormalization)
max_pooling3d_3 (MaxPooling (128, 1, 1, 1, 256) 0
3D)
batch_normalization_11 (Bat (128, 1, 1, 1, 256) 1024
chNormalization)
global_average_pooling3d (G (128, 256) 0
lobalAveragePooling3D)
dense (Dense) (128, 512) 131584
batch_normalization_12 (Bat (128, 512) 2048
chNormalization)
dropout (Dropout) (128, 512) 0
dense_1 (Dense) (128, 1) 513
=================================================================
Total params: 3,653,025
Trainable params: 3,649,121
Non-trainable params: 3,904
_________________________________________________________________
Epoch 1/25
2022-12-15 17:46:04.897341: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:428] Loaded cuDNN version 8401
2022-12-15 17:46:05.829836: I tensorflow/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory
2022-12-15 17:46:06.464508: I tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:630] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.
2022-12-15 17:46:07.214021: I tensorflow/compiler/xla/service/service.cc:173] XLA service 0x2319ed30 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
2022-12-15 17:46:07.214054: I tensorflow/compiler/xla/service/service.cc:181] StreamExecutor device (0): NVIDIA GeForce RTX 3080, Compute Capability 8.6
2022-12-15 17:46:07.217900: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.
2022-12-15 17:46:07.277629: I tensorflow/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory
2022-12-15 17:46:07.317843: I tensorflow/compiler/jit/xla_compilation_cache.cc:477] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.
5898/5898 [==============================] - 1184s 199ms/step - loss: 0.0203 - accuracy: 0.9956 - precision: 0.0807 - recall: 0.1113 - false_negatives: 1381.0000 - false_positives: 1972.0000 - true_negatives: 751418.0000 - true_positives: 173.0000
Epoch 2/25
5898/5898 [==============================] - 1178s 200ms/step - loss: 0.0068 - accuracy: 0.9984 - precision: 0.6869 - recall: 0.3779 - false_negatives: 968.0000 - false_positives: 268.0000 - true_negatives: 753120.0000 - true_positives: 588.0000
Epoch 3/25
5898/5898 [==============================] - 1178s 200ms/step - loss: 0.0052 - accuracy: 0.9986 - precision: 0.7472 - recall: 0.4782 - false_negatives: 813.0000 - false_positives: 252.0000 - true_negatives: 753134.0000 - true_positives: 745.0000
Epoch 4/25
5898/5898 [==============================] - 1181s 200ms/step - loss: 0.0045 - accuracy: 0.9987 - precision: 0.7676 - recall: 0.5540 - false_negatives: 694.0000 - false_positives: 261.0000 - true_negatives: 753127.0000 - true_positives: 862.0000
Epoch 5/25
5898/5898 [==============================] - 1181s 200ms/step - loss: 0.0039 - accuracy: 0.9988 - precision: 0.7913 - recall: 0.5963 - false_negatives: 629.0000 - false_positives: 245.0000 - true_negatives: 753141.0000 - true_positives: 929.0000
Epoch 6/25
5898/5898 [==============================] - 1178s 200ms/step - loss: 0.0033 - accuracy: 0.9990 - precision: 0.8080 - recall: 0.6465 - false_negatives: 550.0000 - false_positives: 239.0000 - true_negatives: 753149.0000 - true_positives: 1006.0000
Epoch 7/25
5898/5898 [==============================] - 1178s 200ms/step - loss: 0.0029 - accuracy: 0.9990 - precision: 0.8178 - recall: 0.6913 - false_negatives: 481.0000 - false_positives: 240.0000 - true_negatives: 753146.0000 - true_positives: 1077.0000
Epoch 8/25
5898/5898 [==============================] - 1181s 200ms/step - loss: 0.0024 - accuracy: 0.9992 - precision: 0.8452 - recall: 0.7530 - false_negatives: 385.0000 - false_positives: 215.0000 - true_negatives: 753170.0000 - true_positives: 1174.0000
Epoch 9/25
5898/5898 [==============================] - 1177s 200ms/step - loss: 0.0018 - accuracy: 0.9993 - precision: 0.8632 - recall: 0.8077 - false_negatives: 299.0000 - false_positives: 199.0000 - true_negatives: 753190.0000 - true_positives: 1256.0000
Epoch 10/25
5898/5898 [==============================] - 1180s 200ms/step - loss: 0.0014 - accuracy: 0.9995 - precision: 0.9055 - recall: 0.8508 - false_negatives: 232.0000 - false_positives: 138.0000 - true_negatives: 753251.0000 - true_positives: 1323.0000
Epoch 11/25
5898/5898 [==============================] - 1181s 200ms/step - loss: 0.0014 - accuracy: 0.9995 - precision: 0.9086 - recall: 0.8678 - false_negatives: 206.0000 - false_positives: 136.0000 - true_negatives: 753250.0000 - true_positives: 1352.0000
Epoch 12/25
5898/5898 [==============================] - 1178s 200ms/step - loss: 0.0011 - accuracy: 0.9996 - precision: 0.9207 - recall: 0.8952 - false_negatives: 163.0000 - false_positives: 120.0000 - true_negatives: 753268.0000 - true_positives: 1393.0000
Epoch 13/25
5898/5898 [==============================] - 1182s 200ms/step - loss: 8.5650e-04 - accuracy: 0.9997 - precision: 0.9382 - recall: 0.9177 - false_negatives: 128.0000 - false_positives: 94.0000 - true_negatives: 753294.0000 - true_positives: 1428.0000
Epoch 14/25
5898/5898 [==============================] - 1179s 200ms/step - loss: 7.9298e-04 - accuracy: 0.9998 - precision: 0.9509 - recall: 0.9326 - false_negatives: 105.0000 - false_positives: 75.0000 - true_negatives: 753312.0000 - true_positives: 1452.0000
Epoch 15/25
5898/5898 [==============================] - 1179s 200ms/step - loss: 7.1897e-04 - accuracy: 0.9998 - precision: 0.9576 - recall: 0.9422 - false_negatives: 90.0000 - false_positives: 65.0000 - true_negatives: 753322.0000 - true_positives: 1467.0000
Epoch 16/25
5898/5898 [==============================] - 1181s 200ms/step - loss: 6.0985e-04 - accuracy: 0.9998 - precision: 0.9567 - recall: 0.9499 - false_negatives: 78.0000 - false_positives: 67.0000 - true_negatives: 753320.0000 - true_positives: 1479.0000
Epoch 17/25
5898/5898 [==============================] - 1182s 200ms/step - loss: 6.1805e-04 - accuracy: 0.9998 - precision: 0.9648 - recall: 0.9499 - false_negatives: 78.0000 - false_positives: 54.0000 - true_negatives: 753332.0000 - true_positives: 1480.0000
Epoch 18/25
5898/5898 [==============================] - 1182s 200ms/step - loss: 4.7617e-04 - accuracy: 0.9998 - precision: 0.9657 - recall: 0.9595 - false_negatives: 63.0000 - false_positives: 53.0000 - true_negatives: 753336.0000 - true_positives: 1492.0000
Epoch 19/25
5898/5898 [==============================] - 1196s 203ms/step - loss: 5.4637e-04 - accuracy: 0.9998 - precision: 0.9637 - recall: 0.9563 - false_negatives: 68.0000 - false_positives: 56.0000 - true_negatives: 753332.0000 - true_positives: 1488.0000
Epoch 20/25
5898/5898 [==============================] - 1748s 296ms/step - loss: nan - accuracy: 0.9979 - precision: 0.0000e+00 - recall: 0.0000e+00 - false_negatives: 1557.0000 - false_positives: 0.0000e+00 - true_negatives: 753387.0000 - true_positives: 0.0000e+00
Epoch 21/25
5898/5898 [==============================] - 1150s 195ms/step - loss: nan - accuracy: 0.9979 - precision: 0.0000e+00 - recall: 0.0000e+00 - false_negatives: 1557.0000 - false_positives: 0.0000e+00 - true_negatives: 753387.0000 - true_positives: 0.0000e+00
Epoch 22/25
5898/5898 [==============================] - 1145s 194ms/step - loss: nan - accuracy: 0.9979 - precision: 0.0000e+00 - recall: 0.0000e+00 - false_negatives: 1558.0000 - false_positives: 0.0000e+00 - true_negatives: 753386.0000 - true_positives: 0.0000e+00
Epoch 23/25
5898/5898 [==============================] - 1145s 194ms/step - loss: nan - accuracy: 0.9979 - precision: 0.0000e+00 - recall: 0.0000e+00 - false_negatives: 1555.0000 - false_positives: 0.0000e+00 - true_negatives: 753389.0000 - true_positives: 0.0000e+00
Epoch 24/25
5898/5898 [==============================] - 1146s 194ms/step - loss: nan - accuracy: 0.9979 - precision: 0.0000e+00 - recall: 0.0000e+00 - false_negatives: 1558.0000 - false_positives: 0.0000e+00 - true_negatives: 753386.0000 - true_positives: 0.0000e+00
Epoch 25/25
5898/5898 [==============================] - 1148s 195ms/step - loss: nan - accuracy: 0.9979 - precision: 0.0000e+00 - recall: 0.0000e+00 - false_negatives: 1557.0000 - false_positives: 0.0000e+00 - true_negatives: 753387.0000 - true_positives: 0.0000e+00
I think this is a gradient exploding.
I've searched the other questions but most of the solutions I already implemented but no luck :
Loss of CNN in Keras becomes nan at some point of training
Why the total loss of a Deep Learning model can abruptly become NaN using Keras (Python)?
I've tried to do regularization (Dropout after every layer, L1, L2) but the reult was poor model (the model didn't learned anything). I've tried adam decay learning rate, batch norm, also didn't solve the problem.
I think this is a gradient exploding.
No. Usually exploding gradients has the loss get worse before returning NaNs.
layers.Dense(units=1, activation="sigmoid")
I thought keras had protections against this, but it's possible the problem here is that the naive loss calculation of sigmoid crossentropy will diverge outside of very limited input range.
Try:
layers.Dense(units=1)
...
model.compile(..., loss=keras.losses.BinaryCrossentropy(from_logits=True))
LMK if that works.
This is from the Keras document example: Train a model to calculate the priority_score and which department to forward for an email.
I implement the model in another way, I can compile it but I cannot train the model. I guess it a model I/O issue, i.e I need to feed the correct format of the I/O data.
ValueError: Failed to find data adapter that can handle input: (<class 'dict'> containing {"<class 'str'>"} keys and {"<class 'numpy.ndarray'>", '(<class \'list\'> containing values of types {"<class \'str\'>"})'} values), (<class 'dict'> containing {"<class 'str'>"} keys and {"<class 'numpy.ndarray'>"} values)
Its too long so I didn't put it into this post's title.
My model has 3 inputs:
title_input: It supposed to be a single string
body_input: It supposed to be a single string
tags_input: An array of 12 0s or 1s. For example, [0,1,0,1,0,0,0,0,0,1]
The output are:
priority: A float
departments: An array of 4 0,1s.
Questions
Can anyone tell me what's wrong in my code?
And generally speaking, how should I think about the I/O of a model? Such as this case. I thought preparing N strings, such as 800 strings, and 800 tags is OK. But I keep getting errors. Well I solved most of them but couldn't overcome this one. Please share your experiences. Thanks!
Appendix
Model summary
Model: "functional_1"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
tags_input (InputLayer) [(None, 12)] 0
__________________________________________________________________________________________________
flatten (Flatten) (None, 12) 0 tags_input[0][0]
__________________________________________________________________________________________________
title_input (InputLayer) [(None, 1)] 0
__________________________________________________________________________________________________
body_input (InputLayer) [(None, 1)] 0
__________________________________________________________________________________________________
dense (Dense) (None, 500) 6500 flatten[0][0]
__________________________________________________________________________________________________
text_vectorization (TextVectori (None, 500) 0 title_input[0][0]
__________________________________________________________________________________________________
text_vectorization_1 (TextVecto (None, 500) 0 body_input[0][0]
__________________________________________________________________________________________________
tf_op_layer_ExpandDims (TensorF [(None, 500, 1)] 0 dense[0][0]
__________________________________________________________________________________________________
embedding (Embedding) (None, 500, 100) 1000100 text_vectorization[0][0]
__________________________________________________________________________________________________
embedding_1 (Embedding) (None, 500, 100) 1000100 text_vectorization_1[0][0]
__________________________________________________________________________________________________
dense_1 (Dense) (None, 500, 100) 200 tf_op_layer_ExpandDims[0][0]
__________________________________________________________________________________________________
concatenate (Concatenate) (None, 500, 300) 0 embedding[0][0]
embedding_1[0][0]
dense_1[0][0]
__________________________________________________________________________________________________
priority (Dense) (None, 500, 1) 301 concatenate[0][0]
__________________________________________________________________________________________________
departments (Dense) (None, 500, 4) 1204 concatenate[0][0]
==================================================================================================
Total params: 2,008,405
Trainable params: 2,008,405
Non-trainable params: 0
__________________________________________________________________________________________________
Full Code
def MultiInputAndOutpt():
max_features = 10000
sequnce_length = 500
embedding_dims = 100
num_departments = 4
num_tags = 12
str = "hello"
title_vect = TextVectorization(max_tokens=max_features, output_mode="int", output_sequence_length=sequnce_length)
body_vect = TextVectorization(max_tokens=max_features, output_mode="int", output_sequence_length=sequnce_length)
title_input = keras.Input(shape=(1,), dtype=tf.string, name="title_input")
x1 = title_vect(title_input)
x1 = layers.Embedding(input_dim=max_features + 1, output_dim=embedding_dims)(x1)
body_input = keras.Input(shape=(1,), dtype=tf.string, name="body_input")
x2 = body_vect(body_input)
x2 = layers.Embedding(input_dim=max_features + 1, output_dim=embedding_dims)(x2)
tags_input = keras.Input(shape=(num_tags,), name="tags_input")
x3 = layers.Flatten()(tags_input)
x3 = layers.Dense(500)(x3)
x3 = tf.expand_dims(x3, axis=-1)
x3 = layers.Dense(100)(x3)
x = layers.concatenate([x1, x2, x3])
priority_score = layers.Dense(1)(x)
priority_score = tf.reshape(priority_score, (-1, 1), name="priority")
departments = layers.Dense(num_departments)(x)
departments = tf.reshape(departments, (-1, num_departments), name="departments")
model = keras.Model(inputs=[title_input, body_input, tags_input], outputs=[priority_score, departments])
model.summary()
model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.1),
loss=[keras.losses.BinaryCrossentropy(from_logits=True),
keras.losses.CategoricalCrossentropy(from_logits=True)],
loss_weights=[1.0, 0.2],
)
# title_data = np.random.randint(num_words, size=(1280, 10))
# body_data = np.random.randint(num_words, size=(1280, 100))
alphabet = np.array(list(string.ascii_lowercase + ' '))
title_data = np.random.choice(alphabet, size=(800, 1000))
body_data = np.random.choice(alphabet, size=(800, 1000))
tags_data = np.random.randint(2, size=(800, num_tags)).astype("float32")
body_data = ["".join(body_data[i]) for i in range(len(body_data))]
title_data = ["".join(title_data[i]) for i in range(len(title_data))]
# Dummy target data
priority_targets = np.random.random(size=(800, 1))
dept_targets = np.random.randint(2, size=(800, num_departments))
model.fit(
{"title_input": title_data, "body_input": body_data, "input3": tags_data},
{"priority": priority_targets, "departments": dept_targets},
epochs=2,
batch_size=32, )
I figured out by myself:
Input
The inputs of the model are correct. I don't need the flatten though.
tags_input (InputLayer) [(None, 12)] 0
__________________________________________________________________________________________________
title_input (InputLayer) [(None, 1)] 0
__________________________________________________________________________________________________
body_input (InputLayer) [(None, 1)] 0
__________________________________________________________________________________________________
output
The outputs are not correct, I don't want (None, 500, 1) and (None, 500, 4) as the output. I only need 1 priority score and the 1 department list of 4 values.
To change the shape from (None, 500, 1) to (None, 1) I need to drop some values. There are many ways do this, here I chose to drop the middle dim directly.
...
departments = layers.Dense(num_departments)(x) # Shape: (None, 500, 4)
departments = tf.slice(departments, [0, 0, 0], [-1, 1, 4]) # Shape (None, 1, 4)
departments = tf.squeeze(departments, [1]) # Shape (None, 4) but its not a squeeze type
departments = layers.Dense(num_departments, name="departments")(departments) # Shape (None, 4)
...
Same to the priority_score output.
And now the outputs become
priority_score (Dense) (None, 1) 2 tf.compat.v1.squeeze[0][0]
__________________________________________________________________________________________________
departments (Dense) (None, 4) 20 tf.compat.v1.squeeze_1[0][0]
Train model
The next step is to prepare the training data. What we need is to construct
title data: N strings, shape (N, 1). here 1 represent a python string.
body data: Same as tiele data
tags data: N array of floats, shape (N, 12)
Targets:
priority_score: N floats, shape (N, 1)
department: N array of floats, shape (N, 4)
where N can be any number.
Then we call the fit function:
model.fit(
{"title_input": title_data, "body_input": body_data, "tags_input": tags_data},
{"priority_score": priority_targets, "departments": dept_targets, },
epochs=50,
batch_size=64, )
Surprisingly the loss keeps growing:
Epoch 1/50
157/157 [==============================] - 5s 28ms/step - loss: 1.3467 - priority_score_loss: 0.6938 - departments_loss: 3.2644 - priority_score_acc: 0.0000e+00 - departments_acc: 0.1267
Epoch 2/50
157/157 [==============================] - 4s 27ms/step - loss: 4.6381 - priority_score_loss: 0.6976 - departments_loss: 19.7023 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2483
Epoch 3/50
157/157 [==============================] - 4s 28ms/step - loss: 16.9411 - priority_score_loss: 0.6984 - departments_loss: 81.2137 - priority_score_acc: 0.0000e+00 - departments_acc: 0.1569
Epoch 4/50
157/157 [==============================] - 5s 29ms/step - loss: 23.8020 - priority_score_loss: 0.7075 - departments_loss: 115.4721 - priority_score_acc: 0.0000e+00 - departments_acc: 0.1427
Epoch 5/50
157/157 [==============================] - 5s 29ms/step - loss: 1.8650 - priority_score_loss: 0.7046 - departments_loss: 5.8019 - priority_score_acc: 0.0000e+00 - departments_acc: 0.1995
Epoch 6/50
157/157 [==============================] - 5s 30ms/step - loss: 3.0613 - priority_score_loss: 0.7025 - departments_loss: 11.7943 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2472
Epoch 7/50
157/157 [==============================] - 5s 30ms/step - loss: 5.2455 - priority_score_loss: 0.7032 - departments_loss: 22.7114 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2402
Epoch 8/50
157/157 [==============================] - 5s 30ms/step - loss: 6.0378 - priority_score_loss: 0.7013 - departments_loss: 26.6828 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2418
Epoch 9/50
157/157 [==============================] - 5s 30ms/step - loss: 10.8300 - priority_score_loss: 0.7033 - departments_loss: 50.6334 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2465
Epoch 10/50
157/157 [==============================] - 4s 27ms/step - loss: 12.1005 - priority_score_loss: 0.7019 - departments_loss: 56.9929 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2627
Epoch 11/50
157/157 [==============================] - 4s 27ms/step - loss: 15.8248 - priority_score_loss: 0.6983 - departments_loss: 75.6328 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2513
Epoch 12/50
157/157 [==============================] - 5s 29ms/step - loss: 19.3059 - priority_score_loss: 0.6940 - departments_loss: 93.0596 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2386
Epoch 13/50
157/157 [==============================] - 5s 29ms/step - loss: 32.6499 - priority_score_loss: 0.6937 - departments_loss: 159.7808 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2526
Epoch 14/50
157/157 [==============================] - 4s 28ms/step - loss: 31.1433 - priority_score_loss: 0.6936 - departments_loss: 152.2486 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2499
Epoch 15/50
157/157 [==============================] - 5s 29ms/step - loss: 41.9199 - priority_score_loss: 0.6932 - departments_loss: 206.1338 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2362
Epoch 16/50
157/157 [==============================] - 5s 30ms/step - loss: 40.2069 - priority_score_loss: 0.6931 - departments_loss: 197.5692 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2300
Epoch 17/50
157/157 [==============================] - 5s 30ms/step - loss: 60.4129 - priority_score_loss: 0.6932 - departments_loss: 298.5986 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2425
Epoch 18/50
157/157 [==============================] - 5s 30ms/step - loss: 75.8330 - priority_score_loss: 0.6932 - departments_loss: 375.6990 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2332
Epoch 19/50
157/157 [==============================] - 5s 29ms/step - loss: 81.5731 - priority_score_loss: 0.6931 - departments_loss: 404.4002 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2568
Epoch 20/50
157/157 [==============================] - 4s 28ms/step - loss: 103.4053 - priority_score_loss: 0.6932 - departments_loss: 513.5608 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2409
Epoch 21/50
157/157 [==============================] - 4s 28ms/step - loss: 106.4842 - priority_score_loss: 0.6932 - departments_loss: 528.9552 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2584
Epoch 22/50
157/157 [==============================] - 4s 28ms/step - loss: 121.2103 - priority_score_loss: 0.6932 - departments_loss: 602.5854 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2332
Epoch 23/50
157/157 [==============================] - 5s 29ms/step - loss: 139.4970 - priority_score_loss: 0.6932 - departments_loss: 694.0189 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2421
Epoch 24/50
157/157 [==============================] - 5s 29ms/step - loss: 180.7346 - priority_score_loss: 0.6933 - departments_loss: 900.2067 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2449
Epoch 25/50
157/157 [==============================] - 4s 28ms/step - loss: 201.8011 - priority_score_loss: 0.6932 - departments_loss: 1005.5396 - priority_score_acc: 0.0000e+00 - departments_acc: 0.2420
Epoch 26/50
I guess this is because the training data is randomly generated, and the model is not well constructed. Anyway, we can train model and predict with some data now.
This was good learning experience.
So I am currently trying to build a race recognition program using a convolution neural network. I'm inputting 200px by 200px versions of the UTKFaceRegonition dataset (put my dataset on a google drive if you want to take a look). Im using 8 different classes (4 races * 2 genders) using keras and tensorflow, each having about 700 images but I have done it with 1000. The problem is when I run the network it gets at best 13.5% accuracy and about 11-12.5% validation accuracy, with a loss around 2.079-2.081, even after 50 epochs or so it won't improve at all. My current hypothesis is that it is randomly guessing stuff/not learning because 8/100=12.5%, which is about what it is getting and on other models I have made with 3 classes it was getting about 33%
I noticed the validation accuracy is different on the first and sometimes second epoch, but after that it ends up staying constant. I've increased the pixel resolution, changed amount of layers, types of layer and neurons per layer, I've tried optimizers (sgd at the normal lr and at very large and small (.1 and 10^-6) and I've tried different loss functions like KLDivergence but nothing seems to have any effect on it except KLDivergence which on one run did pretty well (about 16%) but then it flopped again. Some ideas I had are maybe theres too much noise in the dataset or maybe it has to do with the amount of dense layers, but honestly I dont know why it is not learning.
Heres the code to make the tensors
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import os
import cv2
import random
import pickle
WIDTH_SIZE = 200
HEIGHT_SIZE = 200
CATEGORIES = []
for CATEGORY in os.listdir('./TRAINING'):
CATEGORIES.append(CATEGORY)
DATADIR = "./TRAINING"
training_data = []
def create_training_data():
for category in CATEGORIES:
path = os.path.join(DATADIR, category)
class_num = CATEGORIES.index(category)
for img in os.listdir(path)[:700]:
try:
img_array = cv2.imread(os.path.join(path,img), cv2.IMREAD_COLOR)
new_array = cv2.resize(img_array,(WIDTH_SIZE,HEIGHT_SIZE))
training_data.append([new_array,class_num])
except Exception as error:
print(error)
create_training_data()
random.shuffle(training_data)
X = []
y = []
for features, label in training_data:
X.append(features)
y.append(label)
X = np.array(X).reshape(-1, WIDTH_SIZE, HEIGHT_SIZE, 3)
y = np.array(y)
pickle_out = open("X.pickle", "wb")
pickle.dump(X, pickle_out)
pickle_out = open("y.pickle", "wb")
pickle.dump(y, pickle_out)
Heres my built model
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D
import pickle
pickle_in = open("X.pickle","rb")
X = pickle.load(pickle_in)
pickle_in = open("y.pickle","rb")
y = pickle.load(pickle_in)
X = X/255.0
model = Sequential()
model.add(Conv2D(256, (2,2), activation = 'relu', input_shape = X.shape[1:]))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(256, (2,2), activation = 'relu'))
model.add(Conv2D(256, (2,2), activation = 'relu'))
model.add(Conv2D(256, (2,2), activation = 'relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(256, (2,2), activation = 'relu'))
model.add(Conv2D(256, (2,2), activation = 'relu'))
model.add(Dropout(0.4))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(256, (2,2), activation = 'relu'))
model.add(Conv2D(256, (2,2), activation = 'relu'))
model.add(Dropout(0.4))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(256, (2,2), activation = 'relu'))
model.add(Conv2D(256, (2,2), activation = 'relu'))
model.add(Dropout(0.4))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Flatten())
model.add(Dense(8, activation="softmax"))
model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),metrics=['accuracy'])
model.fit(X, y, batch_size=16,epochs=100,validation_split=.1)
Heres a log of 10 epochs I ran.
5040/5040 [==============================] - 55s 11ms/sample - loss: 2.0803 - accuracy: 0.1226 - val_loss: 2.0796 - val_accuracy: 0.1250
Epoch 2/100
5040/5040 [==============================] - 53s 10ms/sample - loss: 2.0797 - accuracy: 0.1147 - val_loss: 2.0798 - val_accuracy: 0.1161
Epoch 3/100
5040/5040 [==============================] - 53s 10ms/sample - loss: 2.0797 - accuracy: 0.1190 - val_loss: 2.0800 - val_accuracy: 0.1161
Epoch 4/100
5040/5040 [==============================] - 53s 11ms/sample - loss: 2.0797 - accuracy: 0.1173 - val_loss: 2.0799 - val_accuracy: 0.1107
Epoch 5/100
5040/5040 [==============================] - 52s 10ms/sample - loss: 2.0797 - accuracy: 0.1183 - val_loss: 2.0802 - val_accuracy: 0.1107
Epoch 6/100
5040/5040 [==============================] - 52s 10ms/sample - loss: 2.0797 - accuracy: 0.1226 - val_loss: 2.0801 - val_accuracy: 0.1107
Epoch 7/100
5040/5040 [==============================] - 52s 10ms/sample - loss: 2.0797 - accuracy: 0.1238 - val_loss: 2.0803 - val_accuracy: 0.1107
Epoch 8/100
5040/5040 [==============================] - 54s 11ms/sample - loss: 2.0797 - accuracy: 0.1169 - val_loss: 2.0802 - val_accuracy: 0.1107
Epoch 9/100
5040/5040 [==============================] - 52s 10ms/sample - loss: 2.0797 - accuracy: 0.1212 - val_loss: 2.0803 - val_accuracy: 0.1107
Epoch 10/100
5040/5040 [==============================] - 53s 11ms/sample - loss: 2.0797 - accuracy: 0.1177 - val_loss: 2.0802 - val_accuracy: 0.1107
So yeah, any help on why my network seems to be just guessing? Thank you!
The problem lies in the design of you network.
Typically you'd want in the first layers to learn high-level features and use larger kernel with odd size. Currently you're essentially interpolating neighbouring pixels. Why odd size? Read e.g. here.
Number of filters typically increases from small (e.g. 16, 32) number to larger values when going deeper into the network. In your network all layers learn the same number of filters. The reasoning is that the deeper you go, the more fine-grained features you'd like to learn - hence increase in number of filters.
In your ANN each layer also cuts out valuable information from the image (by default you are using valid padding).
Here's a very basic network that gets me after 40 seconds and 10 epochs over 95% training accuracy:
import pickle
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D
pickle_in = open("X.pickle","rb")
X = pickle.load(pickle_in)
pickle_in = open("y.pickle","rb")
y = pickle.load(pickle_in)
X = X/255.0
model = Sequential()
model.add(Conv2D(16, (5,5), activation = 'relu', input_shape = X.shape[1:], padding='same'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(32, (3,3), activation = 'relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(64, (3,3), activation = 'relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Flatten())
model.add(Dense(512))
model.add(Dense(8, activation='softmax'))
model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),metrics=['accuracy'])
Architecture:
Model: "sequential_4"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_19 (Conv2D) (None, 200, 200, 16) 1216
_________________________________________________________________
max_pooling2d_14 (MaxPooling (None, 100, 100, 16) 0
_________________________________________________________________
conv2d_20 (Conv2D) (None, 100, 100, 32) 4640
_________________________________________________________________
max_pooling2d_15 (MaxPooling (None, 50, 50, 32) 0
_________________________________________________________________
conv2d_21 (Conv2D) (None, 50, 50, 64) 18496
_________________________________________________________________
max_pooling2d_16 (MaxPooling (None, 25, 25, 64) 0
_________________________________________________________________
flatten_4 (Flatten) (None, 40000) 0
_________________________________________________________________
dense_7 (Dense) (None, 512) 20480512
_________________________________________________________________
dense_8 (Dense) (None, 8) 4104
=================================================================
Total params: 20,508,968
Trainable params: 20,508,968
Non-trainable params: 0
Training:
Train on 5040 samples, validate on 560 samples
Epoch 1/10
5040/5040 [==============================] - 7s 1ms/sample - loss: 2.2725 - accuracy: 0.1897 - val_loss: 1.8939 - val_accuracy: 0.2946
Epoch 2/10
5040/5040 [==============================] - 6s 1ms/sample - loss: 1.7831 - accuracy: 0.3375 - val_loss: 1.8658 - val_accuracy: 0.3179
Epoch 3/10
5040/5040 [==============================] - 6s 1ms/sample - loss: 1.4857 - accuracy: 0.4623 - val_loss: 1.9507 - val_accuracy: 0.3357
Epoch 4/10
5040/5040 [==============================] - 6s 1ms/sample - loss: 1.1294 - accuracy: 0.6028 - val_loss: 2.1745 - val_accuracy: 0.3250
Epoch 5/10
5040/5040 [==============================] - 6s 1ms/sample - loss: 0.8060 - accuracy: 0.7179 - val_loss: 3.1622 - val_accuracy: 0.3000
Epoch 6/10
5040/5040 [==============================] - 6s 1ms/sample - loss: 0.5574 - accuracy: 0.8169 - val_loss: 3.7494 - val_accuracy: 0.2839
Epoch 7/10
5040/5040 [==============================] - 6s 1ms/sample - loss: 0.3756 - accuracy: 0.8813 - val_loss: 4.9125 - val_accuracy: 0.2643
Epoch 8/10
5040/5040 [==============================] - 6s 1ms/sample - loss: 0.3001 - accuracy: 0.9036 - val_loss: 5.6300 - val_accuracy: 0.2821
Epoch 9/10
5040/5040 [==============================] - 6s 1ms/sample - loss: 0.2345 - accuracy: 0.9337 - val_loss: 5.7263 - val_accuracy: 0.2679
Epoch 10/10
5040/5040 [==============================] - 6s 1ms/sample - loss: 0.1549 - accuracy: 0.9581 - val_loss: 7.3682 - val_accuracy: 0.2732
As you can see, validation score is terrible, but the point was to demonstrate that poor architecture can prevent training altogether.
I'm trying to use this approach in Tensorflow 2.X to load large dataset that does not fit in memory.
I have a folder with X sub-folders that contains images. Each sub-folder is a class.
\dataset
-\class1
-img1_1.jpg
-img1_2.jpg
-...
-\classe2
-img2_1.jpg
-img2_2.jpg
-...
I create my data generator from my folder like this:
train_data_gen = image_generator.flow_from_directory(directory="path\\to\\dataset",
batch_size=100,
shuffle=True,
target_size=(100, 100), # Image H x W
classes=list(CLASS_NAMES)) # list of folder/class names ["class1", "class2", ...., "classX"]
Found 629 images belonging to 2 classes.
I've did a smaller dataset to test the pipeline. Only 629 images in 2 classes.
Now I can create a dummy model like this:
model = tf.keras.Sequential()
model.add(Dense(1, activation=activation, input_shape=(100, 100, 3))) # only 1 layer of 1 neuron
model.add(Dense(2)) # 2classes
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['categorical_accuracy'])
Once compile I try to fit this dummy model:
STEPS_PER_EPOCH = np.ceil(image_count / batch_size) # 629 / 100
model.fit_generator(generator=train_data_gen , steps_per_epoch=STEPS_PER_EPOCH, epochs=2, verbose=1)
1/7 [===>..........................] - ETA: 2s - loss: 1.1921e-07 - categorical_accuracy: 0.9948
2/7 [=======>......................] - ETA: 1s - loss: 1.1921e-07 - categorical_accuracy: 0.5124
3/7 [===========>..................] - ETA: 0s - loss: 1.1921e-07 - categorical_accuracy: 0.3449
4/7 [================>.............] - ETA: 0s - loss: 1.1921e-07 - categorical_accuracy: 0.2662
5/7 [====================>.........] - ETA: 0s - loss: 1.1921e-07 - categorical_accuracy: 0.2130
6/7 [========================>.....] - ETA: 0s - loss: 1.1921e-07 - categorical_accuracy: 0.1808
2020-04-14 20:39:48.629203: W tensorflow/core/framework/op_kernel.cc:1610] Invalid argument: ValueError: generator yielded an element of shape (29, 100, 100, 3) where an element of shape (100, 100, 100, 3) was expected.
From what i understand, the last batch doesn't has the same shape has the previous batches. So it crashes. I've tried to specify a batch_input_shape.
model.add(Dense(1, activation=activation, batch_input_shape=(None, 100, 100, 3)))
I've found here that I should put None to not specify the number of elements in the batch so it can be dynamic. But no success.
Edit: From the comment I had 2 mistakes:
The output shape was bad. I missed the flatten layer in the model.
The previous link does work with the correction of the flatten layer
Missing some code, I actually feed the fit_generator with a tf.data.Dataset.from_generator but I gave here a image_generator.flow_from_directory.
Here is the final code:
train_data_gen = image_generator.flow_from_directory(directory="path\\to\\dataset",
batch_size=1000,
shuffle=True,
target_size=(100, 100),
classes=list(CLASS_NAMES))
train_dataset = tf.data.Dataset.from_generator(
lambda: train_data_gen,
output_types=(tf.float32, tf.float32),
output_shapes=([None, x, y, 3],
[None, len(CLASS_NAMES)]))
model = tf.keras.Sequential()
model.add(Flatten(batch_input_shape=(None, 100, 100, 3)))
model.add(Dense(1, activation=activation))
model.add(Dense(2))
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['categorical_accuracy'])
STEPS_PER_EPOCH = np.ceil(image_count / batch_size) # 629 / 100
model.fit_generator(generator=train_data_gen , steps_per_epoch=STEPS_PER_EPOCH, epochs=2, verbose=1)
For the benefit of community here i am explaining, how to use image_generator in Tensorflow with input_shape (100, 100, 3) using dogs vs cats dataset
If we haven't choose right batch size there is a chance of model struck right after first epoch, hence i am starting my explanation with how to choose batch_size ?
We generally observe that batch size to be the power of 2, this is because of the effective work of optimized matrix operation libraries. This is further elaborated in this research paper.
Check out this blog which describes how to choose the right batch size while comparing the effects of different batch sizes on the accuracy of CIFAR-10 dataset.
Here is the end to end working code with outputs
import os
import numpy as np
from keras import layers
import pandas as pd
from tensorflow.keras.layers import Input, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D
from tensorflow.keras.layers import AveragePooling2D, MaxPooling2D, Dropout, GlobalMaxPooling2D, GlobalAveragePooling2D
from tensorflow.keras.models import Sequential
from tensorflow.keras import regularizers, optimizers
from tensorflow.keras.preprocessing import image
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import keras.backend as K
K.set_image_data_format('channels_last')
train_dir = '/content/drive/My Drive/Dogs_Vs_Cats/train'
test_dir = '/content/drive/My Drive/Dogs_Vs_Cats/test'
img_width, img_height = 100, 100
input_shape = img_width, img_height, 3
train_samples = 2000
test_samples = 1000
epochs = 30
batch_size = 32
train_datagen = ImageDataGenerator(
rescale = 1. /255,
shear_range = 0.2,
zoom_range = 0.2,
horizontal_flip = True)
test_datagen = ImageDataGenerator(
rescale = 1. /255)
train_data = train_datagen.flow_from_directory(
train_dir,
target_size = (img_width, img_height),
batch_size = batch_size,
class_mode = 'binary')
test_data = test_datagen.flow_from_directory(
test_dir,
target_size = (img_width, img_height),
batch_size = batch_size,
class_mode = 'binary')
model = Sequential()
model.add(Conv2D(32, (7, 7), strides = (1, 1), input_shape = input_shape))
model.add(BatchNormalization(axis = 3))
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2)))
model.add(Conv2D(64, (7, 7), strides = (1, 1)))
model.add(BatchNormalization(axis = 3))
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2)))
model.add(Flatten())
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
model.summary()
model.compile(loss = 'binary_crossentropy',
optimizer = 'rmsprop',
metrics = ['accuracy'])
model.fit_generator(
train_data,
steps_per_epoch = train_samples//batch_size,
epochs = epochs,
validation_data = test_data,
verbose = 1,
validation_steps = test_samples//batch_size)
Output:
Found 2000 images belonging to 2 classes.
Found 1000 images belonging to 2 classes.
Model: "sequential_5"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_8 (Conv2D) (None, 94, 94, 32) 4736
_________________________________________________________________
batch_normalization_8 (Batch (None, 94, 94, 32) 128
_________________________________________________________________
activation_8 (Activation) (None, 94, 94, 32) 0
_________________________________________________________________
max_pooling2d_8 (MaxPooling2 (None, 47, 47, 32) 0
_________________________________________________________________
conv2d_9 (Conv2D) (None, 41, 41, 64) 100416
_________________________________________________________________
batch_normalization_9 (Batch (None, 41, 41, 64) 256
_________________________________________________________________
activation_9 (Activation) (None, 41, 41, 64) 0
_________________________________________________________________
max_pooling2d_9 (MaxPooling2 (None, 20, 20, 64) 0
_________________________________________________________________
flatten_4 (Flatten) (None, 25600) 0
_________________________________________________________________
dense_11 (Dense) (None, 64) 1638464
_________________________________________________________________
dropout_4 (Dropout) (None, 64) 0
_________________________________________________________________
dense_12 (Dense) (None, 1) 65
=================================================================
Total params: 1,744,065
Trainable params: 1,743,873
Non-trainable params: 192
_________________________________________________________________
Epoch 1/30
62/62 [==============================] - 14s 225ms/step - loss: 1.8307 - accuracy: 0.4853 - val_loss: 0.6931 - val_accuracy: 0.5000
Epoch 2/30
62/62 [==============================] - 14s 226ms/step - loss: 0.7085 - accuracy: 0.4832 - val_loss: 0.6931 - val_accuracy: 0.5010
Epoch 3/30
62/62 [==============================] - 14s 218ms/step - loss: 0.6955 - accuracy: 0.5300 - val_loss: 0.6894 - val_accuracy: 0.5292
Epoch 4/30
62/62 [==============================] - 14s 221ms/step - loss: 0.6938 - accuracy: 0.5407 - val_loss: 0.7309 - val_accuracy: 0.5262
Epoch 5/30
62/62 [==============================] - 14s 218ms/step - loss: 0.6860 - accuracy: 0.5498 - val_loss: 0.6776 - val_accuracy: 0.5665
Epoch 6/30
62/62 [==============================] - 13s 216ms/step - loss: 0.7027 - accuracy: 0.5407 - val_loss: 0.6895 - val_accuracy: 0.5101
Epoch 7/30
62/62 [==============================] - 13s 216ms/step - loss: 0.6852 - accuracy: 0.5528 - val_loss: 0.6567 - val_accuracy: 0.5887
Epoch 8/30
62/62 [==============================] - 13s 217ms/step - loss: 0.6772 - accuracy: 0.5427 - val_loss: 0.6643 - val_accuracy: 0.5847
Epoch 9/30
62/62 [==============================] - 13s 217ms/step - loss: 0.6709 - accuracy: 0.5534 - val_loss: 0.6623 - val_accuracy: 0.5887
Epoch 10/30
62/62 [==============================] - 14s 219ms/step - loss: 0.6579 - accuracy: 0.5711 - val_loss: 0.6614 - val_accuracy: 0.6058
Epoch 11/30
62/62 [==============================] - 13s 218ms/step - loss: 0.6591 - accuracy: 0.5625 - val_loss: 0.6594 - val_accuracy: 0.5454
Epoch 12/30
62/62 [==============================] - 13s 216ms/step - loss: 0.6419 - accuracy: 0.5767 - val_loss: 1.1041 - val_accuracy: 0.5161
Epoch 13/30
62/62 [==============================] - 13s 215ms/step - loss: 0.6479 - accuracy: 0.5783 - val_loss: 0.6441 - val_accuracy: 0.5837
Epoch 14/30
62/62 [==============================] - 13s 216ms/step - loss: 0.6373 - accuracy: 0.5899 - val_loss: 0.6427 - val_accuracy: 0.6310
Epoch 15/30
62/62 [==============================] - 13s 215ms/step - loss: 0.6203 - accuracy: 0.6133 - val_loss: 0.7390 - val_accuracy: 0.6220
Epoch 16/30
62/62 [==============================] - 13s 217ms/step - loss: 0.6277 - accuracy: 0.6362 - val_loss: 0.6649 - val_accuracy: 0.5786
Epoch 17/30
62/62 [==============================] - 13s 215ms/step - loss: 0.6155 - accuracy: 0.6316 - val_loss: 0.9823 - val_accuracy: 0.5484
Epoch 18/30
62/62 [==============================] - 14s 222ms/step - loss: 0.6056 - accuracy: 0.6408 - val_loss: 0.6333 - val_accuracy: 0.6048
Epoch 19/30
62/62 [==============================] - 14s 218ms/step - loss: 0.6025 - accuracy: 0.6529 - val_loss: 0.6514 - val_accuracy: 0.6442
Epoch 20/30
62/62 [==============================] - 13s 215ms/step - loss: 0.6149 - accuracy: 0.6423 - val_loss: 0.6373 - val_accuracy: 0.6048
Epoch 21/30
62/62 [==============================] - 13s 215ms/step - loss: 0.6030 - accuracy: 0.6519 - val_loss: 0.6086 - val_accuracy: 0.6573
Epoch 22/30
62/62 [==============================] - 13s 217ms/step - loss: 0.5936 - accuracy: 0.6865 - val_loss: 1.0677 - val_accuracy: 0.5605
Epoch 23/30
62/62 [==============================] - 13s 214ms/step - loss: 0.5964 - accuracy: 0.6728 - val_loss: 0.7927 - val_accuracy: 0.5877
Epoch 24/30
62/62 [==============================] - 13s 215ms/step - loss: 0.5866 - accuracy: 0.6707 - val_loss: 0.6116 - val_accuracy: 0.6421
Epoch 25/30
62/62 [==============================] - 13s 214ms/step - loss: 0.5933 - accuracy: 0.6662 - val_loss: 0.8282 - val_accuracy: 0.6048
Epoch 26/30
62/62 [==============================] - 13s 214ms/step - loss: 0.5705 - accuracy: 0.6885 - val_loss: 0.5806 - val_accuracy: 0.6966
Epoch 27/30
62/62 [==============================] - 14s 218ms/step - loss: 0.5709 - accuracy: 0.7017 - val_loss: 1.2404 - val_accuracy: 0.5333
Epoch 28/30
62/62 [==============================] - 13s 216ms/step - loss: 0.5691 - accuracy: 0.7104 - val_loss: 0.6136 - val_accuracy: 0.6442
Epoch 29/30
62/62 [==============================] - 13s 215ms/step - loss: 0.5627 - accuracy: 0.7048 - val_loss: 0.6936 - val_accuracy: 0.6613
Epoch 30/30
62/62 [==============================] - 13s 214ms/step - loss: 0.5714 - accuracy: 0.6941 - val_loss: 0.5872 - val_accuracy: 0.6825
I am trying to build the a very simple model using keras using the Model function, like below, where the input and output of the Model function are [img,labels] and the loss.
I am confused why this code is not working, if the output cannot be the loss. How should the Model function work and when should we use the Model function? Thanks.
sess = tf.Session()
K.set_session(sess)
K.set_learning_phase(1)
img = Input((784,),name='img')
labels = Input((10,),name='labels')
# img = tf.placeholder(tf.float32, shape=(None, 784))
# labels = tf.placeholder(tf.float32, shape=(None, 10))
x = Dense(128, activation='relu')(img)
x = Dropout(0.5)(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.5)(x)
preds = Dense(10, activation='softmax')(x)
from keras.losses import binary_crossentropy
#loss = tf.reduce_mean(categorical_crossentropy(labels, preds))
loss = binary_crossentropy(labels, preds)
print(type(loss))
model = Model([img,labels], loss, name='squeezenet')
model.summary()
As #yu-yang pointed out, the loss is specified with compile().
If you think about it, it makes sense because the real output of your model is your prediction, not the loss, the loss is only used to train the model.
A working example of your network:
import keras
from keras.optimizers import Adam
from keras.models import Model
from keras.layers import Input, Dense, Dropout
from keras.losses import categorical_crossentropy
img = Input((784,),name='img')
x = Dense(128, activation='relu')(img)
x = Dropout(0.5)(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.5)(x)
preds = Dense(10, activation='softmax')(x)
model = Model(inputs=img, outputs=preds, name='squeezenet')
model.compile(optimizer=Adam(),
loss=categorical_crossentropy,
metrics=['acc'])
model.summary()
Output:
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
img (InputLayer) (None, 784) 0
_________________________________________________________________
dense_32 (Dense) (None, 128) 100480
_________________________________________________________________
dropout_21 (Dropout) (None, 128) 0
_________________________________________________________________
dense_33 (Dense) (None, 128) 16512
_________________________________________________________________
dropout_22 (Dropout) (None, 128) 0
_________________________________________________________________
dense_34 (Dense) (None, 10) 1290
=================================================================
Total params: 118,282
Trainable params: 118,282
Non-trainable params: 0
_________________________________________________________________
With MNIST dataset:
from keras.datasets import mnist
from keras.utils import to_categorical
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(-1, 784)
y_train = to_categorical(y_train, num_classes=10)
x_test = x_test.reshape(-1, 784)
y_test = to_categorical(y_test, num_classes=10)
model.fit(x_train, y_train, epochs=10, validation_data=(x_test, y_test))
Output:
Train on 60000 samples, validate on 10000 samples
Epoch 1/10
60000/60000 [==============================] - 4s - loss: 12.2797 - acc: 0.2360 - val_loss: 11.0902 - val_acc: 0.3116
Epoch 2/10
60000/60000 [==============================] - 4s - loss: 10.4161 - acc: 0.3527 - val_loss: 8.7122 - val_acc: 0.4589
Epoch 3/10
60000/60000 [==============================] - 4s - loss: 9.5797 - acc: 0.4051 - val_loss: 8.9226 - val_acc: 0.4460
Epoch 4/10
60000/60000 [==============================] - 4s - loss: 9.2017 - acc: 0.4285 - val_loss: 8.0564 - val_acc: 0.4998
Epoch 5/10
60000/60000 [==============================] - 4s - loss: 8.8558 - acc: 0.4501 - val_loss: 8.0878 - val_acc: 0.4980
Epoch 6/10
60000/60000 [==============================] - 5s - loss: 8.8239 - acc: 0.4521 - val_loss: 8.2495 - val_acc: 0.4880
Epoch 7/10
60000/60000 [==============================] - 4s - loss: 8.7842 - acc: 0.4547 - val_loss: 7.7146 - val_acc: 0.5211
Epoch 8/10
60000/60000 [==============================] - 4s - loss: 8.7395 - acc: 0.4575 - val_loss: 7.7944 - val_acc: 0.5163
Epoch 9/10
60000/60000 [==============================] - 5s - loss: 8.7109 - acc: 0.4593 - val_loss: 7.8235 - val_acc: 0.5145
Epoch 10/10
60000/60000 [==============================] - 4s - loss: 8.4927 - acc: 0.4729 - val_loss: 7.5933 - val_acc: 0.5288