keras triplet loss crashes when training - tensorflow

Question

I am trying to implement a simple face recognition application but I have been stuck with a problem for days now. Hopefully someone having more experience in the subject can help. Fingers crossed.
Here is the program:
import tensorflow as tf
#from keras.applications.vgg16 import VGG16
from keras.applications.resnet50 import ResNet50, preprocess_input
from keras.preprocessing import image
#import PIL
#from PIL import Image
from keras.models import Model
from keras.layers import Dense, Input, subtract, concatenate, Lambda, add, maximum
from keras import backend as K
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import Adam
import numpy as np
def identity_loss(y_true, y_pred):
return K.mean(y_pred - 0 * y_true)
def triplet_loss(inputs, dist='euclidean', margin='maxplus'):
anchor, positive, negative = inputs
positive_distance = K.square(anchor - positive)
negative_distance = K.square(anchor - negative)
if dist == 'euclidean':
positive_distance = K.sqrt(K.sum(positive_distance, axis=-1, keepdims=True))
negative_distance = K.sqrt(K.sum(negative_distance, axis=-1, keepdims=True))
elif dist == 'sqeuclidean':
positive_distance = K.sum(positive_distance, axis=-1, keepdims=True)
negative_distance = K.sum(negative_distance, axis=-1, keepdims=True)
loss = positive_distance - negative_distance
if margin == 'maxplus':
loss = K.maximum(0.0, 1 + loss)
elif margin == 'softplus':
loss = K.log(1 + K.exp(loss))
return K.mean(loss)
model = ResNet50(weights='imagenet')
model.layers.pop()
x = model.get_layer('flatten_1').output
model_out = Dense(128, activation='relu', name='model_out')(x)
model_out = Lambda(lambda x: K.l2_normalize(x,axis=-1))(model_out)
new_model = Model(inputs=model.input, outputs=model_out)
anchor_input = Input(shape=(224, 224, 3), name='anchor_input')
pos_input = Input(shape=(224, 224, 3), name='pos_input')
neg_input = Input(shape=(224, 224, 3), name='neg_input')
encoding_anchor = new_model(anchor_input)
encoding_pos = new_model(pos_input)
encoding_neg = new_model(neg_input)
loss = Lambda(triplet_loss)([encoding_anchor, encoding_pos, encoding_neg])
siamese_network = Model(inputs = [anchor_input, pos_input, neg_input],
outputs = loss)
siamese_network.compile(optimizer=Adam(lr=.00001), loss=identity_loss)
######################### For reading img path info - start ##########
train_path1 = '/home/cesncn/Desktop/github_projects/face_recog_proj_with_triplet_loss/training_img_pairs.csv'
TRAIN_INPUT_PATHS = [train_path1]
RECORD_DEFAULTS_TRAIN = [[0], [''], [''], ['']]
def decode_csv_train(line):
parsed_line = tf.decode_csv(line, RECORD_DEFAULTS_TRAIN)
anchor_path = parsed_line[1]
pos_path = parsed_line[2]
neg_path = parsed_line[3]
return anchor_path, pos_path, neg_path
######################### For reading img path info - end ##########
batch_size = 16
filenames = tf.placeholder(tf.string, shape=[None])
dataset = tf.data.Dataset.from_tensor_slices(filenames)
dataset = dataset.flat_map(lambda filename: tf.data.TextLineDataset(filename).skip(1).map(decode_csv_train))
dataset = dataset.shuffle(buffer_size=1000)
dataset = dataset.batch(batch_size)
iterator = dataset.make_initializable_iterator()
next_element = iterator.get_next()
init_global_var = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_global_var)
nr_epochs = 5
for i in range(0, nr_epochs):
print("\nnr_epoch: ", str(i), "\n")
sess.run(iterator.initializer, feed_dict={filenames: TRAIN_INPUT_PATHS})
while True:
try:
anchor_path, pos_path, neg_path = sess.run(next_element)
anchor_imgs = np.empty((0, 224, 224, 3))
pos_imgs = np.empty((0, 224, 224, 3))
neg_imgs = np.empty((0, 224, 224, 3))
for j in range (0, len(anchor_path)):
anchor_img = image.load_img(anchor_path[j], target_size=(224, 224))
anchor_img = image.img_to_array(anchor_img)
anchor_img = np.expand_dims(anchor_img, axis=0)
anchor_img = preprocess_input(anchor_img)
anchor_imgs = np.append(anchor_imgs, anchor_img, axis=0)
pos_img = image.load_img(pos_path[j], target_size=(224, 224))
pos_img = image.img_to_array(pos_img)
pos_img = np.expand_dims(pos_img, axis=0)
pos_img = preprocess_input(pos_img)
pos_imgs = np.append(pos_imgs, pos_img, axis=0)
neg_img = image.load_img(neg_path[j], target_size=(224, 224))
neg_img = image.img_to_array(neg_img)
neg_img = np.expand_dims(neg_img, axis=0)
neg_img = preprocess_input(neg_img)
neg_imgs = np.append(neg_imgs, neg_img, axis=0)
# HERE IT CRASHES WHEN I CALL THE FIT FUNCTION! ! !
siamese_network.fit([anchor_imgs, pos_imgs, neg_imgs],
batch_size = batch_size,
epochs = 1,
verbose = 2)
except tf.errors.OutOfRangeError:
print("Out of range error triggered (looped through training set 1 time)")
break
When I call the function siamese_network.fit(...), it crashes and gives the following error, which tells me absolutely nothing!!
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-16-65401d04281c> in <module>()
61 batch_size = batch_size,
62 epochs = 1,
---> 63 verbose = 2)
64
65 #siamese_network.fit({'anchor_input': anchor_imgs, 'pos_input': pos_imgs, 'neg_input': neg_imgs},
~/anaconda3/envs/tensorflow/lib/python3.6/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1628 sample_weight=sample_weight,
1629 class_weight=class_weight,
-> 1630 batch_size=batch_size)
1631 # Prepare validation data.
1632 do_validation = False
~/anaconda3/envs/tensorflow/lib/python3.6/site-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)
1485 sample_weights = [_standardize_weights(ref, sw, cw, mode)
1486 for (ref, sw, cw, mode)
-> 1487 in zip(y, sample_weights, class_weights, self._feed_sample_weight_modes)]
1488
1489 if check_array_lengths:
~/anaconda3/envs/tensorflow/lib/python3.6/site-packages/keras/engine/training.py in <listcomp>(.0)
1484 self._feed_output_names)
1485 sample_weights = [_standardize_weights(ref, sw, cw, mode)
-> 1486 for (ref, sw, cw, mode)
1487 in zip(y, sample_weights, class_weights, self._feed_sample_weight_modes)]
1488
~/anaconda3/envs/tensorflow/lib/python3.6/site-packages/keras/engine/training.py in _standardize_weights(y, sample_weight, class_weight, sample_weight_mode)
538 else:
539 if sample_weight_mode is None:
--> 540 return np.ones((y.shape[0],), dtype=K.floatx())
541 else:
542 return np.ones((y.shape[0], y.shape[1]), dtype=K.floatx())
AttributeError: 'NoneType' object has no attribute 'shape'
Does anyone have any idea about how to solve this?

Answer 1

I have been tyring to implement several variations of the triplet loss function. The above version is just one of the tries...
Anyhow, in the version above, the problem appears to be the following:
Here is how I create and compile the model:
siamese_network = Model(inputs = [anchor_input, pos_input, neg_input],
outputs = loss)
siamese_network.compile(optimizer=Adam(lr=.00001), loss=identity_loss)
Obviously, I say here that there is an output from the model, called "loss".
And later when I train the model, I set the y to null in the fit function. Big mistake.
z = [0] # ADDED LINE
siamese_network.fit(x = [anchor_imgs, pos_imgs, neg_imgs],
y = z # ADDED LINE
batch_size = batch_size,
epochs = 1,
verbose = 2)
And this fix solved the problem.
I rather chose to answer rather than deleting the question. Hopefully it helps others who may have a similar problem..
PS. y is set to NULL by default if it is not set to anything in Keras.