Retrieve final (incomplete) batch of custom Data Generator - tensorflow

Question

I have a made a custom data generator that outputs batches of image sequences of shape (batch size, sequence length, image height, image width, channels), along with two labels y1 and y2.
However, I cant seem to retrieve the final (incomplete) batch during training. Any ideas where I am going wrong?
class DataGenerator(tf.keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, labels, training_set=False, batch_size=32, dim=(224, 224), n_channels=3, shuffle=True):
'Initialization'
self.dim = dim
self.batch_size = batch_size
self.labels = labels
self.training_set = training_set
self.list_IDs = list_IDs
self.n_channels = n_channels
self.shuffle = shuffle
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
num_batchs_per_epoch = int(np.floor(len(self.list_IDs) / self.batch_size))
return num_batchs_per_epoch
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
start = index*self.batch_size
end = (index+1)*self.batch_size
indexes = self.indexes[start:end]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
X, y1, y2 = self.__data_generation(list_IDs_temp)
return X, [y1, y2]
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples' # X : (n_samples, 3, *dim, n_channels)
# Initialization
X = np.empty((self.batch_size, 3, *self.dim, self.n_channels))
y1 = np.empty((self.batch_size), dtype=float)
y2 = np.empty((self.batch_size), dtype=int)
# Generate data
for i, ID in enumerate(list_IDs_temp):
sequence = [s for s in ID]
f0, f1, f2 = [self.load_resize_image(image) for image in sequence]
# preprocess steps
f0 = self.preprocess(f0, self.training_set)
f1 = self.preprocess(f1, self.training_set)
f2 = self.preprocess(f2, self.training_set)
triplet = np.concatenate((f0,f1,f2), axis=0)
X[i,:,:,:,:] = triplet
ID = tuple(ID)
y1[i] = self.labels[ID][0]
y2[i] = self.labels[ID][1]
return X, y1, y2
def preprocess(self, img, training_set):
if self.training_set:
# apply transformations
gen = ImageDataGenerator()
img[0,:,:,:] = gen.apply_transform(x=img[0,:,:,:], transform_parameters={'theta':random.uniform(-180, 180),
'brightness': random.uniform(0.8, 1.2),
'flip_horizontal': random.getrandbits(1),
'shear': random.uniform(0,5),
'zx': random.uniform(0.9,1.1),
'zy': random.uniform(0.9,1.1),
'flip_vertical': random.getrandbits(1)
})
return img
def load_resize_image(self, image):
img = cv2.imread(image)
img = cv2.resize(img, dsize=(224, 224), interpolation=cv2.INTER_CUBIC)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img_array = np.array(img)
img_array = np.expand_dims(img_array, 0)
return img_array
And at training...
history = model.fit(
training_generator,
epochs=epochs,
validation_data=validation_generator,
callbacks=callbacks
)

Answer 1

The code will always omit the last batch of data, due to this line of code:
int(np.floor(len(self.list_IDs) / self.batch_size))
See the example below:
number_of_samples = 1002
batch_size = 4
num_batches_per_epoch = int(np.floor(number_of_samples / 4))
num_batches_per_epoch (=250, if number_of_samples == 1000,1001,1002,1003)
The way the dataset is written, it will always omit one batch, which is not a problem, since in essence it is incomplete.
As you are shuffling at the end of each epoch:
if self.shuffle == True:
np.random.shuffle(self.indexes)
the not seen few samples in an epoch will definitely be seen in later epochs.