How to get the filenames of all categories (TP, TN, FP, FN) of a Confusion Matrix in Keras/TensorFlow? - tensorflow

Question

I am working with image data where I am trying to find the list of the files are in TP, TN (true positives, true negatives) and so on. The purpose is to check (visually) whether the files are being identified properly by the model. currntly I am using a sequential image classification model in google colab. Following is my code.
## Model
model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(128, 128, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(32, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Flatten())
model.add(layers.Dense(8, activation='relu'))
model.add(layers.Dense(1,activation='sigmoid'))
from keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale=1./255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1./255)
train_generator = train_datagen.flow_from_directory('./train', target_size=(128, 128), batch_size=batch_size, color_mode = 'grayscale', class_mode='binary')
validation_generator = test_datagen.flow_from_directory('./validation', target_size=(128, 128), batch_size=batch_size, color_mode = 'grayscale', class_mode='binary')
test_generator = test_datagen.flow_from_directory('./test', target_size=(128, 128), batch_size=1,
color_mode = 'grayscale', class_mode='binary', shuffle=False)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
h = model.fit(train_generator, epochs = 50, validation_data=validation_generator)
from sklearn.metrics import confusion_matrix
y_true = test_generator.classes
true_classes = test_generator.classes
class_labels = list(test_generator.class_indices.keys())
confusion_matrix(y_true, yy_pred)
Output of confusion Matrix-
array([[22, 10],
[9, 50]])
Where I am trying to get image file names those are in True Positives (22 images), True Negatives (50 images) and so on. I am not sure whether I can get a list directly or do I have to re-generate the predicted images!

Answer 1

The function below will processes the test_generator and produce a classification report and a confusion matrix as well as a list of filenams the were misclassified.
You can process the classification report to get the metrics you desire
def predictor(test_gen):
y_pred= []
error_list=[]
error_pred_list = []
y_true=test_gen.labels
classes=list(test_gen.class_indices.keys())
class_count=len(classes)
errors=0
preds=model.predict(test_gen, verbose=1)
tests=len(preds)
for i, p in enumerate(preds):
pred_index=np.argmax(p)
true_index=test_gen.labels[i] # labels are integer values
if pred_index != true_index: # a misclassification has occurred
errors=errors + 1
file=test_gen.filenames[i]
error_list.append(file)
error_class=classes[pred_index]
error_pred_list.append(error_class)
y_pred.append(pred_index)
acc=( 1-errors/tests) * 100
msg=f'there were {errors} errors in {tests} tests for an accuracy of {acc:6.2f}'
print_in_color(msg, (0,255,255), (100,100,100)) # cyan foreground
ypred=np.array(y_pred)
ytrue=np.array(y_true)
f1score=f1_score(ytrue, ypred, average='weighted')* 100
if class_count <=30:
cm = confusion_matrix(ytrue, ypred )
# plot the confusion matrix
plt.figure(figsize=(12, 8))
sns.heatmap(cm, annot=True, vmin=0, fmt='g', cmap='Blues', cbar=False)
plt.xticks(np.arange(class_count)+.5, classes, rotation=90)
plt.yticks(np.arange(class_count)+.5, classes, rotation=0)
plt.xlabel("Predicted")
plt.ylabel("Actual")
plt.title("Confusion Matrix")
plt.show()
clr = classification_report(y_true, y_pred, target_names=classes, digits= 4) # create classification report
print("Classification Report:\n----------------------\n", clr)
return errors, tests, error_list, error_pred_list, f1score
errors, tests, error_list, error_pred_list, f1score =predictor(test_gen)