Process finished with exit code 137 (interrupted by signal 9: SIGKILL) - pandas

Question

I have added a large-scale key values pairs in python file which is about 20000 and i got below error after running a code.
Trace
(236167, 3)
Process finished with exit code 137 (interrupted by signal 9: SIGKILL)
File.py
import pandas as pd
import cupy.cuda as np
import itertools
from sklearn import metrics
from sklearn.metrics import confusion_matrix, accuracy_score, roc_curve, auc
import matplotlib.pyplot as plt
from tqdm import tqdm
np.Device(0).use()
# --------------------------
# Data set
# Ref: https://github.com/serengil/deepface/tree/master/tests/dataset
idendities = {
"AnneBancroft": [
"13859_AnneBancroft_25_f.jpg",
"13872_AnneBancroft_73_f.jpg",
"13864_AnneBancroft_31_f.jpg",
"13870_AnneBancroft_61_f.jpg",
"13844_AnneBancroft_67_f.jpg",
"13863_AnneBancroft_22_f.jpg",
"13869_AnneBancroft_72_f.jpg",
"13843_AnneBancroft_60_f.jpg",
"13868_AnneBancroft_71_f.jpg",
"13860_AnneBancroft_66_f.jpg",
"13853_AnneBancroft_49_f.jpg",
"13842_AnneBancroft_51_f.jpg",
"13874_AnneBancroft_73_f.jpg",
"13846_AnneBancroft_44_f.jpg",
"13871_AnneBancroft_35_f.jpg",
"13857_AnneBancroft_24_f.jpg",
"13850_AnneBancroft_53_f.jpg",
"13865_AnneBancroft_41_f.jpg",
"13862_AnneBancroft_46_f.jpg",
"13852_AnneBancroft_69_f.jpg",
"13866_AnneBancroft_68_f.jpg",
"13873_AnneBancroft_25_f.jpg",
"13861_AnneBancroft_23_f.jpg",
"13848_AnneBancroft_52_f.jpg",
"13847_AnneBancroft_33_f.jpg",
"13851_AnneBancroft_28_f.jpg",
"13856_AnneBancroft_25_f.jpg",
"13845_AnneBancroft_31_f.jpg",
"13867_AnneBancroft_70_f.jpg",
"13854_AnneBancroft_70_f.jpg",
"13849_AnneBancroft_61_f.jpg",
"13855_AnneBancroft_28_f.jpg",
"13858_AnneBancroft_22_f.jpg"
],
"RoseMarie": [
"9907_RoseMarie_82_f.jpg",
"9893_RoseMarie_35_f.jpg",
"9911_RoseMarie_88_f.jpg",
"9906_RoseMarie_80_f.jpg",
"9895_RoseMarie_40_f.jpg",
"9901_RoseMarie_57_f.jpg",
"9903_RoseMarie_77_f.jpg",
"9892_RoseMarie_30_f.jpg",
"9909_RoseMarie_85_f.jpg",
"9900_RoseMarie_52_f.jpg",
"9897_RoseMarie_44_f.jpg",
"9904_RoseMarie_78_f.jpg",
"9905_RoseMarie_79_f.jpg",
"9898_RoseMarie_46_f.jpg",
"9908_RoseMarie_83_f.jpg",
"9902_RoseMarie_70_f.jpg",
"9896_RoseMarie_42_f.jpg",
"9899_RoseMarie_50_f.jpg",
"9910_RoseMarie_87_f.jpg",
"9894_RoseMarie_37_f.jpg"
],
"BobDylan": [
"1665_BobDylan_35_m.jpg",
"1651_BobDylan_23_m.jpg",
"1663_BobDylan_33_m.jpg",
"1682_BobDylan_64_m.jpg",
"1678_BobDylan_56_m.jpg",
"1684_BobDylan_68_m.jpg",
"1686_BobDylan_72_m.jpg",
"1645_BobDylan_16_m.jpg",
"1664_BobDylan_34_m.jpg",
"1680_BobDylan_61_m.jpg",
"1674_BobDylan_47_m.jpg",
"1656_BobDylan_26_m.jpg",
"1658_BobDylan_28_m.jpg",
"1667_BobDylan_40_m.jpg",
"1673_BobDylan_46_m.jpg",
"1668_BobDylan_41_m.jpg",
"1657_BobDylan_27_m.jpg",
"1685_BobDylan_71_m.jpg",
"1647_BobDylan_19_m.jpg",
"1660_BobDylan_30_m.jpg",
"1679_BobDylan_57_m.jpg",
"1672_BobDylan_45_m.jpg",
"1666_BobDylan_37_m.jpg",
"1650_BobDylan_22_m.jpg",
"1683_BobDylan_66_m.jpg",
"1652_BobDylan_23_m.jpg",
"1654_BobDylan_24_m.jpg",
"1687_BobDylan_74_m.jpg",
"1649_BobDylan_21_m.jpg",
"1677_BobDylan_54_m.jpg",
"1659_BobDylan_29_m.jpg",
"1675_BobDylan_48_m.jpg",
"1662_BobDylan_32_m.jpg",
"1671_BobDylan_44_m.jpg",
"1669_BobDylan_42_m.jpg",
"1653_BobDylan_24_m.jpg",
"1648_BobDylan_20_m.jpg",
"1681_BobDylan_62_m.jpg",
"1661_BobDylan_31_m.jpg",
"1670_BobDylan_43_m.jpg",
"1655_BobDylan_25_m.jpg",
"1676_BobDylan_49_m.jpg",
"1646_BobDylan_18_m.jpg"
],
"LorneGreene": [
"8137_LorneGreene_25_m.jpg",
"8145_LorneGreene_48_m.jpg",
"8140_LorneGreene_38_m.jpg",
"8138_LorneGreene_28_m.jpg",
"8139_LorneGreene_33_m.jpg",
"8149_LorneGreene_52_m.jpg",
"8154_LorneGreene_58_m.jpg",
"8142_LorneGreene_44_m.jpg",
"8162_LorneGreene_68_m.jpg",
"8155_LorneGreene_61_m.jpg",
"8164_LorneGreene_71_m.jpg",
"8147_LorneGreene_50_m.jpg",
"8151_LorneGreene_54_m.jpg",
"8163_LorneGreene_70_m.jpg",
"8150_LorneGreene_53_m.jpg",
"8156_LorneGreene_62_m.jpg",
"8160_LorneGreene_66_m.jpg",
"8146_LorneGreene_49_m.jpg",
"8144_LorneGreene_46_m.jpg",
"8158_LorneGreene_64_m.jpg",
"8152_LorneGreene_55_m.jpg",
"8159_LorneGreene_65_m.jpg",
"8161_LorneGreene_67_m.jpg",
"8157_LorneGreene_63_m.jpg",
"8141_LorneGreene_43_m.jpg",
"8143_LorneGreene_45_m.jpg",
"8136_LorneGreene_18_m.jpg",
"8153_LorneGreene_57_m.jpg",
"8148_LorneGreene_51_m.jpg"
],
"LaurenBacall": [
"11540_LaurenBacall_26_f.jpg",
"11539_LaurenBacall_25_f.jpg",
"11547_LaurenBacall_45_f.jpg",
"11549_LaurenBacall_72_f.jpg",
"11534_LaurenBacall_20_f.jpg",
"11559_LaurenBacall_31_f.jpg",
"11545_LaurenBacall_35_f.jpg",
"11546_LaurenBacall_40_f.jpg",
"11563_LaurenBacall_64_f.jpg",
"11555_LaurenBacall_82_f.jpg",
"11541_LaurenBacall_31_f.jpg",
"11564_LaurenBacall_27_f.jpg",
"11561_LaurenBacall_57_f.jpg",
"11552_LaurenBacall_75_f.jpg",
"11556_LaurenBacall_83_f.jpg",
"11543_LaurenBacall_31_f.jpg",
"11533_LaurenBacall_19_f.jpg",
"11557_LaurenBacall_85_f.jpg",
"11544_LaurenBacall_34_f.jpg",
"11535_LaurenBacall_21_f.jpg",
"11565_LaurenBacall_26_f.jpg",
"11558_LaurenBacall_42_f.jpg",
"11531_LaurenBacall_28_f.jpg",
"11536_LaurenBacall_22_f.jpg",
"11562_LaurenBacall_46_f.jpg",
"11554_LaurenBacall_81_f.jpg",
"11542_LaurenBacall_31_f.jpg",
"11537_LaurenBacall_22_f.jpg",
"11560_LaurenBacall_56_f.jpg",
"11548_LaurenBacall_65_f.jpg",
"11550_LaurenBacall_73_f.jpg",
"11530_LaurenBacall_17_f.jpg",
"11532_LaurenBacall_18_f.jpg",
"11566_LaurenBacall_20_f.jpg",
"11551_LaurenBacall_77_f.jpg",
"11538_LaurenBacall_23_f.jpg",
"11553_LaurenBacall_80_f.jpg"
],
"SerenaWilliams": [
"16468_SerenaWilliams_32_f.jpg",
"16486_SerenaWilliams_32_f.jpg",
"16479_SerenaWilliams_25_f.jpg",
"16474_SerenaWilliams_18_f.jpg",
"16472_SerenaWilliams_21_f.jpg",
"16008_SerenaWilliams_36_f.jpg",
"16484_SerenaWilliams_31_f.jpg",
"16469_SerenaWilliams_31_f.jpg",
"16478_SerenaWilliams_24_f.jpg",
"16485_SerenaWilliams_32_f.jpg",
"16480_SerenaWilliams_26_f.jpg",
"16481_SerenaWilliams_27_f.jpg",
"16487_SerenaWilliams_33_f.jpg",
"16477_SerenaWilliams_23_f.jpg",
"16010_SerenaWilliams_34_f.jpg",
"16483_SerenaWilliams_30_f.jpg",
"16471_SerenaWilliams_29_f.jpg",
"16009_SerenaWilliams_35_f.jpg",
"16476_SerenaWilliams_20_f.jpg",
"16475_SerenaWilliams_19_f.jpg",
"16482_SerenaWilliams_28_f.jpg",
"16007_SerenaWilliams_36_f.jpg",
"16470_SerenaWilliams_35_f.jpg",
"16473_SerenaWilliams_24_f.jpg"
],
"JohnVernon": [
"6459_JohnVernon_49_m.jpg",
"6447_JohnVernon_33_m.jpg",
"6446_JohnVernon_32_m.jpg",
"6448_JohnVernon_34_m.jpg",
"6454_JohnVernon_40_m.jpg",
"6452_JohnVernon_38_m.jpg",
"6471_JohnVernon_71_m.jpg",
"6468_JohnVernon_60_m.jpg",
"6469_JohnVernon_63_m.jpg",
"6458_JohnVernon_47_m.jpg",
"6463_JohnVernon_53_m.jpg",
"6444_JohnVernon_30_m.jpg",
"6457_JohnVernon_46_m.jpg",
"6456_JohnVernon_42_m.jpg",
"6462_JohnVernon_52_m.jpg",
"6464_JohnVernon_54_m.jpg",
"6451_JohnVernon_37_m.jpg",
"6449_JohnVernon_35_m.jpg",
"6470_JohnVernon_67_m.jpg",
"6445_JohnVernon_31_m.jpg",
"6461_JohnVernon_51_m.jpg",
"6450_JohnVernon_36_m.jpg",
"6460_JohnVernon_50_m.jpg",
"6455_JohnVernon_41_m.jpg",
"6466_JohnVernon_57_m.jpg",
"6465_JohnVernon_56_m.jpg",
"6453_JohnVernon_39_m.jpg",
"6467_JohnVernon_58_m.jpg"
],
"JamesStewart": [
"8647_JamesStewart_45_m.jpg",
"8657_JamesStewart_29_m.jpg",
"8644_JamesStewart_32_m.jpg",
"8639_JamesStewart_28_m.jpg",
"8645_JamesStewart_38_m.jpg",
"8642_JamesStewart_31_m.jpg",
"8643_JamesStewart_32_m.jpg",
"8652_JamesStewart_69_m.jpg",
"8655_JamesStewart_32_m.jpg",
"8638_JamesStewart_26_m.jpg",
"8658_JamesStewart_41_m.jpg",
"8646_JamesStewart_40_m.jpg",
"8641_JamesStewart_31_m.jpg",
"8650_JamesStewart_65_m.jpg",
"8656_JamesStewart_32_m.jpg",
"8651_JamesStewart_68_m.jpg",
"8654_JamesStewart_34_m.jpg",
"8637_JamesStewart_86_m.jpg",
"8640_JamesStewart_30_m.jpg",
"8649_JamesStewart_52_m.jpg",
"8653_JamesStewart_41_m.jpg",
"8648_JamesStewart_51_m.jpg"
],
"JoanLeslie": [
"10177_JoanLeslie_35_f.jpg",
"10181_JoanLeslie_50_f.jpg",
"10182_JoanLeslie_59_f.jpg",
"10167_JoanLeslie_21_f.jpg",
"10184_JoanLeslie_77_f.jpg",
"10175_JoanLeslie_32_f.jpg",
"10170_JoanLeslie_25_f.jpg",
"10166_JoanLeslie_19_f.jpg",
"10188_JoanLeslie_83_f.jpg",
"10168_JoanLeslie_22_f.jpg",
"10174_JoanLeslie_30_f.jpg",
"10173_JoanLeslie_29_f.jpg",
"10165_JoanLeslie_17_f.jpg",
"10190_JoanLeslie_87_f.jpg",
"10171_JoanLeslie_26_f.jpg",
"10183_JoanLeslie_74_f.jpg",
"10163_JoanLeslie_13_f.jpg",
"10189_JoanLeslie_84_f.jpg",
"10172_JoanLeslie_28_f.jpg",
"10185_JoanLeslie_78_f.jpg",
"10187_JoanLeslie_81_f.jpg",
"10169_JoanLeslie_23_f.jpg",
"10164_JoanLeslie_16_f.jpg",
"10179_JoanLeslie_38_f.jpg",
"10180_JoanLeslie_45_f.jpg",
"10178_JoanLeslie_36_f.jpg",
"10176_JoanLeslie_33_f.jpg",
"10186_JoanLeslie_80_f.jpg"
],
"MelindaDillion": [
"12321_MelindaDillion_57_f.jpg",
"12310_MelindaDillion_41_f.jpg",
"12307_MelindaDillion_38_f.jpg",
"12304_MelindaDillion_30_f.jpg",
"12323_MelindaDillion_63_f.jpg",
"12314_MelindaDillion_45_f.jpg",
"12324_MelindaDillion_64_f.jpg",
"12327_MelindaDillion_70_f.jpg",
"12312_MelindaDillion_43_f.jpg",
"12306_MelindaDillion_37_f.jpg",
"12316_MelindaDillion_47_f.jpg",
"12319_MelindaDillion_54_f.jpg",
"12305_MelindaDillion_34_f.jpg",
"12325_MelindaDillion_66_f.jpg",
"12309_MelindaDillion_40_f.jpg",
"12313_MelindaDillion_44_f.jpg",
"12311_MelindaDillion_42_f.jpg",
"12326_MelindaDillion_68_f.jpg",
"12303_MelindaDillion_29_f.jpg",
"12320_MelindaDillion_55_f.jpg",
"12317_MelindaDillion_48_f.jpg",
"12315_MelindaDillion_46_f.jpg",
"12322_MelindaDillion_59_f.jpg",
"12308_MelindaDillion_39_f.jpg",
"12328_MelindaDillion_73_f.jpg",
"12318_MelindaDillion_50_f.jpg"
],
"StephenHawking": [
"1020_StephenHawking_65_m.jpg",
"1004_StephenHawking_43_m.jpg",
"1017_StephenHawking_65_m.jpg",
"1014_StephenHawking_67_m.jpg",
"1006_StephenHawking_36_m.jpg",
"1000_StephenHawking_1_m.jpg",
"1018_StephenHawking_66_m.jpg",
"1005_StephenHawking_23_m.jpg",
"1007_StephenHawking_43_m.jpg",
"1012_StephenHawking_67_m.jpg",
"1024_StephenHawking_54_m.jpg",
"1002_StephenHawking_15_m.jpg",
"1019_StephenHawking_53_m.jpg",
"1022_StephenHawking_48_m.jpg",
"1003_StephenHawking_21_m.jpg",
"1010_StephenHawking_62_m.jpg",
"1009_StephenHawking_46_m.jpg",
"1008_StephenHawking_43_m.jpg",
"1016_StephenHawking_53_m.jpg",
"1001_StephenHawking_3_m.jpg",
"1011_StephenHawking_64_m.jpg",
"1015_StephenHawking_40_m.jpg",
"1021_StephenHawking_64_m.jpg",
"1013_StephenHawking_67_m.jpg",
"1023_StephenHawking_45_m.jpg"
]
}
# --------------------------
# Positives
positives = []
for key, values in idendities.items():
# print(key)
for i in range(0, len(values) - 1):
for j in range(i + 1, len(values)):
# print(values[i], " and ", values[j])
positive = []
positive.append(values[i])
positive.append(values[j])
positives.append(positive)
positives = pd.DataFrame(positives, columns=["file_x", "file_y"])
positives["decision"] = "Yes"
print(positives.shape)
# --------------------------
# Negatives
samples_list = list(idendities.values())
negatives = []
for i in range(0, len(idendities) - 1):
for j in range(i + 1, len(idendities)):
# print(samples_list[i], " vs ",samples_list[j])
cross_product = itertools.product(samples_list[i], samples_list[j])
cross_product = list(cross_product)
# print(cross_product)
for cross_sample in cross_product:
# print(cross_sample[0], " vs ", cross_sample[1])
negative = []
negative.append(cross_sample[0])
negative.append(cross_sample[1])
negatives.append(negative)
negatives = pd.DataFrame(negatives, columns=["file_x", "file_y"])
negatives["decision"] = "No"
negatives = negatives.sample(positives.shape[0])
print(negatives.shape)
# --------------------------
# Merge positive and negative ones
df = pd.concat([positives, negatives]).reset_index(drop=True)
print(df.decision.value_counts())
df.file_x = "deepface/tests/dataset/" + df.file_x
df.file_y = "deepface/tests/dataset/" + df.file_y
# --------------------------
# DeepFace
from deepface import DeepFace
from deepface.basemodels import VGGFace, OpenFace, Facenet, FbDeepFace
pretrained_models = {}
pretrained_models["VGG-Face"] = VGGFace.loadModel()
print("VGG-Face loaded")
pretrained_models["Facenet"] = Facenet.loadModel()
print("Facenet loaded")
pretrained_models["OpenFace"] = OpenFace.loadModel()
print("OpenFace loaded")
pretrained_models["DeepFace"] = FbDeepFace.loadModel()
print("FbDeepFace loaded")
instances = df[["file_x", "file_y"]].values.tolist()
models = ['VGG-Face']
metrics = ['cosine']
if True:
for model in models:
for metric in metrics:
resp_obj = DeepFace.verify(instances
, model_name=model
, model=pretrained_models[model]
, distance_metric=metric)
distances = []
for i in range(0, len(instances)):
distance = round(resp_obj["pair_%s" % (i + 1)]["distance"], 4)
distances.append(distance)
df['%s_%s' % (model, metric)] = distances
df.to_csv("face-recognition-pivot.csv", index=False)
else:
df = pd.read_csv("face-recognition-pivot.csv")
df_raw = df.copy()
# --------------------------
# Distribution
fig = plt.figure(figsize=(15, 15))
figure_idx = 1
for model in models:
for metric in metrics:
feature = '%s_%s' % (model, metric)
ax1 = fig.add_subplot(4, 2, figure_idx)
df[df.decision == "Yes"][feature].plot(kind='kde', title=feature, label='Yes', legend=True)
df[df.decision == "No"][feature].plot(kind='kde', title=feature, label='No', legend=True)
figure_idx = figure_idx + 1
# plt.show()
# --------------------------
# Pre-processing for modelling
columns = []
for model in models:
for metric in metrics:
feature = '%s_%s' % (model, metric)
columns.append(feature)
columns.append("decision")
df = df[columns]
df.loc[df[df.decision == 'Yes'].index, 'decision'] = 1
df.loc[df[df.decision == 'No'].index, 'decision'] = 0
print(df.head())
# --------------------------
# Train test split
from sklearn.model_selection import train_test_split
df_train, df_test = train_test_split(df, test_size=0.30, random_state=17)
target_name = "decision"
y_train = df_train[target_name].values
x_train = df_train.drop(columns=[target_name]).values
y_test = df_test[target_name].values
x_test = df_test.drop(columns=[target_name]).values
# --------------------------
# LightGBM
import lightgbm as lgb
features = df.drop(columns=[target_name]).columns.tolist()
lgb_train = lgb.Dataset(x_train, y_train, feature_name=features)
lgb_test = lgb.Dataset(x_test, y_test, feature_name=features)
params = {
'task': 'train'
, 'boosting_type': 'gbdt'
, 'objective': 'multiclass'
, 'num_class': 2
, 'metric': 'multi_logloss'
}
gbm = lgb.train(params, lgb_train, num_boost_round=250, early_stopping_rounds=15, valid_sets=lgb_test)
gbm.save_model("face-recognition-ensemble-model.txt")
# --------------------------
# Evaluation
predictions = gbm.predict(x_test)
predictions_classes = []
for i in predictions:
prediction_class = np.argmax(i)
predictions_classes.append(prediction_class)
cm = confusion_matrix(y_test, predictions_classes)
print(cm)
tn, fp, fn, tp = cm.ravel()
recall = tp / (tp + fn)
precision = tp / (tp + fp)
accuracy = (tp + tn) / (tn + fp + fn + tp)
f1 = 2 * (precision * recall) / (precision + recall)
print("Precision: ", 100 * precision, "%")
print("Recall: ", 100 * recall, "%")
print("F1 score ", 100 * f1, "%")
print("Accuracy: ", 100 * accuracy, "%")
# --------------------------
# Interpretability
ax = lgb.plot_importance(gbm, max_num_features=20)
# plt.show()
import os
os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin'
plt.rcParams["figure.figsize"] = [20, 20]
for i in range(0, gbm.num_trees()):
ax = lgb.plot_tree(gbm, tree_index=i)
# plt.show()
if i == 2:
break
# --------------------------
# ROC Curve
from sklearn.metrics import confusion_matrix, accuracy_score, roc_auc_score, roc_curve
y_pred_proba = predictions[::, 1]
fpr, tpr, _ = roc_curve(y_test, y_pred_proba)
auc = roc_auc_score(y_test, y_pred_proba)
plt.figure(figsize=(7, 3))
lw = 2
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
fig.savefig('/home/khawar/deepface/tests/VGG-FACE_Cosine_ROC.png', dpi=fig.dpi)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('VGG Face')
plt.plot(fpr, tpr, label="ROC with Cosine auc=" + str(auc))
fig.savefig('/home/khawar/deepface/tests/VGG-FACE_Cosine_ROC_T_F.png', dpi=fig.dpi)
plt.legend(loc=4)
fig.savefig('/home/khawar/deepface/tests/VGG-FACE_Cosine.png', dpi=fig.dpi)
plt.show()
# --------------------------