Defining custom metric using call back in keras [duplicate] - tensorflow

Question

I am trying to create a custom macro for recall = (recall of class1 + recall of class2)/2. I came up with the following code but I am not sure how to calculate the true positive of class 0.
def unweightedRecall():
def recall(y_true, y_pred):
# recall of class 1
true_positives1 = K.sum(K.round(K.clip(y_pred * y_true, 0, 1)))
possible_positives1 = K.sum(K.round(K.clip(y_true, 0, 1)))
recall1 = true_positives1 / (possible_positives1 + K.epsilon())
# --- get true positive of class 0 in true_positives0 here ---
# Also, is there a cleaner way to get possible_positives0
possible_positives0 = K.int_shape(y_true)[0] - possible_positives1
recall0 = true_positives0 / (possible_positives0 + K.epsilon())
return (recall0 + recall1)/2
return recall
It seems I will have to use Keras.backend.equal(x, y), but how do i create a tensor with shape K.int_shape(y_true)[0] and all values, say x?
Edit 1
Based on Marcin's comments, I wanted to create a custom metric based on callback in keras. While browsing issues in Keras, I came across the following code for f1 metric:
class Metrics(keras.callbacks.Callback):
def on_epoch_end(self, batch, logs={}):
predict = np.asarray(self.model.predict(self.validation_data[0]))
targ = self.validation_data[1]
self.f1s=f1(targ, predict)
return
metrics = Metrics()
model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=[X_test,y_test],
verbose=1, callbacks=[metrics])
But how is the callback returning the accuracy? I wanted to implement unweighted recall = (recall class1 + recall class2)/2. I can think of the following code but would appreciate any help to complete it
from sklearn.metrics import recall_score
class Metrics(keras.callbacks.Callback):
def on_epoch_end(self, batch, logs={}):
predict = np.asarray(self.model.predict(self.validation_data[0]))
targ = self.validation_data[1]
# --- what to store the result in?? ---
self.XXXX=recall_score(targ, predict, average='macro')
# we really dont need to return anything ??
return
metrics = Metrics()
model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=[X_test,y_test],
verbose=1, callbacks=[metrics])
Edit 2: model:
def createModelHelper(numNeurons=40, optimizer='adam'):
inputLayer = Input(shape=(data.shape[1],))
denseLayer1 = Dense(numNeurons)(inputLayer)
outputLayer = Dense(1, activation='sigmoid')(denseLayer1)
model = Model(input=inputLayer, output=outputLayer)
model.compile(loss=unweightedRecall, optimizer=optimizer)
return model

Answer 1

keras version (with the mean problem).
Are your two classes actually only one dimension output (0 or 1)?
If so:
def recall(y_true, y_pred):
# recall of class 1
#do not use "round" here if you're going to use this as a loss function
true_positives = K.sum(K.round(y_pred) * y_true)
possible_positives = K.sum(y_true)
return true_positives / (possible_positives + K.epsilon())
def unweightedRecall(y_true, y_pred):
return (recall(y_true,y_pred) + recall(1-y_true,1-y_pred))/2.
Now, if your two classes are actually a 2-element output:
def unweightedRecall(y_true, y_pred):
return (recall(y_true[:,0],y_pred[:,0]) + recall(y_true[:,1],y_pred[:,1]))/2.
Callback version:
For the callback, you can use a LambdaCallback, and you manually print or store the results:
myCallBack = LambdaCallback(on_epoch_end=unweightedRecall)
stored_metrics = []
def unweightedRecall(epoch,logs):
predict = model.predict(self.validation_data[0])
targ = self.validation_data[1]
result = (recall(targ,predict) + recall(1-targ,1-predict))/2.
print("recall for epoch " + str(epoch) + ": " + str(result))
stored_metrics.append(result)
Where recall is a function using np instead of K. And epsilon = np.finfo(float).eps or epsilon = np.finfo(np.float32).eps)