how to make my logistic regression faster - numpy

Question

I have to do simple logistic regression (only in numpy, I can't use pytourch or tensorflow).
Data: part of MNIST
Goal: I should have accuracy about 86%.
Unfortunately i have only about 70%, and my loss function oscillate strangely.
It must be sth wrong with functions: t_cross_entropy or np_cross_entropy_grad
Of Course i tried to change learning rate, but without any satisfying results.
Could you help? (below you have a code and charts)
I CAN CHANGE ONLY functions: np_linear, np_softmax, np_cross_entropy,
np_cross_entropy_grad (and eventually in class NumpyLogisticRegression only forward function)
1. load part of MINST
# Import MNIST dataset
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm
%matplotlib inline
def load_dataset(dataset_name):
data = np.load('data/{}/{}.npz'.format(dataset_name.upper(),
dataset_name))
return data['X_train'], data['y_train'], data['X_test'],
data['y_test']
X_train, y_train, X_test, y_test = load_dataset('mini_mnist')
f, ax = plt.subplots(1, 10, sharex='col', sharey='row',figsize=(18, 16))
for a in ax:
a.imshow(X_train[np.random.randint(X_train.shape[0])].reshape(28,
28), cmap='gray')
plt.show()
X_train = np.c_[np.ones(X_train.shape[0]), X_train]
X_test = np.c_[np.ones(X_test.shape[0]), X_test]
print("train data shape: {}, test data shape:
{}".format(X_train.shape, X_test.shape))
2. main class and functions
def np_linear(x, a):
return np.dot(x, a.transpose())
'''
Calculate l(x;a) in BxK
:param x: Bx(D+1) input data
:param a: Kx(D+1) weight matrix
'''
def np_softmax(l):
exps = np.exp(l - np.max(l))
return exps / np.sum(exps)
'''
Calculate p(l) in BxK
:param l: BxK logits
'''
def np_cross_entropy(p, y):
m = y.shape[0]
log_likelihood = -np.log(p[range(m),y])
loss = np.sum(log_likelihood) / m
return loss
'''
Calculate L(p,y)
:param p: BxK predictions
:param y: B true labels
'''
def np_cross_entropy_grad(p, y, x):
m = y.shape[0]
grad = p
grad[range(m),y] -= 1
grad = grad/m
grad = grad.transpose()
return np.dot(grad, x)
'''
Calculate dL/da in Kx(D+1)
:param p: BxK predictions
:param y: B true labels
:param x: Bx(D+1) input data
'''
class NumpyLogisticRegression:
def __init__(self, n_classes, n_epochs, input_size, learning_rate=0.1, batch_size=256):
self.A = np.zeros((n_classes, input_size))
self.learning_rate = learning_rate
self.batch_size = batch_size
self.input_size = input_size
self.n_classes = n_classes
self.n_epochs = n_epochs
def forward(self, x):
return np_softmax(np_linear(x, self.A))
def train(self, X, Y, X_test=None, y_test=None):
loss, train_accuracy, test_accuracy = [], [], []
for e in tqdm(range(self.n_epochs)):
perm = np.random.permutation(len(X))
X, Y, = X[perm], Y[perm]
for batch in range(len(X) // self.batch_size):
x = X[batch * self.batch_size:(batch + 1) * self.batch_size]
y = Y[batch * self.batch_size:(batch + 1) * self.batch_size]
p = self.forward(x)
l = np_cross_entropy(p, y)
loss.append(l)
train_accuracy.append(self.test(x, y))
if X_test is not None and y_test is not None:
test_accuracy.append(self.test(X_test, y_test))
grad_A = np_cross_entropy_grad(p, y, x)
self.A -= grad_A * self.learning_rate
return loss, train_accuracy, test_accuracy
def test(self, X, Y):
p = np.argmax(self.forward(X), axis=1)
return np.mean(p == Y)
3. Test
clf = NumpyLogisticRegression(n_classes=10, n_epochs=10, input_size=785)
loss, train_accuracy, test_accuracy = clf.train(X_train, y_train, X_test, y_test)
4. Charts (without code, only results)

Answer 1

Problem was in np_softmax function, it should look like this:
def np_softmax(l):
exps = np.exp(l - np.max(l))
return exps / np.sum(exps, axis=1).reshape(-1,1)
Mine was prepered for single vector argument, this is proper version for matrix input.