Implementing a neural network from scratch - numpy

Question

I am using numpy version 1.24.3
I am trying to create a model from scratch to train on the fashion mnist dataset to pu it simply i am creating a neural-network, but i keep getting he following error:
C:\Users\njeng\PycharmProjects\ARTIFICIAL-INTELLIGENCE\nn.py:64: RuntimeWarning: overflow encountered in exp
return np.exp(Z) / np.sum(np.exp(Z))
C:\Users\njeng\PycharmProjects\ARTIFICIAL-INTELLIGENCE\nn.py:64: RuntimeWarning: invalid value encountered in divide
return np.exp(Z) / np.sum(np.exp(Z))
Traceback (most recent call last):
File "C:\Users\njeng\PycharmProjects\ARTIFICIAL-INTELLIGENCE\nn.py", line 145, in <module>
W1, b1, W2, b2 = gradient_descent(X_train, Y_train, 100, 0.1)
File "C:\Users\njeng\PycharmProjects\ARTIFICIAL-INTELLIGENCE\nn.py", line 135, in gradient_descent
dW1, db1, dW2, db2 = back_prop(Z1, A1, Z2, A2, W2, X, Y)
File "C:\Users\njeng\PycharmProjects\ARTIFICIAL-INTELLIGENCE\nn.py", line 97, in back_prop
db2 = 1 / m * np.sum(dZ2, 2)
File "<__array_function__ internals>", line 180, in sum
File "C:\Users\njeng\AppData\Roaming\Python\Python310\site-packages\numpy\core\fromnumeric.py", line 2298, in sum
return _wrapreduction(a, np.add, 'sum', axis, dtype, out, keepdims=keepdims,
File "C:\Users\njeng\AppData\Roaming\Python\Python310\site-packages\numpy\core\fromnumeric.py", line 86, in _wrapreduction
return ufunc.reduce(obj, axis, dtype, out, **passkwargs)
numpy.AxisError: axis 2 is out of bounds for array of dimension 2
The error seems to orriginat5e from he following piece of code:
def softmax(Z):
return np.exp(Z) / np.sum(np.exp(Z))
I can't seem o figure out the cause of he problem, I looked a he documentation and the problem seems to persist.
This is the code I am tring to run:
import numpy as np
import pandas as pd
'''Creating a simple neural network to perform prediction on a fashion mnist dataset using 2 layers and
each layer has 10 nodes:
THE LOGIC: z1 -> is the unactivated first layer -> RELU
z2 -> is the unactivated second layer -> softmax
b1 -> 1st bias for layer 1
b2 -> 2nd bias for layer 2
w1 -> weight layer 1
w2 -> weight layer 2
dZ2 -> error of the second layer
db2 -> average of the absolute error for the 2nd layer
dw1 -> check how much the weights contributed to the error
db1 -> check how much the biases contributed to the error
alpha -> learning rate
lastly we update our parameters
'''
df = pd.read_csv('fashion-mnist_train.csv')
print(df)
print()
# changing the data to a numpy array
df = np.array(df)
print(df)
print()
m, n = df.shape
print(m, n)
np.random.shuffle(df)
# Transpose the data
# Instead of each row being an example we make each column an example
df_dev = df[0:1000].T
Y_dev = df_dev[0]
X_dev = df_dev[1:n]
df_train = df[1000:m].T
Y_train = df_train[0]
X_train = df_train[1:n]
def init_params():
w1 = np.random.randn(10, 784)
b1 = np.random.randn(10, 1)
w2 = np.random.randn(10, 10)
b2 = np.random.randn(10, 1)
return w1, b1, w2, b2
# 1st activation Function for Layer 1
def ReLU(Z):
return np.maximum(0, Z)
# 2nd activation function for Layer 2
def softmax(Z):
return np.exp(Z) / np.sum(np.exp(Z))
# implementing Forward propagation
def forward_prop(W1, b1, W2, b2, X):
Z1 = W1.dot(X) + b1
A1 = ReLU(Z1)
Z2 = W2.dot(A1) + b2
A2 = softmax(Z2)
return Z1, A1, Z2, A2
# perform one hot encoding on y
def one_hot(Y):
one_hot_Y = np.zeros((Y.size, Y.max() + 1))
one_hot_Y[np.arange(Y.size), Y] = 1
one_hot_Y = one_hot_Y.T
return one_hot_Y
# implementing the derivative of RELU
def derive_ReLU(Z):
return Z > 0
# backward propagation
def back_prop(Z1, A1, Z2, A2, W2, X, Y):
m = Y.size
one_hot_Y = one_hot(Y)
dZ2 = A2 - one_hot_Y
dW2 = 1 / m * dZ2.dot(A1.T)
db2 = 1 / m * np.sum(dZ2, 2)
dZ1 = W2.T.dot(dZ2) * derive_ReLU(Z1)
dW1 = 1 / m * dZ1.dot(X.T)
db1 = 1 / m * np.sum(dZ1, 2)
return dW1, db1, dW2, db2
def update_params(W1, b1, W2, b2, dW1, db1, dW2, db2, alpha):
W1 = W1 - alpha * dW1
b1 = b1 - alpha * db1
W2 = W2 - alpha * dW2
b2 = b2 - alpha * db2
return W1, b1, W2, b2
def loss(X, Y, A2):
m = Y.size
log_likelihood = -np.log(A2[Y, range(m)])
return 1 / m * np.sum(log_likelihood)
def get_predictions(A2):
return np.argmax(A2, 0)
def get_accuracy(predictions, Y):
print(predictions, Y)
return np.sum(predictions == Y) / Y.size
def gradient_descent(X, Y, iterations, alpha):
W1, b1, W2, b2 = init_params()
for i in range(iterations):
Z1, A1, Z2, A2 = forward_prop(W1, b1, W2, b2, X)
dW1, db1, dW2, db2 = back_prop(Z1, A1, Z2, A2, W2, X, Y)
W1, b1, W2, b2 = update_params(W1, b1, W2, b2, dW1, db1, dW2, db2, alpha)
if i % 50 == 0:
print('Iterations: ', i)
print('Accuracy: ', get_accuracy(get_predictions(A2), Y))
return W1, b1, W2, b2
W1, b1, W2, b2 = gradient_descent(X_train, Y_train, 100, 0.1)