pytorch conv2d vs numpy results are different - numpy

Question

I'm working on implementing pytorch conv2d with numpy. But pytorch conv2d vs numpy results are different for the same input and conv weight. How to fix it? Thanks for any help.
Code sample below:
Note:
The code contains 4 parts:
Fixed random seed to generate fixed input and conv weight.
Implement conv2d with pytorch.
Implement conv2d with numpy.
Run conv2d and verify the results are different.
import random
import numpy as np
import torch
import torch.nn as nn
fixed_seed = 5179
np.random.seed(fixed_seed)
random.seed(fixed_seed)
torch.manual_seed(fixed_seed)
np.set_printoptions(precision=8, floatmode="fixed")
torch.set_printoptions(precision=8)
def conv_forward_torch(input_image_tensor, weight, stride, pad):
# input_image_tensor B Hi Wi Ci
# weight Hk Wk Ci Co
input_image_tensor = input_image_tensor.permute(0, 3, 1, 2) # B Ci Hi Wi
weight = weight.permute(3, 2, 0, 1) # Co Ci Hk Wk
output = torch.nn.functional.conv2d(input_image_tensor, weight, stride=stride, padding=pad) # B Co Ho Wo
output = output.permute(0, 2, 3, 1).cpu().detach().numpy() # B Ho Wo Co
return output
def conv_forward_naive(x, w, stride, pad, bias = None):
# x B Hi Wi Ci
# w Hk Wk Ci Co
x = np.transpose(x, [0, 3, 1, 2]) # B Ci Hi Wi
w = np.transpose(w, [3, 2, 0, 1]) # Co Ci Kh Kw
if pad != 0:
x = np.pad(x, ((0, 0), (0 ,0), (pad, pad), (pad, pad)),'constant')
b, ci, hi, wi = x.shape
co, ci, hk, wk = w.shape
ho = np.floor(1 + (hi - hk) / stride).astype(int)
wo = np.floor(1 + (wi - wk) / stride).astype(int)
out = np.zeros((b, co, ho, wo), dtype=np.float32) # B Co Ho Wo
x = np.expand_dims(x, axis=1) # B 1 Ci Hi Wi
w = np.expand_dims(w, axis=0) # 1 Co Ci Hk Wk
for i in range(ho):
for j in range(wo):
x_windows = x[:, :, :, i * stride:i * stride + hk, j * stride: j * stride + wk] # B 1 Ci Hk Wk
out[:, :, i, j] = np.sum(x_windows * w, axis=(2, 3, 4)) # B Co
out = np.transpose(out, [0, 2, 3, 1]) # B Ho Wo Co
return out
B = 1 # Batch size
Hi = 2 # Input height
Wi = 2 # Input width
Ci = 1 # Input channel
Co = 1 # Ouput channel
P = 0 # Padding size
Hk = 2 # Kernel height
Wk = 2 # Kernel width
S = 1 # Stride
input_image_tensor = torch.randn(B, Hi, Wi, Ci)
conv_weight_tensor = torch.randn(Hk, Wk, Ci, Co)
input_image = input_image_tensor.detach().numpy()
conv_weight = conv_weight_tensor.detach().numpy()
y_torch = conv_forward_torch(input_image_tensor, conv_weight_tensor, S, P)
y_np = conv_forward_naive(input_image, conv_weight, S, P)
is_same = y_torch == y_np
print(is_same, y_torch, y_np)
Expected output:
is_same should be True.
Actual output for y_torch, y_np:
[[[[False]]]] [[[[-3.62229419]]]] [[[[-3.62229395]]]]
torch.version = '1.11.0'
np.version = '1.20.1'

Answer 1

Your code works as expected. The result you observed is because of a difference in floating-point precisions between NumPy and PyTorch. To compare floating points you should not use a direct equal check, but instead something like np.allclose.
In this case your snippet indeed returns True:
>>> np.allclose(y_torch, y_np)
True