Adding patch distorts alignment - matplotlib

Question

I am working with the following image:
from matplotlib import cbook
import matplotlib.patches as mpatches
from matplotlib.axes._base import _TransformedBoundsLocator
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec
import numpy as np
# a numpy array of 15x15
Z = cbook.get_sample_data("axes_grid/bivariate_normal.npy", np_load=True)
gs = GridSpec(2, 3)
fig = plt.figure(figsize=(3*3,2*3))
ax1 = fig.add_subplot(gs[:2, :2])
ax2 = fig.add_subplot(gs[1, 2])
Z2 = np.zeros((150, 150))
ny, nx = Z.shape
Z2[30:30+ny, 30:30+nx] = Z
ax1.imshow(Z2)
ax1.set_aspect("equal")
ax2.set_aspect("equal")
plt.tight_layout()
plt.show()
output:
As shown in the image, the x-axis of both plots are aligned. However, when I am adding a patch to the first plot the alignment becomes distorted:
Z = cbook.get_sample_data("axes_grid/bivariate_normal.npy", np_load=True)
gs = GridSpec(2, 3)
fig = plt.figure(figsize=(3*3,2*3))
ax1 = fig.add_subplot(gs[:2, :2])
ax2 = fig.add_subplot(gs[1, 2])
Z2 = np.zeros((150, 150))
ny, nx = Z.shape
Z2[30:30+ny, 30:30+nx] = Z
ax1.imshow(Z2)
x, y, width, height = 30, 30, 15, 15
ex, ey = (0,1)
xy_data = x + ex * width, y + ey * height
p = mpatches.ConnectionPatch(
xyA=(0,1), coordsA=ax2.transAxes,
xyB=xy_data, coordsB=ax1.transData)
ax1.add_patch(p)
ax1.set_aspect("equal")
ax2.set_aspect("equal")
plt.tight_layout()
plt.show()
output:
Why is this? How can I add a patch whilst retaining the original layout?