Pyplot modifies zoom when calling plt.draw() - matplotlib

Question

I wrote a function that lets the user pick points from an image and returns the coordinates.
fig, ax = plt.subplots()
ax.imshow(pixel_array, cmap=plt.cm.hot)
plt.ion()
plt.show()
coords = get_points(fig, ax, 4)
where the function get_points is defined as
def get_points(fig, ax, n):
# Request the user to select n points in the figure fig on the axis ax
# If n is zero or negative, allow the selection of as many points as the user would like (minimum abs(n))
# If n is positive, allow only n points to be selected
if n > 0:
print('Select', n, 'points.')
elif n < 0:
print('Select at least', abs(n), 'points.')
else:
print('Select any number of points.')
print("Press 'Esc' to cancel the last point, or 'Enter' when complete.")
coords = []
handles = []
finished = False
def onClick(event):
nonlocal coords
if fig.canvas.manager.toolbar.mode != '': return # Don't take clicks involved in toolbar functions (pan, zoom, ...)
if event.inaxes!=ax: return # Don't take clicks outside the axes
if len(coords) < n or n <= 0:
coords.append((event.xdata, event.ydata))
handles.append(plt.scatter(event.xdata, event.ydata, s=100, c='red', marker='+'))
plt.draw()
else:
print('You have reached the maximum number of points for this operation.')
def onKey(event):
nonlocal finished
if event.key == 'escape':
if len(coords) > 0:
del coords[-1]
handles[-1].remove()
del handles[-1]
plt.draw()
elif event.key == 'enter':
if len(coords) >= abs(n):
finished = True
else:
print('You have selected', len(coords), 'out of a required', n, 'points.')
else:
print('You pressed "', event.key, '" which is unsupported', sep='')
cid = fig.canvas.mpl_connect('button_press_event', onClick)
cid2 = fig.canvas.mpl_connect('key_press_event',onKey)
while not finished:
plt.waitforbuttonpress(-1)
plt.disconnect(cid)
plt.disconnect(cid2)
return np.array(coords)
But the problem is that if I zoom or pan, then each time I select a point it resets the zoom/pan to the default. Is there any way to have it not do this? Is there a better way to do this overall?
Thanks in advance.