Increase of multiprocess time - numpy

Question

I defined a multiprocess script to improve an image analysis. It seems work well but i tried to do several tests in order to define the best processes number.
It consists in varying this processes number. And as there is some dispersion, i add a loop in order to repeat one hundred times my test.
But during the process, time increases significantly. What could be the origin of my problem? Have I to flush memory? but it seems to be no saturation.
A piece of my code :
from multiprocessing import Process, current_process
import multiprocessing
import glob as glob
import matplotlib.pyplot as plt
from skimage import io
import time
import sys
import numpy as np
import numpy.ma as ma
import gc
import os
from PIL import Image
from skimage import exposure
import cv2
Path_input = "E:\\test\\raw\\"
Path_output = "E:\\test\\"
Img_list = glob.glob((Path_input + 'Test_*.tif' ))[:]
size_y,size_x = io.imread(Img_list[0]).shape
#Function for the multi process
def Ajustement(x):
#image reading
img = plt.imread(Img_list[x])
#create a CLAHE object
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))
cl1 = clahe.apply(img_rescale.astype(np.uint16))
cv2.imwrite(Path_output+ '\\Ajusted' + "%05d" % x + '.tif',cl1)
return 'Ajustement OK!'
#create strings list of process
cpu_max = 10
list = ['Process_']*cpu_max
list_process =[]
counter = 1
for u in list:
list_process.append(list[counter-1]+np.str(counter))
counter = counter+1
get_timer = time.clock if sys.platform == "win32" else time.time
time_store = []
time_process = []
if __name__ == '__main__':
range_adjusted = np.arange(0,len(Img_list),cpu_max)
m=0
for m in range(0,100,1): #loop for obtain a mean time for the process
print m
timer = get_timer() # time measuring starts now
for i in range_adjusted:
o = 0
for item in list_process[:cpu_max]: #process creation
globals()[item] = Process(name ='worker1', target=Normalization_and_ajustement, args=(i+o,))
o=o+1
o = 0
for item in list_process[:cpu_max]: #process start
globals()[item].start()
o=o+1
o = 0
for item in list_process[:cpu_max]: #process join
globals()[item].join()
o=o+1
if i == range_adjusted.max():
print("Normalization and Equalization finished")
timer = get_timer() - timer # get delta time as soon as it finishes
time_store.append(timer)
time_process.append(timer/cpu_max)
np.savetxt(Path_output + 'time_tot_normalization.txt',time_store)
np.savetxt(Path_output + 'time_process_normalization.txt',time_process)
print("\tTotal: {:.2f} seconds".format(timer))
print("\tAvg. per process: {:.2f} seconds".format(timer/cpu_max))
m=m+1

Answer 1

I think it was due to a memory leak . Indeed i added gc.collect() command after each loop and the problem was solved.