Issues with Selenium background execution (python) - selenium

Question

I made a little code to scrape a website and save reviews and I'm rather a beginner at scraping.
It's the typical webstore, with a review section.
Comments in that review load dynamically by clicking on the "load more button".
My issue isn't specific to one site.
the idea of the code is to browse through products and for each product it will load their reviews. Some products have more reviews than others, up to 2000 or so...
By default, the website display 5 reviews and if you want to see more, you've got to click on the "load more" button.
Basically, if I launch my code, it's going open my chrome brower and do all that but if I launch my code and then switch deskops (on mac) then it doesn't load the reviews, it detects that there are more reviews but then it keep looking for the load more button without finding it (at least, that's what I assume since it throws no error).
Obviously it's got something to do with the fact that I'm switching desktops but I don't know how to fix it.
Has anyone experience similar issues? Am i missing something here?
MacOs Big Sur
Chrome webdriver
using proxies
driver = webdriver.Chrome()
#####
# function to get the number of reviews
def get_number_of_reviews():
try:
driver.execute_script(
"arguments[0].scrollIntoView(true);",
WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.CSS_SELECTOR, ".bv_numReviews_text")
)
),
)
number_of_reviews = driver.find_element_by_css_selector(
".bv_numReviews_text"
).text
number_of_reviews = int(number_of_reviews[1 : (len(number_of_reviews) - 1)])
except Exception as e:
# A common exception is when there is no review, the element containing the reviews is missing and therefore throws an exception
number_of_reviews = 0
return number_of_reviews
# function load every review in the perfume review section
"""
# While it's possible, click load more
# If it's not clickable anymore or the button disappeared, break out
"""
def load_all_reviews(number_of_reviews, start):
start = start
while True:
try:
driver.execute_script(
"arguments[0].scrollIntoView(true);",
WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.CSS_SELECTOR, ".bv-content-btn-pages-load-more")
)
),
)
driver.execute_script(
"arguments[0].click();",
WebDriverWait(driver, 20).until(
EC.element_to_be_clickable(
(By.CSS_SELECTOR, ".bv-content-btn-pages-load-more")
)
),
)
print("\n New reviews have been loaded !")
except Exception as e:
reviews = driver.find_elements_by_css_selector(".bv-content-review")
print("\n There aren't any new reviews to load")
print("\n Time elapsed : ", (time.time() - start))
if len(reviews) == number_of_reviews or (time.time() - start) > 25:
return reviews
else:
print("encore un tour! ", len(reviews))
return load_all_reviews(number_of_reviews, start)
break
# function returning the user asigned rating from 1 to 5
def get_rating():
try:
return review.find_element_by_class_name("bv-off-screen").text[0]
except Exception as e:
return "NA"
# function returning the review title
def get_title():
try:
return review.find_element_by_class_name("bv-content-title-container").text
except:
return "NA"
# function returning the review
def get_review():
try:
return review.find_element_by_class_name("bv-content-summary-body-text").text
except:
return "NA"
# We create a list containing all perfume links
links = read_perfume_links_from_pickle("data/perfume_list_sephora.pickle")
driver.get(links[random.randint(0, 1300)])
driver.implicitly_wait(10)
driver.find_element_by_xpath('//*[#id="footer_tc_privacy_button_2"]').click()
"""
# Itirating on the list of links to the products
# Checking the number of review
# If there are no reviews, we break out of the while loop
# Else, we collect each review
# Then, we move on to the next product
"""
lines = []
for link in tqdm(links, position=0, leave=True):
url = link
driver.get(url)
# We check the number of reviews
number_of_reviews = get_number_of_reviews()
if number_of_reviews > 0:
print("Nombre de reviews : ", number_of_reviews)
print("on dort")
time.sleep(6)
print("\n on se réveille")
driver.find_element_by_xpath('//*[#id="ratings-summary"]').click()
start = time.time()
reviews = load_all_reviews(number_of_reviews, start)
print("nbr reviews : ", len(reviews))
for review in tqdm(reviews, position=0, leave=True):
unique_ID = url.split("-")[-1].split(".")[0]
rating = get_rating() # Rating
title = get_title()
review_text = get_review() # Review
temp = []
temp.append(unique_ID) # id
temp.append(title) # title
temp.append(review_text) # review
temp.append(rating) # rating
lines.append(temp)
else:
print("")
driver.quit()