I am trying to web-scrape the following site but im stucked at login.
This is what I have been trying. Any ideas??
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
driver = webdriver.Chrome('/Library/Chrome Drivers/chromedriver')
def site_login():
driver.get("https://partners.rappi.com/")
login = driver.find_elements_by_xpath("//span[contains(text(),'INGRESAR')]")
user = driver.find_elements_by_xpath("//input[#id='1-email']")
pass = driver.find_elements_by_xpath("//body/div[#id='auth0-lock-container- 1']/div[1]/div[2]/form[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[2]/span[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/input[1]")
login.click()
user.send_keys('xxx#mail.com')
pass.send_keys('Password123')
driver.find_element_by_id(“submit”).click()
site_login()
There's a random loading time of the web page. You need to introduce webdriverWait.
try out the below code :
Code :
driver = webdriver.Chrome("C:\\Users\\Inc\\Desktop\\Selenium+Python\\chromedriver.exe")
driver.maximize_window()
wait = WebDriverWait(driver, 30)
driver.get("https://partners.rappi.com/")
wait.until(EC.element_to_be_clickable((By.XPATH, "//span[text()='INGRESAR']/.."))).click()
wait.until(EC.element_to_be_clickable((By.NAME, "email"))).send_keys("abc")
wait.until(EC.element_to_be_clickable((By.NAME, "password"))).send_keys("password")
wait.until(EC.element_to_be_clickable((By.NAME, "submit"))).click()
Related
I'am trying to scrape this web page: https://whalewisdom.com/filer/fisher-asset-management-llc#tabholdings_tab_link
I would like to setup the python selenium code, in order to setup correctly multitems in: "50" pages per page
But my code click on wrong button. where is my code error?
options = webdriver.FirefoxOptions()
options.binary_location = r'C:/Users/Mozilla Firefox/firefox.exe'
driver = selenium.webdriver.Firefox(executable_path='C:/geckodriver.exe' , options=options)
driver.execute("get", {'url': 'https://whalewisdom.com/filer/fisher-asset-management-llc#tabholdings_tab_link'})
driver.execute_script("return arguments[0].scrollIntoView(true);", WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//label[#id='qtr-1-label']"))))
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[#class='btn btn-default dropdown-toggle']"))).click()
thank you for your help.
-ag
You code clicked on wrong button because you have multiple elements with exact same class and you are fetching the first one and clicking on it.
Also I see on the page, you sometime get a popup which may make other elements not interactable. SO we would want the popup to close first(if appeared) then move ahead.
Using Chrome driver
Setup and Imports
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
# REPLACE YOUR CHROME PATH HERE
chrome_path = r"C:\Users\hpoddar\Desktop\Tools\chromedriver_win32\chromedriver.exe"
s = Service(chrome_path)
driver = webdriver.Chrome(service=s)
Fetch the page
driver.get(' https://whalewisdom.com/filer/fisher-asset-management-llc#tabholdings_tab_link')
Close the popup(if appeared)
try:
popup = WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, "//a[#id='dfwid-close-184302']")))
popup.click()
except TimeoutException:
print("No Popup appeared on the page")
Click on dropdown and the menu item 50
dropdown = driver.find_element(By.CSS_SELECTOR, '.btn-group.dropdown')
dropdown.click()
ele50 = driver.find_element(By.XPATH, '//li[#role="menuitem"]/a[contains(text(), "50")]')
ele50.click()
Output
The above code clicks on item 50
Using Firefox driver
The imports would be same as above, the following code would also remains some with just a minute change.
# REPLACE YOUR FIREFOX DRIVER PATH HERE
firefoxpath = r'C:\Users\hpoddar\Desktop\Tools\firefoxdriver\geckodriver.exe'
s = Service(firefoxpath)
driver = webdriver.Firefox(service=s)
driver.get(' https://whalewisdom.com/filer/fisher-asset-management-llc#tabholdings_tab_link')
try:
popup = WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, "//a[#id='dfwid-close-184302']")))
popup.click()
except TimeoutException:
print("No Popup appeared on the page")
dropdown = driver.find_element(By.CSS_SELECTOR, '.btn-group.dropdown')
dropdown.click()
ele50 = driver.find_element(By.XPATH, '//li[#role="menuitem"]/a[contains(text(), "50")]')
ele50.click()
Output
which similarly clicks on the desired element
Path = r"C:\WebDriverEdge\chromedriver.exe"
service = Service(Path)
options = Options()
options.add_argument('--user-data-dir=C:\\Users\\Admin\\AppData\\Local\\Google\\Chrome\\User Data\\')
options.add_argument("--profile-directory=Profile 1")
#connect to driver
driver = webdriver.Chrome(service = service, options = options)
driver.get("https://open.spotify.com/search")
x_path = '//*[#id="main"]/div/div[2]/div[1]/header/div[3]/div/div/form/input'
search = driver.find_element(By.XPATH, x_path)
action = webdriver.ActionChains(driver)
action.move_to_element(search).send_keys("Let me").perform()
I try to click on search bar at Spotify and use it to search. My problem is when I already login my code get error "unable to find element" but without sign in I can fill the search bar easily.
I don't know why. Is any one run into this before? Thanks in advance
p/s: XPath still the same
I'd rather use this form[role='search'] input css with explicit wait like below:
driver.get("https://open.spotify.com/search")
driver.maximize_window()
wait = WebDriverWait(driver, 20)
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "form[role='search']"))).send_keys('Let me')
Imports:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
How to long press (Press and Hold) mouse left key using only Selenium in Python
I am trying the answer by zirene-nguyễn in the above page. But it does not click-and-hold the button. How can I fix it so that it works to bypass the captcha in this example?
from selenium import webdriver
driver = webdriver.Firefox()
driver.get('https://www.walmart.com/blocked?url=L2lwL0Nsb3JveC1EaXNpbmZlY3RpbmctV2lwZXMtMjI1LUNvdW50LVZhbHVlLVBhY2stQ3Jpc3AtTGVtb24tYW5kLUZyZXNoLVNjZW50LTMtUGFjay03NS1Db3VudC1FYWNoLzE0ODk4MzY1&uuid=9ed7f800-f288-11eb-ad50-1b3c9c7d7310&vid=9cf07351-f288-11eb-9ab5-ef26d206453b&g=b')
element = driver.find_element_by_xpath("//div[#id='px-captcha']")
from selenium.webdriver import ActionChains
action = ActionChains(driver)
frame_x = element.location['x']
frame_y = element.location['y']
x_move = frame_x + element.size['width']*0.5
y_move = frame_y + element.size['height']*0.5
action.move_to_element_with_offset(element, x_move, y_move).click_and_hold().perform()
import time
time.sleep(10)
action.release(element)
action.perform()
time.sleep(0.2)
action.release(element)
I'm trying to scrape data from webpage.
Below code works on my local window machine but it doesn't work on ec2 linux instance.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
option = webdriver.firefox.options.Options()
option.add_argument("--headless")
driver = webdriver.Firefox(executable_path=dir_firefox_driver, options=option)
driver.get("https://www.mwave.me/en/mcountdown")
time.sleep(3)
driver.set_window_size(600, 800)
temp = driver.find_element_by_css_selector(".chart_view_more button")
I saw the article that wait until element appear, so I tried below code
driver = webdriver.Firefox(executable_path=dir_firefox_driver, options=option)
driver.get("https://www.mwave.me/en/mcountdown")
time.sleep(3)
driver.set_window_size(600, 800)
try:
element = WebDriverWait(driver, 20).until(
EC.presence_of_element_located((By.CSS_SELECTOR, ".chart_view_more button"))
)
except:
print("there is no element")
quit()
temp = driver.find_element_by_css_selector(".chart_view_more button")
It also doesn't work. I can't find difference in my local machine and ec2 instance.
Can somebody give me any suggestion?
few things you can try to solve this issue:-
Instead of
driver.set_window_size(600, 800)
use :
driver.maximize_window()
also use the below code :
driver = webdriver.Firefox(executable_path=dir_firefox_driver, options=option)
driver.get("https://www.mwave.me/en/mcountdown")
time.sleep(3)
driver.maximize_window()
try:
element = WebDriverWait(driver, 20).until(
EC.visibility_of_element_located((By.CSS_SELECTOR, ".chart_view_more button"))
)
except:
print("there is no element")
quit()
temp = driver.find_element_by_css_selector(".chart_view_more button")
What I need: switch to the Reviews tab in description of an extension from Chrome Store (e.g. this one) in order to count the number of reviews.
What I've done: Used BeautifulSoup + Selenium to switch between tabs. I used driver.find_element_by_id('id') BUT it returns an error that it can not find the element.
Here's the code I use:
from selenium import webdriver
driver = webdriver.Chrome()
driver.get(url)
button = driver.find_element_by_id(':22')
button.click()
page = requests.get(driver.current_url)
soup = BeautifulSoup(page.content,'html5lib')
comment_list = soup.find('div', class_ = 'e-f-b-L') #the class of reviews I need to count.
Here's the html-code of the Review button element:
Issues:
How do I make it click the 'Reviews' button so the 'Reviews' tab is displayed?
You can click on that Reviews tab very smoothly If you define a simple xpath like '//div[.="Reviews"]' or so. Check out the script as a proof of concept:
from selenium import webdriver
from selenium.webdriver.support import ui
url = "https://chrome.google.com/webstore/detail/emoji-keyboard-by-emojion/ipdjnhgkpapgippgcgkfcbpdpcgifncb?hl=en"
driver = webdriver.Chrome()
wait = ui.WebDriverWait(driver, 10)
driver.get(url)
wait.until(lambda driver: driver.find_element_by_xpath('//div[.="Reviews"]')).click()
driver.quit()
To make it headless:
from selenium import webdriver
from selenium.webdriver.support import ui
url = "https://chrome.google.com/webstore/detail/emoji-keyboard-by-emojion/ipdjnhgkpapgippgcgkfcbpdpcgifncb?hl=en"
chromeOptions = webdriver.ChromeOptions()
chromeOptions.add_argument("--headless")
driver = webdriver.Chrome(chrome_options=chromeOptions)
wait = ui.WebDriverWait(driver, 10)
driver.get(url)
wait.until(lambda driver: driver.find_element_by_xpath('//div[.="Reviews"]')).click()
print("It's done")