I'm new to automation with selenium webdriver. I'm trying to book a flight on a travel website. (ryanair.com)
I'm getting stuck with a popup datepicker. I can use .sendkeys to enter the day only, after any field on the date inputs is clicked it triggers the popup calendar to appear so if the format of the inputis dd/mm/yyyy and I want to enter "20042019" it only enters 20 in the dd and then automatically selects the current month and year as autocomplete and opens the calendar popup.
I'ver read a few articles saying that these calendars are usually of two types
1. iframe
2. datepicker - I think the one on ryanair is datepickers based on the xpath below
//div[#id='datetimepicker_dateview']//table//tbody//td[not(contains(#class,'k-other-month')
Maybe that's the wrong xpath? But I think it's correct
I have tried to find the list of dates to book using :
List<WebElement> list_AllDateToBook = driver.findElements(By.xpath()
System.out.println("size of list is : " + list_AllDateToBook.size() );
System.out.println("list is : " + list_AllDateToBook );
Output is:
size of list is : 0
list is : []
When I use xpath to enter the date of the date field it works for the first input using:
WebElement day = driver.findElement(By.xpath("//*[#id='row-dates-pax']/div[1]/div/div[1]/div/div[2]/div[2]/div/input[1]"));
However when I change the xpath to the second input it won't enter the second input(month)
WebElement day = driver.findElement(By.xpath("//*[#id='row-dates-pax']/div[1]/div/div[1]/div/div[2]/div[2]/div/input[2]"));
A sample of the datepicker HTML is below (it's too long to add it all!)
<core-datepicker class="start-date" default-date="" start-date="18-03-2019" end-date="28-03-2020" highlight-from="20-03-2019" highlight-to="20-04-2019" end-show="true" fly-out="true" value="dateRange.startDate" cb-event-id="cal-selector:select-start-date" unavailable-dates="dateRange.unavailabiltyStartDates" selected-month-view="dateRange.startDateSelectedMonthView" show-month-toggle="::dateRange.showMonthToggle" show-single-month=""><!----><div ng-class="::{'has-monthly-toggle': isMonthToggleVisible()}"><div bindonce="" class="datepicker-wrapper r scrollable value-selected" ng-class="{ scrollable: !device.isPhone(), mobile: device.isPhone(), 'value-selected': value, 'six-rows': checkIf6Rows()}" style="transition-timing-function: cubic-bezier(0.1, 0.57, 0.1, 1); transition-duration: 0ms; transform: translate(0px, 0px) translateZ(0px);"><!----><!----><ul ng-if="!device.isPhone()"><!----><li bindonce="" ng-repeat="i in _monthViews" ng-class="{ 'starting-month': checkIfIsSame(getDate(i), highlightFrom, 'month'), 'selected-month': checkIfIsSame(getDate(i), value, 'month'), 'highlight-on': canHighlight(i) }" class="calendar-view starting-month selected-month"><h1 class="month-name">March 2019</h1><ul class="week-days"><!---->
I'm really stuck here. Any advice would be great.
Thanks
Here is the working code from python. Please see the approach that I followed and simulate the same in java.
import os
import time
from selenium import webdriver
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
def get_full_path_to_folder(folderName):
folders = os.path.abspath(os.pardir).split(os.sep)
folderPath = ''
for folder in folders:
if folderPath == '':
folderPath = folder
else:
folderPath = folderPath + "\\" +folder
if os.path.isdir(os.path.join(folderPath, folderName)):
return os.path.join(folderPath, folderName)
break
def wait_for_element_present(locator_type, locator):
if locator_type == 'xpath':
return (wait.until(EC.presence_of_element_located((By.XPATH, locator))))
elif locator_type == "css":
return (wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, locator))))
chrome_options = ChromeOptions()
chrome_options.add_argument("--incognito")
driver = webdriver.Chrome(executable_path=os.path.join(get_full_path_to_folder('drivers'), "chromedriver.exe"))
driver.implicitly_wait(10)
wait = WebDriverWait(driver,10)
url = "https://www.ryanair.com/us/en/"
# go to url
driver.get(url)
#================ you should focus the below code=======================
# close the cookie pop up, otherwise the date and country elements not interable for selenium for interaction
wait_for_element_present('css',".cookie-popup__close").click()
#click on destination edit box
driver.find_element_by_css_selector(".route-selector-destination").click()
# select desitnation country
destiCountry = wait_for_element_present('xpath',"//div[#ng-repeat='option in $ctrl.firstOptions track by option.id' and text()=' Italy']")
destiCountry.click()
#select destination airport
desti = wait_for_element_present('xpath',"//span[contains(#ng-if,'secondOptionsEnabled') and .='Naples']")
desti.click()
# select outbound date
dateEle = wait_for_element_present('css',"li[data-id='24-03-2019']")
if dateEle.value_of_css_property('font-size') == '15px':
dateEle.click()
#select in bound date
dateEle = wait_for_element_present('css',"li[data-id='20-04-2019']")
if dateEle.value_of_css_property('font-size') == '15px':
dateEle.click()
#hurry, the date selection is successful.
Please try with :
li[data-id='12-04-2019'] span
Note: Add till span tag. Just tried via console and working.
document.querySelector("li[data-id='12-04-2019'] span").click()
Related
I am trying to scrape a website with scrapy-selenium. I am facing two problem
I applied xpath on chrome developer tool I found all elements but after execution of code it returns only one Selector object.
text() function of xpath expression returns none.
This is the URL I am trying to scrape: http://www.atab.org.bd/Member/Dhaka_Zone
Here is a screenshot of inspector tool:
Here is my code:
# -*- coding: utf-8 -*-
import scrapy
from scrapy.selector import Selector
from scrapy_selenium import SeleniumRequest
from selenium.webdriver.common.keys import Keys
class AtabDSpider(scrapy.Spider):
name = 'atab_d'
def start_requests(self):
yield SeleniumRequest(
url = "https://www.atab.org.bd/Member/Dhaka_Zone",
#url = "https://www.bit2lead.com",
#wait_time = 15,
wait_time = 3,
callback = self.parse
)
def parse(self, response):
companies = response.xpath("//ul[#class='row']/li")
print("Numbers Of Iterable Item: " + str(len(companies)))
for company in companies:
yield {
"company": company.xpath(".//div[#class='card']/div[1]/div/a/h3[#data-bind='text: NameOfOrganization']/text()").get()
#also tried
#"company": company.xpath(".//div[#class='card']/div[1]/div/a/h3/text()").get()
}
Here is a screenshot of my terminal:
And This is the url: ( https://www.algoslab.com ) I was practicing before That worked well. Although it's simple enough.
Why don't you try directly like the following to get everything in one go with the blink of an eye:
import requests
link = 'http://123.253.36.205:8051/API/Member/GetMembersList?searchString=&zone=0&blacklisted=false'
r = requests.get(link)
for item in r.json():
_name = item['NameOfOrganization']
phone = item['Phone']
print(_name,phone)
Output are like (should produce 3160 lines of results):
Aqib Travels & Tours Ltd. +88-029101468, 58151369
4S Tours & Travels Ltd 8954750
5M Logistics And Tours Ltd +880 2 48810030
The xpath you want could be simplified as //h3[#data-bind='text: NameOfOrganization'] to select the element and then view the text
I am trying to scrape the stock data but even though I'm using the "find elements by id" the result is one text.
i have tried various methods such as find elements by xpath and etc..
and i tried to make an array that contains all the IDs by finding "attribute 'target'" so i can loop through it but i wasn't successful so i had to code each ID.
import json
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
url = 'http://www.tsetmc.com/Loader.aspx?ParTree=15131F'
delay = 100
driver = webdriver.Chrome()
driver.get(url)
WebDriverWait(driver, delay)
zapna = driver.find_elements_by_id(id_='43479730079120887')
renik = driver.find_elements_by_id(id_='33854964748757477')
retko = driver.find_elements_by_id(id_='3823243780502959')
rampna = driver.find_elements_by_id(id_='67126881188552864')
mafakher = driver.find_elements_by_id(id_='4247709727327181')
for ii in retko:
print(ii.text , "\n")
driver.close()
and the result is:
رتكوكنترلخوردگيتكينكو2,1512.531M63.044 B25,14523,88824,900-245-0.9724,907-238-0.9523,88825,699-749-33.2512,55324,90024,9035,4601
what i expect is:
رتكو
كنترلخوردگيتكينكو
2,151
2.531M
63.044 B
25,145
23,888
24,900
-245
-0.97
24,907
-238
-0.95
23,888
25,699
-749
-33.25
1
2,553
24,900
24,903
5,460
1
any idea ?
You just have to go one layer deeper (using, for example, xpath) and iterate through the children:
for ii in retko:
targets = ii.find_elements_by_xpath('.//*')
for target in targets:
print(target.text)
Output:
رتكو
رتكو
كنترلخوردگيتكينكو
كنترلخوردگيتكينكو
3,149
3.235M
3.235M
etc.
I'm just started using selenium to scrape the table from webpage. So, I implemented the navigation of webpage using selenium. But, the the result keep looping when I run the code. Pretty sure that I wrote the code wrong. What should I fix the code so the navigation selenium works?
import requests
import csv
from bs4 import BeautifulSoup as bs
from selenium import webdriver
browser=webdriver.Chrome()
browser.get('https://dir.businessworld.com.my/15/posts/16-Computers-The-Internet')
# url = requests.get("https://dir.businessworld.com.my/15/posts/16-Computers-The-Internet/")
soup=bs(browser.page_source)
filename = "C:/Users/User/Desktop/test.csv"
csv_writer = csv.writer(open(filename, 'w'))
pages_remaining = True
while pages_remaining:
for tr in soup.find_all("tr"):
data = []
# for headers ( entered only once - the first time - )
for th in tr.find_all("th"):
data.append(th.text)
if data:
print("Inserting headers : {}".format(','.join(data)))
csv_writer.writerow(data)
continue
for td in tr.find_all("td"):
if td.a:
data.append(td.a.text.strip())
else:
data.append(td.text.strip())
if data:
print("Inserting data: {}".format(','.join(data)))
csv_writer.writerow(data)
try:
#Checks if there are more pages with links
next_link = driver.find_element_by_xpath('//*[#id="content"]/div[3]/table/tbody/tr/td[2]/table/tbody/tr/td[6]/a ]')
next_link.click()
time.sleep(30)
except NoSuchElementException:
rows_remaining = False
Check if there any next button present on the page then click else exit from while loop.
if len(browser.find_elements_by_xpath("//a[contains(.,'Next')]"))>0:
browser.find_element_by_xpath("//a[contains(.,'Next')]").click()
else:
break
No need to use time.sleep() instead use WebDriverWait()
Code:
import csv
from bs4 import BeautifulSoup as bs
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
browser=webdriver.Chrome()
browser.get('https://dir.businessworld.com.my/15/posts/16-Computers-The-Internet')
WebDriverWait(browser, 10).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "table.postlisting")))
soup=bs(browser.page_source)
filename = "C:/Users/User/Desktop/test.csv"
csv_writer = csv.writer(open(filename, 'w'))
pages_remaining = True
while pages_remaining:
WebDriverWait(browser,10).until(EC.visibility_of_element_located((By.CSS_SELECTOR,"table.postlisting")))
for tr in soup.find_all("tr"):
data = []
# for headers ( entered only once - the first time - )
for th in tr.find_all("th"):
data.append(th.text)
if data:
print("Inserting headers : {}".format(','.join(data)))
csv_writer.writerow(data)
continue
for td in tr.find_all("td"):
if td.a:
data.append(td.a.text.strip())
else:
data.append(td.text.strip())
if data:
print("Inserting data: {}".format(','.join(data)))
csv_writer.writerow(data)
if len(browser.find_elements_by_xpath("//a[contains(.,'Next')]"))>0:
browser.find_element_by_xpath("//a[contains(.,'Next')]").click()
else:
break
I have a program to download photos on various websites. Each url is formed at the end of the address by codes, which are accessed in a dataframe. In a dataframe of 8,583 lines
The sites have javascript, so I use selenium to access the src of the photos. And I download it with urllib.request.urlretrieve
Example of a photo site: http://divulgacandcontas.tse.jus.br/divulga/#/candidato/2018/2022802018/PB/150000608817
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
from bs4 import BeautifulSoup
import time
import urllib.request, urllib.parse, urllib.error
# Root URL of the site that is accessed to fetch the photo link
url_raiz = 'http://divulgacandcontas.tse.jus.br/divulga/#/candidato/2018/2022802018/'
# Accesses the dataframe that has the "sequencial" type codes
candidatos = pd.read_excel('candidatos_2018.xlsx',sheet_name='Sheet1', converters={'sequencial': lambda x: str(x), 'cpf': lambda x: str(x),'numero_urna': lambda x: str(x)})
# Function that opens each page and takes the link from the photo
def pegalink(url):
profile = webdriver.FirefoxProfile()
browser = webdriver.Firefox(profile)
browser.get(url)
time.sleep(10)
html = browser.page_source
soup = BeautifulSoup(html, "html.parser")
browser.close()
link = soup.find("img", {"class": "img-thumbnail img-responsive dvg-cand-foto"})['src']
return link
# Function that downloads the photo and saves it with the code name "cpf"
def baixa_foto(nome, url):
urllib.request.urlretrieve(url, nome)
# Iteration in the dataframe
for num, row in candidatos.iterrows():
cpf = (row['cpf']).strip()
uf = (row['uf']).strip()
print(cpf)
print("-/-")
sequencial = (row['sequencial']).strip()
# Creates full page address
url = url_raiz + uf + '/' + sequencial
link_foto = pegalink(url)
baixa_foto(cpf, link_foto)
Please I look guidance for:
Put a try-Exception type to wait for the page to load (I'm having errors reading the src - after many hits the site takes more than ten seconds to load)
And I would like to record all possible errors - in a file or dataframe - to write down the "sequencial" code that gave error and continue the program
Would anyone know how to do it? The guidelines below were very useful, but I was unable to move forward
I put in a folder a part of the data I use and the program, if you want to look: https://drive.google.com/drive/folders/1lAnODBgC5ZUDINzGWMcvXKTzU7tVZXsj?usp=sharing
put your code within :
try:
WebDriverWait(browser, 30).until(wait_for(page_has_loaded))
# here goes your code
except: Exception
print "This is an unexpected condition!"
For waitForPageToLoad :
def page_has_loaded():
page_state = browser.execute_script(
'return document.readyState;'
)
return page_state == 'complete'
30 above is time in seconds. You can adjust it as per your need.
Approach 2 :
class wait_for_page_load(object):
def __init__(self, browser):
self.browser = browser
def __enter__(self):
self.old_page = self.browser.find_element_by_tag_name('html')
def page_has_loaded(self):
new_page = self.browser.find_element_by_tag_name('html')
return new_page.id != self.old_page.id
def __exit__(self, *_):
wait_for(self.page_has_loaded)
def pegalink(url):
profile = webdriver.FirefoxProfile()
browser = webdriver.Firefox(profile)
browser.get(url)
try:
with wait_for_page_load(browser):
html = browser.page_source
soup = BeautifulSoup(html, "html.parser")
browser.close()
link = soup.find("img", {"class": "img-thumbnail img-responsive dvg-cand-foto"})['src']
except Exception:
print ("This is an unexpected condition!")
print("Erro em: ", url)
link = "Erro"
return link
How can I get the all element id's of a screen in selenium?
Please refer to this screenshot
Element ID is getting changed every time the page loads. I used contains#id , start-with#id, but it doesn't work every time. Now I want to get all the element id's from the webpage, so I can select the exact element.
My webpage contains input text, buttons, drop-downs.
Use Xpath instead or cssSelector. If you persist on knowing all page ids as a list of String in java, try to execute a javascript function.
// The Firefox driver supports javascript
WebDriver driver = new FirefoxDriver();
// Go to the Google Suggest home page
driver.get("http://www.google.com/webhp?complete=1&hl=en");
ArrayList ids = (ArrayList)((JavascriptExecutor) driver).executeScript(
" var allElements = document.getElementsByTagName('*'); var allIds = []; "
+ " for (var i = 0, n = allElements.length; i < n; ++i) { "
+ " var el = allElements[i]; "
+ " if (el.id) { allIds.push(el.id); } } return allIds; "
);
Regards,
Alan Mehio
London, UK