take title of sub link on home page - selenium

i've aready got the sub link on the homepage. But try to take title for each sub link.
Can anyone help me! thanks you!
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
tem_data = {'URL':[], #make database,
'Title':[]}
driver = webdriver.Edge()
driver.get("https://m.cafe.naver.com/ca-fe/minivelobike")
URL = ([my_elem.get_attribute("href") for my_elem in WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.XPATH, "//a[#class='txt_area' and #href]")))])
Title = "some code to get 'tit" #get title of each URL[enter image description here][1]
tem_data['URL'].append(URL)
tem_data['Title'].append(Title)
print(tem_data)
Keyword_seach=('xxx.db')

The below should get you the titles:
Title = [my_elem.text for my_elem in driver.find_elements(By.CSS_SELECTOR, "strong.tit")]
The thing about Korean characters is that if you want to view them in your terminal, you will need to set your output encoding to UTF-8:
import sys
sys.stdout.reconfigure(encoding='utf-8')
print(tem_data['Title'])

Related

Selenium: how to see full DOM structure?

I've seen similar post about this question with the resolution being using the WebDriverWait. But I still kept getting an exception error saying the selector is not present.
Even when I printed driver.execute_script("return document.body.innerHTML;" at the end of my code the full DOM didn't show up, it looks exactly like the page source, but I need the rest of the HTML's elements
from multiprocessing.connection import wait
from ntpath import join
import os
from xml.dom.minidom import Element
from selenium import webdriver
from selenium.webdriver.common.by import By ## Used for grabbing elements by
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
import time
os.environ['PATH'] += ";C:\seleniumDrivers"
chrome = webdriver.Chrome()
chrome.get("https://jklm.fun/XSNM")
chrome.implicitly_wait(10)
name = chrome.find_element(By.CLASS_NAME, "nickname")
name.clear()
name.send_keys("Mr.Roboto")
btn = chrome.find_element(By.XPATH, '/html/body/div[2]/div[3]/form/div[2]/button')
btn.click()
join_btn = WebDriverWait(chrome, 1000000).until(EC.presence_of_element_located(
chrome.find_element(
By.XPATH, '/html/body/div[2]/div[3]/div[1]/div[1]/button')))
#join_btn = chrome.find_element(By.XPATH, '/html/body/div[2]/div[3]/div[1]/div[1]/button')
#join_btn = chrome.find_element(By.CSS_SELECTOR, 'button[data-text="joinGame"')
join_btn.click()
Element was in an iframe. I used chrome.switch_to.frame()

Get link and title of sub link in a homepage

I want to get a link inside the homepage of the website,
I tried to get it by selenium but only get the homepage link.
please help me to get all links inside.
Code trials:
from selenium import webdriver
import time
driver = webdriver.Edge()
driver.get('https://m.cafe.naver.com/ca-fe/minivelobike')
time.sleep(7)
elems = driver.find_elements_by_xpath("//a[#href]")
for elem in elems:
print(elem.get_attribute("href"))
driver.close()
Snapshot of the links:
To print the value of the href attributes of the articles you need to induce WebDriverWait for the visibility_of_element_located() and you can use either of the following locator strategies:
Using CSS_SELECTOR:
driver.get("https://m.cafe.naver.com/ca-fe/minivelobike")
print([my_elem.get_attribute("href") for my_elem in WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "a.txt_area[href]")))])
Using XPATH:
driver.get("https://m.cafe.naver.com/ca-fe/minivelobike")
print([my_elem.get_attribute("href") for my_elem in WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.XPATH, "//a[#class='txt_area' and #href]")))])
Console Output:
['https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074719&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074718&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074717&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074716&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074715&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074714&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074713&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074712&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074711&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074710&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074709&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074708&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074707&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074704&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074703&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074702&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074701&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074700&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074699&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074698&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074697&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074696&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074695&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074694&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074693&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074692&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074691&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074690&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074689&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074688&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074687&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074686&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074685&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074682&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074681&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074680&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074679&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074678&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074677&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074676&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074675&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074674&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074673&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074672&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074671&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074670&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074669&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074668&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074667&boardtype=L', 'https://m.cafe.naver.com/ArticleRead.nhn?clubid=11853711&articleid=1074666&boardtype=L']
Note : You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
i think this should work
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
driver = webdriver.Edge()
driver.get('https://m.cafe.naver.com/ca-fe/minivelobike')
time.sleep(7)
wrapper = driver.find_element(By.ID, "ct")
main_ul = wrapper.find_element(By.TAG_NAME, "ul")
for li in main_ul.find_elements(By.TAG_NAME, "li"):
try:
anchor_tag = li.find_element(By.TAG_NAME, "a")
href = anchor_tag.get_attribute("href")
print(href)
except:
print("Anchor tag doesnt exist")

make dataframe by sub link and its title on home page

I want to get sub link and its title on a homepage of the website, I tried to get it by selenium but only get the homepage link. please help me to get all sub links and its title like : (link0,title0),(link1,title1),link2, title2),(link3,title3), (linkn,titlen)
i tried it but not work
enter image description herefrom selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
tem_data = {'URL':[], #make database,
'Title':[]}
driver = webdriver.Edge()
driver.get("https://m.cafe.naver.com/ca-fe/minivelobike")
URL = ([my_elem.get_attribute("href") for my_elem in WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.XPATH, "//a[#class='txt_area' and #href]")))])
Title = "some code to get 'tit" #get title of url
tem_data['URL'].append(URL)
tem_data['Title'].append(Title)
print(tem_data)

Selenium will not click class id

I am trying to have selenium to do the following:
Open a website
Click on the search box
Type "Seattle" in the search box
Select the first result from the suggested results
My code fails at Step 2.
The class id for the search box is "class = input_search ng-pristine ng-valid ng-empty ng-touched"
Here's my code:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
driver = webdriver.Chrome()
url = 'https://wego.here.com/'
driver.get(url)
driver.find_element_by_css_selector('.input_search.ng-pristine.ng-valid.ng-empty.ng-touched').click()
driver.find_element_by_css_selector('.input_search.ng-pristine.ng-valid.ng-empty.ng-touched').send_keys('Seattle')
driver.find_element_by_css_selector('.input_search.ng-pristine.ng-valid.ng-empty.ng-touched').send_keys(Keys.ENTER)
Any suggestions would be greatly appreciated!
ADDITIONAL QUESTION
Thanks to #Prophet I was able to get the first auto-click and auto-fill done, but when I try to do the same task with a different search box, it does not like it. Please refer to the following code that I added to the existing code:
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "button.btn"))).send_keys(Keys.ENTER)
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "input.itinerary_item_input_0"))).click()
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "input.itinerary_item_input_0"))).send_keys('Chicago')
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "input.itinerary_item_input_0"))).send_keys(Keys.ENTER)
button.btn did work, but not the input.itinerary_item_input_0. Here is the source screenshot:
You are using a wrong locator.
ng-empty and ng-touched may not be there all the times.
So instead of
driver.find_element_by_css_selector('.input_search.ng-pristine.ng-valid.ng-empty.ng-touched').click()
Try using this:
driver.find_element_by_css_selector('input.input_search').click()
input.input_search is an unique, stable locator for that element.
Also, you have to add a delay, preferably to use the expected conditions, as following:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 20)
url = 'https://wego.here.com/'
driver.get(url)
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "input.input_search"))).click()
wait=WebDriverWait(driver, 60)
driver.get('https://wego.here.com/')
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,"input.input_search"))).send_keys("Seattle")
wait.until(EC.element_to_be_clickable((By.XPATH,"//div[#class='dropdown_list']/div[1]"))).click()
This will select the first option after sending a string to input tag.
Imports:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

How do I click the Span with Python Selenium

[enter image description here][1]I dont know how to select and click the red underlined. https://i.stack.imgur.com/nEyAb.png
If I run the code the span is already open but I cant select a number. This is the Webiste which I try to automate https://shop.jaegermeister.de/account/index/sTarget/shotmachine
This is my Code so far:
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
import time
PATH = r"C:\Program Files\chromedriver.exe"
driver = webdriver.Chrome(PATH)
driver.get("https://shop.jaegermeister.de/account/index/sTarget/shotmachine")
time.sleep(1)
elem = driver.find_element_by_class_name("field-wrap__checker").click()
time.sleep(1)
elem = driver.find_element_by_id("firstname")
elem.send_keys("First")
time.sleep(1)
elem = driver.find_element_by_id("lastname")
elem.send_keys("Second")
time.sleep(1)
elem = driver.find_element_by_class_name("select2-selection__arrow").click()
Try finding the element by its XPATH, which is by my own experience the most reliable. To do this, right click on the element in the DevTools, go to "copy and select" and click on "copy XPATH".
Also, it would help if you'd use WebDriverWait(driver,10).until(EC.element_to_be_clickable()) before using driver.find_element_by_class_name.
This of course requires the following imports:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC