Unable to extract values linked to data points from an interactive line chart - selenium

Question

I am trying to extract the percentage values (Target Rate Probability) from an interactive chart with preselected conditions "13 Dez23" and "Historical" on the following website: https://www.cmegroup.com/markets/interest-rates/cme-fedwatch-tool.html?redirect=/trading/interest-rates/countdown-to-fomc.html. The percentage probabilities only appear when I hover over the respective data points with a mouse.
I have tried the following:
XPath of tried element: /html/body/form/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/div/div[1]/div/svg/g[5]/g[2]/path[2]
1. Method Approach: Webdriverwait in order for the elements to load on the page
1. Method Error: TimeoutException
2. Method Approach:
driver.execute_script("document.evaluate('/html/body
/form/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/div/div[1]/div/svg/g[5]/g[2]/path[2]', document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.style.display = 'block';")
2. Method Error:
selenium.common.exceptions.JavascriptException: Message: javascript error: Cannot read properties of null (reading 'style')
3. Method Approach:
driver.execute_script("document.evaluate('/html/body/form
/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/div/div[1]/div/svg/g[5]/g[2]/path[2]', document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.style.visibility = 'visible';")
3. Method Error:
selenium.common.exceptions.JavascriptException: Message: javascript error: Cannot read properties of null (reading 'style')
EC: Invisibility of element is True
**Code: **
from webdriver_manager.chrome import ChromeDriverManager
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
import datetime as dt
import time
driver = webdriver.Chrome(executable_path=ChromeDriverManager().install())
driver.get('https://www.cmegroup.com/markets/interest-rates/cme-fedwatch-tool.html?redirect=/trading/interest-rates/countdown-to-fomc.html')
time.sleep(10)
page_source=driver.page_source
time.sleep(5)
driver.switch_to.frame('cmeIframe-jtxelq2f')
time.sleep(5)
driver.find_element(By.XPATH,"/html/body/form/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/ul/li[8]/a").click()
time.sleep(5)
driver.find_element(By.CSS_SELECTOR,"#ctl00_MainContent_ucViewControl_IntegratedFedWatchTool_lbHistorical").click()
time.sleep(10)
# 175-200 target rate probability
action = webdriver.ActionChains(driver)
CSS_Selector = '#highcharts-986wlrn-17 > svg > g.highcharts-series-group > g.highcharts-markers.highcharts-series-0.highcharts-line-series.highcharts-color-0.highcharts-tracker > path.highcharts-halo.highcharts-color-0'
# Wait for the element
wait = WebDriverWait(driver, 10)
# Check, if the element is invisible (IT IS)
print(wait.until(EC.invisibility_of_element_located((By.CSS_SELECTOR, CSS_Selector))))
#Change the data-z-index', '4' to put infront of the other elements (DOESNT WORK!)
driver.execute_script("document.querySelector('#highcharts-986wlrn-17 > svg > g.highcharts-series-group > g.highcharts-markers.highcharts-series-0.highcharts-line-series.highcharts-color-0.highcharts-tracker > path.highcharts-halo.highcharts-color-0').setAttribute('data-z-index', '4')")
time.sleep(10)
#Doesnt find the first element of 175-200 (DOESNT WORK!)
element = driver.find_element(By.CSS_SELECTOR,'#highcharts-986wlrn-17 > svg > g.highcharts-series-group > g.highcharts-markers.highcharts-series-0.highcharts-line-series.highcharts-color-0.highcharts-tracker > path.highcharts-halo.highcharts-color-0')
# Move the mouse over the very left element first element 175-200 (DOESNT WORK!)
action.move_to_element(element)
action.perform()
#Determining the location and size in order for the mouse to move along the interactive chart for 175-200
loc = element.location
size = element.size
#moving to right end of the interactive chart for 175-200 (DOESNT WORK!)
action.move_to_element_with_offset(element,510, 0).perform() #found 510 when inspecting the chart metrics
#first date (found on the very left of the chart) (DOESNT WORK!)
driver.find_element(By.XPATH,'/html/body/form/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/div/div[1]/div/svg/g[10]/g/text/tspan[1]')
#first value (found on the very left of the chart) (DOESNT WORK!)
driver.find_element(By.XPATH,'/html/body/form/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/div/div[1]/div/svg/g[5]/g[2]/path[1]')
#setting the limit of when to break out of the while loop (DOESNT WORK!)
limit = dt.datetime.strptime('Thu, May 5, 2022', '%a, %B %-m, %B')
pace = -5
dictionary = {}
while True:
#moving back to the very left first element of the interactive chart
action.move_by_offset(pace, 0).perform() # moving by a pace of -5 (-5 is trial value)
date = driver.find_element(By.CSS_SELECTOR,'#highcharts-986wlrn-17 > svg > g.highcharts-tooltip > g.highcharts-label.highcharts-tooltip-header.highcharts-tooltip-box > text').text
value = driver.find_element(By.CSS_SELECTOR,'#highcharts-986wlrn-17 > svg > g.highcharts-tooltip > g.highcharts-label.highcharts-tooltip-box.highcharts-color-0 > text').text
if dt.datetime.strptime(date, '%a, %B %-m, %B') < limit:
break
# add results to dictionary
if date in dictionary:
pass
else:
dictionary[date] = value
driver.quit()
[![Interactive Line Chart][1]][1]
[1]: https://i.stack.imgur.com/ZDHQt.jpg

Answer 1

The code below assumes that you already clicked on "13 Dec23" and "Historical":
import time
import pyautogui # pip install pyautogui
# sleep 3 seconds so that you can switch to the browser and move the mouse to the start position
time.sleep(3)
# start position: put mouse on chart's left border, for example between 0% and 20%
start_pos = pyautogui.position()
# this works only if the tooltip containing the percentage values was already been displayed once
old_values = driver.find_element(By.CSS_SELECTOR, 'g.highcharts-tooltip').text.replace('\n','').split('●')
data = []
for k in range(500):
pyautogui.moveTo(start_pos.x + 3*k , start_pos.y)
new_values = driver.find_element(By.CSS_SELECTOR, 'g.highcharts-tooltip').text.replace('\n','').split('●')
if new_values != old_values:
old_values = new_values
data.append(new_values)
# uncomment this if you want the mouse to move slower
# time.sleep(.1)
data will then be a list of lists, where each list contains the day label and the corresponding percentage values (as strings)
[['Thu, May 5, 2022',
'350-375: 17.18%',
'325-350: 28.11%',
'300-325: 27.01%',
'250-275: 4.39%',
'275-300: 14.80%',
'225-250: 0.65%',
'375-400: 6.27%',
'400-425: 1.36%',
'425-450: 0.17%'],
['Fri, May 6, 2022',
'350-375: 15.54%',
'325-350: 26.14%',
'300-325: 26.94%',
'250-275: 6.08%',
'275-300: 16.73%',
'225-250: 1.24%',
'375-400: 5.71%',
'400-425: 1.29%',
'425-450: 0.17%',
'200-225: 0.14%'],
... etc ...