I am trying to extract the percentage values (Target Rate Probability) from an interactive chart with preselected conditions "13 Dez23" and "Historical" on the following website: https://www.cmegroup.com/markets/interest-rates/cme-fedwatch-tool.html?redirect=/trading/interest-rates/countdown-to-fomc.html. The percentage probabilities only appear when I hover over the respective data points with a mouse.
I have tried the following:
XPath of tried element: /html/body/form/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/div/div[1]/div/svg/g[5]/g[2]/path[2]
1. Method Approach: Webdriverwait in order for the elements to load on the page
1. Method Error: TimeoutException
2. Method Approach:
driver.execute_script("document.evaluate('/html/body
/form/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/div/div[1]/div/svg/g[5]/g[2]/path[2]', document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.style.display = 'block';")
2. Method Error:
selenium.common.exceptions.JavascriptException: Message: javascript error: Cannot read properties of null (reading 'style')
3. Method Approach:
driver.execute_script("document.evaluate('/html/body/form
/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/div/div[1]/div/svg/g[5]/g[2]/path[2]', document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.style.visibility = 'visible';")
3. Method Error:
selenium.common.exceptions.JavascriptException: Message: javascript error: Cannot read properties of null (reading 'style')
EC: Invisibility of element is True
**Code: **
from webdriver_manager.chrome import ChromeDriverManager
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
import datetime as dt
import time
driver = webdriver.Chrome(executable_path=ChromeDriverManager().install())
driver.get('https://www.cmegroup.com/markets/interest-rates/cme-fedwatch-tool.html?redirect=/trading/interest-rates/countdown-to-fomc.html')
time.sleep(10)
page_source=driver.page_source
time.sleep(5)
driver.switch_to.frame('cmeIframe-jtxelq2f')
time.sleep(5)
driver.find_element(By.XPATH,"/html/body/form/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/ul/li[8]/a").click()
time.sleep(5)
driver.find_element(By.CSS_SELECTOR,"#ctl00_MainContent_ucViewControl_IntegratedFedWatchTool_lbHistorical").click()
time.sleep(10)
# 175-200 target rate probability
action = webdriver.ActionChains(driver)
CSS_Selector = '#highcharts-986wlrn-17 > svg > g.highcharts-series-group > g.highcharts-markers.highcharts-series-0.highcharts-line-series.highcharts-color-0.highcharts-tracker > path.highcharts-halo.highcharts-color-0'
# Wait for the element
wait = WebDriverWait(driver, 10)
# Check, if the element is invisible (IT IS)
print(wait.until(EC.invisibility_of_element_located((By.CSS_SELECTOR, CSS_Selector))))
#Change the data-z-index', '4' to put infront of the other elements (DOESNT WORK!)
driver.execute_script("document.querySelector('#highcharts-986wlrn-17 > svg > g.highcharts-series-group > g.highcharts-markers.highcharts-series-0.highcharts-line-series.highcharts-color-0.highcharts-tracker > path.highcharts-halo.highcharts-color-0').setAttribute('data-z-index', '4')")
time.sleep(10)
#Doesnt find the first element of 175-200 (DOESNT WORK!)
element = driver.find_element(By.CSS_SELECTOR,'#highcharts-986wlrn-17 > svg > g.highcharts-series-group > g.highcharts-markers.highcharts-series-0.highcharts-line-series.highcharts-color-0.highcharts-tracker > path.highcharts-halo.highcharts-color-0')
# Move the mouse over the very left element first element 175-200 (DOESNT WORK!)
action.move_to_element(element)
action.perform()
#Determining the location and size in order for the mouse to move along the interactive chart for 175-200
loc = element.location
size = element.size
#moving to right end of the interactive chart for 175-200 (DOESNT WORK!)
action.move_to_element_with_offset(element,510, 0).perform() #found 510 when inspecting the chart metrics
#first date (found on the very left of the chart) (DOESNT WORK!)
driver.find_element(By.XPATH,'/html/body/form/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/div/div[1]/div/svg/g[10]/g/text/tspan[1]')
#first value (found on the very left of the chart) (DOESNT WORK!)
driver.find_element(By.XPATH,'/html/body/form/div[3]/div[2]/div[3]/div[1]/div/div/div[1]/div/div[3]/div[3]/div/div/div/div[1]/div/svg/g[5]/g[2]/path[1]')
#setting the limit of when to break out of the while loop (DOESNT WORK!)
limit = dt.datetime.strptime('Thu, May 5, 2022', '%a, %B %-m, %B')
pace = -5
dictionary = {}
while True:
#moving back to the very left first element of the interactive chart
action.move_by_offset(pace, 0).perform() # moving by a pace of -5 (-5 is trial value)
date = driver.find_element(By.CSS_SELECTOR,'#highcharts-986wlrn-17 > svg > g.highcharts-tooltip > g.highcharts-label.highcharts-tooltip-header.highcharts-tooltip-box > text').text
value = driver.find_element(By.CSS_SELECTOR,'#highcharts-986wlrn-17 > svg > g.highcharts-tooltip > g.highcharts-label.highcharts-tooltip-box.highcharts-color-0 > text').text
if dt.datetime.strptime(date, '%a, %B %-m, %B') < limit:
break
# add results to dictionary
if date in dictionary:
pass
else:
dictionary[date] = value
driver.quit()
[![Interactive Line Chart][1]][1]
[1]: https://i.stack.imgur.com/ZDHQt.jpg
The code below assumes that you already clicked on "13 Dec23" and "Historical":
import time
import pyautogui # pip install pyautogui
# sleep 3 seconds so that you can switch to the browser and move the mouse to the start position
time.sleep(3)
# start position: put mouse on chart's left border, for example between 0% and 20%
start_pos = pyautogui.position()
# this works only if the tooltip containing the percentage values was already been displayed once
old_values = driver.find_element(By.CSS_SELECTOR, 'g.highcharts-tooltip').text.replace('\n','').split('●')
data = []
for k in range(500):
pyautogui.moveTo(start_pos.x + 3*k , start_pos.y)
new_values = driver.find_element(By.CSS_SELECTOR, 'g.highcharts-tooltip').text.replace('\n','').split('●')
if new_values != old_values:
old_values = new_values
data.append(new_values)
# uncomment this if you want the mouse to move slower
# time.sleep(.1)
data will then be a list of lists, where each list contains the day label and the corresponding percentage values (as strings)
[['Thu, May 5, 2022',
'350-375: 17.18%',
'325-350: 28.11%',
'300-325: 27.01%',
'250-275: 4.39%',
'275-300: 14.80%',
'225-250: 0.65%',
'375-400: 6.27%',
'400-425: 1.36%',
'425-450: 0.17%'],
['Fri, May 6, 2022',
'350-375: 15.54%',
'325-350: 26.14%',
'300-325: 26.94%',
'250-275: 6.08%',
'275-300: 16.73%',
'225-250: 1.24%',
'375-400: 5.71%',
'400-425: 1.29%',
'425-450: 0.17%',
'200-225: 0.14%'],
... etc ...
I used matplotlib to generate a keyboard. But after I wrote the code, I found that the picture displayed is incomplete. I have tried a few ways to solve the problem, like adjusting the dpi and changing the size of the picture. However, they are all useless.
this is the output
Here is the code:
ax=fig.add_subplot(111)
#ax.set_axis_off()
ax.plot(X,Y)
currentAxis = plt.gca()
generate_keyboard(currentAxis)
plt.text(0,0,"g")
plt.savefig(fname="3.jpg")
plt.show()`
def generate_keyboard(currentAxis):
rect1=patches.Rectangle((-0.5,-1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect2=patches.Rectangle((-1.5,-1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect3=patches.Rectangle((-2.5,-1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect4=patches.Rectangle((-3.5,-1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect5=patches.Rectangle((-4.5,-1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect6=patches.Rectangle((0.5,-1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect7=patches.Rectangle((1.5,-1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect8=patches.Rectangle((2.5,-1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect9=patches.Rectangle((3.5,-1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect10=patches.Rectangle((-1,1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect11=patches.Rectangle((-2,1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect12=patches.Rectangle((-3,1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect13=patches.Rectangle((-4,1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect14=patches.Rectangle((-5,1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect15=patches.Rectangle((0,1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect16=patches.Rectangle((1,1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect17=patches.Rectangle((2,1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect18=patches.Rectangle((3,1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect19=patches.Rectangle((4,1),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect20=patches.Rectangle((-0.5,-3),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect21=patches.Rectangle((-1.5,-3),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect22=patches.Rectangle((-2.5,-3),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect23=patches.Rectangle((-3.5,-3),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect24=patches.Rectangle((0.5,-3),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect25=patches.Rectangle((1.5,-3),1,2,linewidth=2,edgecolor='r',facecolor='none')
rect26=patches.Rectangle((2.5,-3),1,2,linewidth=2,edgecolor='r',facecolor='none')
currentAxis.add_patch(rect1)
currentAxis.add_patch(rect2)
currentAxis.add_patch(rect3)
currentAxis.add_patch(rect4)
currentAxis.add_patch(rect5)
currentAxis.add_patch(rect6)
currentAxis.add_patch(rect7)
currentAxis.add_patch(rect8)
currentAxis.add_patch(rect9)
currentAxis.add_patch(rect10)
currentAxis.add_patch(rect11)
currentAxis.add_patch(rect12)
currentAxis.add_patch(rect13)
currentAxis.add_patch(rect14)
currentAxis.add_patch(rect15)
currentAxis.add_patch(rect16)
currentAxis.add_patch(rect17)
currentAxis.add_patch(rect18)
currentAxis.add_patch(rect19)currentAxis.add_patch(rect20)
currentAxis.add_patch(rect21)
currentAxis.add_patch(rect22)
currentAxis.add_patch(rect23)
currentAxis.add_patch(rect24)
currentAxis.add_patch(rect25)
currentAxis.add_patch(rect26)
I recently updated my QGIS and I noticed the styles now show the band Band 1(Gray)
The issue is it now shows within my print layout:
Using PYQGIS, how can I remove just the Band 1 (Gray)?
For reference, here is how I am currently setting the legend in the layout:
def set_legend(layout: QgsPrintLayout, tree: QgsLayerTree, layer: QgsLayer, item_id: str):
'''Sets the Legend items'''
logging.info(f'setting legend: {item_id}')
item = layout.itemById(item_id)
# set layer as root for legend
tree.addLayer(layer)
item.model().setRootGroup(tree)
node = item.model().rootGroup().findLayer(layer)
# hide the node title
QgsLegendRenderer.setNodeLegendStyle(node, QgsLegendStyle.Hidden)
Thank you!
Here's my solution after digging StackOverflow and the API:
root = model.rootGroup().findLayer(layer)
# hide the node with label: Band 1 (Gray)
if isinstance(layer, QgsRasterLayer):
nodes = model.layerLegendNodes(root)
if nodes[0].data(0) == 'Band 1 (Gray)':
indexes = list(range(1, len(nodes)))
QgsMapLayerLegendUtils.setLegendNodeOrder(root, indexes)
model.refreshLayerLegend(root)
I am trying to display the widgets of a HoloViews dynamic plot (Select, Slider, etc.) over the plot. All I can find is the widget_location argument which takes the location relative to the plot (‘left’ , ‘right’, …). But I want it to be placed over the plot, not next to it. I was wondering if there is a way for doing this?
P.S. for instance there is opts(colorbar_opts={'location':(float,float)}) which can be used to place the colorbar where you want. It would be very useful to have a similar option for widgets.
OK, I found the solution! I have to use custom CSS. The code below shows how to do it.
import holoviews as hv
import panel as pn
pn.extension('ace')
hv.extension("bokeh")
plots = {}
for i in range(5,10):
data = {
"x": list(range(0,i)), "y": [i]*i
}
plots[i]=hv.Curve(data).opts(width=500)
hvmap = hv.HoloMap(plots)
left_pos = pn.widgets.IntSlider(value=5, step=1, start=-1000, end=5, name="x")
top_pos = pn.widgets.IntSlider(value=5, step=1, start=5, end=200, name="y")
style = pn.pane.HTML(height=0, width=0, sizing_mode="fixed", margin=0)
css = pn.widgets.Ace(height=150)
#pn.depends(left_pos=left_pos, top_pos=top_pos, watch=True)
def _update_css(left_pos, top_pos):
value = f"""
.bk.panel-widget-box {{
left: {left_pos}px !important;
top: {top_pos}px !important;
}}
"""
css.value = value
style.object = "<style>" + value + "</style>"
pn.Column("""# How to overlay widgets on HoloViews Map?
We will be using css to overlay and Panel to create the this tool""",
hvmap,
"## Settings",
left_pos,
top_pos,
css,
style,
).servable()
All credits goes to Marc Skov Madsen. Original answer here
I'm trying to migrate from plone 3.3.5 to plone 4.0.7 and I'm stuck on a step that converts all the FileFields to BlobFields.
Plone upgrade script successfully converts all native FileFields but I have several custom AT-based classes which have to be converted manually. I've tried two ways of doing the conversion which leads me to the same error.
Using schemaextender as outlined in Plone migration guide and a source code example
Renaming all FileFields to blob fields and then running this script:
from AccessControl.SecurityManagement import newSecurityManager
from AccessControl import getSecurityManager
from Products.CMFCore.utils import getToolByName
from zope.app.component.hooks import setSite
from Products.contentmigration.migrator import BaseInlineMigrator
from Products.contentmigration.walker import CustomQueryWalker
from plone.app.blob.field import BlobField
admin=app.acl_users.getUserById("admin")
newSecurityManager(None, admin)
portal = app.plone
setSite(portal)
def find_all_types_fields(portal_catalog, type_instance_to_search):
output = {}
searched = []
for k in catalog():
kobj = k.getObject()
if kobj.__class__.__name__ in searched:
continue
searched.append(kobj.__class__.__name__)
for field in kobj.schema.fields():
if isinstance(field, type_instance_to_search):
if kobj.__class__.__name__ in output:
output[kobj.__class__.__name__].append(field.__name__)
else:
output[kobj.__class__.__name__] = [field.__name__]
return output
def produce_migrator(field_map):
source_class = field_map.keys()[0]
fields = {}
for x in field_map.values()[0]: fields[x] = None
class FileBlobMigrator(BaseInlineMigrator):
'''Migrating ExtensionBlobField (which is still a FileField) to BlobField'''
src_portal_type = source_class
src_meta_type = source_class
fields_map = fields
def migrate_data(self):
'''Unfinished'''
for k in self.fields_map.keys():
#print "examining attributes"
#import pdb; pdb.set_trace()
#if hasattr(self.obj, k):
if k in self.obj.schema.keys():
print("***converting attribute:", k)
field = self.obj.getField(k).get(self.obj)
mutator = self.obj.getField(k).getMutator(self.obj)
mutator(field)
def last_migrate_reindex(self):
'''Unfinished'''
self.obj.reindexObject()
return FileBlobMigrator
def consume_migrator(portal_catalog, migrator):
walker = CustomQueryWalker(portal_catalog, migrator, full_transaction=True)
transaction.savepoint(optimistic=True)
walker_status = walker.go()
return walker.getOutput()
def migrate_blobs(catalog, migrate_type):
all_fields = find_all_types_fields(catalog, migrate_type)
import pdb; pdb.set_trace()
for k in [ {k : all_fields[k]} for k in all_fields]:
migrator = produce_migrator(k)
print consume_migrator(catalog, migrator)
catalog = getToolByName(portal, 'portal_catalog')
migrate_blobs(catalog, BlobField)
The problem occurs on self.obj.reindexObject() line where I receive the following traceback:
2011-08-09 17:21:12 ERROR Zope.UnIndex KeywordIndex: unindex_object could not remove documentId -1945041983 from index object_provides. This should not happen.
Traceback (most recent call last):
File "/home/alex/projects/plone4/eggs/Zope2-2.12.18-py2.6-linux-x86_64.egg/Products/PluginIndexes/common/UnIndex.py", line 166, in removeForwardIndexEntry indexRow.remove(documentId)
KeyError: -1945041983
> /home/alex/projects/plone4/eggs/Zope2-2.12.18-py2.6-linux-x86_64.egg/Products/PluginIndexes/common/UnIndex.py(192)removeForwardIndexEntry()
191 str(documentId), str(self.id)),
--> 192 exc_info=sys.exc_info())
193 else:
If I remove the line that triggers reindexing, the conversion completes successfully, but if I try to manually reindex catalog later, every object that's been converted can no longer be found, and I'm a bit at loss of what to do now.
The site has LinguaPlone installed, maybe it has something to do with this?
One option would be to run the migration without the reindexObject() call and do a "Clear and Rebuild" in the catalog ZMI Advanced tab after migrating.