trying to scrape a website using selenium. the following code works fine.
import selenium
import numpy as np
from tqdm import tqdm
import time
import pandas as pd
from selenium import webdriver as wb
time.sleep(2)
driver=wb.Chrome('chromedriver.exe')
driver.maximize_window()
driver.get('https://www.jumia.co.ke/')
button = driver.find_element_by_xpath('//*[#id="jm"]/div[4]/section/button').click();
move=driver.find_element_by_xpath('//*[#id="jm"]/div[3]/button').click();
time.sleep(3)
clickObj=driver.find_element_by_xpath('//*
[#id="jm"]/main/div[1]/div[1]/div[1]/div/a[5]/span')
clickObj.click()
however am getting an error running code below, although the chrome driver clicks on the page: when I rerun again, the error disappears.
gash=driver.find_element_by_xpath('//*[#id="ctlg"]/div/div[2]/div/div/a[2]/p')
gash.click()
the error:
ElementClickInterceptedException Traceback (most recent call last)
<ipython-input-3-79a9b5b24451> in <module>
1 gash=driver.find_element_by_xpath('//*[#id="ctlg"]/div/div[2]/div/div/a[2]/p')
----> 2 gash.click()
~\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in click(self)
78 def click(self):
79 """Clicks the element."""
---> 80 self._execute(Command.CLICK_ELEMENT)
81
82 def submit(self):
~\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in _execute(self,
command, params)
631 params = {}
632 params['id'] = self._id
--> 633 return self._parent.execute(command, params)
634
635 def find_element(self, by=By.ID, value=None):
~\anaconda3\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self,
driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
~\anaconda3\lib\site-packages\selenium\webdriver\remote\errorhandler.py in
check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
ElementClickInterceptedException: Message: element click intercepted: Element is not
clickable at point (313, 748)
(Session info: chrome=87.0.4280.88)
how can I solve this error?
Kindly include an example
Use the following code to resolve this issue
browser.execute_script("arguments[0].click();", "name of button ")
Related
I am trying to run sample codes using selenium and webdriver manager from the following site: https://pypi.org/project/webdriver-manager/. I am using google chrome, i tried the following codes:
Selenium version: 4.1.0
webdriver_manager version: 3.5.2
# code for chrome
# Selenium 4
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
I also tried:
# code for chromium
# selenium 4
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from webdriver_manager.utils import ChromeType
driver = webdriver.Chrome(service=Service(ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install()))
Unfortunately i get the following error (it's similar for both codes):
JSONDecodeError Traceback (most recent call last)
5 from webdriver_manager.utils import ChromeType
6
----> 7 driver = webdriver.Chrome(service=Service(ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install()))
C:\ProgramData\Anaconda3\lib\site-packages\webdriver_manager\chrome.py in install(self)
30
31 def install(self):
---> 32 driver_path = self._get_driver_path(self.driver)
33
34 os.chmod(driver_path, 0o755)
C:\ProgramData\Anaconda3\lib\site-packages\webdriver_manager\manager.py in _get_driver_path(self, driver)
21 driver_name = driver.get_name()
22 os_type = driver.get_os_type()
---> 23 driver_version = driver.get_version()
24
25 binary_path = self.driver_cache.find_driver(browser_version, driver_name, os_type,
C:\ProgramData\Anaconda3\lib\site-packages\webdriver_manager\driver.py in get_version(self)
39 driver_version = self._version
40 if driver_version == "latest":
---> 41 return self.get_latest_release_version()
42 return self._version
43
C:\ProgramData\Anaconda3\lib\site-packages\webdriver_manager\driver.py in get_latest_release_version(self)
72 verify=self.ssl_verify
73 )
---> 74 validate_response(resp)
75 self._version = resp.text.rstrip()
76 return self._version
C:\ProgramData\Anaconda3\lib\site-packages\webdriver_manager\utils.py in validate_response(resp)
81 elif resp.status_code != 200:
82 raise ValueError(
---> 83 f'response body:\n{resp.json()}\n'
84 f'request url:\n{resp.request.url}\n'
85 f'response headers:\n{dict(resp.headers)}\n'
C:\ProgramData\Anaconda3\lib\site-packages\requests\models.py in json(self, **kwargs)
908 # used.
909 pass
--> 910 return complexjson.loads(self.text, **kwargs)
911
912 #property
C:\ProgramData\Anaconda3\lib\json\__init__.py in loads(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
344 parse_int is None and parse_float is None and
345 parse_constant is None and object_pairs_hook is None and not kw):
--> 346 return _default_decoder.decode(s)
347 if cls is None:
348 cls = JSONDecoder
C:\ProgramData\Anaconda3\lib\json\decoder.py in decode(self, s, _w)
335
336 """
--> 337 obj, end = self.raw_decode(s, idx=_w(s, 0).end())
338 end = _w(s, end).end()
339 if end != len(s):
C:\ProgramData\Anaconda3\lib\json\decoder.py in raw_decode(self, s, idx)
353 obj, end = self.scan_once(s, idx)
354 except StopIteration as err:
--> 355 raise JSONDecodeError("Expecting value", s, err.value) from None
356 return obj, end
JSONDecodeError: Expecting value: line 1 column 1 (char 0)
I am trying to load built-in dataset lvis. It turns out that the tfds and lvis should be imported and installed respectively, however, I did possible all, it still does not work.
import os
import tensorflow as tf
from matplotlib import pyplot as plt
%matplotlib inline
!pip install lvis
!pip install tfds-nightly
import tensorflow_datasets as tfds
train_data, info = tfds.load('lvis', split='train', as_supervised=True, with_info=True)
validation_data = tfds.load('lvis', split='validation', as_supervised=True)
test_data = tfds.load('lvis', split='test', as_supervised=True)
There are some odd outputs after running upon codes in colab.
otFoundError Traceback (most recent call last)
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/utils/py_utils.py in try_reraise(*args, **kwargs)
391 try:
--> 392 yield
393 except Exception as e: # pylint: disable=broad-except
15 frames
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/load.py in builder(name, try_gcs, **builder_kwargs)
167 with py_utils.try_reraise(prefix=f'Failed to construct dataset {name}: '):
--> 168 return cls(**builder_kwargs) # pytype: disable=not-instantiable
169
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/dataset_builder.py in __init__(self, file_format, **kwargs)
917 """
--> 918 super().__init__(**kwargs)
919 self.info.set_file_format(file_format)
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/dataset_builder.py in __init__(self, data_dir, config, version)
184 else: # Use the code version (do not restore data)
--> 185 self.info.initialize_from_bucket()
186
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/utils/py_utils.py in __get__(self, obj, objtype)
145 if cached is None:
--> 146 cached = self.fget(obj) # pytype: disable=attribute-error
147 setattr(obj, attr, cached)
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/dataset_builder.py in info(self)
328 "the restored dataset.")
--> 329 info = self._info()
330 if not isinstance(info, dataset_info.DatasetInfo):
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/object_detection/lvis/lvis.py in _info(self)
94 names_file=tfds.core.tfds_path(
---> 95 'object_detection/lvis/lvis_classes.txt'))
96 return tfds.core.DatasetInfo(
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/features/class_label_feature.py in __init__(self, num_classes, names, names_file)
67 else:
---> 68 self.names = _load_names_from_file(names_file)
69
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/features/class_label_feature.py in _load_names_from_file(names_filepath)
198 name.strip()
--> 199 for name in tf.compat.as_text(f.read()).split("\n")
200 if name.strip() # Filter empty names
/usr/local/lib/python3.7/dist-packages/tensorflow/python/lib/io/file_io.py in read(self, n)
116 """
--> 117 self._preread_check()
118 if n == -1:
/usr/local/lib/python3.7/dist-packages/tensorflow/python/lib/io/file_io.py in _preread_check(self)
79 self._read_buf = _pywrap_file_io.BufferedInputStream(
---> 80 compat.path_to_str(self.__name), 1024 * 512)
81
NotFoundError: /usr/local/lib/python3.7/dist-packages/tensorflow_datasets/object_detection/lvis/lvis_classes.txt; No such file or directory
The above exception was the direct cause of the following exception:
RuntimeError Traceback (most recent call last)
<ipython-input-4-b8c819fe5c62> in <module>()
----> 1 train_data, info = tfds.load('lvis', split='train', as_supervised=True, with_info=True)
2 validation_data = tfds.load('lvis', split='validation', as_supervised=True)
3 test_data = tfds.load('lvis', split='test', as_supervised=True)
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/load.py in load(name, split, data_dir, batch_size, shuffle_files, download, as_supervised, decoders, read_config, with_info, builder_kwargs, download_and_prepare_kwargs, as_dataset_kwargs, try_gcs)
315 builder_kwargs = {}
316
--> 317 dbuilder = builder(name, data_dir=data_dir, try_gcs=try_gcs, **builder_kwargs)
318 if download:
319 download_and_prepare_kwargs = download_and_prepare_kwargs or {}
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/load.py in builder(name, try_gcs, **builder_kwargs)
166 if cls:
167 with py_utils.try_reraise(prefix=f'Failed to construct dataset {name}: '):
--> 168 return cls(**builder_kwargs) # pytype: disable=not-instantiable
169
170 # If neither the code nor the files are found, raise DatasetNotFoundError
/usr/lib/python3.7/contextlib.py in __exit__(self, type, value, traceback)
128 value = type()
129 try:
--> 130 self.gen.throw(type, value, traceback)
131 except StopIteration as exc:
132 # Suppress StopIteration *unless* it's the same exception that
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/utils/py_utils.py in try_reraise(*args, **kwargs)
392 yield
393 except Exception as e: # pylint: disable=broad-except
--> 394 reraise(e, *args, **kwargs)
395
396
/usr/local/lib/python3.7/dist-packages/tensorflow_datasets/core/utils/py_utils.py in reraise(e, prefix, suffix)
359 else:
360 exception = RuntimeError(f'{type(e).__name__}: {msg}')
--> 361 raise exception from e
362 # Otherwise, modify the exception in-place
363 elif len(e.args) <= 1:
RuntimeError: NotFoundError: Failed to construct dataset lvis: /usr/local/lib/python3.7/dist-packages/tensorflow_datasets/object_detection/lvis/lvis_classes.txt; No such file or directory
This is what I did to get it to work on Colab Notebook:
!pip install -q tfds-nightly tensorflow tensorflow-datasets matplotlib lvis pycocotools apache_beam
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import tensorflow_datasets as tfds
Since the tfds object detection lvis folder isn't up to date, I deleted that folder then redownloaded it from the tfds github page.
First install github-clone so we can download specific repo subfolders
!pip install github-clone
Then remove the lvis folder and redownload it from github:
!rm -rf ../usr/local/lib/python3.7/dist-packages/tensorflow_datasets/object_detection/lvis
!ghclone https://github.com/tensorflow/datasets/tree/master/tensorflow_datasets/object_detection/lvis
!mv ./lvis ../usr/local/lib/python3.7/dist-packages/tensorflow_datasets/object_detection/
After that I could get it to work, this next chunk of code worked for me:
ds, info = tfds.load('lvis', split='train[:25%]', with_info=True,
data_dir= '../content/tensorflow_datasets/',
decoders=tfds.decode.PartialDecoding({
'image': True,
'features': tfds.features.FeaturesDict({'image/id':True,
'objects':tfds.features.Sequence({
'id': True,
'bbox': True,
'label': tfds.features.ClassLabel(names=['skateboard','shoe'])
})
})
})
)
I have a script python script trying to access a google spreadsheet, but I am getting a 500 error from trying to get the list of worksheets with gc.worksheets()
APIError: {'code': 500, 'message': 'Internal error encountered.', 'status': 'INTERNAL'}
---------------------------------------------------------------------------
APIError Traceback (most recent call last)
/tmp/ipykernel_14907/1619355115.py in <module>
----> 1 gc.worksheets()
~/anaconda3/lib/python3.8/site-packages/gspread/models.py in worksheets(self)
351 in a spreadsheet.
352 """
--> 353 sheet_data = self.fetch_sheet_metadata()
354 return [Worksheet(self, x['properties']) for x in sheet_data['sheets']]
355
~/anaconda3/lib/python3.8/site-packages/gspread/models.py in fetch_sheet_metadata(self, params)
293 url = SPREADSHEET_URL % self.id
294
--> 295 r = self.client.request('get', url, params=params)
296
297 return r.json()
~/anaconda3/lib/python3.8/site-packages/gspread/client.py in request(self, method, endpoint, params, data, json, files, headers)
74 return response
75 else:
---> 76 raise APIError(response)
77
78 def list_spreadsheet_files(self, title=None):
APIError: {'code': 500, 'message': 'Internal error encountered.', 'status': 'INTERNAL'}
The Code I am using is the following:
import pandas as pd
import gspread
from oauth2client.service_account import ServiceAccountCredentials
CREDENTIALS_JSON = 'credentials.json'
SCOPES = ['https://spreadsheets.google.com/feeds','https://www.googleapis.com/auth/drive']
creds = ServiceAccountCredentials.from_json_keyfile_name(CREDENTIALS_JSON, SCOPES)
client = gspread.authorize(creds)
gc = client.open_by_url('<I use the url of the file here>')
worksheets = gc.worksheets() # This like is returning an error
sheet1 = gc.get_worksheet_by_id(<sheet id>) # This line is also returning an error
Everything was working just fine a day before, and I haven't made any changes to the script.
My code, which used to work until yesterday, suddenly stopped working. I'd been running codes from jupyter notebook to auto-collect repeating data from a webpage. It is as follows (URL is changed to Google for privacy reasons):
# use selenium to start Chrome session to open a certain page
dr = webdriver.Chrome()
dr.get("http://www.google.com")
The error message I got:
---------------------------------------------------------------------------
WebDriverException Traceback (most recent call last)
<ipython-input-25-84be89301b0d> in <module>()
1 # use selenium to start Chrome session to open google.
----> 2 dr = webdriver.Chrome()
3 dr.get("http://www.google.com")
/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/chrome/webdriver.py in __init__(self, executable_path, port, chrome_options, service_args, desired_capabilities, service_log_path)
65 command_executor=ChromeRemoteConnection(
66 remote_server_addr=self.service.service_url),
---> 67 desired_capabilities=desired_capabilities)
68 except:
69 self.quit()
/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py in __init__(self, command_executor, desired_capabilities, browser_profile, proxy, keep_alive)
85 self.error_handler = ErrorHandler()
86 self.start_client()
---> 87 self.start_session(desired_capabilities, browser_profile)
88 self._switch_to = SwitchTo(self)
89 self._mobile = Mobile(self)
/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py in start_session(self, desired_capabilities, browser_profile)
139 desired_capabilities['firefox_profile'] = browser_profile.encoded
140 response = self.execute(Command.NEW_SESSION, {
--> 141 'desiredCapabilities': desired_capabilities,
142 })
143 self.session_id = response['sessionId']
/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py in execute(self, driver_command, params)
199 response = self.command_executor.execute(driver_command, params)
200 if response:
--> 201 self.error_handler.check_response(response)
202 response['value'] = self._unwrap_value(
203 response.get('value', None))
/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/errorhandler.py in check_response(self, response)
179 elif exception_class == UnexpectedAlertPresentException and 'alert' in value:
180 raise exception_class(message, screen, stacktrace, value['alert'].get('text'))
--> 181 raise exception_class(message, screen, stacktrace)
182
183 def _value_or_default(self, obj, key, default):
WebDriverException: Message: session not created exception
from unknown error: Runtime.executionContextCreated has invalid 'context': {"auxData":{"frameId":"7600.1","isDefault":true},"id":1,"name":"","origin":"://"}
(Session info: chrome=54.0.2840.71)
(Driver info: chromedriver=2.20.353124 (035346203162d32c80f1dce587c8154a1efa0c3b),platform=Mac OS X 10.11.6 x86_64)
I really have no idea how to solve this and why I'm suddenly getting this message.
Download again new chrome driver from below link and use the same
http://chromedriver.storage.googleapis.com/index.html?path=2.24/
I'm trying to view a 71290x71290 array of numpy.float32s. My computer runs out of memory when I try to use matplotlib to view it interactively, so I'm trying to save it as an image file and view it outside of ipython. The following should work, afaik from reading other SO posts:
import numpy as np
W = np.zeros((71290, 71290), dtype='float32')
from scipy.misc import imsave
imsave('test.png', W)
But the last line gives me an overflow error:
OverflowError: size does not fit in an int
Here's the full stack trace:
---------------------------------------------------------------------------
OverflowError Traceback (most recent call last)
/home/agittens/Documents/langmodel/<ipython-input-4-bf9f2254f869> in <module>()
----> 1 imsave('test.png', W)
/usr/local/lib/python2.7/dist-packages/scipy/misc/pilutil.pyc in imsave(name, arr)
160
161 """
--> 162 im = toimage(arr)
163 im.save(name)
164 return
/usr/local/lib/python2.7/dist-packages/scipy/misc/pilutil.pyc in toimage(arr, high, low, cmin, cmax, pal, mode, channel_axis)
235 if mode in [None, 'L', 'P']:
236 bytedata = bytescale(data,high=high,low=low,cmin=cmin,cmax=cmax)
--> 237 image = Image.frombytes('L',shape,bytedata.tostring())
238 if pal is not None:
239 image.putpalette(asarray(pal,dtype=uint8).tostring())
/usr/lib/python2.7/dist-packages/PIL/Image.pyc in fromstring(mode, size, data, decoder_name, *args)
1795
1796 im = new(mode, size)
-> 1797 im.fromstring(data, decoder_name, args)
1798 return im
1799
/usr/lib/python2.7/dist-packages/PIL/Image.pyc in fromstring(self, data, decoder_name, *args)
589 d = _getdecoder(self.mode, decoder_name, args)
590 d.setimage(self.im)
--> 591 s = d.decode(data)
592
593 if s[0] >= 0:
OverflowError: size does not fit in an int