urllib request.urlretrieve error 400 - urllib

I'm trying to download an image from a url
request.urlretrieve('https://website.com/image.jpg', "test.jpg")
The above works. However when I try to use:
a = 'https://website.com/image.jpg'
request.urlretrieve(a, "test.jpg")
It gives me error 400. How do I get the URL in there without causing an error?
File "/usr/lib/python3.5/urllib/request.py", line 188, in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.5/urllib/request.py", line 472, in open
response = meth(req, response)
File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
'http', request, response, code, msg, hdrs)
File "/usr/lib/python3.5/urllib/request.py", line 510, in error
return self._call_chain(*args)
File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
result = func(*args)
File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 400: Bad Request

A common reason for a 400 Bad Request error is because the URL was typed wrong. Try to print(a) to check if it indeed contains what you think.
the following lines work for me (python 3.5):
import urllib.request
# without variable
urllib.request.urlretrieve("https://www.google.fr/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png", "test.png")
# with variable
google_pic_url = "https://www.google.fr/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png"
urllib.request.urlretrieve(google_pic_url, "test2.png")
You can also try with another URL, like the one above to check if it's specific with your URL (this could be related to DNS issues, or server issues)

Related

python 3.8 urllib file:// issue

I want to open a local file with urllib.request.urlopen with the following code:
urllib.request.urlopen('file:///home/parham/.bashrc')
But it generates the following error:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/home/linuxbrew/.linuxbrew/Cellar/python#3.8/3.8.5/lib/python3.8/urllib/request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "/home/linuxbrew/.linuxbrew/Cellar/python#3.8/3.8.5/lib/python3.8/urllib/request.py", line 525, in open
response = self._open(req, data)
File "/home/linuxbrew/.linuxbrew/Cellar/python#3.8/3.8.5/lib/python3.8/urllib/request.py", line 536, in _open
result = self._call_chain(self.handle_open, 'default',
File "/home/linuxbrew/.linuxbrew/Cellar/python#3.8/3.8.5/lib/python3.8/urllib/request.py", line 502, in _call_chain
result = func(*args)
File "/home/linuxbrew/.linuxbrew/Cellar/python#3.8/3.8.5/lib/python3.8/urllib/request.py", line 806, in <lambda>
meth(r, proxy, type))
File "/home/linuxbrew/.linuxbrew/Cellar/python#3.8/3.8.5/lib/python3.8/urllib/request.py", line 834, in proxy_open
return self.parent.open(req, timeout=req.timeout)
File "/home/linuxbrew/.linuxbrew/Cellar/python#3.8/3.8.5/lib/python3.8/urllib/request.py", line 531, in open
response = meth(req, response)
File "/home/linuxbrew/.linuxbrew/Cellar/python#3.8/3.8.5/lib/python3.8/urllib/request.py", line 640, in http_response
response = self.parent.error(
File "/home/linuxbrew/.linuxbrew/Cellar/python#3.8/3.8.5/lib/python3.8/urllib/request.py", line 569, in error
return self._call_chain(*args)
File "/home/linuxbrew/.linuxbrew/Cellar/python#3.8/3.8.5/lib/python3.8/urllib/request.py", line 502, in _call_chain
result = func(*args)
File "/home/linuxbrew/.linuxbrew/Cellar/python#3.8/3.8.5/lib/python3.8/urllib/request.py", line 649, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 400: Bad Request
Am I doing it right? or the issue is related to urllib?
This issue is related to the shell-proxy plugin of oh-my-zsh. After disabling it everything works as before.

Exception Resource Temporarily unavailable while firing 50 requests for checking snapshot exits or not

I have created snapshots of 50 disks because gcloud does not allow snapshots with similar name. Before firing snapshot create I was checking if snapshot exists or not in gcp and fired 50 requests simultaneously and almost 5-6 requests failed with below exception.
snapshots().get(project=self.project, snapshot=name).execute()
exception :
File "/tmp/cloudpoint/libs/gcp/lib/oauth2client/_helpers.py", line 133, in positional_wrapper return wrapped(*args, **kwargs)
File "/tmp/cloudpoint/libs/gcp/lib/googleapiclient/http.py", line 837, in execute method=str(self.method), body=self.body, headers=self.headers)
File "/tmp/cloudpoint/libs/gcp/lib/googleapiclient/http.py", line 163, in _retry_request resp, content = http.request(uri, method, *args, **kwargs)
File "/tmp/cloudpoint/libs/gcp/lib/oauth2client/transport.py", line 175, in new_request redirections, connection_type)
File "/tmp/cloudpoint/libs/gcp/lib/oauth2client/transport.py", line 282, in request connection_type=connection_type)
File "/usr/lib/python2.7/dist-packages/httplib2/__init__.py", line 1607, in request (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/usr/lib/python2.7/dist-packages/httplib2/__init__.py", line 1349, in _request(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/usr/lib/python2.7/dist-packages/httplib2/__init__.py", line 1305, in _conn_request response = conn.getresponse()
File "/usr/lib/python2.7/httplib.py", line 1136, in getresponse response.begin()
File "/usr/lib/python2.7/httplib.py", line 453, in begin version, status, reason = self._read_status()
File "/usr/lib/python2.7/httplib.py", line 409, in _read_status line = self.fp.readline(_MAXLINE + 1)
File "/usr/lib/python2.7/socket.py", line 480, in readline data = self._sock.recv(self._rbufsize)
File "/usr/lib/python2.7/ssl.py", line 756, in recv return self.read(buflen)
File "/usr/lib/python2.7/ssl.py", line 643, in read v = self._sslobj.read(len)
error: [Errno 11] Resource temporarily unavailable
The error message "Resource temporarily unavailable" means that the Compute Engine API was not available to fulfill the request. Since you made 50 simultaneous requests to check if the snapshots exist, the Compute Engine API was not able to handle all 50 of the requests at the same time so it timed out for 5-6 of the 50 requests.

Socket error when connect to Google Big Query in my local PC

It was working fine for me to run following steps in past two days to connect to Big query in pycharm in my PC
step 1:gcloud auth application-default login
step 2 :then connect to BG in the Pycharm in my local PC.
however, when I used the same method to try today:
Below error occurs:
Since I am behind "Great Wall" in China, so can only use VPN to login in google cloud, so I am not sure if it is caused by the VPN or does it have something to do with the setting of my google'account ? however, I tried with "service-account-key", it happen to be the same issue log:
Traceback (most recent call last):
File "C:/Users/emma/PycharmProjects/GCP/INIT.py", line 134, in <module>
explicit()
File "C:/Users/emma/PycharmProjects/GCP/INIT.py", line 54, in explicit
for dataset in bigquery_client.list_datasets():
File "C:\Python27\lib\site-packages\google\cloud\iterator.py", line 218, in _items_iter
for page in self._page_iter(increment=False):
File "C:\Python27\lib\site-packages\google\cloud\iterator.py", line 247, in _page_iter
page = self._next_page()
File "C:\Python27\lib\site-packages\google\cloud\iterator.py", line 347, in _next_page
response = self._get_next_page_response()
File "C:\Python27\lib\site-packages\google\cloud\iterator.py", line 396, in _get_next_page_response
query_params=params)
File "C:\Python27\lib\site-packages\google\cloud\_http.py", line 299, in api_request
headers=headers, target_object=_target_object)
File "C:\Python27\lib\site-packages\google\cloud\_http.py", line 193, in _make_request
return self._do_request(method, url, headers, data, target_object)
File "C:\Python27\lib\site-packages\google\cloud\_http.py", line 223, in _do_request
body=data)
File "C:\Python27\lib\site-packages\google_auth_httplib2.py", line 187, in request
self._request, method, uri, request_headers)
File "C:\Python27\lib\site-packages\google\auth\credentials.py", line 121, in before_request
self.refresh(request)
File "C:\Python27\lib\site-packages\google\oauth2\service_account.py", line 310, in refresh
request, self._token_uri, assertion)
File "C:\Python27\lib\site-packages\google\oauth2\_client.py", line 143, in jwt_grant
response_data = _token_endpoint_request(request, token_uri, body)
File "C:\Python27\lib\site-packages\google\oauth2\_client.py", line 104, in _token_endpoint_request
method='POST', url=token_uri, headers=headers, body=body)
File "C:\Python27\lib\site-packages\google_auth_httplib2.py", line 116, in __call__
url, method=method, body=body, headers=headers, **kwargs)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1609, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1351, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1272, in _conn_request
conn.connect()
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1075, in connect
raise socket.error, msg
socket.error: [Errno 10060]
I update the python-cloud library, then everything works fine now.
Hope it keep it all the time :(

matplotlib fetch historical works when start year 2015 but not 2016 ?

want to get just a single day ( last trading day) from yahoo finance using matplotlib BUT I get an error on the code below. I REALLY can't see why can anyone help please?
import datetime
import matplotlib.finance as finance
start_date = datetime.date(2016,8,1)
today = end_date = datetime.date.today()
ticker = 'GE'
fh = finance.fetch_historical_yahoo(ticker,start_date,end_date)
the error thrown is
Traceback (most recent call last):
File "/home/dave/Desktop/development/1 10 16 mp single test.py", line
13, in <module>
fh = finance.fetch_historical_yahoo(ticker,start_date,end_date)
File "/usr/lib/python3/dist-packages/matplotlib/finance.py", line
431, in fetch_historical_yahoo
with contextlib.closing(urlopen(url)) as urlfh:
File "/usr/lib/python3.4/urllib/request.py", line 161, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.4/urllib/request.py", line 470, in open
response = meth(req, response)
File "/usr/lib/python3.4/urllib/request.py", line 580, in http_response
'http', request, response, code, msg, hdrs)
File "/usr/lib/python3.4/urllib/request.py", line 508, in error
return self._call_chain(*args)
File "/usr/lib/python3.4/urllib/request.py", line 442, in _call_chain
result = func(*args)
File "/usr/lib/python3.4/urllib/request.py", line 588, in
http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found

SSL3_GET_RECORD:wrong version number when loading to bigquery table from google cloud storage

When loading to a Bigquery table from files in Google cloud storage, I kept getting this SSL3_GET_RECORD:wrong version number exception.
Eventually when I check the job history from the Google Bigquery job history webpage, the load job will show that it has succeeded.
Could you please help? Thank you.
Here is the error message I am getting:
========================================
== Platform ==
CPython:2.7.6:Linux-2.6.18-194.32.1.el5-x86_64-with-redhat-5.5-Final
== bq version ==
2.0.18
== Command line ==
['/opt/google-cloud-sdk/platform/bq/bq.py', '--credential_file', '/offworld/hornet/.config/gcloud/legacy_credentials/clok#vindicotech.com/singlestore.json', '--project', 'formal-cascade-571', 'load', '--source_format=NEWLINE_DELIMITED_JSON', 'dw_sandbox.impressions_20140603', 'gs://dw_sandbox/impressions/20140603/20140604175042285_20140604195938608_20140603_0_*', '/offworld/specificmedia/logsTobq/schemas/impressionsSchema.txt']
== UTC timestamp ==
2014-06-05 01:19:06
== Error trace ==
File "/opt/google-cloud-sdk/platform/bq/bq.py", line 779, in RunSafely
return_value = self.RunWithArgs(*args, **kwds)
File "/opt/google-cloud-sdk/platform/bq/bq.py", line 1020, in RunWithArgs
job = client.Load(table_reference, source, schema=schema, **opts)
File "/opt/google-cloud-sdk/platform/bq/bigquery_client.py", line 2011, in Load
upload_file=upload_file, **kwds)
File "/opt/google-cloud-sdk/platform/bq/bigquery_client.py", line 1611, in ExecuteJob
job_id=job_id)
File "/opt/google-cloud-sdk/platform/bq/bigquery_client.py", line 1599, in RunJobSynchronously
result = self.WaitJob(job_reference)
File "/opt/google-cloud-sdk/platform/bq/bigquery_client.py", line 1713, in WaitJob
done, job = self.PollJob(job_reference, status=status, wait=wait)
File "/opt/google-cloud-sdk/platform/bq/bigquery_client.py", line 1752, in PollJob
job = self.apiclient.jobs().get(**dict(job_reference)).execute()
File "/opt/google-cloud-sdk/platform/bq/bigquery_client.py", line 307, in execute
return super(BigqueryHttp, self).execute(**kwds)
File "/opt/google-cloud-sdk/bin/bootstrapping/../../lib/oauth2client/util.py", line 132, in positional_wrapper
return wrapped(*args, **kwargs)
File "/opt/google-cloud-sdk/bin/bootstrapping/../../lib/apiclient/http.py", line 716, in execute
body=self.body, headers=self.headers)
File "/opt/google-cloud-sdk/bin/bootstrapping/../../lib/oauth2client/util.py", line 132, in positional_wrapper
return wrapped(*args, **kwargs)
File "/opt/google-cloud-sdk/bin/bootstrapping/../../lib/oauth2client/client.py", line 490, in new_request
redirections, connection_type)
File "/opt/google-cloud-sdk/bin/bootstrapping/../../lib/httplib2/__init__.py", line 1586, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/opt/google-cloud-sdk/bin/bootstrapping/../../lib/httplib2/__init__.py", line 1333, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/opt/google-cloud-sdk/bin/bootstrapping/../../lib/httplib2/__init__.py", line 1289, in _conn_request
response = conn.getresponse()
File "/usr/lib64/python2.7/httplib.py", line 1045, in getresponse
response.begin()
File "/usr/lib64/python2.7/httplib.py", line 409, in begin
version, status, reason = self._read_status()
File "/usr/lib64/python2.7/httplib.py", line 365, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "/usr/lib64/python2.7/socket.py", line 476, in readline
data = self._sock.recv(self._rbufsize)
File "/usr/lib64/python2.7/ssl.py", line 241, in recv
return self.read(buflen)
File "/usr/lib64/python2.7/ssl.py", line 160, in read
return self._sslobj.read(len)
========================================
Unexpected exception in load operation: [Errno 1] _ssl.c:1426:
error:1408F10B:SSL routines:SSL3_GET_RECORD:wrong version number
Is there any chance that you're using the same HTTP object in multiple threads? I.e the thread you create the job in is not necessarily the same thread you poll for completion in? If this is the case, this came up internally within google today, and this was the fix:
class _HTTPFactoryWrapper(object):
"""Wraps a request factory so that each request returns a new http object.
API client's Http object is not threadsafe since calls to the same domain will
reuse the same HTTPConnection. If one API call is outstanding then a second
will try to send a request over the same domain. This causes chaos that
seems to surface itself as SSLErrors during processing.
"""
def __init__(self, factory):
self.factory = factory
def request(self, *args, **kwargs):
return self.factory.Create().request(*args, **kwargs)
Then change the creation of the bigquery stub from:
return discovery.build(api_name, api_version, http=http_factory.Create())
To
http_wrapper = _HTTPFactoryWrapper(http_factory)
return discovery.build(api_name, api_version, http=http_wrapper)