I need to build webcrawler for internal usage and I need to login into administration area. I'm trying to use requests lib, tried this ways:
import urllib.parse
import requests
base_url = "https://target.url"
data = ({'login': 'login', 'pass': 'password'})
params = urllib.parse.urlencode(data)
r = requests.post(base_url, data=params)
print(r.text)
and
import requests
base_url = "https://target.url"
r = requests.post(base_url, auth=('login', 'password')
print(r.text)
but in both cases r.text returns me login page content same as if I try to get any other page after auth code:
req = requests.get("https://target.url/smth")
What I lose sight of? I have ideas:
chain of hidden redirections from https://target.url to real login page, so I send auth info to wrong url
I don't send additional required info (like cookies e.g.)
Could you please comment? How can I gather required for login information?
In my case problem was in 'Referer' parameter in headers, which is required but wasn't specified
Related
I am fetching google photos from my account using Google Photo API. Now there is a need for me to execute that php file via terminal, but the problem is that I can't authenticate with Google API in doing so. Is there a way to do this, and if yes, then how shall it be done?
Yes, it is possible, you need an interactive login for the first authentication but then you can save the token and refresh it automatically as required.
I have implemented this class in Python to do just that.
from requests.adapters import HTTPAdapter
from requests_oauthlib import OAuth2Session
from pathlib import Path
from urllib3.util.retry import Retry
from typing import List, Optional
from json import load, dump, JSONDecodeError
import logging
log = logging.getLogger(__name__)
# OAuth endpoints given in the Google API documentation
authorization_base_url = "https://accounts.google.com/o/oauth2/v2/auth"
token_uri = "https://www.googleapis.com/oauth2/v4/token"
class Authorize:
def __init__(
self, scope: List[str], token_file: Path,
secrets_file: Path, max_retries: int = 5
):
""" A very simple class to handle Google API authorization flow
for the requests library. Includes saving the token and automatic
token refresh.
Args:
scope: list of the scopes for which permission will be granted
token_file: full path of a file in which the user token will be
placed. After first use the previous token will also be read in from
this file
secrets_file: full path of the client secrets file obtained from
Google Api Console
"""
self.max_retries = max_retries
self.scope: List[str] = scope
self.token_file: Path = token_file
self.session = None
self.token = None
try:
with secrets_file.open('r') as stream:
all_json = load(stream)
secrets = all_json['installed']
self.client_id = secrets['client_id']
self.client_secret = secrets['client_secret']
self.redirect_uri = secrets['redirect_uris'][0]
self.token_uri = secrets['token_uri']
self.extra = {
'client_id': self.client_id,
'client_secret': self.client_secret}
except (JSONDecodeError, IOError):
print('missing or bad secrets file: {}'.format(secrets_file))
exit(1)
def load_token(self) -> Optional[str]:
try:
with self.token_file.open('r') as stream:
token = load(stream)
except (JSONDecodeError, IOError):
return None
return token
def save_token(self, token: str):
with self.token_file.open('w') as stream:
dump(token, stream)
self.token_file.chmod(0o600)
def authorize(self):
""" Initiates OAuth2 authentication and authorization flow
"""
token = self.load_token()
if token:
self.session = OAuth2Session(self.client_id, token=token,
auto_refresh_url=self.token_uri,
auto_refresh_kwargs=self.extra,
token_updater=self.save_token)
else:
self.session = OAuth2Session(self.client_id, scope=self.scope,
redirect_uri=self.redirect_uri,
auto_refresh_url=self.token_uri,
auto_refresh_kwargs=self.extra,
token_updater=self.save_token)
# Redirect user to Google for authorization
authorization_url, _ = self.session.authorization_url(
authorization_base_url,
access_type="offline",
prompt="select_account")
print('Please go here and authorize,', authorization_url)
# Get the authorization verifier code from the callback url
response_code = input('Paste the response token here:')
# Fetch the access token
self.token = self.session.fetch_token(
self.token_uri, client_secret=self.client_secret,
code=response_code)
self.save_token(self.token)
# note we want retries on POST as well, need to review this once we
# start to do methods that write to Google Photos
retries = Retry(total=self.max_retries,
backoff_factor=0.1,
status_forcelist=[500, 502, 503, 504],
method_whitelist=frozenset(['GET', 'POST']),
raise_on_status=False)
self.session.mount('https://', HTTPAdapter(max_retries=retries))
I have just tried
===============================
# Give user the file requested
url = "http://superhost.gr/data/files/%s" % realfile
username = getpass.getuser()
password = getpass.getpass()
r = requests.get( url, auth = (username, password) ) # ask user for authentication data
r.raise_for_status()
===============================
as well as input() for both user & pass combo but iam not getting in chrome the basic pop-up HTTP auth window.
Any idea why?
How can i ASK the user for http auth data and store them isntead of giving them to the script?
You can try to use below code to ask user for credentials and send them via HTTP request
from requests.auth import HTTPBasicAuth
import requests
import sys
username = sys.argv[1]
password = sys.argv[2]
realfile = sys.argv[3]
url = "http://superhost.gr/data/files/%s" % realfile
r = requests.get( url, auth=HTTPBasicAuth(username, password))
print(r.status_code)
This script user can run as python script.py username password filename
# Give user the file requested
print('''<meta http-equiv="refresh"
content="5;url=http://superhost.gr/data/files/%s">''' % realfile)
authuser = os.environ.get( 'REMOTE_USER', 'Άγνωστος' )
print( authuser )
================================
Trying this, feels liek i'm almost there except that when printing the value of authuser variable it default to "Άγνωστος" meaning not there.
is there any other way i can grab what the user gave a auth login info?
A friend is trying to run a script to check which notebooks are using the most memory, but their server is password protected. I'm trying to figure out how to configure authentication using urllib2 since I don't believe there is a username, only a password.
The #aiguofer answer did not work for me because jupyter now uses '_xsrf' in cookie. The follwoing woked for me:
s = requests.Session()
url='http://127.0.0.1:8888/login/'
resp=s.get(url)
xsrf_cookie = resp.cookies['_xsrf']
params={'_xsrf':xsrf_cookie,'password': password}
s.post(url, data=params)
After that s can be used to call the apis.
After digging into the notebook code and through some trial and error, I figured out how to do this (and I switched to using requests).
I can't guarantee that this is the best way to do it, but it certainly worked for me. I actually set my vars elsewhere in the code but included here for completeness
import requests
hostname = '127.0.0.1'
port = '8888'
password = 'mypassword'
base_url = 'http://{0}:{1}/'.format(hostname, port)
h = {}
if password:
r = requests.post(base_url + 'login', params={
'password': password
})
h = r.request.headers
sessions = requests.get(base_url + 'api/sessions', headers=h).json()
I believe this works because when you hit the /login endpoint, it redirects you with the right headers set. I guess requests keeps the headers of the redirect, so we can reuse those for the other call. It might be better to extract only the cookies and use those, but this works for now :)
It seems there are some changes with new version. url '/login' does not work for me, I need to add next parameters
url='http://localhost:8050/login?next=%2F'
For the login request. The rest just like Hassan answer
i found when use jupyter put api upload file response 403,
need add "X-XSRFToken" header can solve it..
data= json.dumps({
"name": "test.jpg",
"path": "path",
"type":"file",
"format": "base64",
"content": "base64 data"
})
headers["X-XSRFToken"] = xsrf_cookie
s.put(url, data=data, headers=headers)
If someone could point me in the right direction I would be very grateful. I am trying to use the requests module to interact with Reddit's APIs, but for some reason I am getting HTTP status 403 codes. I am able to successfully log in I believe (since I get HTTP 200), but I cannot perform a successful API request. I am very new to this so please use small words. Thanks.
import requests
import json
from pprint import pprint
# URLs
url_base = r'https://www.reddit.com'
url_login = '/api/login'
url_me = '/api/v1/me'
# Credentials
user = '__kitten_mittens__'
passwd = 'password'
params = {'user': user,
'passwd': passwd,
'api_type': 'json',}
headers = {'user-agent': '/u/__kitten_mittens__ practice api bot'}
# Set up the session/headers
client = requests.session()
client.headers = headers
response = client.post(url_base + url_login, data = params)
j = json.loads(response.content.decode('utf-8'))
mh = j['json']['data']['modhash']
print("The modhash for {USER} is {mh}".format(USER=user, mh=mh))
# THIS CODE GIVES HTTP STATUS CODE 403
response = client.get(url_base + url_me)
me_json = json.loads(r.content.decode('utf-8'))
pprint(me_json)
EDIT: I fixed the missing single quote from the password field. That was not part of the problem.
Whats is the python urllib equivallent of
curl -u username:password status="abcd" http://example.com/update.json
I did this:
handle = urllib2.Request(url)
authheader = "Basic %s" % base64.encodestring('%s:%s' % (username, password))
handle.add_header("Authorization", authheader)
Is there a better / simpler way?
The trick is to create a password manager, and then tell urllib about it. Usually, you won't care about the realm of the authentication, just the host/url part. For example, the following:
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
top_level_url = "http://example.com/"
password_mgr.add_password(None, top_level_url, 'user', 'password')
handler = urllib2.HTTPBasicAuthHandler(password_mgr)
opener = urllib2.build_opener(urllib2.HTTPHandler, handler)
request = urllib2.Request(url)
Will set the user name and password to every URL starting with top_level_url. Other options are to specify a host name or more complete URL here.
A good document describing this and more is at http://www.voidspace.org.uk/python/articles/urllib2.shtml#id6.
Yes, have a look at the urllib2.HTTP*AuthHandlers.
Example from the documentation:
import urllib2
# Create an OpenerDirector with support for Basic HTTP Authentication...
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(realm='PDQ Application',
uri='https://mahler:8092/site-updates.py',
user='klem',
passwd='kadidd!ehopper')
opener = urllib2.build_opener(auth_handler)
# ...and install it globally so it can be used with urlopen.
urllib2.install_opener(opener)
urllib2.urlopen('http://www.example.com/login.html')