First time using Locust. I have a Flask App that requires user to login to access most routes.
I cant get Locust to successfully login to my Flask App.
Here is my Locust.py file:
from locust import HttpLocust, TaskSet, task
import re
class UserBehavior(TaskSet):
def on_start(self):
""" on_start is called when a Locust start before any task is scheduled """
self.client.verify = False
self.get_token()
self.login()
def on_stop(self):
""" on_stop is called when the TaskSet is stopping """
self.logout()
def get_token(self):
response = self.client.get("/login")
# Sample string from response:
# <input id="csrf_token" name="csrf_token" type="hidden" value="REDACTED">
self.csrftoken = re.search(' name="csrf_token" .* value="(.+?)"', response.text).group(1)
print(f"DEBUG: self.csrftoken = {self.csrftoken}")
def login(self):
response = self.client.post("/login",
{"email": "REDACTED", "password": "REDACTED"},
headers={"X-CSRFToken": self.csrftoken})
print(f"DEBUG: login response.status_code = {response.status_code}")
def logout(self):
self.client.get("/logout")
#task(5)
def list_domains(self):
response = self.client.get("/domains", headers={"X-CSRFToken": self.csrftoken})
print(f"DEBUG list: response.status_code = {response.status_code}")
class WebsiteUser(HttpLocust):
task_set = UserBehavior
min_wait = 5000
max_wait = 9000
Here is the login function of my Flask App: (with a few debug statements added)
#users.route('/login', methods=['GET', 'POST'])
def login():
if request.method == 'POST': ##DEBUG
logging.debug(f"debug0: inside login func with method == POST") ##DEBUG
if current_user.is_authenticated:
return redirect(url_for('main.home'))
form = LoginForm()
if form.validate_on_submit():
logging.debug(f"debug0.1: inside validate_on_submit") ##DEBUG
user = Users.query.filter_by(email=form.email.data).first()
if user and user.check_password(form.password.data):
login_user(user, remember=form.remember.data)
next_page = request.args.get('next')
if not next_page or url_parse(next_page).netloc != '':
next_page = url_for('main.home')
logging.debug(f"debug1: Login was successful") ##DEBUG
return redirect(next_page)
else:
logging.debug(f"debug2: Login failed") ##DEBUG
flash(f'Login unsuccessful. Please check email and password!', 'danger')
logging.debug(f"debug3: the end of login func") ##DEBUG
return render_template('login.html', title='Login', form=form)
When i run Locust, I get this output:
[2019-09-16 18:03:06,598] Mac-mini-3.local/INFO/locust.main: Starting web monitor at *:8089
[2019-09-16 18:03:06,598] Mac-mini-3.local/INFO/locust.main: Starting Locust 0.11.0
[2019-09-16 18:03:14,069] Mac-mini-3.local/INFO/locust.runners: Hatching and swarming 2 clients at the rate 1 clients/s...
[2019-09-16 18:03:14,138] Mac-mini-3.local/ERROR/stderr: /Users/myuser/.local/share/virtualenvs/locustio-gB1-mbqd/lib/python3.7/site-packages/urllib3/connectionpool.py:851: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings InsecureRequestWarning)
[2019-09-16 18:03:14,162] Mac-mini-3.local/INFO/stdout: DEBUG: self.csrftoken = REDACTED
[2019-09-16 18:03:14,183] Mac-mini-3.local/INFO/stdout: DEBUG: login response.status_code = 200
[2019-09-16 18:03:14,213] Mac-mini-3.local/INFO/stdout: DEBUG list: response.status_code = 200
[2019-09-16 18:03:15,112] Mac-mini-3.local/INFO/stdout: DEBUG: self.csrftoken = REDACTED
[2019-09-16 18:03:15,137] Mac-mini-3.local/INFO/stdout: DEBUG: login response.status_code = 200
I'm not concerned about the 'InsecureRequestWarning' as this is because I am using a self signed cert and i have disabled verification with 'self.client.verify = False'
The csrftoken looks correct.
From the Flask App itself, I get this output:
DEBUG:user:debug0: inside login func with method == POST
INFO:flask_wtf.csrf:The CSRF token is missing.
DEBUG:user:debug3: the end of login func
DEBUG:user:debug3: the end of login func
DEBUG:user:debug3: the end of login func
DEBUG:user:debug3: the end of login func
So, it's hitting the login function (proven by debug0) but it's not getting into the 'form.validate_on_submit()' conditional.
So far I have spent all day on this, reading articles and trying a lot of things, ie adding the X-CSRFToken headers.
I feel I am missing something fundamental, and would really appreciate some help.
thanks,
WJ
Ok, I solved it, so thought I would share the anwser for anyone else that comes across this. As suggested by #user10788336 its is not a Locust issue.
The issue was that when POSTing to the flask route, the form was not being validated (ie form.validate() was not getting set).
So, I made two changes.
1) changed the POST to have an additional form item which i called "test-mode" and I set the value to "locust-test"
here is the new Locust.py file:
from locust import HttpLocust, TaskSet, task
import re
class UserBehavior(TaskSet):
def on_start(self):
""" on_start is called when a Locust start before any task is scheduled """
self.client.verify = False
self.get_token()
self.login()
def on_stop(self):
""" on_stop is called when the TaskSet is stopping """
self.logout()
def get_token(self):
response = self.client.get("/login")
# Sample string from response:
# <input id="csrf_token" name="csrf_token" type="hidden" value="REDACTED">
self.csrftoken = re.search(' name="csrf_token" .* value="(.+?)"', response.text).group(1)
print(f"DEBUG: self.csrftoken = {self.csrftoken}")
def login(self):
response = self.client.post("/login",
{"email": "REDACTED",
"password": "REDACTED",
"test-mode": "locust-test"
},
headers={"X-CSRFToken": self.csrftoken})
print(f"DEBUG: login response.status_code = {response.status_code}")
def logout(self):
self.client.get("/logout")
#task(5)
def list_domains(self):
response = self.client.get("/domains", headers={"X-CSRFToken": self.csrftoken})
print(f"DEBUG list: response.status_code = {response.status_code}")
class WebsiteUser(HttpLocust):
task_set = UserBehavior
min_wait = 5000
max_wait = 9000
The diff between old and new is:
< {"email": "REDACTED", "password": "REDACTED"},
---
> {"email": "REDACTED",
> "password": "REDACTED",
> "test-mode": "locust-test"
> },
2) I changed my login function the Flask app:
The change is that I don't need the form to be validated, so I skip that ONLY when I detect that I am running in test-mode.
here is the new login function:
#users.route('/login', methods=['GET', 'POST'])
def login():
if current_user.is_authenticated:
return redirect(url_for('main.home'))
form = LoginForm()
# shortcut for Locust testing - need to avoid form.validate() (which is within form.validate_on_submit())
form_is_ok = False
if request.method == 'POST':
if request.form.get('test-mode') == 'locust-test':
form_is_ok = True
else:
form_is_ok = form.validate_on_submit()
if form_is_ok:
logging.debug(f"debug0.1: inside validate_on_submit") # DEBUG
user = Users.query.filter_by(email=form.email.data).first()
if user and user.check_password(form.password.data):
login_user(user, remember=form.remember.data)
next_page = request.args.get('next')
if not next_page or url_parse(next_page).netloc != '':
next_page = url_for('main.home')
logging.debug(f"debug1: Login was successful") # DEBUG
return redirect(next_page)
else:
logging.debug(f"debug2: Login failed") # DEBUG
flash(f'Login unsuccessful. Please check email and password!', 'danger')
logging.debug(f"debug3: the end of login func") # DEBUG
return render_template('login.html', title='Login', form=form)
The diff between old and new is:
< if form.validate_on_submit():
---
>
> # shortcut for Locust testing - need to avoid form.validate() (which is within form.validate_on_submit())
> form_is_ok = False
> if request.method == 'POST':
> if request.form.get('test-mode') == 'locust-test':
> form_is_ok = True
> else:
> form_is_ok = form.validate_on_submit()
>
> if form_is_ok:
I think this is still secure ... thoughts on that?
I might add a config variable, that disables/enables this functionality.
And it works!!!!
BTW, Locust is awesome!
Hope this helps.
cheers,
WJ
Can you share the code for what the form.validate_on_submit method is expecting/validating? Chances are there is a hidden field in the login form that is acting as a dynamic token and/or you're missing a required field. Can you also share the HTML source of the login form?
I'd also be curious to see you add an additional debug statement in your login method that outputs the value of the CSRF token, to make sure it is valid, e.g.
def login(self):
print(f"DEBUG: login csrftoken = {self.csrftoken}")
response = self.client.post("/login",
{"email": "REDACTED",
"password": "REDACTED",
"test-mode": "locust-test"
},
headers={"X-CSRFToken": self.csrftoken})
print(f"DEBUG: login response.status_code = {response.status_code}")
I'd also be curious to see if there are any error messages when you submit the form without a valid CSRF token. Does it return a 200 status code and report the invalid token?
My guess is that it's something to do with the CSRF token not being valid or handled correctly.
Is this an older or customized version of Flask login? I'm not seeing an X-CSRFToken header method in the latest version of Flask login docs at https://flask-login.readthedocs.io/en/latest/
Related
I want to run my FastAPI for multiprocessing but when I try to run using uvicorn with reload and workers but it could not find my login endpoint.
It says "detail": "Not Found" when I hit the endpoint this is my object_counter_api.py
app = FastAPI(
title="Object Counter API",
description="REST API for Object Counter Service.",
version="0.1",
contact={ "name": "my_endpoint",
"url": "https://my_endpoint.com",
"email": "sales#my_endpoint.com"
}
)
# Allow cors error mitigation
origins = ["*"]
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
expose_headers=["*"]
)
class ObjectCounterAPI:
def run_object_counter(self, host, port):
ABLogger.info(self, "running object counter API!")
self.setup()
uvicorn.run("object_counter_api:app", host=self.vehicle_config.host,port=self.vehicle_config.port, reload=True, workers=2)
def setup(self):
#app.post("/vehicle/api/v1/login")
async def login(request: LoginRequest):
ABLogger.info(self, "login credentials check!")
access_token = await self.authenticate_user(request.username, request.password)
response = LoginResponse(access_token=access_token, token_type="bearer")
return response
this is my main.py
if __name__ == "__main__":
#hydra.main(config_path=root / "hydra_config", config_name="debug", version_base=None)
def main(hydra_config: DictConfig):
api = ObjectCounterAPI(hydra_config)
mp.Process(target = api.run_object_counter(host=hydra_config.api.vehicle_counter.host, port=hydra_config.api.vehicle_counter.port)).start()
main()
I use this function (https://developers.google.com/youtube/v3/docs/activities/list) to retrieve list of user activities.
I use my channelId and I don't see subscriptions for period before autumn of 2021. Can anyone explain me why? May be you write me when this type of activity (subscription) was added in the type of request, named Activities: list?
Thanks!
See below example code:
import os
import google_auth_oauthlib.flow
import googleapiclient.discovery
import googleapiclient.errors
scopes = ["https://www.googleapis.com/auth/youtube.readonly"]
def main():
# Disable OAuthlib's HTTPS verification when running locally.
# *DO NOT* leave this option enabled in production.
os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
api_service_name = "youtube"
api_version = "v3"
client_secrets_file = "YOUR_CLIENT_SECRET_FILE.json"
# Get credentials and create an API client
flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file(
client_secrets_file, scopes)
credentials = flow.run_console()
youtube = googleapiclient.discovery.build(
api_service_name, api_version, credentials=credentials)
request = youtube.activities().list(
part="snippet,contentDetails",
channelId="yourChannelId", #Note: paste your own channelId
maxResults=300
)
response = request.execute()
print(response)
if __name__ == "__main__":
main()
You can do an example of request in the right side:
https://developers.google.com/youtube/v3/docs/activities/list
I am fetching google photos from my account using Google Photo API. Now there is a need for me to execute that php file via terminal, but the problem is that I can't authenticate with Google API in doing so. Is there a way to do this, and if yes, then how shall it be done?
Yes, it is possible, you need an interactive login for the first authentication but then you can save the token and refresh it automatically as required.
I have implemented this class in Python to do just that.
from requests.adapters import HTTPAdapter
from requests_oauthlib import OAuth2Session
from pathlib import Path
from urllib3.util.retry import Retry
from typing import List, Optional
from json import load, dump, JSONDecodeError
import logging
log = logging.getLogger(__name__)
# OAuth endpoints given in the Google API documentation
authorization_base_url = "https://accounts.google.com/o/oauth2/v2/auth"
token_uri = "https://www.googleapis.com/oauth2/v4/token"
class Authorize:
def __init__(
self, scope: List[str], token_file: Path,
secrets_file: Path, max_retries: int = 5
):
""" A very simple class to handle Google API authorization flow
for the requests library. Includes saving the token and automatic
token refresh.
Args:
scope: list of the scopes for which permission will be granted
token_file: full path of a file in which the user token will be
placed. After first use the previous token will also be read in from
this file
secrets_file: full path of the client secrets file obtained from
Google Api Console
"""
self.max_retries = max_retries
self.scope: List[str] = scope
self.token_file: Path = token_file
self.session = None
self.token = None
try:
with secrets_file.open('r') as stream:
all_json = load(stream)
secrets = all_json['installed']
self.client_id = secrets['client_id']
self.client_secret = secrets['client_secret']
self.redirect_uri = secrets['redirect_uris'][0]
self.token_uri = secrets['token_uri']
self.extra = {
'client_id': self.client_id,
'client_secret': self.client_secret}
except (JSONDecodeError, IOError):
print('missing or bad secrets file: {}'.format(secrets_file))
exit(1)
def load_token(self) -> Optional[str]:
try:
with self.token_file.open('r') as stream:
token = load(stream)
except (JSONDecodeError, IOError):
return None
return token
def save_token(self, token: str):
with self.token_file.open('w') as stream:
dump(token, stream)
self.token_file.chmod(0o600)
def authorize(self):
""" Initiates OAuth2 authentication and authorization flow
"""
token = self.load_token()
if token:
self.session = OAuth2Session(self.client_id, token=token,
auto_refresh_url=self.token_uri,
auto_refresh_kwargs=self.extra,
token_updater=self.save_token)
else:
self.session = OAuth2Session(self.client_id, scope=self.scope,
redirect_uri=self.redirect_uri,
auto_refresh_url=self.token_uri,
auto_refresh_kwargs=self.extra,
token_updater=self.save_token)
# Redirect user to Google for authorization
authorization_url, _ = self.session.authorization_url(
authorization_base_url,
access_type="offline",
prompt="select_account")
print('Please go here and authorize,', authorization_url)
# Get the authorization verifier code from the callback url
response_code = input('Paste the response token here:')
# Fetch the access token
self.token = self.session.fetch_token(
self.token_uri, client_secret=self.client_secret,
code=response_code)
self.save_token(self.token)
# note we want retries on POST as well, need to review this once we
# start to do methods that write to Google Photos
retries = Retry(total=self.max_retries,
backoff_factor=0.1,
status_forcelist=[500, 502, 503, 504],
method_whitelist=frozenset(['GET', 'POST']),
raise_on_status=False)
self.session.mount('https://', HTTPAdapter(max_retries=retries))
My use case is to get some files from company's sharepoint (Online) site. I have been granted read access for my username password to connect through SharePoint API. For the calls I will have to pass proxy and company SSL verification.
I have tried using a number of APIs such as sharepy, Office365-REST-Python-Client, HttpNtlmAuth, HTTPBasicAuth, but all of them giving me [SSL: CERTIFICATE_VERIFY_FAILED] error.
I am not sure if passing certificate to these APIs is possible or not.
Is there any other plugin that I can try for my scenario?
For this plugin, as a work-around I have done monkey patching for the common functions that send requests to the APIs. Following are the examples of few such functions:
class SharePointApi:
"""SharePoint aceess api."""
def __init__(self):
self.base_url = configReader.get('SHAREPOINT', 'URL')
self.ctx_auth = AuthenticationContext(self.base_url)
self.ctx_auth.provider = SamlTokenProvider(self.base_url, username, password)
self.ctx_auth.provider.acquire_service_token = self._patched_acquire_service_token
self.ctx_auth.provider.acquire_authentication_cookie = self._patched_acquire_authentication_cookie
self.ctx_auth.provider.get_realm_from_target_url = self._patched_get_realm_from_target_url
self.ctx_auth.provider.get_app_only_access_token = self._patched_get_app_only_access_token
def _patched_acquire_authentication_cookie(self, options):
"""Retrieve SPO auth cookie"""
url = options['endpoint']
session = requests.session()
session.post(url, data=self.ctx_auth.provider.token, headers={'Content-Type': 'application/x-www-form-urlencoded'}
, verify=False
)
logger.debug_secrets("session.cookies: %s", session.cookies)
cookies = requests.utils.dict_from_cookiejar(session.cookies)
logger.debug_secrets("cookies: %s", cookies)
if 'FedAuth' in cookies and 'rtFa' in cookies:
self.ctx_auth.provider.FedAuth = cookies['FedAuth']
self.ctx_auth.provider.rtFa = cookies['rtFa']
return True
self.ctx_auth.provider.error = "An error occurred while retrieving auth cookies"
logger.error(self.ctx_auth.provider.error)
return False
def _patched_get_realm_from_target_url(self):
response = requests.head(url=self.ctx_auth.provider.url, headers={'Authorization': 'Bearer'}, verify=False, proxies=proxies)
return self.ctx_auth.provider.process_realm_response(response)
def _patched_get_app_only_access_token(self, target_host, target_realm):
resource = self.ctx_auth.provider.get_formatted_principal(self.ctx_auth.provider.SharePointPrincipal, target_host, target_realm)
client_id = self.ctx_auth.provider.get_formatted_principal(self.ctx_auth.provider.client_id, None, target_realm)
sts_url = self.ctx_auth.provider.get_security_token_service_url(target_realm)
oauth2_request = self.ctx_auth.provider.create_access_token_request(client_id, self.ctx_auth.provider.client_secret, resource)
response = requests.post(url=sts_url, headers={'Content-Type': 'application/x-www-form-urlencoded'}, data=oauth2_request, verify=False, proxies=proxies)
return response.json()
I built my own API with flask, but now I'm trying to write a script to consume the API and the Requests module won't make it past the log in.
The Form
The form has jut a 'username' input, 'password', and I'm also using flask-wtf to implement csrf protection, so there is a 'csrf_token' hidden input.
from flask_wtf import FlaskForm
from wtforms import StringField, PasswordField
from wtforms.validators import DataRequired, Length
class LoginForm(FlaskForm):
username = StringField('Username', validators=[DataRequired(), Length(min=1, max=20)])
password = PasswordField('Password', validators=[DataRequired(), Length(min=1, max=25)])
The Route
#auth.route('/login', methods=['GET', 'POST'])
def login():
form = LoginForm()
if request.method == 'GET':
return render_template('login.html', form=form)
elif request.method == 'POST':
if form.validate_on_submit():
credentials = {'username': request.form['username'],
'password': request.form['password']}
try:
r = requests.post('http://api.domain.com:80/authenticate', data=credentials)
response = r.json()
except Exception as e:
print('JSON Auth Error: {}'.format(e))
else:
if r.status_code is 200:
session['access_token'] = response['access_token']
session['refresh_token'] = response['refresh_token']
return redirect('/')
return render_template('login.html', form=form, error='invalid')
The API Endpoint
#api.route('/authenticate', methods=['POST'])
def authenticate():
# form data
username = request.form['username']
password = request.form['password']
# check if user exists
try:
user = User.query.filter_by(username=username).first()
except exc.SQLAlchemyError as e:
print(e)
if user and password == user.password:
tokens = generate_tokens()
return jsonify({'access_token': tokens['access'], 'refresh_token': tokens['refresh']}), 200
else:
return jsonify({'message': 'invalid login'}), 401
The API Resource Endpoint I am trying to access
#api.route('/pending', methods=['GET'])
#login_required
def pending():
The #login_required verification method
def tokens_valid():
if 'access_token' and 'refresh_token' in session:
# retrieve access and refresh tokens from session cookie
tokens = {'access_token': session['access_token'],
'refresh_token': session['refresh_token']}
# validate token endpoint
try:
r = requests.post('http://api.domain.com/validate', data=tokens)
response = r.json()
except Exception as e:
print('JSON Token Error: {}'.format(e))
else:
if r.status_code is 200:
if response['message'] == 'valid access token':
return True
elif response['message'] == 'new access token':
#set new tokens
session['access_token'] = response['access_token']
session['refresh_token'] = response['refresh_token']
return True
return False
What I have tried
v.1
def login():
credentials = {'username': 'username',
'password': 'password'}
s = requests.Session()
r = s.post('http://sub.domain.com/auth/login', data=credentials)
return s
v.2
def login():
login_form = requests.get('http://sub.domain.com/auth/login')
soup = BeautifulSoup(login_form.text, "html.parser")
csrf = soup.find('input', id='csrf_token').get('value')
credentials = {'username': 'username',
'password': 'password',
'csrf_token': csrf}
s = requests.Session()
r = s.post('http://sub.domain.com/auth/login', data=credentials)
return s
v.3
from flask import Flask, session
def login():
credentials = {'username': 'username',
'password': 'password'}
r = requests.post('http://api.domain.com/authenticate', data=credentials)
tokens = r.json()
session['access_token'] = tokens['access_token']
session['refresh_token'] = tokens['refresh_token']
So far I can get the '/authenticate' api endpoint to return my session tokens, but i can't access the protected resource because i can't seem to save them to my session any how.
Can anyone help me figure this out?