I faced a problem how to set parameters for API Gateway to query Amazon Redshift with Lambda function.
My connection is working properly, but I got all the time full table respond.
I need to define variables, that user can query specific parameters, values and schemas
Can someone suggest an examples how to set it up
My config is:
#!/usr/bin/env python
import psycopg2
import logging
import traceback
import json
from os import environ
query="SELECT * from public"
logger=logging.getLogger()
logger.setLevel(logging.INFO)
def make_connection():
conn=psycopg2.connect(dbname= 'database', host='redshift-cluster.amazonaws.com',
port= '5439', user= 'user', password= 'password')
conn.autocommit=True
return conn
def log_err(errmsg):
logger.error(errmsg)
return {"body": errmsg , "headers": {}, "statusCode": 400,
"isBase64Encoded":"false"}
logger.info("Cold start complete.")
print('Loading Function')
def handler(event,context):
try:
cnx = make_connection()
cursor=cnx.cursor()
try:
cursor.execute(query)
except:
return log_err ("ERROR: Cannot execute cursor.\n{}".format(
traceback.format_exc()) )
try:
results_list=[]
for result in cursor: results_list.append(result)
print(results_list)
cursor.close()
except:
return log_err ("ERROR: Cannot retrieve query data.\n{}".format(
traceback.format_exc()))
return {"body": str(results_list), "headers": {}, "statusCode": 200,
"isBase64Encoded":"false"}
except:
return log_err("ERROR: Cannot connect to database from handler.\n{}".format(
traceback.format_exc()))
finally:
try:
cnx.close()
except:
pass
if __name__== "__main__":
handler(None,None)
Rather than using psycopg2, I would recommend Using the Amazon Redshift Data API. It provides a much easier way to run a query on Amazon Redshift.
First, send the query with execute_statement(), then retrieve the results with get_statement_result().
Related
I have followed this article set up, it all seems to work properly, but I would like now to perform a SQL query and pass the result into a pandas data frame, how could I proceed?
This is what I have now;
host_server = os.environ.get('host_server', 'localhost')
db_server_port = urllib.parse.quote_plus(str(os.environ.get('db_server_port', '5432')))
database_name = os.environ.get('database_name', 'my_data_base123')
db_username = urllib.parse.quote_plus(str(os.environ.get('db_username', 'my_user_name123')))
db_password = urllib.parse.quote_plus(str(os.environ.get('db_password', 'my_password123')))
ssl_mode = urllib.parse.quote_plus(str(os.environ.get('ssl_mode','prefer')))
DATABASE_URL = 'postgresql://{}:{}#{}:{}/{}?sslmode={}'.format(db_username, db_password, host_server, db_server_port, database_name, ssl_mode)
database = databases.Database(DATABASE_URL)
metadata = sqlalchemy.MetaData()
engine = sqlalchemy.create_engine(
DATABASE_URL, pool_size=3, max_overflow=0
)
metadata.create_all(database)
app = FastAPI(title="REST API using FastAPI PostgreSQL Async EndPoints")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"]
)
#app.on_event("startup")
async def startup():
await database.connect()
#app.on_event("shutdown")
async def shutdown():
await database.disconnect()
Trying to access the database with a SQL query:
with engine.connect() as conn:
result = conn.execute(text("select 'hello world'))
print(result.all())
as it says in the sqlalchemy documentation , but i get some errors, like:
print(result.all())
AttributeError: 'ResultProxy' object has no attribute 'all'
even if i try to access the tables of my database
with engine.connect() as conn:
result = conn.execute(text("select * FROM users"))
print(result.all())
i get the same error
It is solved, I had to upgrade SQLAlchemy
sudo pip install sqlalchemy --upgrade
I found a XHR request with all the street address info I want to scrape within it.
However, I do not know how to extract it to a pandas dataframe or a python list. Any ideas? Thank you very much!
Since it's graphql, you can formulate the query string however you like, but here I've written it the same way it's sent when the browser makes a request:
def main():
import requests
url = "https://api-endpoint.cons-prod-us-central1.kw.com/graphql"
headers = {
"x-shared-secret": "MjFydHQ0dndjM3ZAI0ZHQCQkI0BHIyM="
}
query = """{
ListOfficeQuery {
id
name
address
subAddress
phone
fax
lat
lng
url
contacts {
name
email
phone
__typename
}
__typename
}
}
"""
payload = {
"operationName": None,
"variables": {},
"query": query
}
response = requests.post(url, headers=headers, json=payload)
response.raise_for_status()
offices = response.json()["data"]["ListOfficeQuery"]
print(f"There are {len(offices)} offices, and the first one's address is \"{offices[0]['address']}\"")
return 0
if __name__ == "__main__":
import sys
sys.exit(main())
Output:
There are 1173 offices, and the first one's address is "1801 South Mo-Pac Expressway, Suite 100"
>>>
You can replicate the XHR using python's requests library using the information in the headers tab (more info here). Then parse the data using json library, and extract information.
from rest_framework import status, response
from rest_framework.test import APITestCase
from lots.models import Lot
class LotsTestCase(APITestCase):
def setUp(self) -> None:
self.lot = Lot.objects.create(name="1",
address="Dont Know",
phone_num="010-4451-2211",
latitude=127.12,
longitude=352.123,
basic_rate=20000,
additional_rate=2000,
partnership=False,
section_count=3,)
def test_delete(self):
response = self.client.delete(f'api/lots/{self.lot["name"]}')
# response = self.client.delete(f'/api/users/{self.users[0].pk}')
# url = reverse(f'/api/lots/{self.lot}', kwargs={'pk': self.lot.pk})
# self.client.delete(url)
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(self.lot.objects.filter(pk=self.lot.pk.count()))
I have problems with the test code above. Why doesn't it work? I know it has to do with calling dictionary values but I just can't figure it out. Thanks for your help.
Lot.objects.create(...) returns a Lot type so you access name by self.lot.name.
I have a question about using apollo-upload-client and graphene-django. Here I've discovered that apollo-upload-client adding operations to formData. But here graphene-django is only trying to get query parameter. And the question is, where and how it should be fixed?
If you're referring to the data that has a header like (when viewing the HTTP from Chrome tools):
Content-Disposition: form-data; name="operations"
and data like
{"operationName":"MyMutation","variables":{"myData"....}, "query":"mutation MyMutation"...},
the graphene-python library interprets this and assembles it into a query for you, inserting the variables and removing the file data from the query. If you are using Django, you can find all of the uploaded files in info.context.FILES when writing a mutation.
Here's my solution to support the latest apollo-upload-client (8.1). I recently had to revisit my Django code when I upgraded from apollo-upload-client 5.x to 8.x. Hope this helps.
Sorry I'm using an older graphene-django but hopefully you can update the mutation syntax to the latest.
Upload scalar type (passthrough, basically):
class Upload(Scalar):
'''A file upload'''
#staticmethod
def serialize(value):
raise Exception('File upload cannot be serialized')
#staticmethod
def parse_literal(node):
raise Exception('No such thing as a file upload literal')
#staticmethod
def parse_value(value):
return value
My upload mutation:
class UploadImage(relay.ClientIDMutation):
class Input:
image = graphene.Field(Upload, required=True)
success = graphene.Field(graphene.Boolean)
#classmethod
def mutate_and_get_payload(cls, input, context, info):
with NamedTemporaryFile(delete=False) as tmp:
for chunk in input['image'].chunks():
tmp.write(chunk)
image_file = tmp.name
# do something with image_file
return UploadImage(success=True)
The heavy lifting happens in a custom GraphQL view. Basically it injects the file object into the appropriate places in the variables map.
def maybe_int(s):
try:
return int(s)
except ValueError:
return s
class CustomGraphqlView(GraphQLView):
def parse_request_json(self, json_string):
try:
request_json = json.loads(json_string)
if self.batch:
assert isinstance(request_json,
list), ('Batch requests should receive a list, but received {}.').format(
repr(request_json))
assert len(request_json) > 0, ('Received an empty list in the batch request.')
else:
assert isinstance(request_json, dict), ('The received data is not a valid JSON query.')
return request_json
except AssertionError as e:
raise HttpError(HttpResponseBadRequest(str(e)))
except BaseException:
logger.exception('Invalid JSON')
raise HttpError(HttpResponseBadRequest('POST body sent invalid JSON.'))
def parse_body(self, request):
content_type = self.get_content_type(request)
if content_type == 'application/graphql':
return {'query': request.body.decode()}
elif content_type == 'application/json':
return self.parse_request_json(request.body.decode('utf-8'))
elif content_type in ['application/x-www-form-urlencoded', 'multipart/form-data']:
operations_json = request.POST.get('operations')
map_json = request.POST.get('map')
if operations_json and map_json:
operations = self.parse_request_json(operations_json)
map = self.parse_request_json(map_json)
for file_id, f in request.FILES.items():
for name in map[file_id]:
segments = [maybe_int(s) for s in name.split('.')]
cur = operations
while len(segments) > 1:
cur = cur[segments.pop(0)]
cur[segments.pop(0)] = f
logger.info('parse_body %s', operations)
return operations
else:
return request.POST
return {}
I am new to IBPy and really interested to know how to get account parameters for multiple accounts. The code below only gives me the output in the command line but I couldn't figure out how to store those information into a dataframe. the function updateAccountValue() doesn't have a unique id that i can use as index for the dataframe.
from ib.opt import Connection, message
import pandas as pd
import time
def error_handler(msg):
"""Handles the capturing of error messages"""
print "Server Error: %s" % msg
def updateAccount_handler(msg):
if msg.key in ['AccountType','NetLiquidation']:
print msg.key, msg.value
if __name__ == "__main__":
conn = Connection.create(port=7497, clientId = 93)
conn.connect()
conn.register(error_handler, 'Error')
conn.register(updateAccount_handler,message.updateAccountValue)
# we can do a loop, i am just giving a simple example for 2 accounts
conn.reqAccountUpdates(1,"Uxxxx008")
time.sleep(0.5)
conn.reqAccountUpdates(1,"Uxxxx765")
time.sleep(0.5)
conn.disconnect()
the output looks like this:
Server Version: 76
TWS Time at connection:20150729 12:46:56 EST
Server Error: <error id=-1, errorCode=2104, errorMsg=Market data farm connection is OK:usfuture>
Server Error: <error id=-1, errorCode=2104, errorMsg=Market data farm connection is OK:usfuture.us>
Server Error: <error id=-1, errorCode=2104, errorMsg=Market data farm connection is OK:usfarm>
Server Error: <error id=-1, errorCode=2106, errorMsg=HMDS data farm connection is OK:ushmds>
AccountType INDIVIDUAL
NetLiquidation 2625.24
AccountType IRA-ROTH NEW
NetLiquidation 11313.83
End goal is to store these information into a pandas dataframe format with an unique id of account number.
Ok, The IBpy works in sender/Receiver architecture. What I would like suggest you is to have a global variable to store the account information.
from ib.opt import Connection, message
import pandas as pd
import time
class account_update :
acc_info = [] # list to store all your account info
# you can have list,tuple,panda dataframe or any data strucure I am using normal list to demonstrate
def error_handler(self,msg):
"""Handles the capturing of error messages"""
print "Server Error: %s" % msg
def updateAccount_handler(self,msg):
if msg.key in ['AccountType','NetLiquidation']:
self.acc_info.append(msg.key, msg.value)
def main(self):
conn = Connection.create(port=7497, clientId = 93)
conn.connect()
conn.register(error_handler, 'Error')
conn.register(updateAccount_handler,message.updateAccountValue)
conn.reqAccountUpdates(1,"Uxxxx008")
time.sleep(0.5)
conn.reqAccountUpdates(1,"Uxxxx765")
time.sleep(0.5)
conn.disconnect()
if __name__ == "__main__" :
account_update().main()
So the logic is simple,create a global variable and make your response handler method to update the global variable whenever it receives a response. Hope it works :)