How to update a different google sheet when pulling information from the first sheet? - pandas

I'm able to pull from googlesheets and update the same sheet I pulled from. (The script will only work for the first sheet for the google spreadsheet.)
After pulling from the first sheet I want to generate a report which works, but I don't want it to overwrite the first sheet, instead I want it to put the information on a separate sheet I made. Here's what I have sofar when trying to change my script to work with a different sheet.
def get_google_sheet(spreadsheet_id, range_name):
# Retrieve sheet data using OAuth credentials and Google Python API.
global scopes
global gsheet
scopes = ['https://www.googleapis.com/auth/spreadsheets']
# Setup the Sheets API
store = file.Storage('credentials1.json')
creds = store.get()
if not creds or creds.invalid:
flow = client.flow_from_clientsecrets('client_secret.json', scopes)
creds = tools.run_flow(flow, store)
service = build('sheets', 'v4', http=creds.authorize(Http()))
# Call the Sheets API
gsheet = service.spreadsheets().values().get(spreadsheetId=spreadsheet_id, range=range_name).execute()
return gsheet
def gsheet2df(gsheet):
# Converts Google sheet data to a Pandas DataFrame.
header = gsheet.get('values', [])[0] # Assumes first line is header!
values = gsheet.get('values', [])[1:] # Everything else is data.
#header = values[0]
if not values:
print('No data found.')
else:
df = pd.DataFrame.from_records(values)
df.columns = header
return df
def Export_Data_To_Sheets():
store = file.Storage('credentials.json')
creds = store.get()
if not creds or creds.invalid:
flow = client.flow_from_clientsecrets('client_secret.json', scopes)
creds = tools.run_flow(flow, store)
service = build('sheets', 'v4', http=creds.authorize(Http()))
RANGE_AND_SHEET = "ReportListA1:E500"
response_date = service.spreadsheets().values().update(
spreadsheetId=SPREADSHEET_ID,
valueInputOption='RAW',
range=RANGE_AND_SHEET,
body=dict(
majorDimension='ROWS',
values=df.T.reset_index().T.values.tolist())
).execute()
print('Sheet successfully Updated')
When I use just the range without identifying the sheet it works no problem. With the range and sheet I get this error.
Traceback (most recent call last):
File "/Users/adaniel/PycharmProjects/EmailAutomation/venv/AutoEmailScript/test.py", line 126, in <module>
Export_Data_To_Sheets()
File "/Users/adaniel/PycharmProjects/EmailAutomation/venv/AutoEmailScript/test.py", line 68, in Export_Data_To_Sheets
response_date = service.spreadsheets().values().update(
File "/Users/adaniels/PycharmProjects/EmailAutomation/venv/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper
return wrapped(*args, **kwargs)
File "/Users/adaniels/PycharmProjects/EmailAutomation/venv/lib/python3.8/site-packages/googleapiclient/http.py", line 840, in execute
raise HttpError(resp, content, uri=self.uri)
googleapiclient.errors.HttpError: <HttpError 400 when requesting https://sheets.googleapis.com/v4/spreadsheets/*******yAABhNJvARr9BzsVLvCYSfpcvQAmpWESL-DJI/values/%28%27ReportList%21%27%2C%20%27A1%3AE500%27%29?valueInputOption=RAW&alt=json returned "Invalid JSON payload received. Unexpected token.
House", "10/19/19", NaN, NaN, NaN], ["Co
^">
I already have two different credential files for different authentication scopes. I also deleted them and reauthenticated, so I'm not sure why I'm getting a 400 error.

Answer:
You need to specify the sheet you wish to update in the range portion of the request.
More Information:
service.open('SheetName').sheet is not a thing - open is not a method of the service, the Sheets service doesn't know at this point what Spreadsheet you are trying to update the values of.
As per the documentation, you need to specify the range of values you wish to update in the range parameter. This includes the Sheet name.
Code Snippet:
RANGE_AND_SHEET = "ReportList!" + RANGE_NAME
response_date = service.spreadsheets().values().update(
spreadsheetId=SPREADSHEET_ID,
range=RANGE_AND_SHEET,
valueInputOption='RAW',
body=value_range_body).execute()
References:
Method: spreadsheets.values.update | Sheets API | Google Developers

Related

Getting the root directory of a file in Google Drive using API

I need to get the full path of folders where a file is located in Google Drive. I'm getting the files themselves using the Google Drive API, but I need information about it's parent folders
I'm using the following code tothe the list of spreadsheets in a Shared Drive:
from googleapiclient import discovery
from httplib2 import Http
from oauth2client import file, client, tools
# Change the value of SCOPES to 'https://www.googleapis.com/auth/drive'
# if you want to be able to read and write to the user's Google Drive.
SCOPES = 'https://www.googleapis.com/auth/drive'
store = file.Storage('storage.json')
creds = store.get()
if not creds or creds.invalid:
flow = client.flow_from_clientsecrets('client_secret.json', SCOPES)
creds = tools.run_flow(flow, store)
DRIVE = discovery.build('drive', 'v3', http=creds.authorize(Http()))
folder_id = "1Z1GzY-D3I3qwQu3oxIW-L1a9nXgD0PXl"
query = "mimeType='application/vnd.google-apps.spreadsheet'"
query+= "and fullText contains 'CLAS' and trashed = false"
# query += " and parents in '" + folder_id + "'"
spreadsheets = []
# Initialize the page token
next_page_token = None
# Loop until all pages of results have been retrieved
while True:
# Execute the list request
response = DRIVE.files().list(
q=query,
corpora='drive',
includeItemsFromAllDrives=True,
driveId='0AEJNMySKcEzsUk9PVA',
supportsAllDrives=True,
# orderBy='folder',
pageSize=1000,
fields='nextPageToken, files(id, name, parents, mimeType, webViewLink)',
pageToken=next_page_token,
).execute()
# Append the results to the list
spreadsheets.extend(response.get('files', []))
# Check if there is another page of results
next_page_token = response.get('nextPageToken', None)
if next_page_token is None:
break
# Set the page token for the next iteration
# parameters['pageToken'] = next_page_token
# Print the number of results
print(f'Last spreadsheet found: {spreadsheets[-1]["name"]}. Number of spreadsheets: {len(spreadsheets)}')
This returns a list of dictionaries with the specified fields. I would like to know the names of the parent folders for each file, for which I'm trying:
from googleapiclient.errors import HttpError
for item in spreadsheets:
if 'parents' in item:
parent_folders_list = []
parent_id = item['parents'][0]
try:
while parent_id:
folder=DRIVE.files().get(fileId=parent_id, fields='name, id, parents').execute()
parent_folders_list.append(folder.get("parents", []))
if parent_id:
parent_id = parent_id[0]
except HttpError as error:
print('An error occurred: %s' % error)
print(f'{item["name"]} is in {parent_folders_list}')
And I've been able to identify that parent_id is correctly retrieved, and that I am able to access it, as I was able to open it in the browser. However, I get back errors 'File Not Found' for all parent_id. I wonder if the DRIVE.files().get(fileId=) is the correct way to get back a folder using the API.
Any help would be greatly appreciated.

Python3/Redis: redis.exceptions.ResponseError: unknown command 'JSON.SET'

I'm trying to run the sample program from this RedisLabs page.
I chose Option A - which was to set up the free Redis cloud server.
(Seems like if you install manually, then you have to add the JSON as a plugin.)
I'm able to connect and use other "set" commands, but getting error on JSON:
File "C:\Users\nwalt\.virtualenvs\TDAmeritradeGetQuotes\lib\site-packages\redis\client.py", line 901, in execute_command
return self.parse_response(conn, command_name, **options)
File "C:\Users\nwalt\.virtualenvs\TDAmeritradeGetQuotes\lib\site-packages\redis\client.py", line 915, in parse_response
response = connection.read_response()
File "C:\Users\nwalt\.virtualenvs\TDAmeritradeGetQuotes\lib\site-packages\redis\connection.py", line 756, in read_response
raise response
redis.exceptions.ResponseError: unknown command 'JSON.SET'
My Python test program (except put in the sample endpoint before posting):
import redis
import json
import pprint
host_info = "redis.us-east-1-1.ec2.cloud.redislabs.com"
redisObj = redis.Redis(host=host_info, port=18274, password='xxx')
print ("Normal call to Redis")
redisObj.set('foo', 'bar')
value = redisObj.get('foo')
print(value)
capitals = {
"Lebanon": "Beirut",
"Norway": "Oslo",
"France": "Paris"
}
print ("capitals - before call to Redis")
pprint.pprint(capitals)
print("JSON call to Redis")
redisObj.execute_command('JSON.SET', 'doc', '.', json.dumps(capitals))
print("Data Saved, now fetch data back from redis")
reply = json.loads(redisObj.execute_command('JSON.GET', 'doc'))
print("reply from Redis get")
pprint.pprint(reply)
This is the screen shot from their website where I created the database. I didn't see any option to enable JSON or add any modules.
Not sure this was available when I created the REDIS database, but it is now. When you create it on redislabs.com, you can turn on the modules, and pick one from the list.
Then use this library: "rejson" from https://pypi.org/project/rejson/ to get the method "jsonset" method, using such code such as this:
rj = Client(host=config_dict['REDIS_CONFIG_HOST'], port=config_dict['REDIS_CONFIG_PORT'], password=config_dict['REDIS_CONFIG_PASSWORD'], decode_responses=True)
out_doc = {}
out_doc['firstname'] = "John"
out_doc['lastname'] = "Doe"
rj.jsonset('config', Path.rootPath(), out_doc)
get_doc = rj.jsonget('config', Path.rootPath())
pprint.pprint(get_doc)
I'm not used the cloud redis, in my local the Python don't load the JSON.SET
I just so make done, in this sample https://onelinerhub.com/python-redis/save-json-to-redis

I want to get the excel file from the data frame created which automatically changes as written in the code

i have tried two methods and both showing different location as given by me in this image
apikey='abcd'
import pandas as pd
from alpha_vantage.timeseries import TimeSeries
import time
ts=TimeSeries(key=apikey,output_format='pandas')
data,metadata=ts.get_intraday(symbol='name',interval='1min',outputsize='full')
data
while True:
data, metadata=ts.get_intraday(symbol='TCS',interval='1min',outputsize='full')
data.to_excel('livedat.xlsx')
time.sleep(60)
The code is running properly but I don't know how to get the data file in excel.
imp- the method should get the file which is updated timely i.e 1min automaticaly.
Also i am using IBM watson studio to write the code.
I am not familiar with the alpha_vantage wrapper that you are using however this is how i would perform your question. The code works and i have included comments.
To get the file in the python script i would do pd.read_excel(filepath).
import requests
import pandas as pd
import time
import datetime
# Your API KEY and the URL we will request from
API_KEY = "YOUR API KEY"
url = "https://www.alphavantage.co/query?"
def Generate_file(symbol="IBM", interval="1min"):
# URL parameters
parameters = {"function": "TIME_SERIES_INTRADAY",
"symbol": symbol,
"interval": interval,
"apikey": API_KEY,
"outputsize": "compact"}
# get the json response from AlphaVantage
response = requests.get(url, params=parameters)
data = response.json()
# filter the response to only get the time series data we want
time_series_interval = f"Time Series ({interval})"
prices = data[time_series_interval]
# convert the filtered reponse to a Pandas DataFrame
df = pd.DataFrame.from_dict(prices, orient="index").reset_index()
df = df.rename(columns={"index": time_series_interval})
# create a timestampe for our excel file. So that the file does not get overriden with new data each time.
current_time = datetime.datetime.now()
file_timestamp = current_time.strftime("%Y%m%d_%H.%M")
filename = f"livedat_{file_timestamp}.xlsx"
df.to_excel(filename)
# sent a limit on the number of calls we make to prevent infinite loop
call_limit = 3
number_of_calls = 0
while(number_of_calls < call_limit):
Generate_file() # our function
number_of_calls += 1
time.sleep(60)

Use gspread and Google sheets API to copy a sheet to multiple workbooks

I have a google sheet file called "template". It has 2 worksheets. I have 100 other files called "Instance01", "Instance02", etc. which have 2 worksheets with the same name as the ones in "template".
I have created a new worksheet in "template", called "sheet3". I want to copy it to each of the 100 "Instance" documents, keeping the name "sheet3" in each Instance.
Can I achieve that with gspread? And how? If not, is there another way to do it programatically (or within reasonable time and effort)?
#Tanaike comment showed the way. I document the code I used below.
# Build the service and gspread agent
sheets_service = build('sheets', 'v4', credentials=credentials)
gc = gspread.authorize(credentials)
# Create functions to rename the sheet after copying
# For renaming sheets
# https://stackoverflow.com/questions/38074069/how-do-i-rename-a-worksheet-in-a-google-sheets-spreadsheet-using-the-api-in-py
def batch(requests, spreadsheetId, service):
body = {
'requests': requests
}
return service.spreadsheets().batchUpdate(spreadsheetId=spreadsheetId, body=body).execute()
def renameSheet(spreadsheetId, sheetId, newName, service):
return batch({
"updateSheetProperties": {
"properties": {
"sheetId": sheetId,
"title": newName,
},
"fields": "title",
}
},
spreadsheetId,
service)
# Now execute the copies
# The ID of the spreadsheet containing the sheet to copy. Everybody has access!!!
spreadsheet_id = ORIGINAL_spreadsheet_id # template workbook
# The ID of the sheet to copy. Everybody has access!!!
sheet_id = original_sheet_id # template sheet id
for workbook_id in COPY_TO: #COPY_TO is a list of workbooks ids
print(workbook_id)
destiny_spreadsheet_id = workbook_id
copy_sheet_to_another_spreadsheet_request_body = {
"destinationSpreadsheetId": destiny_spreadsheet_id
}
request = sheets_service.spreadsheets().sheets().copyTo(spreadsheetId=spreadsheet_id, sheetId=sheet_id, body=copy_sheet_to_another_spreadsheet_request_body)
response = request.execute()
# Rename it
copied_to_sheet = gc.open_by_key(destiny_spreadsheet_id) #template.worksheet(destiny)
sheet = copied_to_sheet.worksheet('Copy of ' + NAME_OF_SHEET_TO_BE_COPIED)
renameSheet(destiny_spreadsheet_id, sheet.id, NAME_OF_SHEET_TO_BE_COPIED, sheets_service)
As the method for copying the sheets, there is the method of "spreadsheets.sheets.copyTo" in Sheets API.
You can see a sample script at this thread. I thought that because the object of credentials can be used for both gspread and googleapiclient, it might help to implement this.
Reference:
spreadsheets.sheets.copyTo

web2py rest api endpoint gives invalid path output

I have made a web2py web application. The api endpoints exposed are as follows.
"/comments[comments]"
"/comments/id/{comments.id}"
"/comments/id/{comments.id}/:field"
"/comments/user-id/{comments.user_id}"
"/comments/user-id/{comments.user_id}/:field"
"/comments/date-commented/{comments.date_commented.year}"
"/comments/date-commented/{comments.date_commented.year}/:field"
"/comments/date-commented/{comments.date_commented.year}/{comments.date_commented.month}"
"/comments/date-commented/{comments.date_commented.year}/{comments.date_commented.month}/:field"
"/comments/date-commented/{comments.date_commented.year}/{comments.date_commented.month}/{comments.date_commented.day}"
"/comments/date-commented/{comments.date_commented.year}/{comments.date_commented.month}/{comments.date_commented.day}/:field"
"/comments/date-commented/{comments.date_commented.year}/{comments.date_commented.month}/{comments.date_commented.day}/{comments.date_commented.hour}"
"/comments/date-commented/{comments.date_commented.year}/{comments.date_commented.month}/{comments.date_commented.day}/{comments.date_commented.hour}/:field"
"/comments/date-commented/{comments.date_commented.year}/{comments.date_commented.month}/{comments.date_commented.day}/{comments.date_commented.hour}/{comments.date_commented.minute}"
"/comments/date-commented/{comments.date_commented.year}/{comments.date_commented.month}/{comments.date_commented.day}/{comments.date_commented.hour}/{comments.date_commented.minute}/:field"
"/comments/date-commented/{comments.date_commented.year}/{comments.date_commented.month}/{comments.date_commented.day}/{comments.date_commented.hour}/{comments.date_commented.minute}/{comments.date_commented.second}"
"/comments/date-commented/{comments.date_commented.year}/{comments.date_commented.month}/{comments.date_commented.day}/{comments.date_commented.hour}/{comments.date_commented.minute}/{comments.date_commented.second}/:field"
"/comments/complaint-id/{comments.complaint_id}"
"/comments/complaint-id/{comments.complaint_id}/:field"
The comments model is as follows
models/db.py
db.define_table(
'comments',
Field('user_id', db.auth_user),
Field('comment_made', 'string', length=2048),
Field('date_commented', 'datetime', default=datetime.now),
Field('complaint_id', db.complaints),
Field('detailed_status', 'string', length=2048),
)
I have been successful in retriving a single comment via the following request
localhost:8000/api/comments/id/1.json
Now I wish to retrieve all the comments. I am not able to figure out how to use /comments[comments] to retrieve all comments.?
I have tried
localhost:8000/api/comments.json
But it gives an output with "invalid path"
I have realized requests such as http://localhost:8000/api/comments/complaint-id/1.json
also give "invalid path" as output.
Please help.
EDIT:
Controllers/default.py
#request.restful()
def api():
response.view='generic.' + request.extension
def GET(*args,**kargs):
patterns='auto'
parser = db.parse_as_rest(patterns,args,kargs)
if parser.status == 200:
return dict(content=parser.response)
else:
raise HTTP(parser.status,parser.error)
def POST(*args,**kargs):
return dict()
return locals()
routes.py in the main web2py folder to change the default application:
routers = dict(
BASE = dict(
default_application='GRS',
)
)
Another observation:
I added another endpoint as below:
def comm():
"""" Comments api substitute"""
rows=db().select(db.comments.ALL) ## this line shows error
# rows = db(db.comments.id > 0).select()
#rows=[[5,6],[3,4],[1,2]]
#for row in rows:
# print row.id
return dict(message=rows)
Even now I am not able to retrieve all comments with "/comm.json". This gives a web2py error ticket which says "need more than 1 value to unpack" on the line "rows=db.select(db.comments.ALL)". Are the above invalid path and this error related in someway?