Has anyone experienced this error before when trying to connect to hive.
Sample code used (https://github.com/telefonicaid/fiware-cygnus/blob/master/cygnus-ngsi/resources/hiveclients/python/hiveserver2-client.py):
import sys
import pyhs2
from pyhs2.error import Pyhs2Exception
# get the input parameters
if len(sys.argv) != 6:
print 'Usage: python hiveserver2-client.py <hive_host> <hive_port> <db_name> <hadoop_user> <hadoop_password>'
sys.exit()
hiveHost = sys.argv[1]
hivePort = sys.argv[2]
dbName = sys.argv[3]
hadoopUser = sys.argv[4]
hadoopPassword = sys.argv[5]
# do the connection
with pyhs2.connect(host=hiveHost,
port=hivePort,
authMechanism="PLAIN",
user=hadoopUser,
password=hadoopPassword,
database=dbName) as conn:
# get a client
with conn.cursor() as client:
# create a loop attending HiveQL queries
while (1):
query = raw_input('remotehive> ')
try:
if not query:
continue
if query == 'exit':
sys.exit()
# execute the query
client.execute(query)
# get the content
for row in client.fetch():
print row
except Pyhs2Exception, ex:
print ex.errorMessage
Error displayed:
[centos#test]$ sudo python hiveserver2-client.py computing.cosmos.lab.fiware.org 10000 default USERNAME TOKEN
Traceback (most recent call last):
File "hiveserver2-client.py", line 42, in <module>
database=dbName) as conn:
File "/usr/lib/python2.7/site-packages/pyhs2/__init__.py", line 7, in connect
return Connection(*args, **kwargs)
File "/usr/lib/python2.7/site-packages/pyhs2/connections.py", line 46, in __init__
transport.open()
File "/usr/lib/python2.7/site-packages/pyhs2/cloudera/thrift_sasl.py", line 74, in open
status, payload = self._recv_sasl_message()
File "/usr/lib/python2.7/site-packages/pyhs2/cloudera/thrift_sasl.py", line 92, in _recv_sasl_message
header = self._trans.readAll(5)
File "/usr/lib/python2.7/site-packages/thrift/transport/TTransport.py", line 60, in readAll
chunk = self.read(sz - have)
File "/usr/lib/python2.7/site-packages/thrift/transport/TSocket.py", line 132, in read
message='TSocket read 0 bytes')
thrift.transport.TTransport.TTransportException: TSocket read 0 bytes
can you post your piece of code ? This looks like some auth mechanism or credentials sent are not Valid.
authMechanism= can be "PLAIN" or "KERBEROS" as per your setup .
Related
I'm getting an error when querying BigQuery from Python using end-user authentication
It works successfully with service account authentication, but fails with end-user authentication.
I am essentially following these instructions https://cloud.google.com/docs/authentication/end-user
The error message is:
ProjectId and DatasetId must be non-empty
I am stumped. Using service account authentication returns the expected data, so it appears to be an authentication related issue, but the authentication step appears to be successful.
Details
from google_auth_oauthlib import flow
from google.cloud import bigquery
appflow = flow.InstalledAppFlow.from_client_secrets_file(
"client_secrets.json", scopes=["https://www.googleapis.com/auth/bigquery"])
appflow.run_local_server()
credentials = appflow.credentials
client = bigquery.Client(project='MyProject', credentials=credentials)
query_string = """SELECT name, SUM(number) as total
FROM `bigquery-public-data.usa_names.usa_1910_current`
WHERE name = 'William'
GROUP BY name;
"""
query_job = client.query(query_string)
for row in query_job.result():
print("{}: {}".format(row["name"], row["total"]))
gives the following errors:
Please visit this URL to authorize this application: https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=...
Traceback (most recent call last):
File "C:\python\test\bqtest3.py", line 15, in <module>
query_job = client.query(query_string)
File "C:\Users\me\AppData\Local\Programs\Python\Python310\lib\site-packages\google\cloud\bigquery\client.py", line 3331, in query
return _job_helpers.query_jobs_insert(
File "C:\Users\me\AppData\Local\Programs\Python\Python310\lib\site-packages\google\cloud\bigquery\_job_helpers.py", line 114, in query_jobs_insert
future = do_query()
File "C:\Users\me\AppData\Local\Programs\Python\Python310\lib\site-packages\google\cloud\bigquery\_job_helpers.py", line 91, in do_query
query_job._begin(retry=retry, timeout=timeout)
File "C:\Users\me\AppData\Local\Programs\Python\Python310\lib\site-packages\google\cloud\bigquery\job\query.py", line 1298, in _begin
super(QueryJob, self)._begin(client=client, retry=retry, timeout=timeout)
File "C:\Users\me\AppData\Local\Programs\Python\Python310\lib\site-packages\google\cloud\bigquery\job\base.py", line 510, in _begin
api_response = client._call_api(
File "C:\Users\me\AppData\Local\Programs\Python\Python310\lib\site-packages\google\cloud\bigquery\client.py", line 756, in _call_api
return call()
File "C:\Users\me\AppData\Local\Programs\Python\Python310\lib\site-packages\google\api_core\retry.py", line 283, in retry_wrapped_func
return retry_target(
File "C:\Users\me\AppData\Local\Programs\Python\Python310\lib\site-packages\google\api_core\retry.py", line 190, in retry_target
return target()
File "C:\Users\me\AppData\Local\Programs\Python\Python310\lib\site-packages\google\cloud\_http\__init__.py", line 494, in api_request
raise exceptions.from_http_response(response)
google.api_core.exceptions.BadRequest: 400 POST https://bigquery.googleapis.com/bigquery/v2/projects/MyProject/jobs?prettyPrint=false: ProjectId and DatasetId must be non-empty
Location: None
Job ID: 9eba1ce9-971a-4495-825a-728aed28fc98
Please add python env to the script like below(marked in bold).
#!/usr/bin/env python
from google_auth_oauthlib import flow
from google.cloud import bigquery
appflow = flow.InstalledAppFlow.from_client_secrets_file("client_secrets.json", scopes=["https://www.googleapis.com/auth/bigquery"])
appflow.run_local_server()
credentials = appflow.credentials
client = bigquery.Client(project='MyProject', credentials=credentials)
query_job = client.query(
"""
SELECT name, SUM(number) as total
FROM bigquery-public-data.usa_names.usa_1910_current
WHERE name = 'William'
GROUP BY name;
"""
)
for row in query_job.result():
print("{}: {}".format(row["name"], row["total"]))
I'm writing discord bot now, so I wanted to know how to save the name of the streamer in a separate file, so when he goes live bot sends a message about that in specific channel.
This is what I tried:
import os
import json
import discord
import requests
from discord.ext import tasks, commands
from twitchAPI.twitch import Twitch
from discord.utils import get
intents = discord.Intents.all()
bot = commands.Bot(command_prefix='$', intents=intents)
Authentication with twitch API:
client_id = os.getenv('client_id')
client_secret = os.getenv('Dweller_token')
twitch = Twitch(client_id, client_secret)
twitch.authenticate_app([])
TWITCH_STREAM_API_ENDPOINT_V5 = "https://api.twitch.tv/dweller/streams/{}"
API_HEADERS = {
'Client-ID': client_id,
'Accept': 'application/vnd.twitchtv.v5+json',
}
Returns True if online, False if not:
def checkuser(user):
try:
userid = twitch.get_users(logins=[user])['data'][0]['id']
url = TWITCH_STREAM_API_ENDPOINT_V5.format(userid)
try:
req = requests.Session().get(url, headers=API_HEADERS)
jsondata = req.json()
if 'stream' in jsondata:
if jsondata['stream'] is not None:
return True
else:
return False
except Exception as e:
print("Error checking user: ", e)
return False
except IndexError:
return False
Bot event. Always checks if streamer is live. Sends a message if so. And adds specific role to the streamer if he is live:
#bot.event
async def on_ready():
# Defines a loop that will run every 10 seconds (checks for live users every 10 seconds).
#tasks.loop(seconds=10)
async def live_notifs_loop():
# Opens and reads the json file
with open('streamers.json', 'r') as file:
streamers = json.loads(file.read())
# Makes sure the json isn't empty before continuing.
if streamers is not None:
# Gets the guild, 'twitch streams' channel, and streaming role.
guild = bot.get_guild(1234567890)
channel = bot.get_channel(1234567890)
role = get(guild.roles, id=1234567890)
# Loops through the json and gets the key,value which in this case is the user_id and twitch_name of
# every item in the json.
for user_id, twitch_name in streamers.items():
# Takes the given twitch_name and checks it using the checkuser function to see if they're live.
# Returns either true or false.
status = checkuser(twitch_name)
# Gets the user using the collected user_id in the json
user = bot.get_user(int(user_id))
# Makes sure they're live
if status is True:
# Checks to see if the live message has already been sent.
async for message in channel.history(limit=200):
# If it has, break the loop (do nothing).
if str(user.mention) in message.content and "is now streaming" in message.content:
break
# If it hasn't, assign them the streaming role and send the message.
else:
# Gets all the members in your guild.
async for member in guild.fetch_members(limit=None):
# If one of the id's of the members in your guild matches the one from the json and
# they're live, give them the streaming role.
if member.id == int(user_id):
await member.add_roles(role)
# Sends the live notification to the 'twitch streams' channel then breaks the loop.
await channel.send(
f":red_circle: **LIVE**\n{user.mention} is now streaming on Twitch!"
f"\nhttps://www.twitch.tv/{twitch_name}")
print(f"{user} started streaming. Sending a notification.")
break
# If they aren't live do this:
else:
# Gets all the members in your guild.
async for member in guild.fetch_members(limit=None):
# If one of the id's of the members in your guild matches the one from the json and they're not
# live, remove the streaming role.
if member.id == int(user_id):
await member.remove_roles(role)
# Checks to see if the live notification was sent.
async for message in channel.history(limit=200):
# If it was, delete it.
if str(user.mention) in message.content and "is now streaming" in message.content:
await message.delete()
# Start your loop.
live_notifs_loop.start()
Command that adds 'chosen' streamers to json file:
# Command to add Twitch usernames to the json.
#bot.command(name='addtwitch', help='Adds your Twitch to the live notifs.', pass_context=True)
async def add_twitch(ctx, twitch_name):
# Opens and reads the json file.
with open('streamers.json', 'r') as file:
streamers = json.loads(file.read())
# Gets the users id that called the command.
user_id = ctx.author.id
# Assigns their given twitch_name to their discord id and adds it to the streamers.json.
streamers[user_id] = twitch_name
# Adds the changes we made to the json file.
with open('streamers.json', 'w') as file:
file.write(json.dumps(streamers))
# Tells the user it worked.
await ctx.send(f"Added {twitch_name} for {ctx.author} to the notifications list.")
print('Server Running')
bot.run(os.getenv('token'))
I want to write '$add_twitch turb4ik' and bot saves streamer turb4ik in streamers.json and checks if streamer is live or not. If True send notification in specific channel. But it doesn't seem to work.
And I get this syntax error:
Unhandled exception in internal background task 'live_notifs_loop'.
Traceback (most recent call last):
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/tasks/__init__.py", line 101, in _loop
await self.coro(*args, **kwargs)
File "main.py", line 62, in live_notifs_loop
streamers = json.loads(file.read())
File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Ignoring exception in command None:
discord.ext.commands.errors.CommandNotFound: Command "add_twitch" is not found
I also tried this piece of code, it gives me the information about the channel, but it doesn't give me streamer status:
client_id = os.getenv('client_id')
oauth_token = os.getenv('Dweller_token')
twitch = Twitch(client_id, oauth_token)
twitch.authenticate_app([])
user_info = twitch.get_users(logins=['turb4ik'])
user_id = user_info['data'][0]['id']
print(user_info)
And there is one more problem: every time I start my bot it says that twitchAPI is not installed and I need to install it every time I start my bot. Sometimes my bot seems to forget about twitchAPI and goes offline and says that I again need to install twitchAPI.
I know this is hard, but please help me. Maybe I should do it witch SQL(sqlite3 library) or so. Much obliget!
Edit:
Another syntax error:
Unhandled exception in internal background task 'live_notifs_loop'.
Traceback (most recent call last):
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/tasks/__init__.py", line 101, in _loop
await self.coro(*args, **kwargs)
File "main.py", line 78, in live_notifs_loop
streamers = json.load(file)
File "/usr/lib/python3.8/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Ignoring exception in command addtwitch:
Traceback (most recent call last):
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/commands/core.py", line 85, in wrapped
ret = await coro(*args, **kwargs)
File "main.py", line 136, in add_twitch
streamers = json.load(file)
File "/usr/lib/python3.8/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/commands/bot.py", line 902, in invoke
await ctx.command.invoke(ctx)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/commands/core.py", line 864, in invoke
await injected(*ctx.args, **ctx.kwargs)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/commands/core.py", line 94, in wrapped
raise CommandInvokeError(exc) from exc
discord.ext.commands.errors.CommandInvokeError: Command raised an exception: JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Unhandled exception in internal background task 'live_notifs_loop'.
Traceback (most recent call last):
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/tasks/__init__.py", line 101, in _loop
await self.coro(*args, **kwargs)
File "main.py", line 62, in live_notifs_loop
streamers = json.loads(file.read())
File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Instead of streamers = json.loads(file.read()) use streamers = json.load(file)
discord.ext.commands.errors.CommandNotFound: Command "add_twitch" is not found
Since you are defining your command with name=addtwitch agrument, you can only call your command with $addtwitch user. To avoid this, add the aliases =['add_twitch'] argument to
#bot.command(name='addtwitch', help='Adds your Twitch to the live notifs.', pass_context=True, aliases =['add_twitch'])
I'm running a Python script to query an AWS-S3 bucket using the AWS-S3-Select tool. I'm importing a variable from a txt file and want to pass it into the S3-Select query. I also want to count all imported variable recurrences (within a specified column) by querying the entire S3 directory instead of just a single file.
This is what I have so far:
import boto3
from boto3.session import Session
with open('txtfile.txt', 'r') as myfile:
variable = myfile.read()
ACCESS_KEY='accessKey'
SECRET_KEY='secredtKey'
session = Session(aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY)
s3b = session.client('s3')
r = s3b.select_object_content(
Bucket='s3BucketName',
Key='directory/fileName',
ExpressionType='SQL',
Expression="'select count(*)from S3Object s where s.columnName = %s;', [variable]",
InputSerialization={'CSV': {"FileHeaderInfo": "Use"}},
OutputSerialization={'CSV': {}},
)
for event in r['Payload']:
if 'Records' in event:
records = event['Records']['Payload'].decode('utf-8')
print(records)
elif 'Stats' in event:
statsDetails = event['Stats']['Details']
print("Stats details bytesScanned: ")
When I run this script I get back the following error:
Traceback (most recent call last):
File "s3_query.py", line 20, in <module>
OutputSerialization={'CSV': {}},
File "/root/anaconda3/lib/python3.6/site-packages/botocore/client.py", line 314, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/root/anaconda3/lib/python3.6/site-packages/botocore/client.py", line 612, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (ParseUnexpectedToken) when calling the SelectObjectContent operation: Unexpected token found COMMA:',' at line 1, column 67.
This line looks quite strange:
Expression="'select count(*)from S3Object s where s.columnName = %s;', [variable]"
That is not normal SQL or Python syntax.
You should probably use:
Expression='select count(*)from S3Object s where s.columnName = %s;' % [variable]
I have a function that runs 3 queries and returns the result of the last (using the previous ones to create the last) when I get to the 3rd query, it get a list index our of range error. I have ran this exact query as the first query (with manually entered variables) and it worked fine.
This is my code:
import pypyodbc
def sql_conn():
conn = pypyodbc.connect(r'Driver={SQL Server};'
r'Server=HPSQL31\ni1;'
r'Database=tq_hp_prod;'
r'Trusted_Connection=yes;')
cursor = conn.cursor()
return conn, cursor
def get_number_of_jobs(ticket):
# Get Connection
conn, cursor = sql_conn()
# Get asset number
sqlcommand = "select top 1 item from deltickitem where dticket = {} and cat_code = 'Trq sub'".format(ticket)
cursor.execute(sqlcommand)
asset = cursor.fetchone()[0]
print(asset)
# Get last MPI date
sqlcommand = "select last_test from prevent where item = {} and description like '%mpi'".format(asset)
cursor.execute(sqlcommand)
last_recal = cursor.fetchone()[0]
print(last_recal)
# Get number of jobs since last recalibration
sqlcommand = """select count(i.item)
from deltickhdr as d
join deltickitem as i
on d.dticket = i.dticket
where i.start_rent >= '2017-03-03 00:00:00'
and i.meterstart <> i.meterstop
and i.item = '002600395'""" #.format(last_recal, asset)
cursor.execute(sqlcommand)
num_jobs = cursor.fetchone()[0]
print(num_jobs)
cursor.close()
conn.close()
return num_jobs
ticketnumber = 14195 # int(input("Ticket: "))
get_number_of_jobs(ticketnumber)
Below is the error(s) i get when i get to the 3rd cursor.execute(sqlcommand)
Traceback (most recent call last):
File "C:\Program Files\JetBrains\PyCharm Community Edition 2016.3.2\helpers\pydev\pydevd.py", line 1596, in <module>
globals = debugger.run(setup['file'], None, None, is_module)
File "C:\Program Files\JetBrains\PyCharm Community Edition 2016.3.2\helpers\pydev\pydevd.py", line 974, in run
pydev_imports.execfile(file, globals, locals) # execute the script
File "C:\Program Files\JetBrains\PyCharm Community Edition 2016.3.2\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "C:/Users/bdrillin/PycharmProjects/Torque_Turn_Data/tt_sub_ui.py", line 56, in <module>
get_number_of_jobs(ticketnumber)
File "C:/Users/bdrillin/PycharmProjects/Torque_Turn_Data/tt_sub_ui.py", line 45, in get_number_of_jobs
cursor.execute(sqlcommand)
File "C:\ProgramData\Anaconda3\lib\site-packages\pypyodbc.py", line 1470, in execute
self._free_stmt(SQL_CLOSE)
File "C:\ProgramData\Anaconda3\lib\site-packages\pypyodbc.py", line 1994, in _free_stmt
check_success(self, ret)
File "C:\ProgramData\Anaconda3\lib\site-packages\pypyodbc.py", line 1007, in check_success
ctrl_err(SQL_HANDLE_STMT, ODBC_obj.stmt_h, ret, ODBC_obj.ansi)
File "C:\ProgramData\Anaconda3\lib\site-packages\pypyodbc.py", line 972, in ctrl_err
state = err_list[0][0]
IndexError: list index out of range
Any help would be great
I've had the same error.
Even though I haven't come to the definite conclusion about what this error means I thought my guessing might help anyone else ending up here.
In my case, the problem was a conflict with a datatype length (NVARCHAR(24) and CHAR(10)).
So I guess this IndexError in ctrl_err function just means there is an error in your SQL code that pypyodbc does not know how to handle.
I know this is not much of an answer, but I know it would have saved me a couple of hours had I known this was not some bug in pypyodbc but an inconsistency in the data I was inserting.
Kind regards,
Luka
I have the following wlst script:
import wlstModule
from com.bea.wli.sb.management.configuration import SessionManagementMBean
from com.bea.wli.sb.management.configuration import ALSBConfigurationMBean
from com.bea.wli.config import Ref
#=======================================================================================
# Utility function to read a binary file
#=======================================================================================
def readBinaryFile(fileName):
file = open(fileName, 'rb')
bytes = file.read()
return bytes
#=======================================================================================
# Utility function to create an arbitrary session name
#=======================================================================================
def createSessionName():
sessionName = String("SessionScript"+Long(System.currentTimeMillis()).toString())
return sessionName
def getSessionManagementMBean(sessionName):
SessionMBean = findService("SessionManagement", "com.bea.wli.sb.management.configuration.SessionManagementMBean")
SessionMBean.createSession(sessionName)
return SessionMBean
SessionMBean = None
importJar='C:\\OSB_PROJECT.jar'
theBytes = readBinaryFile(importJar)
sessionName = createSessionName()
SessionMBean = getSessionManagementMBean(sessionName)
The result is an error:
wls:/offline> execfile('C:\script.py') Traceback (innermost last):
File "", line 1, in ? File "C:\script.py", line 31, in ?
File "C:\script.py", line 22, in get SessionManagementMBean
NameError: findService
How can I fix this?
Are you ever connecting to your server and accessing the domain runtime? You should be doing something like the following:
connect("weblogic", "weblogic", "t3://localhost:7001")
domainRuntime()
# obtain session management mbean to create a session.
# This mbean instance can be used more than once to
# create/discard/commit many sessions
sessionMBean = findService(SessionManagementMBean.NAME,SessionManagementMBean.TYPE)
See more here:
http://docs.oracle.com/cd/E13171_01/alsb/docs25/javadoc/com/bea/wli/sb/management/configuration/SessionManagementMBean.html