How to solve the consumer error in Rabbitmq using python - rabbitmq

when I run Consumer.py for headersExchange in rabbitmq using python,it is getting error like below
I have mentioned consumer and publish program below
Traceback (most recent call last):
File "headersConsumer.py", line 32, in <module>
main()
File "headersConsumer.py", line 14, in main
channel.exchange_declare(exchange = 'headers_logs',exchange_type='headers',durable=True)
File "C:\Python38\lib\site-packages\pika\adapters\blocking_connection.py", line 2387, in
exchange_declare
self._flush_output(declare_ok_result.is_ready)
File "C:\Python38\lib\site-packages\pika\adapters\blocking_connection.py", line 1339, in
_flush_output
raise self._closing_reason # pylint: disable=E0702
pika.exceptions.ChannelClosedByBroker: (406, "PRECONDITION_FAILED
- inequivalent arg 'type' for exchange 'headers_logs' in vhost '/': received 'headers' but
current is 'fanout'")
I have written consumer code like this
#!/usr/bin/env python
import pika, sys, os
def main():
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.exchange_declare(exchange = 'headers_logs',exchange_type='headers',durable=True)
channel.queue_declare(queue = "HeaderQueue1", durable=True)
channel.queue_bind(exchange = 'headers_logs', queue="HeadersQueue1", routing_key='',
arguments={'x-match': 'any', 'key1': 'one', 'key2': 'two'})
def callback(ch, method, properties, body):
print(" [x] %r" % body.decode())
print(' [*] Waiting for logs. To exit press CTRL+C')
channel.basic_consume(
queue="HeadersQueue1", on_message_callback=callback, auto_ack=True)
channel.start_consuming()
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
print('Interrupted')
try:
sys.exit(0)
except SystemExit:
os._exit(0)
I have written publish program likethis
import pika
import sys
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.exchange_declare(exchange='headers_logs',exchange_type='headers')
message = ' '.join(sys.argv[1:]) or "Hello World!"
channel.basic_publish((exchange='headers_logs',routing_key="",body=message,properties=pika.BasicProperties(
delivery_mode = 2, # make message persistent
headers = {'key1':'one', 'key2': 'three'}
))
print(" [x] Sent %r" % message)
connection.close()
I am not understanding this error,Can anyone please suggest this error

PRECONDITION_FAILED means that you declared an exchange with set of parameters then you are trying to create the same queue name using different parameters.
in your case:
headers_logs' in vhost '/': received 'headers' but
current is 'fanout'")
so you are trying to change the exchange type from fanout to headers
Se here for more detail (this is for the queues but exchanges work in the same way).
Before a queue can be used it has to be declared. Declaring a queue
will cause it to be created if it does not already exist. The
declaration will have no effect if the queue does already exist and
its attributes are the same as those in the declaration. When the
existing queue attributes are not the same as those in the declaration
a channel-level exception with code 406 (PRECONDITION_FAILED) will be
raised.

Related

How to know if twitch streamer is live and send message about that? Discord.py

I'm writing discord bot now, so I wanted to know how to save the name of the streamer in a separate file, so when he goes live bot sends a message about that in specific channel.
This is what I tried:
import os
import json
import discord
import requests
from discord.ext import tasks, commands
from twitchAPI.twitch import Twitch
from discord.utils import get
intents = discord.Intents.all()
bot = commands.Bot(command_prefix='$', intents=intents)
Authentication with twitch API:
client_id = os.getenv('client_id')
client_secret = os.getenv('Dweller_token')
twitch = Twitch(client_id, client_secret)
twitch.authenticate_app([])
TWITCH_STREAM_API_ENDPOINT_V5 = "https://api.twitch.tv/dweller/streams/{}"
API_HEADERS = {
'Client-ID': client_id,
'Accept': 'application/vnd.twitchtv.v5+json',
}
Returns True if online, False if not:
def checkuser(user):
try:
userid = twitch.get_users(logins=[user])['data'][0]['id']
url = TWITCH_STREAM_API_ENDPOINT_V5.format(userid)
try:
req = requests.Session().get(url, headers=API_HEADERS)
jsondata = req.json()
if 'stream' in jsondata:
if jsondata['stream'] is not None:
return True
else:
return False
except Exception as e:
print("Error checking user: ", e)
return False
except IndexError:
return False
Bot event. Always checks if streamer is live. Sends a message if so. And adds specific role to the streamer if he is live:
#bot.event
async def on_ready():
# Defines a loop that will run every 10 seconds (checks for live users every 10 seconds).
#tasks.loop(seconds=10)
async def live_notifs_loop():
# Opens and reads the json file
with open('streamers.json', 'r') as file:
streamers = json.loads(file.read())
# Makes sure the json isn't empty before continuing.
if streamers is not None:
# Gets the guild, 'twitch streams' channel, and streaming role.
guild = bot.get_guild(1234567890)
channel = bot.get_channel(1234567890)
role = get(guild.roles, id=1234567890)
# Loops through the json and gets the key,value which in this case is the user_id and twitch_name of
# every item in the json.
for user_id, twitch_name in streamers.items():
# Takes the given twitch_name and checks it using the checkuser function to see if they're live.
# Returns either true or false.
status = checkuser(twitch_name)
# Gets the user using the collected user_id in the json
user = bot.get_user(int(user_id))
# Makes sure they're live
if status is True:
# Checks to see if the live message has already been sent.
async for message in channel.history(limit=200):
# If it has, break the loop (do nothing).
if str(user.mention) in message.content and "is now streaming" in message.content:
break
# If it hasn't, assign them the streaming role and send the message.
else:
# Gets all the members in your guild.
async for member in guild.fetch_members(limit=None):
# If one of the id's of the members in your guild matches the one from the json and
# they're live, give them the streaming role.
if member.id == int(user_id):
await member.add_roles(role)
# Sends the live notification to the 'twitch streams' channel then breaks the loop.
await channel.send(
f":red_circle: **LIVE**\n{user.mention} is now streaming on Twitch!"
f"\nhttps://www.twitch.tv/{twitch_name}")
print(f"{user} started streaming. Sending a notification.")
break
# If they aren't live do this:
else:
# Gets all the members in your guild.
async for member in guild.fetch_members(limit=None):
# If one of the id's of the members in your guild matches the one from the json and they're not
# live, remove the streaming role.
if member.id == int(user_id):
await member.remove_roles(role)
# Checks to see if the live notification was sent.
async for message in channel.history(limit=200):
# If it was, delete it.
if str(user.mention) in message.content and "is now streaming" in message.content:
await message.delete()
# Start your loop.
live_notifs_loop.start()
Command that adds 'chosen' streamers to json file:
# Command to add Twitch usernames to the json.
#bot.command(name='addtwitch', help='Adds your Twitch to the live notifs.', pass_context=True)
async def add_twitch(ctx, twitch_name):
# Opens and reads the json file.
with open('streamers.json', 'r') as file:
streamers = json.loads(file.read())
# Gets the users id that called the command.
user_id = ctx.author.id
# Assigns their given twitch_name to their discord id and adds it to the streamers.json.
streamers[user_id] = twitch_name
# Adds the changes we made to the json file.
with open('streamers.json', 'w') as file:
file.write(json.dumps(streamers))
# Tells the user it worked.
await ctx.send(f"Added {twitch_name} for {ctx.author} to the notifications list.")
print('Server Running')
bot.run(os.getenv('token'))
I want to write '$add_twitch turb4ik' and bot saves streamer turb4ik in streamers.json and checks if streamer is live or not. If True send notification in specific channel. But it doesn't seem to work.
And I get this syntax error:
Unhandled exception in internal background task 'live_notifs_loop'.
Traceback (most recent call last):
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/tasks/__init__.py", line 101, in _loop
await self.coro(*args, **kwargs)
File "main.py", line 62, in live_notifs_loop
streamers = json.loads(file.read())
File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Ignoring exception in command None:
discord.ext.commands.errors.CommandNotFound: Command "add_twitch" is not found
I also tried this piece of code, it gives me the information about the channel, but it doesn't give me streamer status:
client_id = os.getenv('client_id')
oauth_token = os.getenv('Dweller_token')
twitch = Twitch(client_id, oauth_token)
twitch.authenticate_app([])
user_info = twitch.get_users(logins=['turb4ik'])
user_id = user_info['data'][0]['id']
print(user_info)
And there is one more problem: every time I start my bot it says that twitchAPI is not installed and I need to install it every time I start my bot. Sometimes my bot seems to forget about twitchAPI and goes offline and says that I again need to install twitchAPI.
I know this is hard, but please help me. Maybe I should do it witch SQL(sqlite3 library) or so. Much obliget!
Edit:
Another syntax error:
Unhandled exception in internal background task 'live_notifs_loop'.
Traceback (most recent call last):
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/tasks/__init__.py", line 101, in _loop
await self.coro(*args, **kwargs)
File "main.py", line 78, in live_notifs_loop
streamers = json.load(file)
File "/usr/lib/python3.8/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Ignoring exception in command addtwitch:
Traceback (most recent call last):
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/commands/core.py", line 85, in wrapped
ret = await coro(*args, **kwargs)
File "main.py", line 136, in add_twitch
streamers = json.load(file)
File "/usr/lib/python3.8/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/commands/bot.py", line 902, in invoke
await ctx.command.invoke(ctx)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/commands/core.py", line 864, in invoke
await injected(*ctx.args, **ctx.kwargs)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/commands/core.py", line 94, in wrapped
raise CommandInvokeError(exc) from exc
discord.ext.commands.errors.CommandInvokeError: Command raised an exception: JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Unhandled exception in internal background task 'live_notifs_loop'.
Traceback (most recent call last):
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/ext/tasks/__init__.py", line 101, in _loop
await self.coro(*args, **kwargs)
File "main.py", line 62, in live_notifs_loop
streamers = json.loads(file.read())
File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Instead of streamers = json.loads(file.read()) use streamers = json.load(file)
discord.ext.commands.errors.CommandNotFound: Command "add_twitch" is not found
Since you are defining your command with name=addtwitch agrument, you can only call your command with $addtwitch user. To avoid this, add the aliases =['add_twitch'] argument to
#bot.command(name='addtwitch', help='Adds your Twitch to the live notifs.', pass_context=True, aliases =['add_twitch'])

Communication between multiple processes with zmq

I am having n processes that have their own local data and actions and I want each process to send a "snapshot" of it's local data to the rest running nodes-processes.
The code that I have so far looks like that:
def node1():
Process(target=sync_1).start()
sleep(4)
data = {'node': 1, 'data': 'node 1 data'}
context_b = zmq.Context()
socket_b = context_b.socket(zmq.PUB)
connnected = False
try:
socket_b.bind("tcp://*:%s" % 5560)
connnected = True
except Exception as e:
print(e)
if connnected:
topic = "101"
try:
socket_b.send_string(topic + ' ' + json.dumps(data))
except Exception as e:
print(e)
socket_b.close()
context_b.term()
def node2():
Process(target=sync_2).start()
def sync_1():
context_c = zmq.Context()
socket_c = context_c.socket(zmq.SUB)
_port = 5560
try:
socket_c.connect("tcp://localhost:%s" % _port)
except Exception as e:
print(e)
topicfilter = "101"
socket_c.setsockopt_string(zmq.SUBSCRIBE, topicfilter, encoding='utf-8')
try:
raw = socket_c.recv().decode("utf-8")
json0 = raw.find('{')
topic = raw[0:json0].strip()
msg = json.loads(raw[json0:])
print("[SYNC 1] received {}-{}]".format(topic, msg))
except Exception as e:
print(e)
def sync_2():
context_c = zmq.Context()
socket_c = context_c.socket(zmq.SUB)
_port = 5560
try:
socket_c.connect("tcp://localhost:%s" % _port)
except Exception as e:
print(e)
topicfilter = "101"
socket_c.setsockopt_string(zmq.SUBSCRIBE, topicfilter, encoding='utf-8')
try:
raw = socket_c.recv().decode("utf-8")
json0 = raw.find('{')
topic = raw[0:json0].strip()
msg = json.loads(raw[json0:])
print("[SYNC 2] received {}-{}]".format(topic, msg))
except Exception as e:
print(e)
if __name__ == '__main__':
Process(target=node1).start()
Process(target=node2).start()
Each node has one "listener" process running at the background (the sync function) in order to receive each node data and use it accordingly, and it works fine when all the sub sockets are connected to one node (node 1 in that case) but I want each node to send data to all the listeners , so I am not sure how to implement that since the listener processes can connect to one port.
Also, the nodes will have to send the local data snapshot every time there is an update , so this cannot be an one time communication , therefore I thought of having listener processes actively waiting for updates all the times.
I believe a diagram could be useful for that problem:
There can be a way easier way to solve this issue , so any help would be highly appreciated!
Update:
The solution was to use the XPUB-XSUB pattern.
By using this pattern I created a proxy thread that allowed me to do exactly what I wanted.
The most useful example I could find for Python is this .

Wrote an errback for my Scrapy spider, but tracebacks also keep happening, why?

I am using Scrapy 1.1 and I call Scrapy from within a script. My spider launching method looks like this:
def run_spider(self):
runner = CrawlerProcess(get_project_settings())
spider = SiteSpider()
configure_logging()
d = runner.crawl(spider, websites_file=self.raw_data_file)
d.addBoth(lambda _: reactor.stop())
reactor.run()
Here is an extract of my spider with an errback written as in the documentation, but it only prints when catches a failure.
class SiteSpider(scrapy.Spider):
name = 'SiteCrawler'
custom_settings = {
'FEED_FORMAT': 'json',
'FEED_URI': 'result.json',
}
def __init__(self, websites_file=None, *args, **kwargs):
super().__init__(*args, **kwargs)
self.websites_file = websites_file
print('***********')
print(self.websites_file)
def start_requests(self):
.....
if is_valid_url(website_url):
yield scrapy.Request(url=website_url, callback=self.parse, errback=self.handle_errors, meta={'url': account_id})
def parse(self, response):
.....
yield item
def handle_errors(self, failure):
if failure.check(HttpError):
# these exceptions come from HttpError spider middleware
# you can get the non-200 response
response = failure.value.response
print('HttpError on ' + response.url)
elif failure.check(DNSLookupError):
# this is the original request
request = failure.request
print('DNSLookupError on ' + request.url)
elif failure.check(TimeoutError, TCPTimedOutError):
request = failure.request
print('TimeoutError on ' + request.url)
My problem is that I get errors I expect, like:
TimeoutError on http://www.example.com
But also get tracebacks for the same websites:
2016-08-05 13:40:55 [scrapy] ERROR: Error downloading <GET http://www.example.com/robots.txt>: TCP connection timed out: 60: Operation timed out.
Traceback (most recent call last):
File ".../anaconda/lib/python3.5/site-packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File ".../anaconda/lib/python3.5/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File ".../anaconda/lib/python3.5/site-packages/scrapy/core/downloader/middleware.py", line 43, in process_request
defer.returnValue((yield download_func(request=request,spider=spider)))
twisted.internet.error.TCPTimedOutError: TCP connection timed out: 60: Operation timed out.
The written exception handling messages and the tracebacks can often be traced to the same websites. After searching a lot on stackoverflow, in the docs and the likes I still dont know why I see the tracebacks.
This also occurs with DNSLookupErrors for example.
Excuse me, my Scrapy knowledge is juvenile. Is this normal behavior?
Also, I added this to settings.py, which is under my crawler. Other entires (for example the item_pipelines) most exactly work.
LOG_LEVEL = 'WARNING'
But I still see debug messages, not only warnings and everything above that. (if configure_logging() is added to the spider launch) I am running this from terminal on mac os x.
I would be very happy to get any help with this.
Try this in a script:
if __name__ == '__main__':
runner = CrawlerProcess(get_project_settings())
spider = SiteSpider()
configure_logging()
d = runner.crawl(spider, websites_file=self.raw_data_file)
d.addBoth(lambda _: reactor.stop())
reactor.run()

Twisted error in server with mqtt client

I'm running a Twisted TCP server. Each protocol instance has an mqtt pub/sub client. I've reduced the actual production code to the simplest possible form below. I've stripped out a lot of irrelevant complexity to simplify the bug-finding process. The server works for multiple client connections and produces the data received from the mqtt client, but after any client connects/disconnects/reconnects a few times I get an exception that I haven't been able to understand or trap. I'm hoping that Jean-Paul or someone can point me at the error of my ways.
Once the exception occurs, no new clients can connect to the server. Each new connect attempt produces the exception.
Clients that are already connected continue to receive data ok.
The exception is
Unhandled Error
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/twisted/python/log.py", line 73, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/usr/lib/python2.7/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python2.7/dist-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
File "/usr/lib/python2.7/dist-packages/twisted/internet/posixbase.py", line 614, in _doReadOrWrite
why = selectable.doRead()
--- <exception caught here> ---
File "/usr/lib/python2.7/dist-packages/twisted/internet/tcp.py", line 1069, in doRead
transport = self.transport(skt, protocol, addr, self, s, self.reactor)
File "/usr/lib/python2.7/dist-packages/twisted/internet/tcp.py", line 786, in __init__
self.startReading()
File "/usr/lib/python2.7/dist-packages/twisted/internet/abstract.py", line 429, in startReading
self.reactor.addReader(self)
File "/usr/lib/python2.7/dist-packages/twisted/internet/epollreactor.py", line 256, in addReader
_epoll.EPOLLIN, _epoll.EPOLLOUT)
File "/usr/lib/python2.7/dist-packages/twisted/internet/epollreactor.py", line 240, in _add
self._poller.modify(fd, flags)
exceptions.IOError: [Errno 2] No such file or directory
The basic server code is:
(this example will run and does generate the exception)
from twisted.internet import reactor
from twisted.internet.protocol import Factory
from twisted.protocols import basic
from paho.mqtt import client # The most recent version of the legacy Mosquitto client
from random import randint
class MsgReceiver(basic.LineReceiver):
def __init__(self, factory): # new (factory)
self.factory = factory # new (factory)
def connectionMade(self):
self.mqClient = self.startMQ()
if self.mqClient:
self.factory.clients.append(self)
else:
self.transport.loseConnection()
def connectionLost(self, reason):
pass
def lineReceived(self, line):
pass
def on_message(self, mosq, obj, msg):
try:
self.sendLine(msg.payload)
except Exception, err:
print(err.message)
def startMQ(self):
mqName = "-".join(["myAppName", str(randint(0, 99999))])
mqClient = client.Client(mqName)
if mqClient.connect("localhost", 1883, 60) != 0:
print('Could not connect to mq server')
return False
mqClient.on_message = self.on_message
(success, mid) = mqClient.subscribe("myTopic", 0)
if success != 0:
return False
mqClient.loop_start()
return mqClient
class MsgReceiverFactory(Factory):
allow_reuse_address = True
def __init__(self, clients):
self.clients = clients
def buildProtocol(self, addr):
return MsgReceiver(self)
if __name__ == "__main__":
try:
clients = []
reactor.listenTCP(43217, MsgReceiverFactory(clients))
reactor.run()
except Exception, err:
print(err.message)
if reactor.running:
reactor.stop()
A simple client that will induce the error when run twice (the first time it runs fine):
Interesting that if I enable the time.sleep(3) it runs fine and doesn't seem to induce the error
#!/usr/bin/python
from __future__ import print_function
import socket
import time
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("localhost", 43217))
data = s.recv(1024)
print(data)
#time.sleep(3)
s.close()

getting error Received unregistered task of type 'mytasks.add'

I have written a file mytasks.py
from celery import Celery
celery = Celery("tasks",
broker='redis://localhost:6379/0',
backend='redis')
#celery.task
def add(x,y):
return x+y
and task.py as follow
from mytasks import add
add.delay(1,1)
I have started redis server and I have started celery server. but when i m running task.py then i am getting the following error:
Received unregistered task of type 'mytasks.add'.
The message has been ignored and discarded.
Did you remember to import the module containing this task?
Or maybe you are using relative imports?
Please see http://bit.ly/gLye1c for more information.
The full contents of the message body was:
{'retries': 0, 'task': 'mytasks.add', 'eta': None, 'args': (1, 1), 'expires': None, 'callbacks': None, 'errbacks': None, 'kwargs': {}, 'id': 'a4792308-d575-4de4-8b67-26982cae2fa4', 'utc': True} (173b)
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/celery/worker/consumer.py", line 411, in on_task_received
strategies[name](message, body, message.ack_log_error)
KeyError: 'mytasks.add'
what may be the possibel reason
Hey I have solved the problem
i did one thing i add
CELERY_IMPORTS=("mytasks")
in my celeryconfig.py file and i got succeed.
also can use include param in Celery class: http://docs.celeryproject.org/en/latest/getting-started/next-steps.html#proj-celery-py