I have created a RabbitMQ 3-node cluster using three aws ec2 servers. I'm trying to access the quorum queue I created using celery. When I connect it gives the error
raise error_for_code(reply_code, reply_text,
amqp.exceptions.AMQPNotImplementedError: Basic.consume: (540) NOT_IMPLEMENTED - queue 'Replica_que' in vhost '/' does not support global qos
I suppose it will work if I disabled global qos but I couldn't find where I can do it. How do I disable global qos in celery?
my celery code
from celery import Celery
from time import sleep
import kombu
broker_uri=['amqp://xxxx:5672/', 'amqp://xxxx:5672/','amqp://xxx:5672/']
backend_uri="mongodb+srv://xxxxx"
app = Celery('TestApp', broker=broker_uri,backend=backend_uri)
app.config_from_object('celeryconfig')
app.conf.task_default_exchange='Replica_que'
app.conf.task_default_routing_key='Replica'
#app.task
def reverse(text):
sleep(10)
return text[:-1]
and the config code
from kombu import Queue
task_queues = [Queue(name="Replica_que", queue_arguments={"x-queue-type": "quorum"})]
task_routes = {
'tasks.add': 'Replica_que',
}
This was possible by adding a celeryconfig.py file,
from kombu import Queue
task_queues = [Queue(name="Replica_que", queue_arguments={"x-queue-type": "quorum"})]
task_routes = {
'tasks.add': 'Replica_que',
}
and creating a custom QoS class: https://github.com/celery/celery/issues/6067
So I added the QoS class
class NoChannelGlobalQoS(bootsteps.StartStopStep):
requires = {'celery.worker.consumer.tasks:Tasks'}
def start(self, c):
qos_global = False
c.connection.default_channel.basic_qos(0, c.initial_prefetch_count, qos_global)
def set_prefetch_count(prefetch_count):
return c.task_consumer.qos(
prefetch_count=prefetch_count,
apply_global=qos_global,
)
c.qos = QoS(set_prefetch_count, c.initial_prefetch_count)
app.steps['consumer'].add(NoChannelGlobalQoS)
Currently this is an issue in celery related to quorum queue but this works.
Related
I'm using Celery 4.4.7 with Redis as my message broker.
I want to use celery as a consumer only, as the external producer is a java application.
The java application pushes messages to a channel on redis. But my celery application is not picking up the messages.
I have simulated the java producer in python using redis-py (redis_producer.py) to publish to a channel. The redis_consumer.py is able to pickup the messages from the producer.
But my celery_consumer.py seems to be blind to these messages.
Messages from redis_producer.py is picked up by the redis_consumer.py, but not from celery.
Messages from kombu_producer.py is picked up by the celery worker, but not from my redis_consumer.py
redis_producer.py
import json
import redis
r = redis.Redis(host='localhost', port=6379)
for i in range(10):
body = {
'id': i,
'message': f'Hello {i}',
}
r.publish(channel='redis.test.topic', message=json.dumps(body))
redis_consumer.py
import json
import multiprocessing
import os
import signal
import redis
redis_conn = redis.Redis(charset='utf-8', decode_responses=True)
def sub(name: str):
pubsub = redis_conn.pubsub()
pubsub.subscribe('redis.test.topic')
for message in pubsub.listen():
print(message)
if message.get('type') == 'message':
data = message.get('data')
print('%s: %s' % (name, data))
def on_terminate(signum, stack):
wait_for_current_scp_operation()
if __name__ == '__main__':
multiprocessing.Process(target=sub, args=('consumer',)).start()
signal.signal(signal.SIGTERM, on_terminate)
celery_consumer.py
import os
import ssl
from celery import Celery
from celery import shared_task
from celery.utils.log import get_task_logger
from kombu import Exchange
from kombu import Queue
logger = get_task_logger(__name__)
# Celery broker Url
broker_url = os.environ.get('CELERY_BROKER_URL', None)
if broker_url is None:
broker_url = 'redis://localhost:6379/0'
# store to use to store task results, default=None
result_backend = os.environ.get('CELERY_RESULT_BACKEND', None)
if result_backend is None:
result_backend = 'redis://localhost:6379/0'
config = dict(
broker_url=broker_url,
result_backend=result_backend,
# maximum number of connections that can be open in the connection pool
broker_pool_limit=20,
broker_transport_options={
'visibility_timeout': 3600,
'confirm_publish': True,
},
# Serializer method
task_serializer='json',
result_serializer='json',
accept_content=['json'],
# Below two settings just reserves one task at a time
# Task acknowledgement mode, default=False
task_acks_late=True,
# How many message to prefetch, default=4, value=1 disable prefetch
worker_prefetch_multiplier=1,
# Dates and times in messages will be converted to use the UTC timezone
timezone='UTC',
enable_utc=True,
# if true, store the task return values
task_ignore_result=False,
# if true, result messages will be persistent, messages won't be lost after a broker restart
result_persistent=False,
# Celery tasks expiry (in secs), default=1d, value=0/None never expire
result_expires=900,
# Message compression setting
task_compression='gzip',
# Task execution marker, default=False
task_track_started=True,
# rate limits on tasks (Disable all rate limits, even if tasks has explicit rate limits set)
worker_disable_rate_limits=True,
# is True, all tasks will be executed locally by blocking until the task returns
task_always_eager=False,
# Send events so the worker can be monitored by tools like celerymon
worker_send_task_events=False,
# Expiry time in seconds for when a monitor clients event queue will be deleted, default=never
event_queue_expires=60,
# Default queue, exchange, routing keys configuration
# task_default_queue = 'default.queue',
# task_default_exchange = 'default.exchange',
# task_default_exchange_type='topic',
# task_default_routing_key = 'default.route',
# task_create_missing_queues = True
task_queues=(
# Default configuration
Queue('redis.test.topic',
Exchange('redis.test.topic'),
routing_key='redis.test.topic'),
),
)
def create_celery_app() -> Celery:
logger.info('Initializing Celery...')
celery_app = Celery(name=__name__)
celery_app.config_from_object(config)
return celery_app
# create a celery app
app = create_celery_app()
#app.task(name='task_process_message', bind=True, max_retries=3)
def task_process_message(self, message):
try:
logger.info(f'{message}: Triggered task: task_process_message')
except Exception as e:
logger.exception(
f'Error executing task_process_message({message}')
# We do not have to reset timestamp since the job always looks back by 1 hr
self.retry(exc=e, countdown=utils.get_retry_delay(self.request.retries))
#shared_task(name='shared_task_process_message', bind=True, max_retries=3)
def shared_task_process_message(self, message):
try:
logger.info(f'{message}: Triggered task: shared_task_process_message')
except Exception as e:
logger.exception(
f'Error executing shared_task_process_message({message}')
# We do not have to reset timestamp since the job always looks back by 1 hr
self.retry(exc=e, countdown=utils.get_retry_delay(self.request.retries))
kombu_producer.py
from kombu import Producer, Consumer, Queue, Connection
import json
redis_url = 'redis://localhost:6379/0'
conn = Connection(redis_url)
producer = Producer(conn.channel())
channel = 'redis.test.topic'
for i in range(10):
body = {
'task': 'task_process_message',
'id': f'{i}',
'kwargs': {'message': f'Hello {i}',
}
}
producer.publish(body=body, routing_key='redis.test.topic')
The below picture shows activity on redis using a regular redis producer/consumer
Below picture shows activity on redis while running kombu producer and celery consumer.
I want to add multiprocessing to my Flask-SocketIO server so I am trying to add a Redis message queue as per the Flask-SocketIO docs. Even without adding multiprocessing, the client is not receiving any events. Everything else is working fine (e.g. the web page is being served, HTTP requests are being made, database calls are being made). There are no error messages on the front or back end. Before I added the Redis queue it was working. I verified that the 'addname' SocketIO route is being hit and that the request.sid looks right. What am I doing wrong?
Very simplified server code:
external_sio = SocketIO(message_queue='redis://')
def requester(user, sid):
global external_sio
external_sio.emit('addname', {'data': 'hello'}, room=sid)
# do some stuff with requests and databases
external_sio.emit('addname', {'data': 'goodbye'}, room=sid)
def main():
app = Flask(__name__,
static_url_path='',
static_folder='dist',
template_folder='dist')
socketio = SocketIO(app)
#socketio.on('addname')
def add_name(user):
global external_sio
external_sio.emit('addname', {'data': 'test'}, room=request.sid)
requester(user.data, request.sid)
socketio.run(app, host='0.0.0.0', port=8000)
if __name__ == '__main__':
main()
Simplified client code (React Javascript):
const socket = SocketIOClient('ipaddress:8000')
socket.emit('addname', {data: 'somename'})
socket.on('addname', ({data}) => console.log(data))
The main server also needs to be connected to the message queue. In your main server do this:
socketio = SocketIO(app, message_queue='redis://')
In your external process do this:
external_sio = SocketIO(message_queue='redis://') # <--- no app on this one
I am trying to set up Django & Celery & Rabbit for the first time following this tutorial. I am using Django 2.0 Celery 4.2.0 and Rabbit on Windows
I am getting the error: celery.exceptions.NotRegistered: 'GeneratePDF'
I have set up as follows:
in my init.py:
from __future__ import absolute_import, unicode_literals
import celery
from .celery import app as celery_app
__all__ = ['celery_app']
in my celery.py:
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
from django.conf import settings
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'abc.settings')
app = Celery('abc')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
in my tasks.py:
from celery import shared_task
from abc.celery import app
#shared_task(name='GeneratePDF')
class GeneratePDF(View):
def get(self, request, *args, **kwargs):
....
in my views.py:
from abc.tasks import GeneratePDF
#method_decorator(login_required, name='dispatch')
class ClientProfilePDF(RedirectView):
def get(self, request, *args, **kwargs):
GeneratePDF.delay(request)
return HttpResponseRedirect('/home/')
in my settings.py:
CELERY_BROKER_URL = 'amqp://localhost'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_RESULT_BACKEND = 'django-db'
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Australia/Sydney'
CELERY_IMPORTS = ('abc.tasks',)
Can anyone point me in the right direction as to where I am going wrong and why I am getting this error? Any help is much appreciated!
Two quick things:
No need for any parameters to app.autodiscover_tasks() Celery alreayd knows how to use settings.INSTALLED_APPS.
The #shared_task decorator is for tasks that live in apps that do not have their own celery.py file that instantiates an app. From the looks of it, your tasks.py file lives in the same django app as the celery.py file. In this case, you should use #app.task and not #shared_task.
before you start, you can get a list of registered tasks by doing celery -A myapp inspect registered. That will let you see if your GeneratePDF task is registered or not.
After running a celery task, I get the following message piped to stdout:
[2015-11-22 21:08:52,158: INFO/MainProcess] Task jobs.tasks.some_task[9c37f17b-dec4-4cb0-ab2b-fb5e997e430a]
succeeded in 2.9128301960008685s: 0.9731072908536255
I would like to programmatically extract the completion time (here: 2015-11-22 21:08:52) and the runtime (2.9128301960008685s) and display these to the user.
How can these values be extracted from the AsyncResult?
Broker is RabbitMQ.
Use celery task events to get the task runtime and time of completion. For more details about task events, refer this link.
from celery import Celery
import datetime
taskId_startTime = {}
taskId_addedTime = {}
def my_monitor():
app = Celery('vwadaptor', broker='redis://workerdb:6379/0',backend='redis://workerdb:6379/0')
state = app.events.State()
def announce_task_succeeded(event):
state.event(event)
task = state.tasks.get(event['uuid'])
print "runtime: ", task.runtime
print "time of completion: ", datetime.datetime.now()
with app.connection() as connection:
recv = app.events.Receiver(connection, handlers={
'task-succeeded': announce_task_succeeded,
})
recv.capture(limit=None, timeout=None, wakeup=True)
my_monitor()
Using Kombu with RabbitMQ to implement a classic publish/subscribe design pattern. I have created a producer that creates a topic:
from kombu import Connection, Exchange, Queue
media_exchange = Exchange('media', 'topic', durable=False)
video_queue = Queue('video', exchange=media_exchange, routing_key='video')
with Connection('amqp://guest:guest#localhost//') as conn:
producer = conn.Producer(serializer='json')
producer.publish('Hello World!',
exchange=media_exchange, routing_key='video',
declare=[video_queue])
I then created a consumer to consume from the publisher:
from kombu import Connection, Exchange, Queue
media_exchange = Exchange('media', type='topic', durable=False)
video_queue = Queue('video', exchange=media_exchange, routing_key='video')
def process_media(body, message):
print(body)
#message.ack()
with Connection('amqp://guest:guest#localhost//') as conn:
with conn.Consumer(video_queue, callbacks=[process_media]) as consumer:
# Process messages and handle events on all channels
while True:
conn.drain_events()
In then launch two consumers, each one in a separate terminal; both wait for a message:
terminal 1: python consumer.py
terminal 2: python consumer.py
When I run the producer, only one consumer receives the message.
The producer publishes in an exchange, not in a queue. The queues are defined by the consumers. When using different queue name for each consumer then all will get the message.
When using many consumers for the same queue then it is load balancing, that's why only one of your consumers gets the message.
To clarify, the messages in the queue are 'consumed' i.e. the first consumer consumes it, and the message is no more in the queue, that's why the second consumer isn't getting anything.
To have 2 separate consumers for same message - use 2 separate queues i.e.
video_queue1 and video_queue2, declared and bound to the exchange media_exchange, using same key video.
producer.py
from kombu import Connection, Exchange, Queue
media_exchange = Exchange('media', 'topic', durable=False)
video_queue1 = Queue('video1', exchange=media_exchange, routing_key='video')
video_queue2 = Queue('video2', exchange=media_exchange, routing_key='video')
with Connection('amqp://guest:guest#localhost//') as conn:
producer = conn.Producer(serializer='json')
producer.publish('Hello World!',
exchange=media_exchange, routing_key='video',
declare=[video_queue1, video_queue2])
consumer1.py
from kombu import Connection, Exchange, Queue
media_exchange = Exchange('media', type='topic', durable=False)
video_queue = Queue('video1', exchange=media_exchange, routing_key='video')
def process_media(body, message):
print(body)
#message.ack()
with Connection('amqp://guest:guest#localhost//') as conn:
with conn.Consumer(video_queue, callbacks=[process_media]) as consumer:
# Process messages and handle events on all channels
while True:
conn.drain_events()
consumer2.py
from kombu import Connection, Exchange, Queue
media_exchange = Exchange('media', type='topic', durable=False)
video_queue = Queue('video2', exchange=media_exchange, routing_key='video')
def process_media(body, message):
print(body)
#message.ack()
with Connection('amqp://guest:guest#localhost//') as conn:
with conn.Consumer(video_queue, callbacks=[process_media]) as consumer:
# Process messages and handle events on all channels
while True:
conn.drain_events()