Communication between multiple processes with zmq - python-multiprocessing

I am having n processes that have their own local data and actions and I want each process to send a "snapshot" of it's local data to the rest running nodes-processes.
The code that I have so far looks like that:
def node1():
Process(target=sync_1).start()
sleep(4)
data = {'node': 1, 'data': 'node 1 data'}
context_b = zmq.Context()
socket_b = context_b.socket(zmq.PUB)
connnected = False
try:
socket_b.bind("tcp://*:%s" % 5560)
connnected = True
except Exception as e:
print(e)
if connnected:
topic = "101"
try:
socket_b.send_string(topic + ' ' + json.dumps(data))
except Exception as e:
print(e)
socket_b.close()
context_b.term()
def node2():
Process(target=sync_2).start()
def sync_1():
context_c = zmq.Context()
socket_c = context_c.socket(zmq.SUB)
_port = 5560
try:
socket_c.connect("tcp://localhost:%s" % _port)
except Exception as e:
print(e)
topicfilter = "101"
socket_c.setsockopt_string(zmq.SUBSCRIBE, topicfilter, encoding='utf-8')
try:
raw = socket_c.recv().decode("utf-8")
json0 = raw.find('{')
topic = raw[0:json0].strip()
msg = json.loads(raw[json0:])
print("[SYNC 1] received {}-{}]".format(topic, msg))
except Exception as e:
print(e)
def sync_2():
context_c = zmq.Context()
socket_c = context_c.socket(zmq.SUB)
_port = 5560
try:
socket_c.connect("tcp://localhost:%s" % _port)
except Exception as e:
print(e)
topicfilter = "101"
socket_c.setsockopt_string(zmq.SUBSCRIBE, topicfilter, encoding='utf-8')
try:
raw = socket_c.recv().decode("utf-8")
json0 = raw.find('{')
topic = raw[0:json0].strip()
msg = json.loads(raw[json0:])
print("[SYNC 2] received {}-{}]".format(topic, msg))
except Exception as e:
print(e)
if __name__ == '__main__':
Process(target=node1).start()
Process(target=node2).start()
Each node has one "listener" process running at the background (the sync function) in order to receive each node data and use it accordingly, and it works fine when all the sub sockets are connected to one node (node 1 in that case) but I want each node to send data to all the listeners , so I am not sure how to implement that since the listener processes can connect to one port.
Also, the nodes will have to send the local data snapshot every time there is an update , so this cannot be an one time communication , therefore I thought of having listener processes actively waiting for updates all the times.
I believe a diagram could be useful for that problem:
There can be a way easier way to solve this issue , so any help would be highly appreciated!

Update:
The solution was to use the XPUB-XSUB pattern.
By using this pattern I created a proxy thread that allowed me to do exactly what I wanted.
The most useful example I could find for Python is this .

Related

How to solve the consumer error in Rabbitmq using python

when I run Consumer.py for headersExchange in rabbitmq using python,it is getting error like below
I have mentioned consumer and publish program below
Traceback (most recent call last):
File "headersConsumer.py", line 32, in <module>
main()
File "headersConsumer.py", line 14, in main
channel.exchange_declare(exchange = 'headers_logs',exchange_type='headers',durable=True)
File "C:\Python38\lib\site-packages\pika\adapters\blocking_connection.py", line 2387, in
exchange_declare
self._flush_output(declare_ok_result.is_ready)
File "C:\Python38\lib\site-packages\pika\adapters\blocking_connection.py", line 1339, in
_flush_output
raise self._closing_reason # pylint: disable=E0702
pika.exceptions.ChannelClosedByBroker: (406, "PRECONDITION_FAILED
- inequivalent arg 'type' for exchange 'headers_logs' in vhost '/': received 'headers' but
current is 'fanout'")
I have written consumer code like this
#!/usr/bin/env python
import pika, sys, os
def main():
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.exchange_declare(exchange = 'headers_logs',exchange_type='headers',durable=True)
channel.queue_declare(queue = "HeaderQueue1", durable=True)
channel.queue_bind(exchange = 'headers_logs', queue="HeadersQueue1", routing_key='',
arguments={'x-match': 'any', 'key1': 'one', 'key2': 'two'})
def callback(ch, method, properties, body):
print(" [x] %r" % body.decode())
print(' [*] Waiting for logs. To exit press CTRL+C')
channel.basic_consume(
queue="HeadersQueue1", on_message_callback=callback, auto_ack=True)
channel.start_consuming()
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
print('Interrupted')
try:
sys.exit(0)
except SystemExit:
os._exit(0)
I have written publish program likethis
import pika
import sys
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.exchange_declare(exchange='headers_logs',exchange_type='headers')
message = ' '.join(sys.argv[1:]) or "Hello World!"
channel.basic_publish((exchange='headers_logs',routing_key="",body=message,properties=pika.BasicProperties(
delivery_mode = 2, # make message persistent
headers = {'key1':'one', 'key2': 'three'}
))
print(" [x] Sent %r" % message)
connection.close()
I am not understanding this error,Can anyone please suggest this error
PRECONDITION_FAILED means that you declared an exchange with set of parameters then you are trying to create the same queue name using different parameters.
in your case:
headers_logs' in vhost '/': received 'headers' but
current is 'fanout'")
so you are trying to change the exchange type from fanout to headers
Se here for more detail (this is for the queues but exchanges work in the same way).
Before a queue can be used it has to be declared. Declaring a queue
will cause it to be created if it does not already exist. The
declaration will have no effect if the queue does already exist and
its attributes are the same as those in the declaration. When the
existing queue attributes are not the same as those in the declaration
a channel-level exception with code 406 (PRECONDITION_FAILED) will be
raised.

Make USB device visible with different vendor and product ID

I'm looking for a way to make a USB device show up as if it has different vendor and product IDs. I'm trying to make a proprietary piece of software to work with a USB device that should be supported but gets rejected solely because of its ID.
The software is for Windows, but I can run it in a VM in Linux. So I'll be fine with either approach, whatever works:
Changing USB ID in Linux
Changing USB ID in Windows
Making Qemu (or perhaps some other equivalent) change USB ID in passthrough
There may be a simpler way to do this, but I was faced with a similar problem and was able to create a process in which I could change the device descriptor information for development purposes. The process is summarized in this diagram:
First configure a static IP address for you Raspberry Pi and configure your PC ethernet TCP/IPv4 settings so you are able to communicate with you Raspberry Pi over the LAN connection.
Download the Virtual Here Raspberry Pi server and the client software for your PC from the Virtual Here website. The trial version will work for this use case.
Move the Virtual Here server software to you Raspberry Pi. In order to run the USB server you need to change the privileges of the file with $ sudo chmod +x vhusbdarm then run with $ sudo ./vhusbdarm.
Run the client software on your local machine. You will see that the client detects the USB device on the USB device server at <Your Raspberry Pi IP address>:7575. Connecting to the device at this point will give no advantage and mimic a direct connection.
Run the python file bellow, which was modified from a solution I found here, but utilizes Scapy sniff to capture the incoming packets before forwarding the raw data. The original script in the linked solution should work fine as well. In the script you can see that I used port 12345.
#!/usr/bin/env python
from scapy.all import *
import socket
import threading
import select
from queue import Queue
main_queue = Queue()
terminateAll = False
class ClientThread(threading.Thread):
def __init__(self, clientSocket, targetHost, targetPort):
threading.Thread.__init__(self)
self.__clientSocket = clientSocket
self.__targetHost = targetHost
self.__targetPort = targetPort
def run(self):
print("Client Thread started")
self.__clientSocket.setblocking(0)
targetHostSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
targetHostSocket.connect((self.__targetHost, self.__targetPort))
targetHostSocket.setblocking(0)
clientData = b""
targetHostData = b""
terminate = False
while not terminate and not terminateAll:
inputs = [self.__clientSocket, targetHostSocket]
outputs = []
if len(clientData) > 0:
outputs.append(self.__clientSocket)
if len(targetHostData) > 0:
outputs.append(targetHostSocket)
try:
inputsReady, outputsReady, errorsReady = select.select(inputs, outputs, [], 1.0)
except Exception as e:
print(e)
break
for inp in inputsReady:
if inp == self.__clientSocket:
try:
data = self.__clientSocket.recv(4096)
#print(data)
#data = b""
#while not main_queue.empty():
# data += main_queue.get()
except Exception as e:
print(e)
if data != None:
if len(data) > 0:
targetHostData += data
#else:
# terminate = True
elif inp == targetHostSocket:
try:
data = b""
while not main_queue.empty():
data += main_queue.get()
except Exception as e:
print(e)
if data != None:
if len(data) > 0:
clientData += data
for out in outputsReady:
if out == self.__clientSocket and len(clientData) > 0:
#pck = Ether(clientData)
#pck.show()
bytesWritten = self.__clientSocket.send(clientData)
if bytesWritten > 0:
clientData = clientData[bytesWritten:]
elif out == targetHostSocket and len(targetHostData) > 0:
#pck = Ether(targetHostData)
#pck.show()
bytesWritten = targetHostSocket.send(targetHostData)
if bytesWritten > 0:
targetHostData = targetHostData[bytesWritten:]
self.__clientSocket.close()
targetHostSocket.close()
print ("ClientThread terminating")
def handle_sniff(pck):
if IP in pck:
if pck[IP].src == "192.168.1.48":
if Raw in pck:
payload = pck[Raw].load
if b'\x12\x01\x00\x01\x00\x00\x00\x08$\x07\x04\x00\x88#\x01\x02\x00\x01' in payload:
payload = payload.replace(b'\x00\x08$\x07\x04\x00\x88#\x01\x02\x00', b'\x00\x08$\x07\x04\x00\x88\x15\x01\x02\x00')
print(payload)
main_queue.put(payload)
if __name__ == '__main__':
localHost = "localhost"
localPort = 12345
targetHost = "192.168.1.12"
targetPort = 7575
serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serverSocket.bind((localHost, localPort))
serverSocket.listen(5)
print("Waiting for client...")
while True:
try:
clientSocket, address = serverSocket.accept()
except KeyboardInterrupt:
print("\nTerminating...")
terminateAll = True
break
print("starting client")
ClientThread(clientSocket, targetHost, targetPort).start()
sniff(iface="Ethernet", prn=lambda pck: handle_sniff(pck))
serverSocket.close()
Once the script is running, configure the Virtual Here client to listen for USB servers at localhost:12345. The handle_sniff function is where the USB device descriptor information is being changed. Once connected you should be able to double click on the USB device in the dropdown tree. You will see the USB data begin to be printed in the your python console.
In the above example I changed the device bcdDevice bytes of the USB Descriptor.
Another helpful script I used to identify the packet that contained the information I was targeting is below. I modified a script I found in this solution. It is modified to print the raw data along with the unpacked device descriptor information, which can then be searched for in the TCP raw data to identify which bytes to replace.
#!/usr/bin/env python
from __future__ import print_function
import argparse
import string
import struct
import sys
import win32api
import win32file
import pywintypes
BUFF=b""
def CTL_CODE(DeviceType, Function, Method, Access):
return (DeviceType << 16) | (Access << 14) | (Function << 2) | Method
def USB_CTL(id):
# CTL_CODE(FILE_DEVICE_USB, (id), METHOD_BUFFERED, FILE_ANY_ACCESS)
return CTL_CODE(0x22, id, 0, 0)
IOCTL_USB_GET_ROOT_HUB_NAME = USB_CTL(258) # HCD_GET_ROOT_HUB_NAME
IOCTL_USB_GET_NODE_INFORMATION = USB_CTL(258) # USB_GET_NODE_INFORMATION
IOCTL_USB_GET_NODE_CONNECTION_INFORMATION = USB_CTL(259) # USB_GET_NODE_CONNECTION_INFORMATION
IOCTL_USB_GET_NODE_CONNECTION_DRIVERKEY_NAME = USB_CTL(264) # USB_GET_NODE_CONNECTION_DRIVERKEY_NAME
IOCTL_USB_GET_NODE_CONNECTION_NAME = USB_CTL(261) # USB_GET_NODE_CONNECTION_NAME
IOCTL_USB_GET_DESCRIPTOR_FROM_NODE_CONNECTION = USB_CTL(260) # USB_GET_DESCRIPTOR_FROM_NODE_CONNECTION
USB_CONFIGURATION_DESCRIPTOR_TYPE = 2
USB_STRING_DESCRIPTOR_TYPE = 3
USB_INTERFACE_DESCRIPTOR_TYPE = 4
MAXIMUM_USB_STRING_LENGTH = 255
def open_dev(name):
try:
handle = win32file.CreateFile(name,
win32file.GENERIC_WRITE,
win32file.FILE_SHARE_WRITE,
None,
win32file.OPEN_EXISTING,
0,
None)
except pywintypes.error as e:
return None
return handle
def get_root_hub_name(handle):
buf = win32file.DeviceIoControl(handle,
IOCTL_USB_GET_ROOT_HUB_NAME,
None,
6,
None)
act_len, _ = struct.unpack('LH', buf)
buf = win32file.DeviceIoControl(handle,
IOCTL_USB_GET_ROOT_HUB_NAME,
None,
act_len,
None)
return buf[4:].decode('utf-16le')
def get_driverkey_name(handle, index):
key_name = bytes(chr(index) + '\0'*9, 'utf-8')
try:
buf = win32file.DeviceIoControl(handle,
IOCTL_USB_GET_NODE_CONNECTION_DRIVERKEY_NAME,
key_name,
10,
None)
except pywintypes.error as e:
print(e.strerror, index)
sys.exit(1)
_, act_len, _ = struct.unpack('LLH', buf)
buf = win32file.DeviceIoControl(handle,
IOCTL_USB_GET_NODE_CONNECTION_DRIVERKEY_NAME,
key_name,
act_len,
None)
return buf[8:].decode('utf-16le')
def get_ext_hub_name(handle, index):
hub_name = chr(index) + '\0'*9
buf = win32file.DeviceIoControl(handle,
IOCTL_USB_GET_NODE_CONNECTION_NAME,
hub_name,
10,
None)
_, act_len, _ = struct.unpack('LLH', buf)
buf = win32file.DeviceIoControl(handle,
IOCTL_USB_GET_NODE_CONNECTION_NAME,
hub_name,
act_len,
None)
return buf[8:].decode('utf-16le')
def get_str_desc(handle, conn_idx, str_idx):
req = struct.pack('LBBHHH',
conn_idx,
0,
0,
(USB_STRING_DESCRIPTOR_TYPE<<8) | str_idx,
win32api.GetSystemDefaultLangID(),
MAXIMUM_USB_STRING_LENGTH)
try:
buf = win32file.DeviceIoControl(handle,
IOCTL_USB_GET_DESCRIPTOR_FROM_NODE_CONNECTION,
req,
12+MAXIMUM_USB_STRING_LENGTH,
None)
except pywintypes.error as e:
return 'ERROR: no String Descriptor for index {}'.format(str_idx)
if len(buf) > 16:
return buf[14:].decode('utf-16le')
return ''
def exam_hub(name, verbose, level):
handle = open_dev(r'\\.\{}'.format(name))
if not handle:
print('Failed to open device {}'.format(name))
return
buf = win32file.DeviceIoControl(handle,
IOCTL_USB_GET_NODE_INFORMATION,
None,
76,
None)
print_hub_ports(handle, ord(buf[6]), verbose, level)
handle.close()
def print_str_or_hex(to_be_print):
if all(c in string.printable for c in to_be_print):
print('"{}"'.format(to_be_print))
return
print('Hex: ', end='')
for x in to_be_print:
print('{:02x} '.format(ord(x)), end='')
print('')
def print_hub_ports(handle, num_ports, verbose, level):
print(handle, num_ports, verbose, level)
for idx in range(1, num_ports+1):
info = bytes(chr(idx) + '\0'*34, 'utf-8')
try:
buf = win32file.DeviceIoControl(handle,
IOCTL_USB_GET_NODE_CONNECTION_INFORMATION,
info,
34 + 11*30,
None)
except pywintypes.error as e:
print(e)
print(e.winerror, e.funcname, e.strerror)
return
print(buf)
_, vid, pid, vers, manu, prod, seri, _, ishub, _, stat = struct.unpack('=12sHHHBBB3s?6sL', buf[:35])
if ishub:
if verbose:
print('{} [Port{}] {}'.format(' '*level, idx, 'USB Hub'))
exam_hub(get_ext_hub_name(handle, idx), verbose, level)
elif stat == 0 and verbose:
print('{} [Port{}] {}'.format(' '*level, idx, 'NoDeviceConnected'))
elif stat == 1:
if verbose or (manu != 0 or prod != 0 or seri != 0):
print('{} [Port{}] {}'.format(' '*level, idx, get_driverkey_name(handle, idx)))
print('{} Vendor ID: 0x{:04X}'.format(' '*level, vid))
print('{} Product ID: 0x{:04X}'.format(' '*level, pid))
print('{} Device BCD: 0x{:04X}'.format(' '*level, vers))
print(vers)
if manu != 0:
print('{} Manufacturer (0x{:x}) -> '.format(' '*level, manu), end='')
print_str_or_hex(get_str_desc(handle, idx, manu))
if prod != 0:
print('{} Product (0x{:x}) -> '.format(' '*level, prod), end='')
print_str_or_hex(get_str_desc(handle, idx, prod))
if seri != 0:
print('{} Serial No (0x{:x}) -> '.format(' '*level, seri), end='')
print_str_or_hex(get_str_desc(handle, idx, seri))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true',
help="Increase output verbosity.")
args = parser.parse_args()
for i in range(10):
name = r"\\.\HCD{}".format(i)
handle = open_dev(name)
if not handle:
continue
root = get_root_hub_name(handle)
print('{}RootHub: {}'.format('\n' if i != 0 else '', root))
dev_name = r'\\.\{}'.format(root)
dev_handle = open_dev(dev_name)
if not dev_handle:
print('Failed to open device {}'.format(dev_name))
continue
buf = win32file.DeviceIoControl(dev_handle,
IOCTL_USB_GET_NODE_INFORMATION,
None,
76,
None)
global BUFF
BUFF=buf
print_hub_ports(dev_handle, buf[6], args.verbose, 0)
dev_handle.close()
handle.close()
if __name__ == '__main__':
main()
P.S. This is also really helpful for filtering and modifying any of the USB data being transferred not just the device descriptor.

Correct way of passing dataframe to ray

I am trying to do the simplest thing with Ray, but no matter what I do it just never releases memory and fails.
The usage case is simply
read parquet files to DF -> pass to pool of actors -> make changes to DF -> return DF
class Main_func:
def calculate(self,data):
#do some things with the DF
return df.copy(deep=True) <- one of many attempts to fix the problem, but didnt work
cpus = 24
actors = []
for _ in range(cpus):
actors.append(Main_func.remote())
from ray.util import ActorPool
pool = ActorPool(actors)
import os
arr = os.listdir("/some/files")
def to_ray():
try:
filename = arr.pop(0)
pf = ParquetFile("/some/files/" + filename)
df = pf.to_pandas()
pool.submit(lambda a,v:a.calculate.remote(v),df.copy(deep=True)
except Exception as e:
print(e)
for _ in range(cpus):
to_ray()
while(True):
res = pool.get_next_unordered()
write('./temp/' + random_filename, res,compression='GZIP')
del res
to_ray()
I have tried other ways of doing the same thing, manually submitting rather than the map command, but whatever i do it always locks memory and fails after a few 100 dataframes.
Does each task needs to preserve state among different files? Ray has tasks abstraction that should simplify things:
import ray
ray.init()
#ray.remote
def read_and_write(path):
df = pd.read_parquet(path)
... do things
df.to_parquet("./temp/...")
import os
arr = os.listdir("/some/files")
results = ray.get([read_and_write.remote(path) for path in arr])

Redis not returning result after upgrading Celery from 3.1 to 4.0

I recently upgraded my Celery installation to 4.0. After a few days of wrestling with the upgrade process, I finally got it to work... sort of. Some tasks will return, but the final task will not.
I have a class, SFF, that takes in and parses a file:
# Constructor with I/O file
def __init__(self, file):
# File data that's gonna get used a lot
sffDescriptor = file.fileno()
fileName = abspath(file.name)
# Get the pointer to the file
filePtr = mmap.mmap(sffDescriptor, 0, flags=mmap.MAP_SHARED, prot=mmap.PROT_READ)
# Get the header info
hdr = filePtr.read(HEADER_SIZE)
self.header = SFFHeader._make(unpack(HEADER_FMT, hdr))
# Read in the palette maps
print self.header.onDemandDataSize
print self.header.onLoadDataSize
palMapsResult = getPalettes.delay(fileName, self.header.palBankOff - HEADER_SIZE, self.header.onDemandDataSize, self.header.numPals)
# Read the sprite list nodes
nodesStart = self.header.sprListOff
nodesEnd = self.header.palBankOff
print nodesEnd - nodesStart
sprNodesResult = getSprNodes.delay(fileName, nodesStart, nodesEnd, self.header.numSprites)
# Get palette data
self.palettes = palMapsResult.get()
# Get sprite data
spriteNodes = sprNodesResult.get()
# TESTING
spritesResultSet = ResultSet([])
numSpriteNodes = len(spriteNodes)
# Split the nodes into chunks of size 32 elements
for x in xrange(0, numSpriteNodes, 32):
spritesResult = getSprites.delay(spriteNodes, x, x+32, fileName, self.palettes, self.header.palBankOff, self.header.onDemandDataSizeTotal)
spritesResultSet.add(spritesResult)
break # REMEMBER TO REMOVE FOR ENTIRE SFF
self.sprites = spritesResultSet.join_native()
It doesn't matter if it's a single task that returns the entire spritesResult, or if I split it using a ResultSet, the outcome is always the same: the Python console I'm using just hangs at either spritesResultSet.join_native() or spritesResult.get() (depending on how I format it).
Here is the task in question:
#task
def getSprites(nodes, start, end, fileName, palettes, palBankOff, onDemandDataSizeTotal):
sprites = []
with open(fileName, "rb") as file:
sffDescriptor = file.fileno()
sffData = mmap.mmap(sffDescriptor, 0, flags=mmap.MAP_SHARED, prot=mmap.PROT_READ)
for node in nodes[start:end]:
sprListNode = dict(SprListNode._make(node)._asdict()) # Need to convert it to a dict since values may change.
#print node
#print sprListNode
# If it's a linked sprite, the data length is 0, so get the linked index.
if sprListNode['dataLen'] == 0:
sprListNodeTemp = SprListNode._make(nodes[sprListNode['index']])
sprListNode['dataLen'] = sprListNodeTemp.dataLen
sprListNode['dataOffset'] = sprListNodeTemp.dataOffset
sprListNode['compression'] = sprListNodeTemp.compression
# What does the offset need to be?
dataOffset = sprListNode['dataOffset']
if sprListNode['loadMode'] == 0:
dataOffset += palBankOff #- HEADER_SIZE
elif sprListNode['loadMode'] == 1:
dataOffset += onDemandDataSizeTotal #- HEADER_SIZE
#print sprListNode
# Seek to the data location and "read" it in. First 4 bytes are just the image length
start = dataOffset + 4
end = dataOffset + sprListNode['dataLen']
#sffData.seek(start)
compressedSprite = sffData[start:end]
# Create the sprite
sprite = Sprite(sprListNode, palettes[sprListNode['palNo']], np.fromstring(compressedSprite, dtype=np.uint8))
sprites.append(sprite)
return json.dumps(sprites, cls=SpriteJSONEncoder)
I know it reaches the return statement, because if I put a print right above it, it will print in the Celery window. I also know that the task is running to completion because I get the following message from the worker:
[2016-11-16 00:03:33,639: INFO/PoolWorker-4] Task framedatabase.tasks.getSprites[285ac9b1-09b4-4cf1-a251-da6212863832] succeeded in 0.137236133218s: '[{"width": 120, "palNo": 30, "group": 9000, "xAxis": 0, "yAxis": 0, "data":...'
Here are my celery settings in settings.py:
# Celery settings
BROKER_URL='redis://localhost:1717/1'
CELERY_RESULT_BACKEND='redis://localhost:1717/0'
CELERY_IGNORE_RESULT=False
CELERY_IMPORTS = ("framedatabase.tasks", )
... and my celery.py:
from __future__ import absolute_import
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'framedatabase.settings')
from django.conf import settings # noqa
app = Celery('framedatabase', backend='redis://localhost:1717/1', broker="redis://localhost:1717/0",
include=['framedatabase.tasks'])
# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
Found the problem. Apparently it was leading to deadlock as mentioned in the section "Avoid launching synchronous subtasks" in the Celery documentation here: http://docs.celeryproject.org/en/latest/userguide/tasks.html#tips-and-best-practices
So I got rid of the line:
sprNodesResult.get()
And changed the final result to a chain:
self.sprites = chain(getSprNodes.s(fileName, nodesStart, nodesEnd, self.header.numSprites),
getSprites.s(0,32,fileName,self.palettes,self.header.palBankOff,self.header.onDemandDataSizeTotal))().get()
And it works! Now I just have to find a way to split this the way I want!

Using pytest with Jython

I'm trying to use pytest on Jython. And I'm getting stuck right at the beginning.
I've successfully installed the pytest package with easy_install:
$ ./jython easy_install pytest
When I try to run example from this page, things go wrong. I receive an extremely long failure report, like the one bellow. Does anybody have any idea why this is happening?
py.test-jython
============================= test session starts ==============================
platform java1.6.0_37 -- Python 2.5.3 -- pytest-2.3.2
collected 1 items
test_sample.py F
=================================== FAILURES ===================================
_________________ test_answer __________________
def test_answer():
assert func(3) == 5
test_sample.py:5:
self = AssertionError()
def __init__(self, *args):
BuiltinAssertionError.__init__(self, *args)
if args:
try:
self.msg = str(args[0])
except py.builtin._sysex:
raise
except:
self.msg = "<[broken __repr__] %s at %0xd>" %(
args[0].__class__, id(args[0]))
else:
f = py.code.Frame(sys._getframe(1))
try:
source = f.code.fullsource
if source is not None:
try:
source = source.getstatement(f.lineno, assertion=True)
except IndexError:
source = None
else:
source = str(source.deindent()).strip()
except py.error.ENOENT:
source = None
# this can also occur during reinterpretation, when the
# co_filename is set to "<run>".
if source:
self.msg = reinterpret(source, f, should_fail=True)
../jython2.5.3/Lib/site-packages/pytest-2.3.2-py2.5.egg/_pytest/assertion/reinterpret.py:32:
source = 'assert func(3) == 5', frame =
should_fail = True
def interpret(source, frame, should_fail=False):
mod = ast.parse(source)
visitor = DebugInterpreter(frame)
try:
visitor.visit(mod)
../jython2.5.3/Lib/site-packages/pytest-2.3.2-py2.5.egg/_pytest/assertion/newinterpret.py:49:
.
.
.
self = <_pytest.assertion.newinterpret.DebugInterpreter object at 0x4>
name = Name
def visit_Name(self, name):
explanation, result = self.generic_visit(name)
../jython2.5.3/Lib/site-packages/pytest-2.3.2-py2.5.egg/_pytest/assertion/newinterpret.py:147:
self = <_pytest.assertion.newinterpret.DebugInterpreter object at 0x4>
node = Name
def generic_visit(self, node):
# Fallback when we don't have a special implementation.
if _is_ast_expr(node):
mod = ast.Expression(node)
co = self._compile(mod)
try:
result = self.frame.eval(co)
except Exception:
raise Failure()
explanation = self.frame.repr(result)
return explanation, result
elif _is_ast_stmt(node):
mod = ast.Module([node])
co = self._compile(mod, "exec")
try:
self.frame.exec_(co)
except Exception:
raise Failure()
return None, None
else:
raise AssertionError("can't handle %s" %(node,))
E AssertionError: can't handle Name
../jython2.5.3/Lib/site-packages/pytest-2.3.2-py2.5.egg/_pytest/assertion/newinterpret.py:134: AssertionError
=========================== 1 failed in 0.55 seconds ===========================
Pytest has a workaround for jython's lacking AST implementation, see issue1479. I just extended the workaround on the pytest side to work on jython-2.5.3. You can install a dev-candidate of pytest with:
pip install -i http://pypi.testrun.org -U pytest
and should get at least version 2.3.4.dev1 with "py.test-jython --version" and get assertions working with jython-2.5.3.
Currently pytest does not support Jython2.5.3, works only on Jython2.5.1.