I have following code that use python API of IB, it should display both price and commission:
from ib.ext.ExecutionFilter import ExecutionFilter
from ib.ext.CommissionReport import CommissionReport
from ib.opt import ibConnection, message
from time import sleep
#-- message handlers -------------------------------------------------
# print all messages from TWS
def watcher(msg):
pass
def ExecutionDetailsHandler(msg):
global execDetails
execDetails = msg.execution
#print execDetails.m_price
#print execDetails.m_side
def CommissionDetailsHandler(msg):
global commission
commission = msg.commissionReport
# global variable that stores the last Execution seen by
ExecutionDetailsHandler
CommissionDetailsHandler
execDetails = None
commission = None
#-- factories
#-----------------------------------------------------------
def makeExecFilter():
filter=ExecutionFilter()
return filter
#-- utilities --------------------------------------------------------
def getExecutionPrice():
filter=makeExecFilter()
con.reqExecutions(744,filter)
# wait for TWS message to come back to message handler
while execDetails is None:
print 'waiting'
sleep(1)
return execDetails.m_price
def getCommission():
filter=CommissionReport()
con.commissionReport(filter)
# wait for TWS message to come back to message handler
while commission is None:
print 'waiting'
sleep(1)
return commission.m_commission
con = ibConnection()
con.registerAll(watcher)
con.register(ExecutionDetailsHandler, 'ExecDetails')
con.register(CommissionDetailsHandler, 'commissionDetails')
con.connect()
price=getExecutionPrice()
c = getCommission()
con.disconnect()
print 'The price of one execution is:', price
print 'The commission fee is:', c
however this is only working for execution price, as it displays the price information after print out. But it does not show the commission information (in my terminal it keeps waiting forever), is there anything wrong in my code?
thx for brian's answer, this does the trick:
commission = None
def commReport(msg):
global commission
#print('ID',msg.commissionReport.m_execId,'COM',msg.commissionReport.m_commission)
commission = msg.commissionReport.m_commission
conn = Connection.create(port=7496, clientId=222)
conn.register(commReport, message.commissionReport)
conn.connect()
and now I am happy to use commission anywhere i want
Related
In my scrapy code I'm trying to yield the following figures from parliament's website where all the members of parliament (MPs) are listed. Opening the links for each MP, I'm making parallel requests to get the figures I'm trying to count. I didn't use metas here because my code doesn't just make consecutive requests but it makes parallel requests for the figures after the individual page of the MP is requested. Thus I thought item containers would fit my purpose better.
Here are the figures I'm trying to scrape
How many bill proposals that each MP has their signature on
How many question proposals that each MP has their signature on
How many times that each MP spoke on the parliament
In order to count and yield out how many bills has each member of parliament has their signature on, I'm trying to write a scraper on the members of parliament which works with 3 layers:
Starting with the link where all MPs are listed
From (1) accessing the individual page of each MP where the three information defined above is displayed
3a) Requesting the page with bill proposals and counting the number of them by len function
3b) Requesting the page with question proposals and counting the number of them by len function
3c) Requesting the page with speeches and counting the number of them by len function
What I want: I want to yield the inquiries of 3a,3b,3c with the name and the party of the MP
Problem: My code above just doesn't yield anything but empty dictionaries for each request
Note: Because my parse functions doesn't work like parse => parse2 => parse3 but rather I have 3 parallel parse functions after parse2, I failed to use the meta because I'm not yielding all the values at parse three. Therefore I preferred using the pipelines which apparently doesn't work.
Main code:
'''
from scrapy import Spider
from scrapy.http import Request
from ..items import MeclisItem
import logging
class MvSpider(Spider):
name = 'mv'
allowed_domains = ['tbmm.gov.tr']
start_urls = ['https://www.tbmm.gov.tr/Milletvekilleri/liste']
def parse(self, response):
items = MeclisItem()
mv_list = mv_list = response.xpath("//ul[#class='list-group list-group-flush']") #taking all MPs listed
for mv in mv_list:
items['name'] = mv.xpath("./li/div/div/a/text()").get() # MP's name taken
items['party'] = mv.xpath("./li/div/div[#class='col-md-4 text-right']/text()").get().strip() #MP's party name taken
partial_link = mv.xpath('.//div[#class="col-md-8"]/a/#href').get()
full_link = response.urljoin(partial_link)
yield Request(full_link, callback = self.mv_analysis)
pass
def mv_analysis(self, response):
items = MeclisItem()
billprop_link_path = response.xpath(".//a[contains(text(),'İmzası Bulunan Kanun Teklifleri')]/#href").get()
billprop_link = response.urljoin(billprop_link_path)
questionprop_link_path = response.xpath(".//a[contains(text(),'Sahibi Olduğu Yazılı Soru Önergeleri')]/#href").get()
questionprop_link = response.urljoin(questionprop_link_path)
speech_link_path = response.xpath(".//a[contains(text(),'Genel Kurul Konuşmaları')]/#href").get()
speech_link = response.urljoin(speech_link_path)
yield Request(billprop_link, callback = self.bill_prop_counter) #number of bill proposals to be requested
yield Request(questionprop_link, callback = self.quest_prop_counter) #number of question propoesals to be requested
yield Request(speech_link, callback = self.speech_counter) #number of speeches to be requested
yield items
# COUNTING FUNCTIONS
def bill_prop_counter(self,response):
items = MeclisItem()
billproposals = response.xpath("//tr[#valign='TOP']")
items['bill_prop_count'] = len(billproposals)
pass
def quest_prop_counter(self, response):
items = MeclisItem()
questionproposals = response.xpath("//tr[#valign='TOP']")
items['res_prop_count'] = len(questionproposals)
pass
def speech_counter(self, response):
items = MeclisItem()
speeches = response.xpath("//tr[#valign='TOP']")
items['speech_count'] = len(speeches)
pass
'''
items.py code:
import scrapy
class MeclisItem(scrapy.Item):
name = scrapy.Field()
party = scrapy.Field()
bill_prop_count = scrapy.Field()
res_prop_count = scrapy.Field()
speech_count = scrapy.Field()
pass
What's displayed at scrapy:
I checked many questions on stackoverflow but still couldn't figure a way out. Thanks in advance.
ps: Spent ten minutes seperately to colour the code above and couldn't make it either :(
Note: Because my parse functions doesn't work like parse => parse2 => parse3 but rather I have 3 parallel parse functions after parse2, I failed to use the meta because I'm not yielding all the values at parse three.
You can do it like this:
Edit:
import scrapy
from scrapy import Spider
from scrapy.http import Request
# from ..items import MeclisItem
import logging
class MeclisItem(scrapy.Item):
name = scrapy.Field()
party = scrapy.Field()
bill_prop_count = scrapy.Field()
res_prop_count = scrapy.Field()
speech_count = scrapy.Field()
class MvSpider(Spider):
name = 'mv'
allowed_domains = ['tbmm.gov.tr']
start_urls = ['https://www.tbmm.gov.tr/Milletvekilleri/liste']
def parse(self, response):
mv_list = mv_list = response.xpath("//ul[#class='list-group list-group-flush']") #taking all MPs listed
for mv in mv_list:
item = MeclisItem()
item['name'] = mv.xpath("./li/div/div/a/text()").get() # MP's name taken
item['party'] = mv.xpath("./li/div/div[#class='col-md-4 text-right']/text()").get().strip() #MP's party name taken
partial_link = mv.xpath('.//div[#class="col-md-8"]/a/#href').get()
full_link = response.urljoin(partial_link)
yield Request(full_link, callback=self.mv_analysis, cb_kwargs={'item': item})
def mv_analysis(self, response, item):
billprop_link_path = response.xpath(".//a[contains(text(),'İmzası Bulunan Kanun Teklifleri')]/#href").get()
billprop_link = response.urljoin(billprop_link_path)
questionprop_link_path = response.xpath(".//a[contains(text(),'Sahibi Olduğu Yazılı Soru Önergeleri')]/#href").get()
questionprop_link = response.urljoin(questionprop_link_path)
speech_link_path = response.xpath(".//a[contains(text(),'Genel Kurul Konuşmaları')]/#href").get()
speech_link = response.urljoin(speech_link_path)
yield Request(billprop_link,
callback=self.bill_prop_counter,
cb_kwargs={'item': item, 'questionprop_link': questionprop_link, 'speech_link': speech_link}) #number of bill proposals to be requested
# COUNTING FUNCTIONS
def bill_prop_counter(self, response, item, questionprop_link, speech_link):
billproposals = response.xpath("//tr[#valign='TOP']")
item['bill_prop_count'] = len(billproposals)
yield Request(questionprop_link,
callback=self.quest_prop_counter,
cb_kwargs={'item': item, 'speech_link': speech_link}) #number of question propoesals to be requested
def quest_prop_counter(self, response, item, speech_link):
questionproposals = response.xpath("//tr[#valign='TOP']")
item['res_prop_count'] = len(questionproposals)
yield Request(speech_link,
callback=self.speech_counter,
cb_kwargs={'item': item}) #number of speeches to be requested
def speech_counter(self, response, item):
speeches = response.xpath("//tr[#valign='TOP']")
item['speech_count'] = len(speeches)
yield item
I am currently following this tutorial on threading in PyQt (code from here). As it was written in PyQt4 (and Python2), I adapted the code to work with PyQt5 and Python3.
Here is the gui file (newdesign.py):
# -*- coding: utf-8 -*-
# Form implementation generated from reading ui file 'threading_design.ui'
#
# Created by: PyQt5 UI code generator 5.6
#
# WARNING! All changes made in this file will be lost!
from PyQt5 import QtCore, QtGui, QtWidgets
class Ui_MainWindow(object):
def setupUi(self, MainWindow):
MainWindow.setObjectName("MainWindow")
MainWindow.resize(526, 373)
self.centralwidget = QtWidgets.QWidget(MainWindow)
self.centralwidget.setObjectName("centralwidget")
self.verticalLayout = QtWidgets.QVBoxLayout(self.centralwidget)
self.verticalLayout.setObjectName("verticalLayout")
self.subreddits_input_layout = QtWidgets.QHBoxLayout()
self.subreddits_input_layout.setObjectName("subreddits_input_layout")
self.label_subreddits = QtWidgets.QLabel(self.centralwidget)
self.label_subreddits.setObjectName("label_subreddits")
self.subreddits_input_layout.addWidget(self.label_subreddits)
self.edit_subreddits = QtWidgets.QLineEdit(self.centralwidget)
self.edit_subreddits.setObjectName("edit_subreddits")
self.subreddits_input_layout.addWidget(self.edit_subreddits)
self.verticalLayout.addLayout(self.subreddits_input_layout)
self.label_submissions_list = QtWidgets.QLabel(self.centralwidget)
self.label_submissions_list.setObjectName("label_submissions_list")
self.verticalLayout.addWidget(self.label_submissions_list)
self.list_submissions = QtWidgets.QListWidget(self.centralwidget)
self.list_submissions.setBatchSize(1)
self.list_submissions.setObjectName("list_submissions")
self.verticalLayout.addWidget(self.list_submissions)
self.progress_bar = QtWidgets.QProgressBar(self.centralwidget)
self.progress_bar.setProperty("value", 0)
self.progress_bar.setObjectName("progress_bar")
self.verticalLayout.addWidget(self.progress_bar)
self.buttons_layout = QtWidgets.QHBoxLayout()
self.buttons_layout.setObjectName("buttons_layout")
self.btn_stop = QtWidgets.QPushButton(self.centralwidget)
self.btn_stop.setEnabled(False)
self.btn_stop.setObjectName("btn_stop")
self.buttons_layout.addWidget(self.btn_stop)
self.btn_start = QtWidgets.QPushButton(self.centralwidget)
self.btn_start.setObjectName("btn_start")
self.buttons_layout.addWidget(self.btn_start)
self.verticalLayout.addLayout(self.buttons_layout)
MainWindow.setCentralWidget(self.centralwidget)
self.retranslateUi(MainWindow)
QtCore.QMetaObject.connectSlotsByName(MainWindow)
def retranslateUi(self, MainWindow):
_translate = QtCore.QCoreApplication.translate
MainWindow.setWindowTitle(_translate("MainWindow", "Threading Tutorial - nikolak.com "))
self.label_subreddits.setText(_translate("MainWindow", "Subreddits:"))
self.edit_subreddits.setPlaceholderText(_translate("MainWindow", "python,programming,linux,etc (comma separated)"))
self.label_submissions_list.setText(_translate("MainWindow", "Submissions:"))
self.btn_stop.setText(_translate("MainWindow", "Stop"))
self.btn_start.setText(_translate("MainWindow", "Start"))
and the main script (main.py):
from PyQt5 import QtWidgets
from PyQt5.QtCore import QThread, pyqtSignal, QObject
import sys
import newdesign
import urllib.request
import json
import time
class getPostsThread(QThread):
def __init__(self, subreddits):
"""
Make a new thread instance with the specified
subreddits as the first argument. The subreddits argument
will be stored in an instance variable called subreddits
which then can be accessed by all other class instance functions
:param subreddits: A list of subreddit names
:type subreddits: list
"""
QThread.__init__(self)
self.subreddits = subreddits
def __del__(self):
self.wait()
def _get_top_post(self, subreddit):
"""
Return a pre-formatted string with top post title, author,
and subreddit name from the subreddit passed as the only required
argument.
:param subreddit: A valid subreddit name
:type subreddit: str
:return: A string with top post title, author,
and subreddit name from that subreddit.
:rtype: str
"""
url = "https://www.reddit.com/r/{}.json?limit=1".format(subreddit)
headers = {'User-Agent': 'nikolak#outlook.com tutorial code'}
request = urllib.request.Request(url, header=headers)
response = urllib.request.urlopen(request)
data = json.load(response)
top_post = data['data']['children'][0]['data']
return "'{title}' by {author} in {subreddit}".format(**top_post)
def run(self):
"""
Go over every item in the self.subreddits list
(which was supplied during __init__)
and for every item assume it's a string with valid subreddit
name and fetch the top post using the _get_top_post method
from reddit. Store the result in a local variable named
top_post and then emit a pyqtSignal add_post(QString) where
QString is equal to the top_post variable that was set by the
_get_top_post function.
"""
for subreddit in self.subreddits:
top_post = self._get_top_post(subreddit)
self.emit(pyqtSignal('add_post(QString)'), top_post)
self.sleep(2)
class ThreadingTutorial(QtWidgets.QMainWindow, newdesign.Ui_MainWindow):
"""
How the basic structure of PyQt GUI code looks and behaves like is
explained in this tutorial
http://nikolak.com/pyqt-qt-designer-getting-started/
"""
def __init__(self):
super(self.__class__, self).__init__()
self.setupUi(self)
self.btn_start.clicked.connect(self.start_getting_top_posts)
def start_getting_top_posts(self):
# Get the subreddits user entered into an QLineEdit field
# this will be equal to '' if there is no text entered
subreddit_list = str(self.edit_subreddits.text()).split(',')
if subreddit_list == ['']: # since ''.split(',') == [''] we use that to check
# whether there is anything there to fetch from
# and if not show a message and abort
QtWidgets.QMessageBox.critical(self, "No subreddits",
"You didn't enter any subreddits.",
QtWidgets.QMessageBox.Ok)
return
# Set the maximum value of progress bar, can be any int and it will
# be automatically converted to x/100% values
# e.g. max_value = 3, current_value = 1, the progress bar will show 33%
self.progress_bar.setMaximum(len(subreddit_list))
# Setting the value on every run to 0
self.progress_bar.setValue(0)
# We have a list of subreddits which we use to create a new getPostsThread
# instance and we pass that list to the thread
self.get_thread = getPostsThread(subreddit_list)
# Next we need to connect the events from that thread to functions we want
# to be run when those pyqtSignals get fired
# Adding post will be handeled in the add_post method and the pyqtSignal that
# the thread will emit is pyqtSignal("add_post(QString)")
# the rest is same as we can use to connect any pyqtSignal
self.connect(self.get_thread, pyqtSignal("add_post(QString)"), self.add_post)
# This is pretty self explanatory
# regardless of whether the thread finishes or the user terminates it
# we want to show the notification to the user that adding is done
# and regardless of whether it was terminated or finished by itself
# the finished pyqtSignal will go off. So we don't need to catch the
# terminated one specifically, but we could if we wanted.
self.connect(self.get_thread, pyqtSignal("finished()"), self.done)
# We have all the events we need connected we can start the thread
self.get_thread.start()
# At this point we want to allow user to stop/terminate the thread
# so we enable that button
self.btn_stop.setEnabled(True)
# And we connect the click of that button to the built in
# terminate method that all QThread instances have
self.btn_stop.clicked.connect(self.get_thread.terminate)
# We don't want to enable user to start another thread while this one is
# running so we disable the start button.
self.btn_start.setEnabled(False)
def add_post(self, post_text):
"""
Add the text that's given to this function to the
list_submissions QListWidget we have in our GUI and
increase the current value of progress bar by 1
:param post_text: text of the item to add to the list
:type post_text: str
"""
self.list_submissions.addItem(post_text)
self.progress_bar.setValue(self.progress_bar.value()+1)
def done(self):
"""
Show the message that fetching posts is done.
Disable Stop button, enable the Start one and reset progress bar to 0
"""
self.btn_stop.setEnabled(False)
self.btn_start.setEnabled(True)
self.progress_bar.setValue(0)
QtWidgets.QMessageBox.information(self, "Done!", "Done fetching posts!")
def main():
app = QtWidgets.QApplication(sys.argv)
form = ThreadingTutorial()
form.show()
app.exec_()
if __name__ == '__main__':
main()
Now I'm getting the following error:
AttributeError: 'ThreadingTutorial' object has no attribute 'connect'
Can anyone please tell me how to fix this? Any help would be, as always, very much appreciated.
Using QObject.connect() and similar in PyQt4 is known as "Old style signals", and is not supported in PyQt5 anymore, it supports only "New style signals", which already in PyQt4 was the recommended way to connect signals.
In PyQt5 you need to use the connect() and emit() methods of the bound signal directly, e.g. instead of:
self.emit(pyqtSignal('add_post(QString)'), top_post)
...
self.connect(self.get_thread, pyqtSignal("add_post(QString)"), self.add_post)
self.connect(self.get_thread, pyqtSignal("finished()"), self.done)
use:
self.add_post.emit(top_post)
...
self.get_thread.add_post.connect(self.add_post)
self.get_thread.finished.connect(self.done)
However for this to work you need to explicitly define the add_post signal on your getPostsThread first, otherwise you'll get an attribute error.
class getPostsThread(QThread):
add_post = pyqtSignal(str)
...
In PyQt4 with old style signals when a signal was used it was automatically defined, this now needs to be done explicitly.
I'm going to be using rxandroid in an android app. I'm trying to model the behavior right now in rxpy because it was the easiest for me to set up and play with. In the example below, source3 is emitting the correct data; which is a concatenation of an initialization that takes some time and a permanent subscription which I have just faked out. I want the BehaviorSubject because I need the last value immediately for field initialization.
I cannot figure out how to chain the BehaviorSubject on top of source3 so that it emits source 3 while remembering the last value. I have searched the internet for two days and not found a clear direction on this use case. Here is my code, and the question is why I don't get any emissions from the observer.
from rx import Observable, Observer
from rx.subjects import BehaviorSubject
import time, random
def fake_initialization(observer):
time.sleep(5) # It takes some time
observer.on_next("Alpha")
observer.on_completed()
def fake_subscription(observer):
iter = 0 # Subscription emits forever
while True:
observer.on_next("message %02d"%(iter))
time.sleep(random.randrange(2,5))
iter += 1
class PrintObserver(Observer):
def on_next(self, value):
print("Received {0}".format(value))
#bsubject.on_next(value)
def on_completed(self):
print("Done!")
def on_error(self, error):
print("Error Occurred: {0}".format(error))
source1 = Observable.create(fake_initialization)
source2 = Observable.create(fake_subscription)
source3 = source1 + source2
bsubject = BehaviorSubject(False)
source4 = source3.multicast(bsubject)
source4.connect()
source4.subscribe(PrintObserver())
This was actually a fairly easy answer for someone. I'm posting this in case anyone else ends up in this situation. Admittedly, I didn't read the rxpy page closely enough. You need to add concurrency on your own, presumably because there are so many concurrent solutions in Python. Here is the final working code:
import random
import time
import multiprocessing
from rx import Observable,Observer
from rx.concurrency import ThreadPoolScheduler
from rx.subjects import Subject
class PrintObserver1(Observer):
def on_next(self, value):
print("Received 1 {0}".format(value))
#bsubject.on_next(value)
def on_completed(self):
print("Done 1!")
def on_error(self, error):
print("Error Occurred: 1 {0}".format(error))
class PrintObserver2(Observer):
def on_next(self, value):
print("Received 2 {0}".format(value))
#bsubject.on_next(value)
def on_completed(self):
print("Done 2!")
def on_error(self, error):
print("Error Occurred: 2 {0}".format(error))
def fake_initialization(observer):
time.sleep(5) # It takes some time
observer.on_next("Alpha")
observer.on_completed()
def fake_subscription(observer):
iter = 0 # Subscription emits forever
while True:
observer.on_next("message %02d"%(iter))
time.sleep(random.randrange(2,5))
iter += 1
optimal_thread_count = multiprocessing.cpu_count()
pool_scheduler = ThreadPoolScheduler(optimal_thread_count)
source1 = Observable.create(fake_initialization).subscribe_on(pool_scheduler)
source2 = Observable.create(fake_subscription).subscribe_on(pool_scheduler)
catted_source = source1 + source2
native_source = Observable.interval(1000)
print native_source,catted_source
#source = source3
subject = Subject()
# native_source = works
# catted_source = not works
subSource = catted_source.subscribe(subject)
#####
subSubject1 = subject.subscribe(PrintObserver1())
subSubject2 = subject.subscribe(PrintObserver2())
time.sleep(30)
subject.on_completed()
subSubject1.dispose()
subSubject2.dispose()
Also note that you have to install the 'futures' package for concurrency to work on Python 2.7.
If you get this error:
from concurrent.futures import ThreadPoolExecutor
ImportError: No module named concurrent.futures
Read this (link is for slightly different error but solution works):
ImportError: No module named concurrent.futures.process
After struggling with inlineCallbacks and yield of twisted/txredisapi, I can save my data into redis. Thanks to author of txredisapi. Now I met a new issue, socket server will not send back to client before/after saving into DB.
Twisted offers simple socket server as following:
from twisted.internet import protocol, reactor
class Echo(protocol.Protocol):
def dataReceived(self, data):
self.transport.write(data) ### write back
class EchoFactory(protocol.Factory):
def buildProtocol(self, addr):
return Echo()
reactor.listenTCP(8000, EchoFactory)
recctor.run()
My code is similiar, only with additional DB ops.
#!/usr/bin/env python
import time
import binascii
import txredisapi
from twisted.internet import defer
from twisted.internet import protocol, reactor
from twisted.internet.protocol import Factory
from twisted.enterprise import adbapi
from twisted.python import log
from dmpack import Dmpack
from dmdb import Dmdb
from dmconfig import DmConf
dm = Dmpack()
conf = DmConf().loadConf()
rcs = txredisapi.lazyConnection(password=conf['RedisPassword'])
dbpool = adbapi.ConnectionPool("MySQLdb",db=conf['DbName'],user=conf['DbAccount'],\
passwd=conf['DbPassword'],host=conf['DbHost'],\
use_unicode=True,charset=conf['DbCharset'])
def getDataParsed(data):
realtime = None
period = None
self.snrCode = dm.snrToAscii(data[2:7])
realtime = data[7:167] # save it into redis
period = data[167:-2] # save it into SQL
return (snrCode, realtime, period)
class PlainTCP(protocol.Protocol):
def __init__(self, factory):
self.factory = factory
self.factory.numConnections = 0
self.snrCode = None
self.rData = None
self.pData = None
self.err = None
def connectionMade(self):
self.factory.numConnections += 1
print "Nr. of connections: %d\n" %(self.factory.numConnections)
self.transport.write("Hello remote\r\n") # it only prints very 5 connections.
def connectionLost(self, reason):
self.factory.numConnections -= 1
print "Nr. of connections: %d\n" %(self.factory.numConnections)
#defer.inlineCallbacks
def dataReceived(self, data):
global dbpool, rcs
(self.snrCode,rDat,pDat) = getDataParsed(data)
if self.snrCode == None or rDat == None or pDat == None:
err = "Bad format"
else:
err = "OK"
print "err:%s"%(err) # debug print to show flow control
self.err = err
self.transport.write(self.snrCode)
self.transport.write(self.err)
self.transport.write(rDat)
self.transport.write(pDat)
self.transport.loseConnection()
if self.snrCode != None and rDat != None and pDat != None:
res = yield self.saveRealTimeData(rcs, rDat)
res = yield self.savePeriodData(dbpool, pDat, conf)
print "err2:%s"%(err) # debug print to show flow control
#defer.inlineCallbacks
def saveRealTimeData(self, rc, dat):
key = "somekey"
val = "somedata"
yield rc.set(key,val)
yield rc.expire(key,30)
#defer.inlineCallbacks
def savePeriodData(self,rc,dat,conf):
query = "some SQL statement"
yield rc.runQuery(query)
class PlainTCPFactory(protocol.Factory):
def buildProtocol(self, addr):
return PlainTCP(self)
def main():
dmdb = Dmdb()
if not dmdb.detectDb():
print "Please run MySQL RDBS first."
sys.exit()
log.startLogging(sys.stdout)
reactor.listenTCP(8080, PlainTCPFactory())
reactor.run()
if __name__ == "__main__":
main()
And clip of my client, which is a simple client:
def connectSend(host="127.0.0.1",port=8080):
global packet
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((host, port))
s.sendall(''.join(packet))
data = s.recv(1024)
s.close()
print 'Received', repr(data)
except socket.error, err:
print "Remote socket is not available: %s"%str(err)
sys.exit(1)
The current status is:
If disable #defer.inlineCallbacks and yield opertions of dataReceived(), both self.transport.write() inside of connectionMode() and dataReceived() can output data to clients.
If we enabled #defer.inlineCallbacks and two yield DB ops of SQL/Redis, then self.transport.write() inside of connectionMode() prints every 5 connections, and dataReceived() will not output any data to clients.
the debug print statements will print on log regardless of #defer.inlineCallbacks anyway.
I was told that dataReceived() should not be #defer.inlineCallbacks. But it doesn't change anything if I removed that decoration.
I am thinking if gevent can help me out of this unpredicted behavior. I am twisted into an endless tornado, cyclone .....
Anyone who has similiar experience, please help me. Thanks.
By changing function as following, the code works.
#COMMENT OUT decorator of #defer.inlineCallbacks
def dataReceived(self, data):
global dbpool, rcs
(self.snrCode,rDat,pDat) = getDataParsed(data)
if self.snrCode == None or rDat == None or pDat == None:
err = "Bad format"
else:
err = "OK"
print "err:%s"%(err) # debug print to show flow control
self.err = err
self.transport.write(self.snrCode)
self.transport.write(self.err)
self.transport.write(rDat)
self.transport.write(pDat)
self.transport.loseConnection()
if self.snrCode != None and rDat != None and pDat != None:
self.saveRealTimeData(rcs, rDat)
self.savePeriodData(dbpool, pDat, conf)
# Removing yield before DB ops
print "err2:%s"%(err) # debug print to show flow control
#defer.inlineCallbacks
def saveRealTimeData(self, rc, dat):
print "saveRedis"
key = "somekey"
val = "somedata"
yield rc.set(key,val)
yield rc.expire(key,30)
#defer.inlineCallbacks
def savePeriodData(self,rc,dat,conf):
print "save SQL"
query = "some SQL statement"
yield rc.runQuery(query)
If we keep #defer.inlineCallbacks and yield in dataReceived. The connection is closed before second DB op. Therefore no data is output to connection. Maybe is caused by inlineCallbacks decorator.
By removing this, the flow control is simple and straightforward.
However, I still can get why I can not add inlineCallbacks if there are two deferred DB ops. This time they don't need deferred?
I am using multiprocessing module to generate 35 dataframes. I guess this will save my time. But the problem is that the class does not return anything. I expect the list of dataframes to be returned from self.dflist
Here is how to create dfnames list.
urls=[]
fnames=[]
dfnames=[]
for x in xrange(100,3600,100):
y = str(x)
i = y.zfill(4)
filename='DCHB_Town_Release_'+i+'.xlsx'
url = "http://www.censusindia.gov.in/2011census/dchb/"+filename
urls.append(url)
fnames.append(filename)
dfnames.append((filename, 'DCHB_Town_Release_'+i))
This is the class that uses the dfnames generated by above code.
import pandas as pd
import multiprocessing
class mydf1():
def __init__(self, dflist, jobs, dfnames):
self.dflist=list()
self.jobs=list()
self.dfnames=dfnames
def dframe_create(self, filename, dfname):
print 'abc', filename, dfname
dfname=pd.read_excel(filename)
self.dflist.append(dfname)
print self.dflist
return self.dflist
def mp(self):
for f,d in self.dfnames:
p = multiprocessing.Process(target=self.dframe_create, args=(f,d))
self.jobs.append(p)
p.start()
#return self.dflist
for j in self.jobs:
j.join()
print '%s.exitcode = %s' % (j.name, j.exitcode)
This class when called like this...
dflist=[]
jobs=[]
x=mydf1(dflist, jobs, dfnames)
y=x.mp()
Prints the self.dflist correctly. But does not return anything.
I can collect all datafarmes sequentially. But in order to save time, I need to use multiple processes simultaneously to generate and add dataframes to a list.
In your case I prefer to write as less code as possible and use Pool:
import pandas as pd
import logging
import multiprocessing
def dframe_create(filename):
try:
return pd.read_excel(filename)
except Exception as e:
logging.error("Something went wrong: %s", e, exc_info=1)
return None
p = multiprocessing.Pool()
excel_files = p.map(dframe_create, dfnames)
for f in excel_files:
if f is not None:
print 'Ready to work'
else:
print ':('
Prints the self.dflist correctly. But does not return anything.
That's because you don't have a return statement in the mp method, e.g.
def mp(self):
...
return self.dflist
It's not entirely clear what you're issue is, however, you have to take some care here in that you can't just pass objects/lists across processes. That's why you have special objects (which lock while they make modifications to a list), that way you don't get tripped up when two processes try to make a change at the same time (and you only get one update).
That is, you have to use multiprocessing's list.
class mydf1():
def __init__(self, dflist, jobs, dfnames):
self.dflist = multiprocessing.list() # perhaps should be multiprocessing.list(dflist or ())
self.jobs = list()
self.dfnames = dfnames
However you have a bigger problem: the whole point of multiprocessing is that they may run/finish out of order, so keeping two lists like this is doomed to fail. You should use a multiprocessing.dict that way the DataFrame is saved unambiguously with the filename.
class mydf1():
def __init__(self, dflist, jobs, dfnames):
self.dfdict = multiprocessing.dict()
...
def dframe_create(self, filename, dfname):
print 'abc', filename, dfname
df = pd.read_excel(filename)
self.dfdict[dfname] = df