How to implement QThread correctly with matplotlib and pyplot - pandas

Question

I understand that there have been one or two other questions posted that are related but not exactly what I need. I'm building this gui that activates a module by clicking a button. This python module that gets activated by pushing the button generates heatmaps from multiple pandas dataframes and saves those images, which in turn is then saved into an xlsx using pandas ExcelWriter.
I've tried to implement QThread, as other stackoverflow examples tried to explain similar problems but I continue getting this error: "It is not safe to use pixmaps outside the GUI thread". I understand that technically I'm not creating the heatmap inside the MAIN gui thread but I thought with QThread that I am still inside "a" gui thread. These dataframes that the heatmaps are based off of can be of a large size at times and I am somewhat grasping the concept of sending a signal to the main gui thread when a heatmap is to be created and have the heatmap function inside the main gui class...but I fear that will be troublesome later in passing so much data around..this is more like pipelining than threading. I just want this working thread to create these images and save them and then take those saved files and save them into an xlsx without interrupting the main gui..
(NOTE: This is a simplified version, in the real program there will be several of these threads created almost simultaneously and inside each thread several heatmaps will be created)
---main.py---
import sys
from MAIN_GUI import *
from PyQt4 import QtGui, QtCore
from excel_dummy import *
if __name__=="__main__":
app = QtGui.QApplication(sys.argv)
class MAIN_GUI(QtGui.QMainWindow):
def __init__(self):
super(MAIN_GUI, self).__init__()
self.uiM = Ui_MainWindow()
self.uiM.setupUi(self)
self.connect(self.uiM.updateALL_Button,QtCore.SIGNAL('clicked()'),self.newThread)
def newThread(self):
Excelify = excelify()
Excelify.start()
self.connect(Excelify,QtCore.SIGNAL('donethread(QString)'),(self.done))
def done(self):
print('done')
main_gui = MAIN_GUI()
main_gui.show()
main_gui.raise_()
sys.exit(app.exec_())
---excel_dummy.py---
import os, pandas as pd
from pandas import ExcelWriter
import numpy as np
import seaborn.matrix as sm
from PyQt4 import QtCore
from PyQt4.QtCore import QThread
from matplotlib.backends.backend_agg import FigureCanvas
from matplotlib.figure import Figure
import time
class excelify(QThread):
def __init__(self):
QThread.__init__(self)
def run(self):
path = 'home/desktop/produced_files'
with ExcelWriter(path + '/final.xlsx', engine='xlsxwriter') as writer:
workbook = writer.book
worksheet = workbook.add_worksheet()
heatit = self.heatmap()
worksheet.insert_image('C3',path + '/' + 'heat.jpg')
worksheet.write(2, 2, 'just write something')
writer.save()
print('file size: %s "%s"' % (os.stat(path).st_size, path))
time.slee(0.3)
self.emit(QtCore.SIGNAL('donethread(QString)'),'')
def heatmap(self):
df = pd.DataFrame(np.array([[1,22222,33333],[2,44444,55555],[3,44444,22222],[4,55555,33333]]),columns=['hour','in','out'])
dfu = pd.DataFrame(df.groupby([df.in,df.hour]).size())
dfu.reset_index(inplace=True)
dfu.rename(columns={'0':'Count'})
dfu.columns=['in','hour','Count']
dfu_2 = dfu.copy()
mask=0
fig = Figure()
ax = fig.add_subplot(1,1,1)
canvas = FigureCanvas(fig)
df_heatmap = dfu_2.pivot('in','hour','Count').fillna(0)
sm.heatmap(df_heatmap,ax=ax,square=True,annot=False,mask=mask)
fig.savefig(path + '/' + heat.jpg')
---MAIN_GUI.py---
from PyQt4 import QtCore,QtGui
try:
_fromUtf8 = QtCore.QString.fromUtf8
except AttributeError:
def _fromUtf8(s):
return s
try:
_encoding = QtGui.QApplication.unicodeUTF8
def _translate(context, text, disambig):
return QtGui.QApplication.translate(context, text, disambig, _encoding)
except AttributeError:
def _translate(context, text, disambig):
return QtGui.QApplication.translate(context, text, disambig)
class Ui_MainWindow(object):
def setupUi(self, MainWindow):
MainWindow.setObjectName(_fromUtf8("MainWindow"))
MainWindow.resize(320,201)
self.centralwidget = QtGui.QWidget(MainWindow)
self.centralwidget.setObjectName(_fromUtf8("centralwidget"))
self.updateALL_Button = QtGui.QPushButton(self.centralwidget)
self.updateALL_Button.setGeometry(QtCore.QRect(40,110,161,27))
self.updateALL_Button.setFocusPolicy(QtCore.Qt.NoFocus)
self.updateALL_Button.setObjectName(_fromUtf8("Options_updateALL_Button"))
MainWindow.setCentralWidget(self.centralwidget)
self.menubar = QtGui.QMenuBar(MainWindow)
self.menubar.setGeometry(QtCore.QRect(0, 0, 320, 24))
self.menubar.setObjectName(_fromUtf8("menubar"))
MainWindow.setMenuBar(self.menubar)
self.statusbar = QtGui.QStatusBar(MainWindow)
self.statusbar.setObjectName(_fromUtf8("statusbar"))
MainWindow.setStatusBar(self.statusbar)
self.retranslateUi(MainWindow)
QtCore.QMetaObject.connectSlotsByName(MainWindow)
def retranslateUi(self,MainWindow):
MainWindow.setWindowTitle(_translate("MainWindow", "MainWindow", None))
self.updateALL_Button.setText(_translate("MainWindow", "updateALL", None))

Answer 1

Even though you are explicitely using the Agg backend to generate your figure, it looks like Seaborn is still using the default backend on your system, which is most likely Qt4Agg, an interactive backend. We want Seaborn to use a non-interactive backend instead to avoid any error (see matplotlib documentation for more details about backends). To do so, tell Matplotlib in your imports to use the Agg backend and import Seaborn after Matplotlib.
You will also need to save your figure as a png, since jpg is not supported by the Agg backend. Unless you have some specific reasons for using jpg, png is usually a better format for graphs.
Finally, you could use a memory buffer instead of saving your images to a temporary file before saving them in an Excel Workbook. I haven't tested it, but it will probably be faster if you are working with large files.
Below is a MWE I've written which includes the aformentioned points and which does not give any error on my system in Python3.4:
import pandas as pd
import time
from pandas import ExcelWriter
import numpy as np
from PyQt4 import QtCore, QtGui
import matplotlib as mpl
mpl.use('Agg')
from matplotlib.backends.backend_agg import FigureCanvas
import seaborn.matrix as sm
try: # Python 2 (not tested)
from cStringIO import StringIO as BytesIO
except ImportError: # Python 3
from io import BytesIO
class MAIN_GUI(QtGui.QWidget):
def __init__(self):
super(MAIN_GUI, self).__init__()
self.worker = Excelify()
btn = QtGui.QPushButton('Run')
disp = QtGui.QLabel()
self.setLayout(QtGui.QGridLayout())
self.layout().addWidget(btn, 0, 0)
self.layout().addWidget(disp, 2, 0)
self.layout().setRowStretch(1, 100)
btn.clicked.connect(self.worker.start)
self.worker.figSaved.connect(disp.setText)
class Excelify(QtCore.QThread):
figSaved = QtCore.pyqtSignal(str)
def run(self):
self.figSaved.emit('Saving figure to Workbook.')
t1 = time.clock()
image_data = self.heatmap()
with ExcelWriter('final.xlsx', engine='xlsxwriter') as writer:
wb = writer.book
ws = wb.add_worksheet()
ws.insert_image('C3', 'heat.png', {'image_data': image_data})
writer.save()
t2 = time.clock()
self.figSaved.emit('Done in %f sec.' % (t2-t1))
def heatmap(self):
df = pd.DataFrame(np.array([[1, 22222, 33333], [2, 44444, 55555],
[3, 44444, 22222], [4, 55555, 33333]]),
columns=['hour', 'in', 'out'])
dfu = pd.DataFrame(df.groupby([df.out, df.hour]).size())
dfu.reset_index(inplace=True)
dfu.rename(columns={'0': 'Count'})
dfu.columns = ['in', 'hour', 'Count']
fig = mpl.figure.Figure()
fig.set_canvas(FigureCanvas(fig))
ax = fig.add_subplot(111)
df_heatmap = dfu.pivot('in', 'hour', 'Count').fillna(0)
sm.heatmap(df_heatmap, ax=ax, square=True, annot=False, mask=0)
buf= BytesIO()
fig.savefig(buf, format='png')
return(buf)
if __name__ == '__main__':
import sys
app = QtGui.QApplication(sys.argv)
w = MAIN_GUI()
w.show()
w.setFixedSize(200, 100)
sys.exit(app.exec_())