pytest-qt waitSignal for long running computation running in a thread pool - pyqt5

I have successfully implemented a Python Qt app based off this very nice tutorial.
I am now writing tests using pytest-qt to specifically test a button that triggers a long running computation that eventually emits a signal when finished. I would like to use waitSignal as documented here.
def test_long_computation(qtbot):
app = Application()
# Watch for the app.worker.finished signal, then start the worker.
with qtbot.waitSignal(app.worker.finished, timeout=10000) as blocker:
blocker.connect(app.worker.failed) # Can add other signals to blocker
app.worker.start()
# Test will block at this point until either the "finished" or the
# "failed" signal is emitted. If 10 seconds passed without a signal,
# TimeoutError will be raised.
assert_application_results(app)
When the button is clicked, this function is executed:
def on_button_click_function(self):
"""
start thread pool to run function
"""
# Pass the function to execute
worker = Worker(self.execute_check_urls)
worker.signals.result.connect(self.print_output) # type: ignore
worker.signals.finished.connect(self.thread_complete) # type: ignore
worker.signals.progress.connect(self.progress_fn) # type: ignore
# Execute
log_info("Starting thread pool worker ...")
self.threadpool.start(worker).
And when the thread completes, a signal is emitted
def thread_complete(self):
main_window = self.find_main_window()
if main_window:
main_window.button_click_signal.emit(
self.results
)
log_info(
f"Emitted signal for button click function: {self.results}"
)
Below is the init function of the main class:
class MyClass:
def __init__(
self,
*args,
**kwargs
):
super(MyClass, self).__init__(*args, **kwargs)
self.threadpool = QThreadPool()
print(
"Multithreading with max %d threads"
% self.threadpool.maxThreadCount()
).
And the worker class which is QRunnable:
def __init__(
self,
fn,
*args,
**kwargs
):
super(Worker, self).__init__()
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = WorkerSignals()
self.kwargs['progress_callback'] = self.signals.progress
#Slot()
def run(self):
try:
result = self.fn(*self.args, **self.kwargs)
except (Exception):
traceback.print_exc()
exctype, value = sys.exc_info()[:2]
self.signals.error.emit(
(exctype, value, traceback.format_exc())
)
else:
self.signals.result.emit(result)
finally:
self.signals.finished.emit().
I would appreciate some guidance on how to access the worker object from the threadpool. I also tested qtbot.mouseClick() which triggers the function but never emits the signal.

Related

PyQt5 QTimer in not working after moving it to a thread

I am currently trying to separate my PyQT5-GUI from my serial communication to prevent a freezing GUI.
Therefore, I tried to implement threading. So when I am pressing a button "Open Port", a new thread starts which handles this incoming and outgoing data of the Port.
Now, this works fine, but I am having problems with a timer not fulfilling its function. Essentially, I want to close the port after a 'no valid' message has been received for x seconds.
I tried to create a minimum example:
Upon starting, the GUI and the thread are created. When pressing "Open", the port is opened and the timer in the thread should start. After 3000 milliseconds, the port should be closed by the timer overflow.
from PyQt5 import QtGui, QtWidgets
from PyQt5.QtCore import *
from PyQt5.QtWidgets import QApplication, QMainWindow
from PyQt5 import QtSerialPort
import sys
class AECRobotControl(QMainWindow):
signal_open_port = pyqtSignal(str)
signal_close_port = pyqtSignal(int)
def __init__(self):
super().__init__()
# main-window setup
self.ui = Ui_RobotControls_Main()
self.ui.setupUi(self)
# Port
self.port_gcode_timer_sending = QTimer() # Timer for sending Gcode to prevent robot queue overflow
# END Port
# Threads
self.thread_port = QThread()
self.thread_port_worker = None
# END Threads
# END Variables
# Functions
# Init the ui with standard values
# self.init_ui()
# Init COM-worker
self.init_worker()
# Init signal callbacks
self.init_signals()
# show GUI
self.show()
# END Functions
def init_ui(self):
self.layout.addWidget(self.button_open)
self.layout.addWidget(self.button_close)
self.setLayout(self.layout)
def init_signals(self):
# Button Open Port
self.ui.pushButton_open_port.clicked.connect(self.button_open_comport)
# END Button Open Port
# Button Close Port
self.ui.pushButton_close_port.clicked.connect(self.button_close_comport)
# END Button Close Port
def init_worker(self):
self.thread_port_worker = RobotMessageThread()
self.thread_port_worker.moveToThread(self.thread_port)
self.thread_port.started.connect(self.thread_port_worker.start_thread)
self.thread_port.finished.connect(self.thread_port.deleteLater)
self.thread_port_worker.finished.connect(self.thread_port.quit)
self.thread_port_worker.finished.connect(self.thread_port_worker.deleteLater)
self.signal_open_port.connect(self.thread_port_worker.open_port_slot)
self.signal_close_port.connect(self.thread_port_worker.close_comport)
self.thread_port.start()
def button_open_comport(self):
self.signal_open_port.emit("COM4")
def button_close_comport(self):
if (self.thread_port.isRunning() == True):
self.signal_close_port.emit(0)
def parse_com_message(self, message):
try:
print(message)
except Exception as e:
print(e)
class RobotMessageThread(QObject):
finished = pyqtSignal()
progress = pyqtSignal(int)
com_message_parsed = pyqtSignal(QByteArray)
com_ascii_message_parsed = pyqtSignal(str)
def __init__(self):
super().__init__()
self.port_com_port = QtSerialPort.QSerialPort()
self.port_name = None
self.port_is_alive_timer = QTimer() # Interprets valid received messages as alive-sign from robot.
""" Functions to be called upon start of thread"""
def start_thread(self):
print("Thread started")
self.port_is_alive_timer.timeout.connect(lambda: self.close_comport(1))
""" Inits the port"""
def _init_port(self):
self.port_com_port.setPortName(self.port_name)
self.port_com_port.setBaudRate(QtSerialPort.QSerialPort.BaudRate.Baud115200)
self.port_com_port.setParity(QtSerialPort.QSerialPort.Parity.NoParity)
self.port_com_port.setDataBits(QtSerialPort.QSerialPort.DataBits.Data8)
self.port_com_port.setStopBits(QtSerialPort.QSerialPort.StopBits.OneStop)
#pyqtSlot(bytearray)
def message_slot(self, message: bytearray):
self._write_to_port(message)
#pyqtSlot(str)
def open_port_slot(self, com_name: str):
self.port_name = com_name
self._init_port()
self._open_comport()
#pyqtSlot()
def close_port_slot(self, message: bytearray):
self.close_comport(0)
""" Tries to open the selected comport"""
def _open_comport(self):
# Check whether port is already open
if self.port_com_port.open(QIODevice.ReadWrite) == True:
self.port_com_port.setDataTerminalReady(True)
print("COM Opened")
# Reset message-buffer
self.port_is_alive_timer.start(3000)
else:
print("opening failed")
""" Closes the selected comport"""
def close_comport(self, source):
if self.port_com_port.isOpen() == True:
# Close port and delete queue
self.port_com_port.clear()
self.port_com_port.close()
# Stop timers
self.port_is_alive_timer.stop()
print("Closed by " + str(source))
else:
print("Closing failed")
# GUI
class Ui_RobotControls_Main(object):
def setupUi(self, RobotControls_Main):
RobotControls_Main.setObjectName("RobotControls_Main")
RobotControls_Main.resize(1024, 900)
RobotControls_Main.setMinimumSize(QSize(1024, 900))
RobotControls_Main.setMaximumSize(QSize(1600, 1200))
self.centralwidget = QtWidgets.QWidget(RobotControls_Main)
self.centralwidget.setObjectName("centralwidget")
self.gridLayout_12 = QtWidgets.QGridLayout(self.centralwidget)
self.gridLayout_12.setObjectName("gridLayout_12")
self.QGroupBox_port_settings = QtWidgets.QGroupBox(self.centralwidget)
self.QGroupBox_port_settings.setObjectName("QGroupBox_port_settings")
self.gridLayout_15 = QtWidgets.QGridLayout(self.QGroupBox_port_settings)
self.gridLayout_15.setObjectName("gridLayout_15")
self.horizontalLayout_21 = QtWidgets.QHBoxLayout()
self.horizontalLayout_21.setObjectName("horizontalLayout_21")
self.pushButton_open_port = QtWidgets.QPushButton(self.QGroupBox_port_settings)
self.pushButton_open_port.setMaximumSize(QSize(100, 50))
self.pushButton_open_port.setObjectName("pushButton_open_port")
self.horizontalLayout_21.addWidget(self.pushButton_open_port)
self.gridLayout_15.addLayout(self.horizontalLayout_21, 0, 0, 1, 1)
self.horizontalLayout_4 = QtWidgets.QHBoxLayout()
self.horizontalLayout_4.setObjectName("horizontalLayout_4")
self.pushButton_close_port = QtWidgets.QPushButton(self.QGroupBox_port_settings)
self.pushButton_close_port.setObjectName("pushButton_close_port")
self.horizontalLayout_4.addWidget(self.pushButton_close_port)
self.gridLayout_15.addLayout(self.horizontalLayout_4, 0, 1, 1, 1)
self.gridLayout_12.addWidget(self.QGroupBox_port_settings, 0, 0, 1, 1)
RobotControls_Main.setCentralWidget(self.centralwidget)
self.statusbar = QtWidgets.QStatusBar(RobotControls_Main)
self.statusbar.setObjectName("statusbar")
RobotControls_Main.setStatusBar(self.statusbar)
self.retranslateUi(RobotControls_Main)
QMetaObject.connectSlotsByName(RobotControls_Main)
def retranslateUi(self, RobotControls_Main):
_translate = QCoreApplication.translate
RobotControls_Main.setWindowTitle(_translate("RobotControls_Main", "RobotControls"))
self.QGroupBox_port_settings.setTitle(_translate("RobotControls_Main", "Port settings"))
self.pushButton_open_port.setText(_translate("RobotControls_Main", "Open Port"))
self.pushButton_close_port.setText(_translate("RobotControls_Main", "Close Port"))
if __name__ == '__main__':
app = QApplication(sys.argv)
main_window = AECRobotControl()
app.exec() # QT main-threadtimer.start(TIME)
I found a solution. In the above code, just one line needs to be added: after moving the worker to the separate thread, the QTimer also has to be moved to the same thread. So, after self.thread_port_worker.moveToThread(self.thread_port), the line self.thread_port_worker.port_is_alive_timer.moveToThread(self.thread_port) needs to be added

pyQt5 handling multiple windows

A simple issue which I can't fix as I'm pretty newbie in using pyQt...
I have an application which has a window with a QPushButton1 and when it's clicked it opens up a second window with another QPushButton2. What is happening is when I close the second window pressing 'x' and I come back to the first one, if I click again the QPushButton1 the second window does not open. Only when I click one more the QPushButton1 the second window is opened. Maybe I would need to set self.w(second window) = None when the second window close, but I don't know where to put it. Thanks for any help
class MainWindow(QWidget, Ui_f_tabella):
def __init__(self):
super().__init__()
self.setupUi(self)
self.w = None # No external window yet.
self.pb_Ins.clicked.connect(self.inserimento_mat)
def inserimento_mat(self):
if self.w is None:
self.w = InserimentoMateriali()
self.w.show()
self.w.pb_conf.clicked.connect(self.conferma_inserimento_materiali)
else:
self.w.close()
self.w = None # Discard reference, close window.
def conferma_inserimento_materiali(self):
"""Aggiunta nuova risorsa"""
self.data = []
for field in (self.w.lE_descMat, self.w.dSB_pesoSp):
if not field.text():
QMessageBox.critical(
self,
"Errore!",
f"Inserimento non consentito per mancanza di informazioni"
#f"Inserire il valore {field.objectName()}",
)
self.data = None # Reset .data
return
self.data.append(field.text())
if not self.data:
return
rec = self.model.record()
rows = self.model.rowCount()
for column_index, field in enumerate(self.data):
if rec.fieldName(column_index)=="ps_mat":
x = field.replace(",", ".")
field=x
rec.setValue(rec.fieldName(column_index), field)
if(self.model.insertRecord(-1, rec)):
self.model.submitAll()
self.model.select()
QMessageBox.information(self, "Conferma inserimento",
f"Inserimento nuovo materiale effettuato")
self.w.close()
self.w = None
else:
msg="Errore inserimento nuovo materiale: <br> <br>" + self.model.lastError().text()
QMessageBox.critical(self, "ERRORE", msg)
class InserimentoMateriali(QDialog,Ui_InsMateriali):
def __init__(self):
super().__init__()
self.setupUi(self)
app = QApplication(sys.argv)
window = MainWindow()
window.show()
sys.exit(app.exec_())enter code here
The problem is that when closing the window using the system buttons doesn't reset your self.w, so the if self.w is None condition is not met.
A possible solution is to set the DeleteOnClose attribute and connect the destroyed signal to a function that restores the variable.
def inserimento_mat(self):
if self.w is None:
self.w = InserimentoMateriali()
self.w.setAttribute(Qt.WA_DeleteOnClose)
self.w.destroyed.connect(self.resetFlag)
self.w.show()
self.w.pb_conf.clicked.connect(self.conferma_inserimento_materiali)
else:
self.w.close()
def resetFlag(self):
self.w = None

TypeError: one() takes 1 positional argument but 2 were given

PyCharm return "TypeError: one() takes 1 positional argument but 2 were given"
I've searched for the whole night but still can't figure it out T T
I think the bug is from mpl_connect(), because when I use the connect() from pyqtBoundSignal it works
"""test.py"""
class forTest(QWidget):
signalTest = pyqtSignal()
def __init__(self):
super(forTest, self).__init__()
canvas = FigureCanvas(figure(facecolor="blue"))
# I got the following from other answer, but still not working
self.cid = canvas.mpl_connect('button_press_event', self.one)
layout = QHBoxLayout()
self.setLayout(layout)
layout.addWidget(canvas)
def one(self):
self.signalTest.emit()
print("emit()")
"""receice.py"""
class RTest(object):
def handle_signal(self):
print("get successfully")
"""main.py"""
if __name__ == "__main__":
app = QApplication(sys.argv)
tt = forTest()
rr = RTest()
tt.signalTest.connect(rr.handle_signal)
tt.show()
sys.exit(app.exec_())
From the mpl_connect() documentation:
func : callable
The callback function to be executed, which must have the signature:
def func(event: Event) -> Any
So, one() must have a further argument:
def one(self, event):
self.signalTest.emit()
print("emit()")
The reason for which the direct signal connection works is that PyQt is able to discard positional arguments when they exceed the connected signal (or function) argument count.

Dequeueing from RandomShuffleQueue does not reduce size

In order to train a model I have encapsulated my model in a class.
I use a tf.RandomShuffleQueue to enqueue a list of filenames to.
However when I dequeue the elements they get dequeued but the size of the queue does not reduce.
Following are more specific questions followed by the code snippet :
If I have only 5 images for example, but steps range upto 100, would this result in the addfilenames called repeatedly automatically ? It does not give me any error on dequeuing so I am thinking that it is getting called automatically.
Why the size of the tf.RandomShuffleQueue is not changing ? It remains constant.
import os
import time
import functools
import tensorflow as tf
from Read_labelclsloc import readlabel
def ReadTrain(traindir):
# Returns a list of training images, their labels and a dictionay.
# The dictionary maps label names to integer numbers.
return trainimgs, trainlbls, classdict
def ReadVal(valdir, classdict):
# Reads the validation image labels.
# Returns a dictionary with filenames as keys and
# corresponding labels as values.
return valdict
def lazy_property(function):
# Just a decorator to make sure that on repeated calls to
# member functions, ops don't get created repeatedly.
# Acknowledgements : https://danijar.com/structuring-your-tensorflow-models/
attribute= '_cache_' + function.__name__
#property
#functools.wraps(function)
def decorator(self):
if not hasattr(self, attribute):
setattr(self, attribute, function(self))
return getattr(self, attribute)
return decorator
class ModelInitial:
def __init__(self, traindir, valdir):
self.graph
self.traindir = traindir
self.valdir = valdir
self.traininginfo()
self.epoch = 0
def traininginfo(self):
self.trainimgs, self.trainlbls, self.classdict = ReadTrain(self.traindir)
self.valdict = ReadVal(self.valdir, self.classdict)
with self.graph.as_default():
self.trainimgs_tensor = tf.constant(self.trainimgs)
self.trainlbls_tensor = tf.constant(self.trainlbls, dtype=tf.uint16)
self.trainimgs_dict = {}
self.trainimgs_dict["ImageFile"] = self.trainimgs_tensor
return None
#lazy_property
def graph(self):
g = tf.Graph()
with g.as_default():
# Layer definitions go here
return g
#lazy_property
def addfilenames (self):
# This is the function where filenames are pushed to a RandomShuffleQueue
filename_queue = tf.RandomShuffleQueue(capacity=len(self.trainimgs), min_after_dequeue=0,\
dtypes=[tf.string], names=["ImageFile"],\
seed=0, name="filename_queue")
sz_op = filename_queue.size()
dq_op = filename_queue.dequeue()
enq_op = filename_queue.enqueue_many(self.trainimgs_dict)
return filename_queue, enq_op, sz_op, dq_op
def Train(self):
# The function for training.
# I have not written the training part yet.
# Still struggling with preprocessing
with self.graph.as_default():
filename_q, filename_enqueue_op, sz_op, dq_op= self.addfilenames
qr = tf.train.QueueRunner(filename_q, [filename_enqueue_op])
filename_dequeue_op = filename_q.dequeue()
init_op = tf.global_variables_initializer()
sess = tf.Session(graph=self.graph)
sess.run(init_op)
coord = tf.train.Coordinator()
enq_threads = qr.create_threads(sess, coord=coord, start=True)
counter = 0
for step in range(100):
print(sess.run(dq_op["ImageFile"]))
print("Epoch = %d "%(self.epoch))
print("size = %d"%(sess.run(sz_op)))
counter+=1
names = [n.name for n in self.graph.as_graph_def().node]
coord.request_stop()
coord.join(enq_threads)
print("Counter = %d"%(counter))
return None
if __name__ == "__main__":
modeltrain = ModelInitial(<Path to training images>,\
<Path to validation images>)
a = modeltrain.graph
print(a)
modeltrain.Train()
print("Success")
The mystery is caused by the tf.train.QueueRunner that you created for the queue, which causes it to be filled in the background.
The following lines cause a background "queue runner" thread to be created:
qr = tf.train.QueueRunner(filename_q, [filename_enqueue_op])
# ...
enq_threads = qr.create_threads(sess, coord=coord, start=True)
This thread calls filename_enqueue_op in a loop, which causes the queue to be filled up as you remove elements from it.
The background thread from step 1 will almost always have a pending enqueue operation (filename_enqueue_op) on the queue. This means that after you dequeue a filename, the pending enqueue will run add fill the queue back up to capacity. (Technically there is a race condition here and you could see a size of capacity - 1, but this is quite unlikely).

Celery: abort task on connection error

I have to implement a Task subclass that gracefully fails if the broker is not running - currently I'm using RabbitMQ.
I could probably just use a try statement to catch the exception:
try:
Mytask.delay(arg1, arg2)
except socket.error:
# Send an notice to an admin
pass
but I'd like to create a subclass of Task that can handle that.
I've tried something like that:
class MyTask(Task):
ignore_result = True
def __call__(self, *args, **kwargs):
try:
return self.run(*args, **kwargs)
except socket.error:
# Send an notice to an admin
return None
but the workflow is clearly wrong. I think I need to inject maybe a backend subclass or a failure policy somehow.
Do you have any suggestion?
A possible solution I came up with:
import socket
from celery.decorators import task
from celery.task import Task
from celery.backends.base import BaseBackend
UNDELIVERED = 'UNDELIVERED'
class DummyBackend(BaseBackend):
"""
Dummy queue backend for undelivered messages (due to the broker being down).
"""
def store_result(self, *args, **kwargs):
pass
def get_status(self, *args, **kwargs):
return UNDELIVERED
def _dummy(self, *args, **kwargs):
return None
wait_for = get_result = get_traceback = _dummy
class SafeTask(Task):
"""
A task not raising socket errors if the broker is down.
"""
abstract = True
on_broker_error = None
errbackend = DummyBackend
#classmethod
def apply_async(cls, *args, **kwargs):
try:
return super(SafeTask, cls).apply_async(*args, **kwargs)
except socket.error, err:
if cls.on_broker_error is not None:
cls.on_broker_error(err, cls, *args, **kwargs)
return cls.app.AsyncResult(None, backend=cls.errbackend(),
task_name=cls.name)
def safetask(*args, **kwargs):
"""
Task factory returning safe tasks handling socket errors.
When a socket error occurs, the given callable *on_broker_error*
is called passing the exception object, the class of the task
and the original args and kwargs.
"""
if 'base' not in kwargs:
on_broker_error = kwargs.pop('on_broker_error', SafeTask.on_broker_error)
errbackend = kwargs.pop('errbackend', SafeTask.errbackend)
kwargs['base'] = type('SafeTask', (SafeTask,), {
'on_broker_error': staticmethod(on_broker_error),
'errbackend': errbackend,
'abstract': True,
})
return task(*args, **kwargs)
You can both subclass SafeTask or use the decorator #safetask.
If you can think of an improvement, don't hesitate to contribute.