Iam working on a script using PYQT5 GUI to do network script using paramiko and iam using pyQT Qthread as below in my main application class i created below function to run a script which reads csv file get IPs and hostnames and then call another QObject Class:
class MemoryUtilizWidget(QDialog):
def __init__(self):
super(MemoryUtilizWidget, self).__init__()
self.setStyleSheet("background-color: rgb(35, 35, 35);")
self.setWindowIcon(QtGui.QIcon('Logo.png'))
self.setWindowTitle('Memory Utilization Script Options')
layout = QGridLayout()
self.setLayout(layout)
def MemoryRunScript(self):
InventoryFile = self.InventoryCombo.currentText()
if InventoryFile:
Routers_List = []
Routers_Names = []
line_count = 0
with open(path + '\\Script Data\\Inventory Database\\' + InventoryFile) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
for row in csv_reader:
if line_count == 0:
pass
else:
Routers_Names.append(row[1])
Routers_List.append(row[2])
line_count+=1
Not_Accessible_Routers = open(path + "\\Script Data\\Routers Summary\\Not Accessible Routers.txt" , 'w+' , encoding='utf-8')
Login_Incorrect_Routers = open(path + "\\Script Data\\Routers Summary\\Login Routers Routers.txt" , 'w+' , encoding='utf-8')
TACACS_Down_Routers = open(path + "\\Script Data\\Routers Summary\\TACACS Down Routers.txt" , 'w+' , encoding='utf-8')
Wrong_Hostname_Routers = open(path + "\\Script Data\\Routers Summary\\Wrong Hostname Routers.txt" , 'w+' , encoding='utf-8')
Username = 'user'
Password = 'pass'
self.MemoryProgressBar.setMinimum(0)
self.MemoryProgressBar.setMaximum(len(Routers_List))
self.RunScriptButton.setEnabled(False)
self.InventoryCombo.setEnabled(False)
self.InventoryButton.setEnabled(False)
self.InventorytLineEdit.setEnabled(False)
self.MemoryThread = QtCore.QThread()
self.MemoryObject = MemoryThreading(Routers_List,Routers_Names,Not_Accessible_Routers,Login_Incorrect_Routers,TACACS_Down_Routers,Wrong_Hostname_Routers,Username,Password)
self.MemoryObject.moveToThread(self.MemoryThread)
self.MemoryObject.Finished.connect(self.MemoryFinished)
self.MemoryThread.started.connect(self.MemoryObject.MainThreading)
self.MemoryThread.start()
self.MemoryObject.Progressvalue.connect(self.ProgressValue)
the QObject class i used multithreading technique of ThreadPoolExecuter to run fast as the IPs are over 3000 IP which will take forever:
class MemoryThreading(QtCore.QObject):
Finished = QtCore.pyqtSignal()
Progressvalue = QtCore.pyqtSignal(int)
def __init__(self,Routers_List, Routers_Names, Login_Incorrect_Routers,TACACS_Down_Routers,Not_Accessible_Routers,Wrong_Hostname_Routers, Username, Password):
super().__init__()
self.Routers_List = Routers_List
self.Routers_Names = Routers_Names
self.Login_Incorrect_Routers = Login_Incorrect_Routers
self.TACACS_Down_Routers = TACACS_Down_Routers
self.Not_Accessible_Routers = Not_Accessible_Routers
self.Wrong_Hostname_Routers = Wrong_Hostname_Routers
self.Username = Username
self.Password = Password
def MainThreading(self):
i = 0
with concurrent.futures.ThreadPoolExecutor() as self.executor:
self.results = self.executor.map(RouterConnection, self.Routers_List,self.Routers_Names,itertools.repeat(self.Login_Incorrect_Routers),itertools.repeat(self.TACACS_Down_Routers),itertools.repeat(self.Not_Accessible_Routers),itertools.repeat(self.Wrong_Hostname_Routers), itertools.repeat(self.Username), itertools.repeat(self.Password))
for result in self.results:
i+=1
self.Progressvalue.emit(i)
try:
if result:
print(result)
except Exception as exc:
print(exc)
self.Progressvalue.emit(0)
self.Finished.emit()
my concern is how to force this Created threads during execution of the script to stop so that the user can run the script and also stop it if possible i searched and googled a lot but didn't find any solution to my case as iam using both QThread and ThreadpoolExecutor
Related
I'm trying to get a server for an app working, but I'm getting an error upon login:
[!] Object '<User at 0x7f12bc185a90>' is already attached to session '2' (this is '3')
It seems the session I'm adding is already on the database. This is the snippet of code that is causing the problem:
#app.route('/login', methods=['POST'])
def login():
u = User.query.filter(User.username == request.form["username"]).first()
if not u or u.password != request.form["password"]:
return error("E1")
s = Session.get_by_user(u)
if s is not None:
db_session.delete(s)
db_session.commit()
print db_session.execute("SELECT * FROM sessions").fetchall()
s = Session(u)
db_session.add(s)
db_session.commit()
return jsonify(s.values)
As you can see, I'm printing the content from the sessions table before trying to add anything, and it is empty! ([])
What else could be causing this?
Here is the 'Session' implementation:
class Session(Base):
__tablename__ = "sessions"
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey('users.id'), unique=True)
user = relationship(User)
key = Column(String(50), unique=True)
created = Column(DateTime)
def __init__(self, user=None):
self.user = user
self.key = base64.encodestring(os.urandom(24)).strip()
self.created = datetime.now()
def __repr__(self):
return '<Session %r>' % (self.key)
#property
def values(self):
return {"username" : self.user.username,
"key" : self.key,
"created" : str(self.created),
}
#classmethod
def get_by_key(cls, key):
s = cls.query.filter(cls.key == key).first()
#print datetime.now() - s.created
if s and datetime.now() - s.created > settings.SESSION_LIFETIME:
s = None
return s
#classmethod
def get_by_user(cls, user):
s = cls.query.filter(cls.user == user).first()
if s and datetime.now() - s.created > settings.SESSION_LIFETIME:
s.query.delete()
db_session.commit()
s = None
return s
As #marcinkuzminski mentioned, you can't add an object that is already attached to another session. Just pulling in the original session from the object with object_session() is risky, though, if you aren't sure that session originated in the same thread context you're currently operating in. A thread-safe method is to use merge():
local_object = db_session.merge(original_object)
db_session.add(local_object)
db_session.commit()
Object you're trying to modify is already attached to another session.
Maybe you have wrong imports, and db_session is a new instance.
A good workaround to this is to extract the current bound session and use it:
Instead of:
db_session.add(s)
Do:
current_db_sessions = db_session.object_session(s)
current_db_sessions.add(s)
This db session issue will arise if you are having server.py and model.py importing each other
server.py
from flask import Flask
import os
import models as appmod #################### importing models here in server.py<----------
app = Flask(__name__) # L1
app.config.from_object(os.environ['APP_SETTINGS']) # L2
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False # L3
database = SQLAlchemy(app) # L4
db = database # L5
#app.route('/item_delete/<id>', methods=['DELETE'])
def remove_method(id = None):
data_rec = appmod.Employee.query.get(id)
db.session.delete(data_rec)
db.session.commit()
return "DELETE"
if __name__ == '__main__':
app.run(port=5000, host='0.0.0.0',debug=True,threaded=True)
models.py
from server import db #################### importing server in models.py here <------------
from sqlalchemy.dialects.mysql import JSON
class Employee(db.Model):
__tablename__ = 'employe_flask'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(128))
datetime = db.Column(db.DateTime)
designation = db.Column(db.String(128))
def __init__(self, name, datetime, designation):
self.name = name
self.datetime = datetime
self.designation = designation
#staticmethod
def delete_rec(data_rec):
db.session.delete(data_rec)#.delete
db.session.commit()
def __repr__(self):
record = {"name":self.name,"date":self.datetime.ctime(),"designation":self.designation}.__str__()
return record
Remove the line L1 to L5 from server.py and place it in common file like settings.py
and import 'app' and 'db' to server.py and import db in models.py
like this files below
server.py
from flask import Flask
import os
import models as appmod
from settings import app, db
#app.route('/item_delete/<id>', methods=['DELETE'])
def remove_method(id = None):
data_rec = appmod.Employee.query.get(id)
db.session.delete(data_rec)
db.session.commit()
return "DELETE"
if __name__ == '__main__':
app.run(port=5000, host='0.0.0.0',debug=True,threaded=True)
settings.py
import os
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
app = Flask(__name__) # L1
app.config.from_object(os.environ['APP_SETTINGS']) # L2
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False # L3
database = SQLAlchemy(app) # L4
db = database # L5
models.py
from settings import db
from sqlalchemy.dialects.mysql import JSON
class Employee(db.Model):
__tablename__ = 'employe_flask'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(128))
datetime = db.Column(db.DateTime)
designation = db.Column(db.String(128))
def __init__(self, name, datetime, designation):
self.name = name
self.datetime = datetime
self.designation = designation
#staticmethod
def delete_rec(data_rec):
db.session.delete(data_rec)#.delete
db.session.commit()
def __repr__(self):
record = {"name":self.name,"date":self.datetime.ctime(),"designation":self.designation}.__str__()
return record
This error means the record you are handling is attached to 2 different session(db)!
One of the reasons is that you may define your model with one db = SQLAlchemy(app) and add/insert/modify the database with another!
My solution is UNIFORMING THE DB!
try this:
u = db.session.query(User).filter(User.username == request.form["username"]).first()
Instead of this:
u = User.query.filter(User.username == request.form["username"]).first()
I had this problem too.
I created a test_file.py and added this code:
from app import app
from models import Tovar
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy(app)
tovardel = Tovar.query.filter(Tovar.region == 1 and Tovar.price == 12).first()
db.session.delete(tovardel)
tovar = Tovar.query.filter(Tovar.region == 1 and Tovar.price == 12).first()
print(tovar.description)
and when I ran the code I got this error:
Object '<Tovar at 0x7f09cbf74208>' is already attached to session '1' (this is '2')
PROBLEM SOLVING:
If you have db = SQLAlchemy(app) in, for example, text_file.py, and in app.py, you get this problem all time. You should del db = SQLAlchemy(app), and import db from app from app import db
I faced the same issue. I was defining models in a separate file and I had to call SQLAlchemy twice. That's why there were two different sessions were running.
I solved this by doing following:
In case you are trying to remove an object from db:
Just create the removeObject function inside the model
I'm attempting to write a fairly simple class to handle some relays for a robot costume. The user should be able to push some buttons to activate different sets of lights/EL wire. A basic thread seemed the best way to handle this, but maybe I'm missing something about the micropython implementation...
Here's the relevant two functions:
def run(self):
"""Start the default state"""
self.current_state.start_new_thread(self.activate_emotion, (self.state,))
def change_state(self):
"""Kill the old state (if applicable) and start a new one"""
print('Exiting thread...')
self.current_state.exit()
print('Starting new thread...')
self.current_state.start_new_thread(self.activate_emotion, (self.state,))
print('Done.')
When my script starts, run() throws "OSError: core1 in use", but the "default" state begins to run. It does this even after a fresh startup. When my button press is detected and activates change_state(), I get the output: "Exiting thread...", and it just hangs there indefinitely. What am I missing here? Any help would be greatly appreciated.
Here's the entirety of my script:
from machine import Pin
import utime
import _thread
import random
class LowRelay(Pin):
def turn_on(self):
self.low()
def turn_off(self):
self.high()
def test(self):
self.turn_on()
utime.sleep(0.5)
self.turn_off()
class Ariel():
def __init__(self):
self.happy_button = Pin(10, Pin.IN, Pin.PULL_DOWN)
self.sad_button = Pin(11, Pin.IN, Pin.PULL_DOWN)
self.scared_button = Pin(12, Pin.IN, Pin.PULL_DOWN)
self.thinking_button = Pin(13, Pin.IN, Pin.PULL_DOWN)
self.happy_relay = LowRelay(2, Pin.OUT)
self.sad_relay = LowRelay(3, Pin.OUT)
self.scared_relay = LowRelay(4, Pin.OUT)
self.thinking_relay = LowRelay(5, Pin.OUT)
self.group1_relay = LowRelay(6, Pin.OUT)
self.group2_relay = LowRelay(7, Pin.OUT)
self.group3_relay = LowRelay(8, Pin.OUT)
self.group4_relay = LowRelay(9, Pin.OUT)
self.led_groups = ['group1', 'group2',
'group3', 'group4']
self.off_led_groups = ['group1', 'group2',
'group3', 'group4']
self.on_led_groups = []
self.state = 'default'
self.current_state = _thread
def full_test(self):
""" Self test """
print('Testing....')
self.group1_relay.test()
self.group2_relay.test()
self.group3_relay.test()
self.group4_relay.test()
self.happy_relay.test()
self.sad_relay.test()
self.scared_relay.test()
self.thinking_relay.test()
print('Test complete.')
utime.sleep(1)
def test(self):
"""Quick Test"""
self.all_relays_off()
utime.sleep(1)
self.all_relays_on()
utime.sleep(1)
self.all_relays_off()
def run(self):
"""Start the default state"""
self.current_state.start_new_thread(self.activate_emotion, (self.state,))
def change_state(self):
"""Kill the old state (if applicable) and start a new one"""
print('Exiting thread...')
self.current_state.exit()
print('Starting new thread...')
self.current_state.start_new_thread(self.activate_emotion, (self.state,))
print('Done.')
def watch_and_wait(self, seconds):
"""Sleep while an effect processes, watching for a state change."""
start_state = self.state
ticks = int(float(seconds)/0.25)
for x in range(ticks):
self.check_buttons()
if self.state != start_state:
print('Changing state...')
self.change_state()
continue
utime.sleep(0.25)
def check_buttons(self):
"""Set states based on button presses"""
if self.happy_button.value():
self.state = 'happy'
if self.sad_button.value():
self.state = 'sad'
if self.scared_button.value():
self.state = 'scared'
if self.thinking_button.value():
self.state = 'thinking'
def swap_groups(self):
"""
Pick one of the two LED groups that is not on, and one that is.
Swap them. If none are on, turn on two.
"""
off_lg = self.off_led_groups
on_lg = self.on_led_groups
on_candidate = off_lg.pop(random.randrange(len(off_lg)))
off_candidate = on_lg.pop(random.randrange(len(on_lg))) if on_lg else None
on_lg.append(on_candidate)
if off_candidate:
off_lg.append(off_candidate)
if len(on_lg) < 2:
on_lg.append(off_lg.pop(random.randrange(len(off_lg))))
for x in self.led_groups:
r = getattr(self, '%s_relay' % x)
if x in on_lg:
r.turn_on()
else:
r.turn_off()
def all_relays_on(self):
"""Turn off all relays"""
self.happy_relay.turn_on()
self.sad_relay.turn_on()
self.scared_relay.turn_on()
self.thinking_relay.turn_on()
self.group1_relay.turn_on()
self.group2_relay.turn_on()
self.group3_relay.turn_on()
self.group4_relay.turn_on()
def all_relays_off(self):
"""Turn off all relays"""
self.happy_relay.turn_off()
self.sad_relay.turn_off()
self.scared_relay.turn_off()
self.thinking_relay.turn_off()
self.group1_relay.turn_off()
self.group2_relay.turn_off()
self.group3_relay.turn_off()
self.group4_relay.turn_off()
####################
# define states
# default - turn off EL wire, swap LED groups every second
# emotions - turn off other LEDs, and turn on relevant EL wire
####################
def activate_emotion(self, emotion):
"""Activates the selected emotional state"""
print('Entering %s state...' % emotion)
self.all_relays_off()
if self.state == 'default':
while self.state == 'default':
self.swap_groups()
self.watch_and_wait(1)
else:
r = getattr(self, '%s_relay' % emotion)
r.turn_on()
self.watch_and_wait(5)
self.state = 'default'
print('Changing state to default.')
self.change_state()
ariel = Ariel()
ariel.test()
while True:
ariel.run()
pass
I'm trying to make a drag and drop behavior in QFileSystemModel but because I have no experience in making a drag and drop before, I tried it first on QTreeView. (I attached the video of the behavior)
Now that I'm fine with the behavior I want, I then just changed the model to QFileSystemModel but sadly It's not working. So I tried to read the QFileSystemModel, QTreeView, and Drag and Drop from Qt and I ended up with the code below:
The code I ended up with:
import os
import sys
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
class MQTreeView(QTreeView):
def __init__(self, model):
super().__init__()
self.setSelectionMode(QAbstractItemView.ExtendedSelection)
# self.setDragDropMode(QAbstractItemView.InternalMove)
self.setModel(model)
self.setDragDropMode(QAbstractItemView.DragDrop)
self.setRootIndex(model.index(os.path.dirname(os.path.abspath("__file__"))))
self.setDefaultDropAction(Qt.MoveAction)
self.viewport().setAcceptDrops(True)
def dragEnterEvent(self, event):
m = event.mimeData()
if m.hasUrls():
event.accept()
return
event.ignore()
# return super().dragEnterEvent(event)
def dropEvent(self, event):
print("[drop event] - dropped")
if event.source():
QTreeView.dropEvent(self, event)
else:
ix = self.indexAt(event.pos())
model = self.model()
if ix.isValid():
if not model.isDir(ix):
ix = ix.parent() # In case of folder/Dir
pathDir = model.filePath(ix)
else:
# for empty drag and drop
pathDir = model.rootPath()
m = event.mimeData()
if m.hasUrls():
urlLocals = [url for url in m.urls() if url.isLocalFile()]
accepted = False
for urlLocal in urlLocals:
path = urlLocal.toLocalFile()
info = QFileInfo(path)
n_path = QDir(pathDir).filePath(info.fileName())
o_path = info.absoluteFilePath()
if n_path == o_path:
continue
if info.isDir():
QDir().rename(o_path, n_path)
else:
qfile = QFile(o_path)
if QFile(n_path).exists():
n_path += "(copy)"
qfile.rename(n_path)
print(f"added -> {info.fileName()}")
accepted = True
if accepted:
event.acceptProposedAction()
# return super().dropEvent(event)
class AppDemo(QWidget):
def __init__(self):
super().__init__()
# -- right -- #
self.model1 = QFileSystemModel()
self.model1.setRootPath(os.path.dirname(os.path.abspath("__file__")))
self.view1 = MQTreeView(self.model1)
# -- left -- #
self.model2 = QFileSystemModel()
self.model2.setRootPath(os.path.dirname(os.path.abspath("__file__")))
self.view2 = MQTreeView(self.model2)
# -- layout -- #
layout = QHBoxLayout(self)
layout.addWidget(self.view1)
layout.addWidget(self.view2)
app = QApplication(sys.argv)
main = AppDemo()
main.show()
app.exec_()
The code above is still not doing the behavior I want but I'm pretty sure that something else is wrong and it is not with the overridden function (dragEnterEvent and dropEvent). My best guess is that I didn't set properly the correct way QTreeView accepts the drops although I'm not really sure.
My Question: What is wrong with my Implementation? Is it the way I accept drops or it is something else?
Found what's wrong! I didn't override the dragMoveEvent method. I need to override the dragMoveEvent to make sure that the drag will not be forbidden.
I need to accept all drag event in the dragEnterEvent:
def dragEnterEvent(self, event):
event.accept()
Then I need to filter the events in the dragMoveEvent:
def dragMoveEvent(self, event):
m = event.mimeData()
if m.hasUrls():
event.accept()
print("[dropEnterEvent] - event accepted")
return
event.ignore()
I attached the video and code of the working behavior below.
The final implementation:
import os
import sys
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
class MQTreeView(QTreeView):
def __init__(self, model, path):
super().__init__()
self.setSelectionMode(QAbstractItemView.ExtendedSelection)
self.setModel(model)
self.setDragDropMode(QAbstractItemView.DragDrop)
self.setRootIndex(model.index(path))
self.setDefaultDropAction(Qt.MoveAction)
self.viewport().setAcceptDrops(True)
def dragEnterEvent(self, event):
event.accept()
def dragMoveEvent(self, event):
m = event.mimeData()
if m.hasUrls():
event.accept()
print("[dropEnterEvent] - event accepted")
return
event.ignore()
def dropEvent(self, event):
print("[drop event] - dropped")
if event.source():
ix = self.indexAt(event.pos())
model = self.model()
if ix.isValid():
if not model.isDir(ix):
ix = ix.parent()
pathDir = model.filePath(ix)
else:
# for empty drag and drop
pathDir = model.rootPath()
m = event.mimeData()
if m.hasUrls():
urlLocals = [url for url in m.urls() if url.isLocalFile()]
accepted = False
for urlLocal in urlLocals:
path = urlLocal.toLocalFile()
info = QFileInfo(path)
destination = QDir(pathDir).filePath(info.fileName())
source = info.absoluteFilePath()
if destination == source:
continue # means they are in the same folder
if info.isDir():
QDir().rename(source, destination)
else:
qfile = QFile(source)
if QFile(destination).exists():
n_info = QFileInfo(destination)
destination = n_info.canonicalPath() + QDir.separator() + n_info.completeBaseName() + " (copy)"
if n_info.completeSuffix(): # for moving files without suffix
destination += "." + n_info.completeSuffix()
qfile.rename(destination)
print(f"added -> {info.fileName()}") # for debugging
accepted = True
if accepted:
event.acceptProposedAction()
class AppDemo(QWidget):
def __init__(self):
super().__init__()
self.setAcceptDrops(True)
cwd = "test/"
nw = "test copy/"
# -- right -- #
self.model1 = QFileSystemModel()
self.model1.setRootPath(os.path.dirname(cwd))
self.view1 = MQTreeView(self.model1, cwd)
# -- left -- #
self.model2 = QFileSystemModel()
self.model2.setRootPath(os.path.dirname(nw))
self.view2 = MQTreeView(self.model2, nw)
# -- layout -- #
layout = QHBoxLayout(self)
layout.addWidget(self.view1)
layout.addWidget(self.view2)
app = QApplication(sys.argv)
main = AppDemo()
main.show()
app.exec_()
To start with, I am not a developer, but a mere automation engineer that have worked a bit with coding in Java, python, C#, C++ and C.
I am trying to make a prototype that take pictures and stores them using a digital pin on the board. Atm I can take pictures using a switch, but it is really slow(around 3 seconds pr image).
My complete system is going to be like this:
A product passes by on a conveyor and a photo cell triggers the board to take an image and store it. If an operator removes a product(because of bad quality) the image is stored in a different folder.
I started with the snapshot function shipped with Mendel and have tried to get rid off the overhead, but the Gstream and pipeline-stuff confuses me a lot.
If someone could help me with how to understand the supplied code, or how to write a minimalistic solution to take an image i would be grateful :)
I have tried to understand and use project-teachable and examples-camera from Google coral https://github.com/google-coral, but with no luck. I have had the best luck with the snapshot tool that uses snapshot.py that are referenced here https://coral.withgoogle.com/docs/camera/datasheet/#snapshot-tool
from periphery import GPIO
import time
import argparse
import contextlib
import fcntl
import os
import select
import sys
import termios
import threading
import gi
gi.require_version('Gst', '1.0')
gi.require_version('GstBase', '1.0')
from functools import partial
from gi.repository import GLib, GObject, Gst, GstBase
from PIL import Image
GObject.threads_init()
Gst.init(None)
WIDTH = 2592
HEIGHT = 1944
FILENAME_PREFIX = 'img'
FILENAME_SUFFIX = '.png'
AF_SYSFS_NODE = '/sys/module/ov5645_camera_mipi_v2/parameters/ov5645_af'
CAMERA_INIT_QUERY_SYSFS_NODE = '/sys/module/ov5645_camera_mipi_v2/parameters/ov5645_initialized'
HDMI_SYSFS_NODE = '/sys/class/drm/card0/card0-HDMI-A-1/status'
# No of initial frames to throw away before camera has stabilized
SCRAP_FRAMES = 1
SRC_WIDTH = 2592
SRC_HEIGHT = 1944
SRC_RATE = '15/1'
SRC_ELEMENT = 'v4l2src'
SINK_WIDTH = 2592
SINK_HEIGHT = 1944
SINK_ELEMENT = ('appsink name=appsink sync=false emit-signals=true '
'max-buffers=1 drop=true')
SCREEN_SINK = 'glimagesink sync=false'
FAKE_SINK = 'fakesink sync=false'
SRC_CAPS = 'video/x-raw,format=YUY2,width={width},height={height},framerate={rate}'
SINK_CAPS = 'video/x-raw,format=RGB,width={width},height={height}'
LEAKY_Q = 'queue max-size-buffers=1 leaky=downstream'
PIPELINE = '''
{src_element} ! {src_caps} ! {leaky_q} ! tee name=t
t. ! {leaky_q} ! {screen_sink}
t. ! {leaky_q} ! videoconvert ! {sink_caps} ! {sink_element}
'''
def on_bus_message(bus, message, loop):
t = message.type
if t == Gst.MessageType.EOS:
loop.quit()
elif t == Gst.MessageType.WARNING:
err, debug = message.parse_warning()
sys.stderr.write('Warning: %s: %s\n' % (err, debug))
elif t == Gst.MessageType.ERROR:
err, debug = message.parse_error()
sys.stderr.write('Error: %s: %s\n' % (err, debug))
loop.quit()
return True
def on_new_sample(sink, snapinfo):
if not snapinfo.save_frame():
# Throw away the frame
return Gst.FlowReturn.OK
sample = sink.emit('pull-sample')
buf = sample.get_buffer()
result, mapinfo = buf.map(Gst.MapFlags.READ)
if result:
imgfile = snapinfo.get_filename()
caps = sample.get_caps()
width = WIDTH
height = HEIGHT
img = Image.frombytes('RGB', (width, height), mapinfo.data, 'raw')
img.save(imgfile)
img.close()
buf.unmap(mapinfo)
return Gst.FlowReturn.OK
def run_pipeline(snapinfo):
src_caps = SRC_CAPS.format(width=SRC_WIDTH, height=SRC_HEIGHT, rate=SRC_RATE)
sink_caps = SINK_CAPS.format(width=SINK_WIDTH, height=SINK_HEIGHT)
screen_sink = FAKE_SINK
pipeline = PIPELINE.format(
leaky_q=LEAKY_Q,
src_element=SRC_ELEMENT,
src_caps=src_caps,
sink_caps=sink_caps,
sink_element=SINK_ELEMENT,
screen_sink=screen_sink)
pipeline = Gst.parse_launch(pipeline)
appsink = pipeline.get_by_name('appsink')
appsink.connect('new-sample', partial(on_new_sample, snapinfo=snapinfo))
loop = GObject.MainLoop()
# Set up a pipeline bus watch to catch errors.
bus = pipeline.get_bus()
bus.add_signal_watch()
bus.connect('message', on_bus_message, loop)
# Connect the loop to the snaphelper
snapinfo.connect_loop(loop)
# Run pipeline.
pipeline.set_state(Gst.State.PLAYING)
try:
loop.run()
except:
pass
# Clean up.
pipeline.set_state(Gst.State.NULL)
while GLib.MainContext.default().iteration(False):
pass
class SnapHelper:
def __init__(self, sysfs, prefix='img', oneshot=True, suffix='jpg'):
self.prefix = prefix
self.oneshot = oneshot
self.suffix = suffix
self.snap_it = oneshot
self.num = 0
self.scrapframes = SCRAP_FRAMES
self.sysfs = sysfs
def get_filename(self):
while True:
filename = self.prefix + str(self.num).zfill(4) + '.' + self.suffix
self.num = self.num + 1
if not os.path.exists(filename):
break
return filename
#def check_af(self):
#try:
# self.sysfs.seek(0)
# v = self.sysfs.read()
# if int(v) != 0x10:
# print('NO Focus')
#except:
# pass
# def refocus(self):
# try:#
# self.sysfs.write('1')
# self.sysfs.flush()
# except:
# pass
def save_frame(self):
# We always want to throw away the initial frames to let the
# camera stabilize. This seemed empirically to be the right number
# when running on desktop.
if self.scrapframes > 0:
self.scrapframes = self.scrapframes - 1
return False
if self.snap_it:
self.snap_it = False
retval = True
else:
retval = False
if self.oneshot:
self.loop.quit()
return retval
def connect_loop(self, loop):
self.loop = loop
def take_picture(snap):
start_time = int(round(time.time()))
run_pipeline(snap)
print(time.time()- start_time)
def main():
button = GPIO(138, "in")
last_state = False
with open(AF_SYSFS_NODE, 'w+') as sysfs:
snap = SnapHelper(sysfs, 'test', 'oneshot', 'jpg')
sysfs.write('2')
while 1:
button_state = button.read()
if(button_state==True and last_state == False):
snap = SnapHelper(sysfs, 'test', 'oneshot', 'jpg')
take_picture(snap)
last_state = button_state
if __name__== "__main__":
main()
sys.exit()
Output is what i expect, but it is slow.
I switched to a USB-webcam and used the pygame library instead.
I know that several questions have been created with people asking about non-responsive GUIs and the ultimate answer is that Tkinter is not thread safe. However, it is my understanding that queues can be utilized to overcome this problem. Therefore, I have been looking into using the multiprocessing module with queues such that my code can be utilized on hyperthreaded and multicore systems.
What I would like to do is to try and do a very complex least squares fitting of multiple imported spectra in different tabs whenever a button is pressed.
The problem is that my code is still hanging up on the long process that I initialize by a button in my GUI. I have knocked the code down to something that still may run and has most of the objects of my original program, yet still suffers from the problem of not being responsive.
I believe my problem is in the multiprocessing portion of my program.
Therefore my question is regarding the multiprocessing portion of the code and if there is a better way to organize the process_spectra() function shown here:
def process_spectra(self):
process_list = []
queue = mp.Queue()
for tab in self.tab_list:
process_list.append(mp.Process(target=Deconvolution(tab).deconvolute(), args=(queue,)))
process_list[-1].start()
process_list[-1].join()
return
At the moment it appears that this is not actually making the deconvolution process into a different thread. I would like the process_spectra function to process all of the spectra with the deconvolution function simultaneously while still being able to interact with and see the changes in the spectra and GUI.
Here is the full code which can be run as a .py file directly to reproduce my problem:
from Tkinter import *
import Tkinter
import tkFileDialog
import matplotlib
from matplotlib import *
matplotlib.use('TKAgg')
from matplotlib import pyplot, figure, backends
import numpy as np
import lmfit
import multiprocessing as mp
# lots of different peaks can appear
class peak:
def __init__(self, n, m):
self.n = n
self.m = m
def location(self, i):
location = i*self.m/self.n
return location
def NM(self):
return str(self.n) + str(self.m)
# The main function that is given by the user has X and Y data and peak data
class Spectra:
def __init__(self, spectra_name, X, Y):
self.spectra_name = spectra_name
self.X = X
self.Y = Y
self.Y_model = Y*0
self.Y_background_model = Y*0
self.Y_without_background_model = Y*0
self.dYdX = np.diff(self.Y)/np.diff(self.X)
self.peak_list = self.initialize_peaks(3, 60)
self.params = lmfit.Parameters()
def peak_amplitude_dictionary(self):
peak_amplitude_dict = {}
for peak in self.peak_list:
peak_amplitude_dict[peak] = self.params['P' + peak.NM() + '_1_amp'].value
return peak_amplitude_dict
def peak_percentage_dictionary(self):
peak_percentage_dict = {}
for peak in self.peak_list:
peak_percentage_dict[peak] = self.peak_amplitude_dictionary()[peak]/np.sum(self.peak_amplitude_dictionary().values())
return peak_percentage_dict
# Function to create all of the peaks and store them in a list
def initialize_peaks(self, lowestNM, highestNM):
peaks=[]
for n in range(0,highestNM+1):
for m in range(0,highestNM+1):
if(n<lowestNM and m<lowestNM): break
elif(n<m): break
else: peaks.append(peak(n,m))
return peaks
# This is just a whole bunch of GUI stuff
class Spectra_Tab(Frame):
def __init__(self, parent, spectra):
self.spectra = spectra
self.parent = parent
Frame.__init__(self, parent)
self.tab_name = spectra.spectra_name
self.canvas_frame = Frame(self, bd=3, bg= 'WHITE', relief=SUNKEN)
self.canvas_frame.pack(side=LEFT, fill=BOTH, padx=0, pady=0, expand=1)
self.results_frame = Frame(self, bd=3, bg= 'WHITE', relief=SUNKEN, width=600)
self.results_frame.pack(side=RIGHT, fill=BOTH, padx=0, pady=0, expand=1)
self.top_canvas_frame = Frame(self.canvas_frame, bd=0, bg= 'WHITE', relief=SUNKEN)
self.top_canvas_frame.pack(side=TOP, fill=BOTH, padx=0, pady=0, expand=1)
self.original_frame = Frame(self.top_canvas_frame, bd=1, relief=SUNKEN)
self.original_frame.pack(side=LEFT, fill=BOTH, padx=0, pady=0, expand=1)
self.scrollbar = Scrollbar(self.results_frame)
self.scrollbar.pack(side=RIGHT, fill=BOTH,expand=1)
self.sidebar = Listbox(self.results_frame)
self.sidebar.pack(fill=BOTH, expand=1)
self.sidebar.config(yscrollcommand=self.scrollbar.set)
self.scrollbar.config(command=self.sidebar.yview)
self.original_fig = figure.Figure()
self.original_plot = self.original_fig.add_subplot(111)
init_values = np.zeros(len(self.spectra.Y))
self.original_line, = self.original_plot.plot(self.spectra.X, self.spectra.Y, 'r-')
self.original_background_line, = self.original_plot.plot(self.spectra.X, init_values, 'k-', animated=True)
self.original_canvas = backends.backend_tkagg.FigureCanvasTkAgg(self.original_fig, master=self.original_frame)
self.original_canvas.get_tk_widget().pack(side=TOP, fill=BOTH, expand=1)
self.original_canvas._tkcanvas.pack(side=TOP, fill=BOTH, expand=1)
self.original_canvas.show()
self.original_canvas.draw()
self.original_canvas_BBox = self.original_plot.figure.canvas.copy_from_bbox(self.original_plot.bbox)
ax1 = self.original_plot.figure.axes[0]
ax1.set_xlim(self.spectra.X.min(), self.spectra.X.max())
ax1.set_ylim(0, self.spectra.Y.max() + .05*self.spectra.Y.max())
self.step=0
self.update()
# This just refreshes the GUI stuff everytime that the parameters are fit in the least squares method
def refreshFigure(self):
self.step=self.step+1
if(self.step==1):
self.original_canvas_BBox = self.original_plot.figure.canvas.copy_from_bbox(self.original_plot.bbox)
self.original_plot.figure.canvas.restore_region(self.original_canvas_BBox)
self.original_background_line.set_data(self.spectra.X, self.spectra.Y_background_model)
self.original_plot.draw_artist(self.original_line)
self.original_plot.draw_artist(self.original_background_line)
self.original_plot.figure.canvas.blit(self.original_plot.bbox)
# show percentage of peaks on the side bar
self.sidebar.delete(0, Tkinter.END)
peak_dict = self.spectra.peak_percentage_dictionary()
for peak in sorted(peak_dict.iterkeys()):
self.sidebar.insert(0, peak.NM() + ' ' + str(peak_dict[peak]) + '%' )
return
# just a tab bar
class TabBar(Frame):
def __init__(self, master=None):
Frame.__init__(self, master)
self.tabs = {}
self.buttons = {}
self.current_tab = None
def show(self):
self.pack(side=BOTTOM, expand=0, fill=X)
def add(self, tab):
tab.pack_forget()
self.tabs[tab.tab_name] = tab
b = Button(self, text=tab.tab_name, relief=RAISED, command=(lambda name=tab.tab_name: self.switch_tab(name)))
b.pack(side=LEFT)
self.buttons[tab.tab_name] = b
def switch_tab(self, name):
if self.current_tab:
self.buttons[self.current_tab].config(relief=RAISED)
self.tabs[self.current_tab].pack_forget()
self.tabs[name].pack(side=BOTTOM)
self.current_tab = name
self.buttons[name].config(relief=SUNKEN)
class Deconvolution:
def __init__(self, spectra_tab):
self.spectra_tab = spectra_tab
self.spectra = spectra_tab.spectra
self.model = [0 for x in self.spectra.X]
self.model_without_background = [0 for x in self.spectra.X]
self.residual_array = [0 for x in self.spectra.X]
# Amplitudes for backgrounds
self.pi_plasmon_amp = np.interp(4.3, self.spectra.X, self.spectra.Y)
self.graphite_amp = np.interp(5, self.spectra.X, self.spectra.Y)
self.spectra.params.add('PPAmp', value=self.pi_plasmon_amp, vary=True, min=0.0, max=None)
self.spectra.params.add('PPCenter', value=4.3, vary=True)
self.spectra.params.add('PPFWHM', value=.4, vary=True)
self.spectra.params.add('GLAmp', value=self.graphite_amp, vary=True, min=0.0, max=None)
self.spectra.params.add('GLCenter', value=5, vary=True)
self.spectra.params.add('GLFWHM', value=.4, vary=True)
self.background_model = self.pseudoVoigt(self.spectra.X, self.spectra.params['PPAmp'].value, self.spectra.params['PPCenter'].value, self.spectra.params['PPFWHM'].value, 1)+\
self.pseudoVoigt(self.spectra.X, self.spectra.params['GLAmp'].value, self.spectra.params['GLCenter'].value, self.spectra.params['GLFWHM'].value, 1)
for peak in self.spectra.peak_list:
for i in range(1,4):
param_prefix = 'P' + peak.NM() + '_' + str(i)
center = peak.location(i)
amp = np.interp(center, self.spectra.X, self.spectra.Y - self.background_model)
width = 0.02
self.spectra.params.add(param_prefix + '_amp', value = 0.8*amp, vary=False, min=0.0, max=None)
self.spectra.params.add(param_prefix + '_center', value = center, vary=False, min=0.0, max=None)
self.spectra.params.add(param_prefix + '_width', value = width, vary=False, min=0.0, max=None)
self.model_without_background += self.pseudoVoigt(self.spectra.X, self.spectra.params[param_prefix + '_amp'].value, self.spectra.params[param_prefix + '_center'].value, self.spectra.params[param_prefix + '_width'].value, 1)
def deconvolute(self):
for State in range(0,3):
# Make each voigt profile for each tube
for peak in self.spectra.peak_list:
for i in range(1,4):
param_prefix = 'P' + peak.NM() + '_' + str(i)
if(State==1):
self.spectra.params[param_prefix + '_amp'].vary = True
if(State==2):
self.spectra.params[param_prefix + '_width'].vary = True
result = lmfit.Minimizer(self.residual, self.spectra.params, fcn_args=(State,))
result.prepare_fit()
result.leastsq()#lbfgsb()
def residual(self, params, State):
self.model = self.background_model
if(State>0):
self.model += self.model_without_background
for x in range(0, len(self.spectra.X)):
if(self.background_model[x]>self.spectra.Y[x]):
self.residual_array[x] = -999999.-9999.*(self.spectra.Y[x]-self.background_model[x])
else:
self.residual_array[x] = self.spectra.Y[x]-self.model[x]
self.spectra.Y_model = self.model
self.spectra.Y_background_model = self.background_model
self.spectra.Y_without_background_model = self.model_without_background
self.spectra_tab.refreshFigure()
return self.residual_array
def pseudoVoigt(self, x, amp, center, width, shapeFactor):
LorentzPortion = (width**2/((x-center)**2+width**2))
GaussianPortion = 1/(np.sqrt(2*np.pi*width**2))*np.e**(-(x-center)**2/(2*width**2))
try:
Voigt = amp*(shapeFactor*LorentzPortion+(1-shapeFactor)*GaussianPortion)
except ZeroDivisionError:
width = width+0.01
LorentzPortion = (width**2/((x-center)**2+width**2))
GaussianPortion = 1/(np.sqrt(2*np.pi*width**2))*np.e**(-(x-center)**2/(2*width**2))
Voigt = amp*(shapeFactor*LorentzPortion+(1-shapeFactor)*GaussianPortion)
return Voigt
class MainWindow(Tk):
def __init__(self, parent):
Tk.__init__(self, parent)
self.parent = parent
self.wm_state('zoomed')
self.spectra_list = []
self.tab_list = []
self.button_frame = Frame(self, bd=3, relief=SUNKEN)
self.button_frame.pack(side=TOP, fill=BOTH)
self.tab_frame = Frame(self, bd=3, relief=SUNKEN)
self.tab_frame.pack(side=BOTTOM, fill=BOTH, expand=1)
open_spectra_button = Button(self.button_frame, text='open spectra', command=self.open_spectra)
open_spectra_button.pack(side=LEFT, fill=Y)
process_spectra_button = Button(self.button_frame, text='process spectra', command=self.process_spectra)
process_spectra_button.pack(side=LEFT, fill=Y)
self.tab_bar = TabBar(self.tab_frame)
self.tab_bar.show()
self.resizable(True,False)
self.update()
def open_spectra(self):
# This will prompt user for file input later, but here is an example
file_name_list = ['spectra_1', 'spectra_2']
for file_name in file_name_list:
# Just make up functions that may be imported
X_values = np.arange(1240.0/1350.0, 1240./200., 0.01)
if(file_name=='spectra_1'):
Y_values = np.array(np.e**.2*X_values + np.sin(10*X_values)+np.cos(4*X_values))
if(file_name=='spectra_2'):
Y_values = np.array(np.e**.2*X_values + np.sin(10*X_values)+np.cos(3*X_values)+.3*np.cos(.5*X_values))
self.spectra_list.append(Spectra(file_name, X_values, Y_values))
self.tab_list.append(Spectra_Tab(self.tab_frame, self.spectra_list[-1]))
self.tab_bar.add(self.tab_list[-1])
self.tab_bar.switch_tab(self.spectra_list[0].spectra_name)
self.tab_bar.show()
return
def process_spectra(self):
process_list = []
queue = mp.Queue()
for tab in self.tab_list:
process_list.append(mp.Process(target=Deconvolution(tab).deconvolute(), args=(queue,)))
process_list[-1].start()
process_list[-1].join()
return
if __name__ == "__main__":
root = MainWindow(None)
root.mainloop()
EDIT:
I am editing this question because I realized that my question did not regard the real problem. I think the code I have supplied has problems with having a Tkinter Frame passed as a parameter to something that needs to be pickled, ? and it can't because it's not thread safe?? It gives a pickle error that points to Tkinter in some way.
However, I am not sure how to reorganize this code such that the only part that is pickled is the data part since the threads or processes must access the Tkinter frames in order to update them via refreshFigure().
Does anyone have any ideas regarding how to do this? I have researched it but everyone's examples are usually simple with only one figure or that only refreshes after the process is completed.
The segment target=Deconvolution(tab).deconvolute() will actually be evaluated instead of passed to a subprocess. You could replace this with a wrapper function
def mp_deconvolute(tab):
return Deconvolution(tab).deconvolute()
I'm not sure if your queue is actually be used at all but I believe that would be more appropriate for a worker Pool scenario.
Edit:
Oh, and you would call it like so
process_list.append(mp.Process(target=mp_deconvolute, args=(tab)))
Edit again:
You could just define that as a lambda function too unless you to to add more complexity
mp_deconv = lambda x: Deconvolution(tab).deconvolute()
process_list.append(mp.Process(target=mp_deconv, args=(tab)))