Using multiprocessing, stopping a subprocess from an PyQt5 application, where the subprocess is not using an event loop
-
I am building a GUI application running neural networks. Starting them off works fine, but I need the ability to abort the calculation, while the net is already running and working.
I built a small prototype, which shows you the main mechanics of my problem within my application.
Architecture
Window Class:
Contains all GUI Elements and is the main Object which initiates and controls everything. It contains a QThreadPool self.__pool and the QRunnable Object self.__runner. The QRunnable Object contains everything needed for the neural net. The reason why I use a QRunnable object is, to not block the GUI while in another thread the neural net is being handled. Also I need to have communication between the neural net and my application.Runner Class:
The runner class handles communication between the Main Window and the neural net itself. The neural net is a Process Object from multiprocessing put into self.__net. For communication I use a Queue self.__queue. When the QRunnable object is being started, the process is starting with self.__net.start(). I observe the Queue with a infinite loop. The loop is terminated on certain signals. In this example I use only the Signal NetSignal.finished. When the net is finished I send a signal to the Window object.RunnerSignal Class:
Since QRunnables cannot make use of Signals, this class is needed to package some Signals into the QRuannable Object.Mnist Class:
The Mnist class is the Subprocess itself, inheriting from Process. In this example it runs an really easy example of the mnist dataset being processed in a neural network. There is no Loop surrounding the process, which normally can be used to stop such a subprocess. When the neural network is finished, the queue transmits a signal to the QRunnable object, to signal the process having finished calculating, which therefore sends a signal to the main window.Question
I need to be able to stop the process somehow. I thought of trying to kill it off with os.kill, which does not really work well in my application. I tried also self.__net.terminate(), self.__net.kill(). I was thinking of somehow passing an object into the callback parameter of the neural net, to maybe abort the processing there, but I am not really sure if it is the way to go.Code:
from sys import exit, argv from multiprocessing import Process, Queue from PyQt5.QtWidgets import QPushButton, QApplication, QHBoxLayout, QWidget, QLabel from PyQt5.QtCore import QRunnable, QObject, pyqtSignal, QThreadPool class Window(QWidget): def __init__(self): QWidget.__init__(self) self.__btn_run = QPushButton("Start") self.__btn_stp = QPushButton("Stop") self.__label = QLabel("Idle") self.__runner = Runner() self.__pool = QThreadPool.globalInstance() self.__btn_run.clicked.connect(self.__run_net) self.__btn_stp.clicked.connect(self.__stp_net) self.__runner.signals.finished.connect(self.__on_finished) self.__btn_stp.setDisabled(True) self.setLayout(QHBoxLayout()) self.layout().addWidget(self.__btn_run) self.layout().addWidget(self.__btn_stp) self.layout().addWidget(self.__label) def __run_net(self): self.__btn_run.setDisabled(True) self.__btn_stp.setEnabled(True) self.__label.setText("Running") self.__pool.start(self.__runner) def __stp_net(self): pass # What to do here? def __on_finished(self): self.__btn_run.setEnabled(True) self.__btn_stp.setDisabled(True) self.__label.setText("Finished") self.__runner = Runner() class Runner(QRunnable): def __init__(self): QRunnable.__init__(self) self.__queue = Queue() self.__net = Mnist(self.__queue) self.signals = RunnerSignal() def run(self): self.__net.start() while True: data = self.__queue.get() if data == NetSignal.finished: self.signals.finished.emit() break class RunnerSignal(QObject): finished = pyqtSignal() class Mnist(Process): def __init__(self, queue: Queue): Process.__init__(self) self.__queue = queue def run(self): mnist = tf.keras.datasets.mnist # 28x28 Bilder hangeschriebener Ziffern von 0-9 (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = tf.keras.utils.normalize(x_train, axis=1) model = tf.keras.models.Sequential() model.add(tf.keras.layers.Flatten()) model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu)) model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu)) model.add(tf.keras.layers.Dense(10, activation=tf.nn.softmax)) model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=['accuracy']) model.fit(x_train, y_train, epochs=8) self.__queue.put(NetSignal.finished) self.__queue.close() class NetSignal: finished = "finished" if __name__ == "__main__": main_thread = QApplication(argv) main_window = Window() main_window.show() exit(main_thread.exec()) -
Ahhhh nice, yes
The problem was, there has always been a zombie process. That is way I thought the termination did not work properly. What I forgot to do is to also make the QRunnable quit, which remained in the main thread as still running.
This is the solution:
import tensorflow as tf from sys import exit, argv from multiprocessing import Process, Queue from PyQt5.QtWidgets import QPushButton, QApplication, QHBoxLayout, QWidget, QLabel from PyQt5.QtCore import QRunnable, QObject, pyqtSignal, QThreadPool class Window(QWidget): def __init__(self): QWidget.__init__(self) self.__btn_run = QPushButton("Start") self.__btn_stp = QPushButton("Stop") self.__label = QLabel("Idle") self.__runner = Runner() self.__pool = QThreadPool.globalInstance() self.__btn_run.clicked.connect(self.__run_net) self.__btn_stp.clicked.connect(self.__stp_net) self.__runner.signals.finished.connect(self.__on_finished) self.__btn_stp.setDisabled(True) self.setLayout(QHBoxLayout()) self.layout().addWidget(self.__btn_run) self.layout().addWidget(self.__btn_stp) self.layout().addWidget(self.__label) def __run_net(self): self.__btn_run.setDisabled(True) self.__btn_stp.setEnabled(True) self.__label.setText("Running") self.__pool.start(self.__runner) def __stp_net(self): self.__runner.close() # What to do here? def __on_finished(self): self.__btn_run.setEnabled(True) self.__btn_stp.setDisabled(True) self.__label.setText("Finished") self.__runner = Runner() class Runner(QRunnable): def __init__(self): QRunnable.__init__(self) self.__queue = Queue() self.__net = Mnist(self.__queue) self.signals = RunnerSignal() def run(self): self.__net.start() while True: data = self.__queue.get() if data == NetSignal.finished: self.signals.finished.emit() break def close(self): self.__net.end_process() class RunnerSignal(QObject): finished = pyqtSignal() class Mnist(Process): def __init__(self, queue: Queue): Process.__init__(self) self.__queue = queue def run(self): mnist = tf.keras.datasets.mnist # 28x28 Bilder hangeschriebener Ziffern von 0-9 (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = tf.keras.utils.normalize(x_train, axis=1) model = tf.keras.models.Sequential() model.add(tf.keras.layers.Flatten()) model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu)) model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu)) model.add(tf.keras.layers.Dense(10, activation=tf.nn.softmax)) model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=['accuracy']) model.fit(x_train, y_train, epochs=8) self.__queue.put(NetSignal.finished) self.__queue.close() def end_process(self): self.terminate() self.__queue.put(NetSignal.finished) class NetSignal: finished = "finished" if __name__ == "__main__": main_thread = QApplication(argv) main_window = Window() main_window.show() exit(main_thread.exec()) -
I am building a GUI application running neural networks. Starting them off works fine, but I need the ability to abort the calculation, while the net is already running and working.
I built a small prototype, which shows you the main mechanics of my problem within my application.
Architecture
Window Class:
Contains all GUI Elements and is the main Object which initiates and controls everything. It contains a QThreadPool self.__pool and the QRunnable Object self.__runner. The QRunnable Object contains everything needed for the neural net. The reason why I use a QRunnable object is, to not block the GUI while in another thread the neural net is being handled. Also I need to have communication between the neural net and my application.Runner Class:
The runner class handles communication between the Main Window and the neural net itself. The neural net is a Process Object from multiprocessing put into self.__net. For communication I use a Queue self.__queue. When the QRunnable object is being started, the process is starting with self.__net.start(). I observe the Queue with a infinite loop. The loop is terminated on certain signals. In this example I use only the Signal NetSignal.finished. When the net is finished I send a signal to the Window object.RunnerSignal Class:
Since QRunnables cannot make use of Signals, this class is needed to package some Signals into the QRuannable Object.Mnist Class:
The Mnist class is the Subprocess itself, inheriting from Process. In this example it runs an really easy example of the mnist dataset being processed in a neural network. There is no Loop surrounding the process, which normally can be used to stop such a subprocess. When the neural network is finished, the queue transmits a signal to the QRunnable object, to signal the process having finished calculating, which therefore sends a signal to the main window.Question
I need to be able to stop the process somehow. I thought of trying to kill it off with os.kill, which does not really work well in my application. I tried also self.__net.terminate(), self.__net.kill(). I was thinking of somehow passing an object into the callback parameter of the neural net, to maybe abort the processing there, but I am not really sure if it is the way to go.Code:
from sys import exit, argv from multiprocessing import Process, Queue from PyQt5.QtWidgets import QPushButton, QApplication, QHBoxLayout, QWidget, QLabel from PyQt5.QtCore import QRunnable, QObject, pyqtSignal, QThreadPool class Window(QWidget): def __init__(self): QWidget.__init__(self) self.__btn_run = QPushButton("Start") self.__btn_stp = QPushButton("Stop") self.__label = QLabel("Idle") self.__runner = Runner() self.__pool = QThreadPool.globalInstance() self.__btn_run.clicked.connect(self.__run_net) self.__btn_stp.clicked.connect(self.__stp_net) self.__runner.signals.finished.connect(self.__on_finished) self.__btn_stp.setDisabled(True) self.setLayout(QHBoxLayout()) self.layout().addWidget(self.__btn_run) self.layout().addWidget(self.__btn_stp) self.layout().addWidget(self.__label) def __run_net(self): self.__btn_run.setDisabled(True) self.__btn_stp.setEnabled(True) self.__label.setText("Running") self.__pool.start(self.__runner) def __stp_net(self): pass # What to do here? def __on_finished(self): self.__btn_run.setEnabled(True) self.__btn_stp.setDisabled(True) self.__label.setText("Finished") self.__runner = Runner() class Runner(QRunnable): def __init__(self): QRunnable.__init__(self) self.__queue = Queue() self.__net = Mnist(self.__queue) self.signals = RunnerSignal() def run(self): self.__net.start() while True: data = self.__queue.get() if data == NetSignal.finished: self.signals.finished.emit() break class RunnerSignal(QObject): finished = pyqtSignal() class Mnist(Process): def __init__(self, queue: Queue): Process.__init__(self) self.__queue = queue def run(self): mnist = tf.keras.datasets.mnist # 28x28 Bilder hangeschriebener Ziffern von 0-9 (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = tf.keras.utils.normalize(x_train, axis=1) model = tf.keras.models.Sequential() model.add(tf.keras.layers.Flatten()) model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu)) model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu)) model.add(tf.keras.layers.Dense(10, activation=tf.nn.softmax)) model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=['accuracy']) model.fit(x_train, y_train, epochs=8) self.__queue.put(NetSignal.finished) self.__queue.close() class NetSignal: finished = "finished" if __name__ == "__main__": main_thread = QApplication(argv) main_window = Window() main_window.show() exit(main_thread.exec())@lvlanson said in Using multiprocessing, stopping a subprocess from an PyQt5 application, where the subprocess is not using an event loop:
self.__net.terminate()
Should work. What happens if you call it?
-
Ahhhh nice, yes
The problem was, there has always been a zombie process. That is way I thought the termination did not work properly. What I forgot to do is to also make the QRunnable quit, which remained in the main thread as still running.
This is the solution:
import tensorflow as tf from sys import exit, argv from multiprocessing import Process, Queue from PyQt5.QtWidgets import QPushButton, QApplication, QHBoxLayout, QWidget, QLabel from PyQt5.QtCore import QRunnable, QObject, pyqtSignal, QThreadPool class Window(QWidget): def __init__(self): QWidget.__init__(self) self.__btn_run = QPushButton("Start") self.__btn_stp = QPushButton("Stop") self.__label = QLabel("Idle") self.__runner = Runner() self.__pool = QThreadPool.globalInstance() self.__btn_run.clicked.connect(self.__run_net) self.__btn_stp.clicked.connect(self.__stp_net) self.__runner.signals.finished.connect(self.__on_finished) self.__btn_stp.setDisabled(True) self.setLayout(QHBoxLayout()) self.layout().addWidget(self.__btn_run) self.layout().addWidget(self.__btn_stp) self.layout().addWidget(self.__label) def __run_net(self): self.__btn_run.setDisabled(True) self.__btn_stp.setEnabled(True) self.__label.setText("Running") self.__pool.start(self.__runner) def __stp_net(self): self.__runner.close() # What to do here? def __on_finished(self): self.__btn_run.setEnabled(True) self.__btn_stp.setDisabled(True) self.__label.setText("Finished") self.__runner = Runner() class Runner(QRunnable): def __init__(self): QRunnable.__init__(self) self.__queue = Queue() self.__net = Mnist(self.__queue) self.signals = RunnerSignal() def run(self): self.__net.start() while True: data = self.__queue.get() if data == NetSignal.finished: self.signals.finished.emit() break def close(self): self.__net.end_process() class RunnerSignal(QObject): finished = pyqtSignal() class Mnist(Process): def __init__(self, queue: Queue): Process.__init__(self) self.__queue = queue def run(self): mnist = tf.keras.datasets.mnist # 28x28 Bilder hangeschriebener Ziffern von 0-9 (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = tf.keras.utils.normalize(x_train, axis=1) model = tf.keras.models.Sequential() model.add(tf.keras.layers.Flatten()) model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu)) model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu)) model.add(tf.keras.layers.Dense(10, activation=tf.nn.softmax)) model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=['accuracy']) model.fit(x_train, y_train, epochs=8) self.__queue.put(NetSignal.finished) self.__queue.close() def end_process(self): self.terminate() self.__queue.put(NetSignal.finished) class NetSignal: finished = "finished" if __name__ == "__main__": main_thread = QApplication(argv) main_window = Window() main_window.show() exit(main_thread.exec())
