Important: Please read the Qt Code of Conduct - https://forum.qt.io/topic/113070/qt-code-of-conduct

PySide2 - multi-threading cases Python crashs



  • The program is trying to do some work at a newly created qthread and send the data to GUI thread to show.
    Refer this link manipulating-widget-in-pyside2-qthread-causes-python3-not-responding for some background.
    Does anyone know this is the coding issue or an pyside2 bug?

    Crash after a long run (100% reproduce, a few minutes or more, it depends) at env:
    Python3.6.8 32bit, WIN10 X64, Pyside2 5.12.1

    Fatal Python error: GC object already tracked
    
    Current thread 0x000019a8 (most recent call first):
      File "C:\Python36-32\lib\ntpath.py", line 89 in join
      File "D:/project/demo.py", line 25 in list_folder
      File "D:/project/demo.py", line 29 in list_folder
      File "D:/project/demo.py", line 35 in run
    
    Thread 0x000034b8 (most recent call first):
      File "D:/project/demo.py", line 74 in addTopItem
      File "D:/project/demo.py", line 105 in <module>
    
    Process finished with exit code -1073740791 (0xC0000409)
    

    Work with no problem (tested at least half an hour, try several times) at env:
    Python3.6.8 32bit, WIN10 X64, PyQt5 5.12

    Modify USE_PYSIDE2 = True/False to switch between pyside2 and pyqt5.

    # -*- coding: utf-8 -*-
    import os
    
    USE_PYSIDE2 = True
    if USE_PYSIDE2:
        from PySide2 import QtWidgets, QtGui, QtCore
        from PySide2.QtCore import Signal
    else:
        from PyQt5 import QtWidgets, QtGui, QtCore
        from PyQt5.QtCore import pyqtSignal as Signal
    
    
    class ShowFolderTreeThread(QtCore.QThread):
        addTopItem = Signal(str, str, tuple)
    
        def __init__(self, p, root_dir: str = "."):
            super().__init__(p)
            self.root_dir = root_dir
    
        def list_folder(self, parent_path: str, max_depth: int = 3, parent_item_ids=tuple()):
            if max_depth <= 0:
                return
            try:
                for row, content in enumerate(os.listdir(parent_path)):
                    content_path = os.path.join(parent_path, content)
                    is_dir: bool = os.path.isdir(content_path)
                    self.addTopItem.emit(content, "Folder" if is_dir else os.path.splitext(content)[1], parent_item_ids)
                    if is_dir:
                        self.list_folder(content_path, max_depth - 1, (*parent_item_ids, row))
            except Exception as e:
                print("list_folder", type(e), e)
    
        def run(self):
            try:
                self.list_folder(self.root_dir)
            except Exception as e:
                print(type(e), e)
    
    
    class MainWindow(QtWidgets.QWidget):
        def __init__(self):
            super().__init__()
            self.treeWidget = QtWidgets.QTreeWidget(self)
            self.treeWidget.setColumnCount(2)
            self.treeWidget.setHeaderLabels(["name", "type", ])
            self.treeWidget.setColumnWidth(0, 250)
    
            self.pushButton = QtWidgets.QPushButton(self)
            self.pushButton.setText("&Refresh")
    
            layout = QtWidgets.QVBoxLayout(self)
            layout.addWidget(self.treeWidget)
            layout.addWidget(self.pushButton)
            self.setLayout(layout)
    
            self.resize(400, 600)
    
            home = os.path.expanduser('~')
            self.workThread = ShowFolderTreeThread(self, home)
    
            self.workThread.finished.connect(self.thread_finished)
            self.workThread.addTopItem.connect(self.addTopItem)
            self.pushButton.clicked.connect(self.start_thread)
    
            self.timer = QtCore.QTimer()
            self.timer.setInterval(10)
            self.timer.setSingleShot(True)
            self.timer.timeout.connect(self.pushButton.click)
            self.timer.start()
    
        def addTopItem(self, content: str, content_type: str, parent_item_ids: tuple):
            item = QtWidgets.QTreeWidgetItem()
            item.setText(0, content)
            item.setText(1, content_type)
            if parent_item_ids:
                index, *item_ids = parent_item_ids
                parent_item = self.treeWidget.topLevelItem(index)
                for i in item_ids:
                    parent_item = parent_item.child(i)
                parent_item.addChild(item)
            else:
                self.treeWidget.addTopLevelItem(item)
    
        def start_thread(self):
            self.pushButton.setDisabled(True)
            self.treeWidget.clear()
            self.workThread.start()
    
        def thread_finished(self):
            self.pushButton.setEnabled(True)
            self.timer.start()
    
        def closeEvent(self, event: QtGui.QCloseEvent):
            self.timer.stop()
            super().closeEvent(event)
    
    
    if __name__ == '__main__':
        import sys
    
        try:
            app = QtWidgets.QApplication(sys.argv)
            w = MainWindow()
            w.show()
            sys.exit(app.exec_())
        except Exception as e:
            print(type(e), e)
            raise e
    
    


  • Just a guess, instead of subclassing QThread, perhaps it would work better with the moveToThread method.

    https://gitlab.com/snippets/1732597



  • You should use the default threading available as a standard python library for your threads. Its called "threading", just "threading". It's very, very simple.



  • The Python Global Interpreter Lock or GIL, in simple words, is a mutex (or a lock) that allows only one thread to hold the control of the Python interpreter. All the GIL does is make sure only one thread is executing Python code at a time; control still switches between threads. What the GIL prevents then, is making use of more than one CPU core or separate CPUs to run threads in parallel.

    Python threading is great for creating a responsive GUI, or for handling multiple short web requests where I/O is the bottleneck more than the Python code. It is not suitable for parallelizing computationally intensive Python code, stick to the multiprocessing module for such tasks or delegate to a dedicated external library. For actual parallelization in Python, you should use the multiprocessing module to fork multiple processes that execute in parallel (due to the global interpreter lock, Python threads provide interleaving, but they are in fact executed serially, not in parallel, and are only useful when interleaving I/O operations). However, threading is still an appropriate model if you want to run multiple I/O-bound tasks simultaneously.


  • Banned

    Okay first you are using QThread wrong as denoted by @Alfalfa you should not be sub-classing QThread in Python-Qt5 (be it Pyside2 or PyQt5) as doing it that way will encounter issues (perhaps like the one you are having) -- I say this from experience and while researching those issues I discovered that the documentation on QThread has not been updated since PyQt4 (at least the last time I checked anyway) which required me to rewrite my whole project.

    Next as @walemark points out QThreads have to work with the GIL limitation which means if you are going to have some kind of long running process you have to do one of two things. Create a multi-process and run it from there --or-- make sure to periodically pass control back to the Event Handler

    Now I am looking at your code and plan to rewrite it in the proper manner but that might take me a bit of time -- lots of elements to implementing a QThread properly as there are quite a few hidden issues with QThreads that are not explained well or at all.

    However, if you need/want a more interactive response, now or in the future, and/or want to see sample code on QThreading just shoot me a PM (I think this forum supports that but if not then just ask here and I will get you a means to PM me)



  • Pyside2 currently does have a bug with QThread where your entire program will lag if you have a thread running. I haven't been able to make anything crash though, for me it will just not respond to the OS.
    Here for the bug report, Pyside-803


  • Banned

    Okay first while QThread in either Ptyhon-Qt (PyQt5 or PySide2) does not have a bug when used properly nor improperly, it does not function well when used improperly but will seem to function just fine. Still while I did not experience any delays when I used it improperly, I did experience other subtle issues which I no longer have. Oh and that bug report -- is not a bug it is someone who does not understand how Events work in conjunction with the GIL which is to say they are handling Events improperly

    Below is an example of using QThread properly with a continuous long running task. Note if you plan to have tasks that are not continuous until you shut them down but are very long running you have to figure out how to break it up into smaller manageable chunks so you can allow for other Events to take place while it is running. Either that or you will need to push it out into its own process by using multiprocessing.

    Note of those 2 options learning to properly handle Events will be much easier than creating the multiprocess but it all depends on the needs of the application. Another option when using disposable Threads is to use the Thread Pool option as it creates a pool of Threads that it reuses so you are not creating and disposing of Threads each time you need to push a task off into its own Thread but do not use QThreadPool for long continuous running tasks that are meant to run until you shut them down.

    USE_PYSIDE2 = True
    if USE_PYSIDE2:
        from PySide2.QtCore    import QCoreApplication, QObject, QThread, Signal, Slot
        from PySide2.QtWidgets import QApplication, QWidget, QVBoxLayout
        from PySide2.QtWidgets import QPushButton, QComboBox
    else:
        from PyQt5.QtCore    import pyqtSignal as Signal
        from PyQt5.QtCore    import pyqtSlot   as Slot
        from PyQt5.QtCore    import QCoreApplication, QObject, QThread
        from PyQt5.QtWidgets import QApplication, QWidget, QVBoxLayout
        from PyQt5.QtWidgets import QPushButton, QComboBox
    
    from time import sleep as tmSleep
    
    class Task(QObject):
        sigOutput = Signal(int)
      # Initialization after Signals declared
        def __init__(self, CountBy):
            QObject.__init__(self)
            self.CntBy = CountBy
            self.Running = True
            self.Delay = 2
    
        def StartTask(self):
            CurCnt = 0
            while self.Running:   # Basically always True until ShutDown called
                CurCnt += 1 * self.CntBy
                self.sigOutput.emit(CurCnt)
              # If you do not give control to the Event Handler with a continuous
              # running process like this it does not matter if it is in its own
              # thread or not as due to rules of the GIL it will lock up the Event
              # Handler causing everything else to "Freeze" up especially in 
              # regards to sending any signals into the Thread
                QCoreApplication.processEvents()
              # Using this to give time for things to occur in the GUI
                tmSleep(self.Delay)
    
        @Slot(int)
        def ReCountr(self, NewCntBy):
            self.CntBy = NewCntBy
    
        @Slot()
        def ShutDown(self):
            self.Running = False
          # All the Signals within a QThread must be disconnected from within
          # to prevent issues occuring on shutdown
            self.sigOutput.disconnect()
          # Make sure you pass control back to the Event Handler
            QCoreApplication.processEvents()
    
    class Threader(QObject):
        sigNewCntBy = Signal(int)
        sigShutDwn  = Signal()
      # Initialization after Signals declared
        def __init__(self, CntBy):
            QObject.__init__(self)
            self.CountBy = CntBy
    
          # Create the Listener
            self.Tasker = Task(self.CountBy)
          # Assign the Task Signals to Threader Slots
            self.Tasker.sigOutput.connect(self.ProcessOut)
          # Assign Threader Signals to the Task Slots
            self.sigNewCntBy.connect(self.Tasker.ReCountr)
            self.sigShutDwn.connect(self.Tasker.ShutDown)
    
          # Create the Thread
            self.Thred = QThread()
          # Move the Task Object into the Thread
            self.Tasker.moveToThread(self.Thred)
          # Assign the Task Starting Function to the Thread Call
            self.Thred.started.connect(self.Tasker.StartTask)
          # Start the Thread which launches Listener.Connect( )
            self.Thred.start()
    
        @Slot(int)
        def ProcessOut(self, Number):
            print('Current Count :',Number)
            
        @Slot(int)
        def NewCountBy(self, Number):
            self.sigNewCntBy.emit(Number)
    
        @Slot(int)
        def ShutDown(self):
            self.sigShutDwn.emit()
            self.Thred.quit()
          # All the Signals outside a QThread must be disconnected from
          # the outside to prevent issues occuring on shutdown
            self.sigNewCntBy.disconnect()
            self.sigShutDwn.disconnect()
          # This waits for the process within the Thread to stop were 
          # upon it finalizes the Quit referenced above
            self.Thred.wait()
    
    class Gui(QWidget):
        def __init__(self):
            QWidget.__init__(self)
            self.Started = False
            self.CountTask = QObject()
    
            self.btnStart = QPushButton('Click')
            self.btnStart.clicked.connect(self.DoTask)
    
          # Note if you use the X to close the window you 
          # need to capture that event and direct it to do
          # self.ShutDown as well this is just a MUC
            self.btnQuit = QPushButton('Quit')
            self.btnQuit.clicked.connect(self.ShutDown)
    
            self.cbxNums = QComboBox()
            self.cbxNums.addItems(['1', '2', '3', '4', '5'])
    
            VBox = QVBoxLayout()
            VBox.addWidget(self.btnQuit)
            VBox.addWidget(self.btnStart)
            VBox.addWidget(self.cbxNums)
    
            self.setLayout(VBox)
    
        @Slot()
        def DoTask(self):
            Num = int(self.cbxNums.currentText())
            
            if self.Started:
                self.CountTask.NewCountBy(Num)
            else:
                self.Started = True
                self.CountTask = Threader(Num)
    
        @Slot()
        def ShutDown(self):
            self.CountTask.ShutDown()
            self.close()
    
    if __name__ == '__main__':
        MainEventThread = QApplication([])
    
        MainApp = Gui()
        MainApp.show()
    
        MainEventThread.exec()
    

Log in to reply