Video streaming from Python 3



  • I am running a webcam image processing script using Python3 and OpenCV and then send the images to QtQuick 5.9 via a QQuickImageProvider class to display the video stream to the user. This code runs and the GUI and video stream appear normal, but the process has a serious memory leak. It consumes all the available memory before the garbage collector starts cleaning anything up. It appears that even though caching is disabled, the QML is still caching the images, but without any handle or means to access them, so I cannot destroy them manually. Here is an example of my problem.

    main.py

    import sys
    from PyQt5.QtCore import QThread
    from PyQt5.QtQml import QQmlApplicationEngine
    from PyQt5.QtWidgets import QApplication
    from gui import GUI
    from webcam_controller import WebcamControl
    
    if __name__ == '__main__':
        # create the main Qt application objects
        app = QApplication(sys.argv)
        engine = QQmlApplicationEngine()
        context = engine.rootContext()
    
        webcam_control = WebcamControl()
        webcam_control_thread = QThread()
        webcam_control.moveToThread(webcam_control_thread)
        webcam_control_thread.started.connect(webcam_control.start)
    
        # start the main thread
        webcam_control_thread.start()
        gui = GUI()
    
        # make GUI signal connections
        webcam_control.status_signal.connect(gui.handle_connection_status)
        webcam_control.webcammanager.video.connect(gui.handle_image)
        gui.webcam_connect.connect(webcam_control.command_connect)
    
        context.setContextProperty("gui", gui)
        engine.addImageProvider("webcam", gui.webcam_provider)
    
        app.setQuitOnLastWindowClosed(False)
        app.lastWindowClosed.connect(webcam_control.shutdown)
        webcam_control.shutdown_event.connect(app.quit)
    
        engine.load('main.qml')
        status = app.exec_()
        sys.exit(status)
    

    webcam_controller.py

    from PyQt5.QtCore import QTimer, QThread, QObject, pyqtSignal
    from PyQt5.QtWidgets import QApplication
    from PyQt5.QtGui import QImage, QPixmap
    import cv2
    
    # from src.webcam_manager import WebcamManager
    
    class WebcamControl(QObject):
        command_signal = pyqtSignal(bool)
        status_signal = pyqtSignal(bool)
        shutdown_event = pyqtSignal()
    
        def __init__(self):
            super(WebcamControl, self).__init__()
    
            self.webcammanager = WebcamManager()
            self.webcammanager_thread = QThread()
            self.webcammanager.moveToThread(self.webcammanager_thread)
            self.webcammanager.video_timer.moveToThread(self.webcammanager_thread)
    
            self.command_signal.connect(self.webcammanager.command_connection)
            self.webcammanager.connection_status.connect(self.handle_connection)
    
        def start(self):
            self.webcammanager_thread.start()
    
        def shutdown(self):
            self.webcammanager.disconnect()
            self.shutdown_event.emit()
    
        def command_connect(self, connect):
            self.command_signal.emit(connect)
    
        def handle_connection(self, connected):
            self.status_signal.emit(connected)
    
    
    class WebcamManager(QObject):
        connection_status = pyqtSignal(bool)
        recording_status = pyqtSignal(bool)
        video = pyqtSignal(QPixmap)
        video_timer = None
    
        def __init__(self):
            QObject.__init__(self)
    
            self.connected = False
            self.video_width = 1280
            self.video_height = 720
            self.capture = None
            self.monitor_loop_running = None
    
            # Set up the video stream loop connection
            self.video_timer = QTimer()
    
        def command_connection(self, connect):
            """ Control connect/disconnect """
            if connect:
                self.connect()
            else:
                self.disconnect()
    
        def connect(self):
            """ connect to webcam """
            if not self.connected:
                self.video_timer.timeout.connect(self.stream_loop)
                self.capture = cv2.VideoCapture(0)
    
                if self.capture.isOpened():
                    self.capture.set(cv2.CAP_PROP_FRAME_WIDTH, self.video_width)
                    self.capture.set(cv2.CAP_PROP_FRAME_HEIGHT, self.video_height)
                    self.monitor_loop_running = True
                    self.video_timer.start(33) # delay for approx. 30 fps.
                    self.connected = True
    
            self.connection_status.emit(self.connected)
    
        def disconnect(self):
            """ disconnect the webcam """
            if self.connected:
                self.connected = False
                self.monitor_loop_running = False
                self.capture.release()
                self.capture = None
    
            self.video_timer.stop()
            self.connection_status.emit(self.connected)
    
        def stream_loop(self):
            if self.monitor_loop_running:
                ret, frame = self.capture.read()
                if ret:
                    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                    image = QImage(frame, 1280, 720, QImage.Format_RGB888)
                    pixmap = QPixmap.fromImage(image)
                    self.video.emit(pixmap)
                else:
                    self.disconnect()
    

    gui.py

    from PyQt5.QtCore import QMutex, QObject, QPointF, pyqtSignal, pyqtSlot
    from PyQt5.QtChart import QAbstractSeries
    from PyQt5.QtGui import QColor, QPixmap
    from PyQt5.QtQuick import QQuickImageProvider
    
    class GUI(QObject):
        webcam_status = pyqtSignal(bool, arguments=['connected'])
        webcam_refresh = pyqtSignal(QPixmap, arguments=['image'])
        webcam_connect = pyqtSignal(bool)
    
        def __init__(self):
            super(GUI, self).__init__()
            self.webcam_provider = WebcamProvider()
    
        def handle_connection_status(self, connected):
            self.webcam_status.emit(connected)
    
        def handle_image(self, image):
            self.webcam_provider.set_image(image)
            self.webcam_refresh.emit(image)
    
        @pyqtSlot(bool)
        def cmd_connect(self, connected):
            self.webcam_connect.emit(connected)
    
    class WebcamProvider(QQuickImageProvider):
        def __init__(self):
            super(WebcamProvider, self).__init__(QQuickImageProvider.Pixmap)
            self.next_image = QPixmap(1280, 720)
    
        def requestPixmap(self, requested_id, requested_size):
            return self.next_image, self.next_image.size()
    
        def set_image(self, image):
            self.next_image = image
    

    main.qml

    import QtQuick 2.7
    import QtQuick.Controls 2.0
    import QtQuick.Layouts 1.0
    
    ApplicationWindow {
        visible: true
        width: 1080
        height: 768
        title: qsTr("Robot Control")
    
    
        ColumnLayout {
            Layout.fillHeight: true
            Layout.fillWidth: true
    
            Button {
                Layout.fillHeight: true
                Layout.fillWidth: true
                id: button
                text: qsTr("Connect Webcam")
                checked: false
                font.pixelSize: 20
                checkable: true
                background: Rectangle {
                    border.width: 0
                    color: "#DDDDDD"
                    width: button.width
                    height: button.height
                }
                onClicked: {
                    if (button.checked) {
                        button.background.color = "#0074D9";
                    }
                    else {
                        button.background.color = "#DDDDDD";
                    }
                    gui.cmd_connect(button.checked);
                }
            }
    
            Image {
                Layout.fillHeight: true
                Layout.fillWidth: true
                Layout.alignment: Qt.AlignLeft
                fillMode: Image.PreserveAspectFit
                id: webcamImage
                cache: false
                asynchronous: true
                source: "image://webcam/video"
            }
    
            Connections {
                target: gui
    
                onWebcam_status: {
                    button.checked = connected;
                    if (connected) {
                        button.background.color = "#2ECC40";
                    }
                    else {
                        button.background.color = "#DDDDDD";
                    }
                }
    
                onWebcam_refresh: {
                    webcamImage.source = "";
                    webcamImage.source = "image://webcam/video";
                }
            }
        }
    }
    

    I have tried calling the garbage collector in both Python and JavaScript, but neither has any effect on the memory usage, so I am pretty certain that the problem is with the Image item.

    I have tried several workarounds, but none of them have been successful. I have tried drawing the frames on a Canvas, but have not been able to get any images to display. I also tried using Flask and streaming the frames through localhost (https://blog.miguelgrinberg.com/post/video-streaming-with-flask) and then using a MediaPlayer and VideoOutput to request the video stream and display it. Here is my code for that:

    main.py

    from flask import Flask, render_template, Response
    from camera import Camera
    
    app = Flask(__name__)
    
    @app.route('/')
    def index():
        return render_template('index.html')
    
    def gen(camera):
        while True:
            frame = camera.get_frame()
            yield (b'--frame\r\n'
    			   b'Content-Type: video/mpeg\r\n\r\n' + frame + b'\r\n')
    
    @app.route('/video_feed')
    def video_feed():
        return Response(gen(Camera()), mimetype='multipart/x-mixed-replace; boundary=frame')
    
    if __name__ == '__main__':
        app.run(host='localhost', debug=True)
    

    camera.py

    import cv2
    
    class Camera(object):
        def __init__(self):
            # Using OpenCV to capture from device 0. If you have trouble capturing
            # from a webcam, comment the line below out and use a video file
            # instead.
            self.video = cv2.VideoCapture(0)
        
        def __del__(self):
            self.video.release()
        
        def get_frame(self):
            success, image = self.video.read()
            # We are using Motion JPEG, but OpenCV defaults to capture raw images,
            # so we must encode it into JPEG in order to correctly display the
            # video stream.
            ret, jpeg = cv2.imencode('.jpg', image)
            return jpeg.tobytes()
    

    MainForm.ui.qml

    import QtQuick 2.7
    import QtMultimedia 5.0
    
    Rectangle {
        width: 300
        height: 300
        color: "black"
    
        MediaPlayer {
            id: mediaPlayer
            source: "http://127.0.0.1:5000/video_feed"
            autoPlay: true
        }
    
        VideoOutput {
            id: video
            anchors.fill: parent
            source: mediaPlayer
        }
    }
    

    I have also tried just using the Image item and setting its source to http://localhost:5000/video_feed.

    MainForm.ui.qml

    import QtQuick 2.7
    
    Item {
        anchors.fill: parent
    
        Image {
            id: img
            anchors.fill: parent
            source: "http://localhost:5000/video_feed"
            fillMode: Image.Stretch
        }
    }
    

    I also tried just embedded an HTML image tag in a QtQuick Text item and to display the video.

    MainForm.ui.qml

    import QtQuick 2.7
    
    Item {
        anchors.fill: parent
    
        Text {
            id: textArea
            anchors.fill: parent
            baseUrl: "http://127.0.0.1:5000/"
            text: qsTr("<img src=\"video_feed\">")
            renderType: Text.QtRendering
            textFormat: Text.RichText
        }
    }
    

    All of these streaming options confuse me because I can open my browser and go to http://localhost:5000/video_feed and it streams the video just fine, but when I try it in the QML nothing displays even though the server gets the request and turns the camera on.

    Is there something very simple that I have wrong with these attempts, or is there a better way to be able to capture images from the webcam using Python 3 and OpenCV and then display them in a QtQuick application on Windows 10?


  • Lifetime Qt Champion

    Hi,

    What OS are you on ?

    Using GStreamer throughout the QtGStreamer module might be worth considering.

    It can do both "showing on screen" and also stream video.



  • Windows 10.

    Thanks, I will checkout GStreamer.



  • I looked into the QtGStreamer module, but do not understand how to apply that to my project as I am using Python 3 instead of C++. I did some research on GStreamer and came across these two repos: https://github.com/GStreamer/gst-python, https://github.com/GStreamer/qt-gstreamer.

    I assumed that I could create a video stream from Python using the gst-python and then receive the video stream in QML with the qt-gstreamer, but I have not been able to install gst-python. I cloned the repo and tried mkdir build && meson build && ninja -C build following the installation process outlined at https://github.com/GStreamer/gst-build but it failed giving me an error about compilers:

    Meson encountered an error in file meson.build, line 1, column 0:
    Unknown compiler(s): ['cl', 'cc', 'gcc', 'clang']
    The follow exceptions were encountered:
    Running "cl /?" gave "[WinError 2] The system cannot find the file specified"
    Running "cc --version" gave "[WinError 2] The system cannot find the file specified"
    Running "gcc --version" gave "[WinError 2] The system cannot find the file specified"
    Running "clang --version" gave "[WinError 2] The system cannot find the file specified"
    

    I do not really understand this error message and can only assume that I do not have any of these compilers installed. I looked up installing gcc on Windows 10 and found that I can install it via MinGW. After installing gcc I tried again and got this error:

    The Meson build system
    Version: 0.42.0
    Source dir: C:\Users\Documents\gstreamer\gst-python
    Build dir: C:\Users\Documents\gstreamer\gst-python\build
    Build type: native build
    Project name: gst-python
    Native C compiler: gcc (gcc 5.3.0)
    Native C++ compiler: c++ (gcc 5.3.0)
    Build machine cpu family: x86
    Build machine cpu: x86
    Found Pkg-config: NO
    Also couldn't find a fallback subproject in subprojects\gstreamer for the dependency gstreamer-1.0
    
    Meson encountered an error in file meson.build, line 16, column 0:
    Pkg-config not found.
    

    Any suggestions?

    Also, this seems like a lot of overhead to maintain when I distribute this to a couple other computers. Are there other workarounds?

    Thanks!


  • Lifetime Qt Champion

    How did you install PyQt and Qt ?




  • Lifetime Qt Champion

    What about PyQt ?



  • I believe that I installed PyQt using pip install pyqt5.


  • Lifetime Qt Champion

    A possible alternative to ease things might be to consider conda for your python package management.


Log in to reply
 

Looks like your connection to Qt Forum was lost, please wait while we try to reconnect.