Parallel pipelines with qml6glsink

[robgu]

Hello,
I have recently compiled the GStreamer 1.24 with the Qt6 and wayland support (I needed to deactivated qml6glmixer). I'm able to successfully capture and show the camera streams with the following commands (for camera 1 and 2, respectively):

gst-launch-1.0 v4l2src device=/dev/video7 ! waylandsink
gst-launch-1.0 v4l2src device=/dev/video0 ! waylandsink

This works perfectly fine.
Basing on the example I switched to Qt code, create a pipeline (v4l2src + glupload + qml6glsink) and the corresponding GstGLQt6VideoItem QML's item (see below for code). This also works perfectly.
Now I want to show the streams of 2 cameras in parallel. For that I duplicated the pipeline and the QML item, but the 2nd GstGLQt6VideoItem show exactly the same stream as the 1st one. Why is it so? What am I doing wrong?
Note that once my application is running (showing duplicated stream of the 1st camera) I checked with the gst-launch commands and both devices are busy, so I suppose that the 2nd pipeline is reading the stream of the 2nd camera but it is not uploaded and shown...
Note also that this code but adapted to Qt5 framework works perfectly fine and streams of both cameras are displayed.

Main.qml

import QtQuick 2.12
import QtQuick.Window 2.12

import org.freedesktop.gstreamer.Qt6GLVideoItem 1.0

Window {
    width: 1280
    height: 800
    visible: true
    title: qsTr("Hello World")

    GstGLQt6VideoItem {
        id: video1
        objectName: "cam1"
        anchors.left: parent.left
        anchors.top: parent.top
        width: 720
        height: 576
    }

    GstGLQt6VideoItem {
        id: video2
        objectName: "cam2"
        anchors.right: parent.right
        anchors.bottom: parent.bottom
        width: 720
        height: 576
    }
}

main.cpp

#include <QGuiApplication>
#include <QQmlApplicationEngine>
#include <QQuickItem>
#include <QQuickWindow>
#include <QRunnable>

#include <gst/gst.h>

class SetPlaying : public QRunnable
{
public:
    SetPlaying(GstElement* pipeline);
    virtual ~SetPlaying();

    void run();

private:
    GstElement* m_pipeline = nullptr;
};

SetPlaying::SetPlaying(GstElement *pipeline)
{
    m_pipeline = pipeline ? static_cast<GstElement*>(pipeline) : nullptr;
}

SetPlaying::~SetPlaying()
{
    if (m_pipeline) gst_object_unref(m_pipeline);
}

void SetPlaying::run()
{
    if (m_pipeline) gst_element_set_state(m_pipeline, GST_STATE_PLAYING);
}

int main(int argc, char *argv[])
{
    int ret = 0;
    gst_init(&argc, &argv);

    {
        QGuiApplication app(argc, argv);

        // Create pipelines before loading QML
        GstElement* pipeline1 = gst_pipeline_new(nullptr);
        GstElement* src1 = gst_element_factory_make("v4l2src", nullptr);
        GstElement* glupload1 = gst_element_factory_make("glupload", nullptr);
        GstElement* sink1 = gst_element_factory_make("qml6glsink", nullptr);
        g_assert(src1 && glupload1 && sink1);

        GstElement* pipeline2 = gst_pipeline_new(nullptr);
        GstElement* src2 = gst_element_factory_make("v4l2src", nullptr);
        GstElement* glupload2 = gst_element_factory_make("glupload", nullptr);
        GstElement* sink2 = gst_element_factory_make("qml6glsink", nullptr);
        g_assert(src2 && glupload2 && sink2);

        // set source cameras
        g_object_set(src1, "device", "/dev/video7", nullptr);
        g_object_set(src1, "io-mode", 2, nullptr);
        g_object_set(src2, "device", "/dev/video0", nullptr);
        g_object_set(src2, "io-mode", 2, nullptr);

        // bind pipeline
        gst_bin_add_many(GST_BIN(pipeline1), src1, glupload1, sink1, nullptr);
        gst_bin_add_many(GST_BIN(pipeline2), src2, glupload2, sink2, nullptr);
        gst_element_link_many(src1, glupload1, sink1, nullptr);
        gst_element_link_many(src2, glupload2, sink2, nullptr);

        QQmlApplicationEngine engine;
        QObject::connect(
            &engine,
            &QQmlApplicationEngine::objectCreationFailed,
            &app,
            []() { QCoreApplication::exit(-1); },
            Qt::QueuedConnection);
        engine.loadFromModule("qmlSinkQt6", "Main");

        // Set the sink
        QQuickWindow* rootObject = static_cast<QQuickWindow*>(engine.rootObjects().first());
        QQuickItem* cam1 = rootObject->findChild<QQuickItem*>("cam1");
        QQuickItem* cam2 = rootObject->findChild<QQuickItem*>("cam2");
        g_assert(cam1 && cam2 && cam1 != cam2);
        g_object_set(sink1, "widget", cam1, nullptr);
        g_object_set(sink2, "widget", cam2, nullptr);

        rootObject->scheduleRenderJob(new SetPlaying(pipeline1), QQuickWindow::BeforeSynchronizingStage);
        rootObject->scheduleRenderJob(new SetPlaying(pipeline2), QQuickWindow::BeforeSynchronizingStage);

        ret = app.exec();

        gst_element_set_state(pipeline1, GST_STATE_NULL);
        gst_element_set_state(pipeline2, GST_STATE_NULL);
        gst_object_unref(pipeline1);
        gst_object_unref(pipeline2);
    }

    gst_deinit();
    return ret;
}

CMakeList.txt:

cmake_minimum_required(VERSION 3.16)

project(qmlSinkQt6 VERSION 0.1 LANGUAGES CXX)

set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(B_SYSROOT "/opt/we-wayland-qt5/2.7.4/sysroots/armv7at2hf-neon-poky-linux-gnueabi/")

find_package(Qt6 6.5 REQUIRED COMPONENTS Quick)

qt_standard_project_setup(REQUIRES 6.5)

qt_add_executable(appqmlSinkQt6
    main.cpp
)

qt_add_qml_module(appqmlSinkQt6
    URI qmlSinkQt6
    VERSION 1.0
    QML_FILES Main.qml
)

# Qt for iOS sets MACOSX_BUNDLE_GUI_IDENTIFIER automatically since Qt 6.1.
# If you are developing for iOS or macOS you should consider setting an
# explicit, fixed bundle identifier manually though.
set_target_properties(appqmlSinkQt6 PROPERTIES
#    MACOSX_BUNDLE_GUI_IDENTIFIER com.example.appqmlSinkQt6
    MACOSX_BUNDLE_BUNDLE_VERSION ${PROJECT_VERSION}
    MACOSX_BUNDLE_SHORT_VERSION_STRING ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}
    MACOSX_BUNDLE TRUE
    WIN32_EXECUTABLE TRUE
)

target_include_directories(appqmlSinkQt6
    PRIVATE ${B_SYSROOT}/usr/include/gstreamer-1.0
    PRIVATE ${B_SYSROOT}/usr/include/glib-2.0
    PRIVATE ${B_SYSROOT}/usr/lib/glib-2.0/include
)

target_link_libraries(appqmlSinkQt6
    PRIVATE Qt6::Quick
    -lglib-2.0 -lgstreamer-1.0 -lgobject-2.0
)

include(GNUInstallDirs)
install(TARGETS appqmlSinkQt6
    BUNDLE DESTINATION .
    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
)

[robgu]

Hello,
I made an additional observation: it seems that the stream that is being duplicated is always the one that is affected to the first GstGLQt6VideoItem component from the QML file.
To be more precise: if video1/cam1 is affected to sink1 and video2/cam2 to sink2 then:

• if video1 is first in QML file both video1 and video2 will output sink1
• if video2 is first in QML file both video1 and video2 will output sink2

Why is it so? Can I have multiple GstGLQt6VideoItem components in the same QML file/module?

[JoeCFD]

@robgu Can you switch to Xorg and try your code again? If it works, it is a wayland issue. I guess your problem is not related to Qt because the QML sink code is maintained by gstreamer folks, not Qt.

//* version number is not needed in Qt 6 */

import QtQuick 
import QtQuick.Window

Why is it so? Can I have multiple GstGLQt6VideoItem components in the same QML file/module?

there should not be any problem to use multiple GstGLQt6VideoItem items for different URLs. If your problem can not be solved, you can work around with gstreamer compositor to merge two streams into one and display them in only one single GstGLQt6VideoItem.

[robgu]

@JoeCFD Thanks for your comments.
Unfortunately the gstreamer compositor will not be useful for our needs, we need several QML components playing different video streams in parallel.
I recompiled Qt6 and gstreamer to add Xorg support on an Ubuntu VM (usually I work with an embedded device with wayland). I made sure I have all the paths (to gstreamer and libs) set up correctly and the plugins/sinks available.
I also changed the video sources to videotestsrc with snow and smpte patterns for retrospectively source 1 and source 2.
I get a very similar result (up left corner: pipeline1, down right: pipeline2):

However, the application crashes with a segmentation fault with the following backtrace:

Thread 6 "QSGRenderThread" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fffd901c640 (LWP 22329)]
0x00007ffff5da47c1 in QObject::~QObject() () from /home/ubuntu/qt6/bin_host_x11/lib/libQt6Core.so.6

(gdb) backtrace 
#0  0x00007ffff5da47c1 in QObject::~QObject() () at /home/ubuntu/qt6/bin_host_x11/lib/libQt6Core.so.6
#1  0x00007ffff1051826 in GstQSGTexture::~GstQSGTexture() (this=0x7fffcc8b0f70, __in_chrg=<optimized out>) at ../subprojects/gst-plugins-good/ext/qt6/gstqsg6material.cc:223
#2  GstQSGMaterialShader::updateUniformData(QSGMaterialShader::RenderState&, QSGMaterial*, QSGMaterial*) (this=0x555555958220, state=..., newMaterial=0x7fffcc8a4ee0, oldMaterial=<optimized out>)
    at ../subprojects/gst-plugins-good/ext/qt6/gstqsg6material.cc:338
#3  0x00007ffff7ad40ef in QSGBatchRenderer::Renderer::updateMaterialDynamicData(QSGBatchRenderer::ShaderManagerShader*, QSGMaterialShader::RenderState&, QSGMaterial*, QSGBatchRenderer::Batch const*, QSGBatchRenderer::Element*, int, int) () at /home/ubuntu/qt6/bin_host_x11/lib/libQt6Quick.so.6
#4  0x00007ffff7ad64ac in QSGBatchRenderer::Renderer::prepareRenderUnmergedBatch(QSGBatchRenderer::Batch*, QSGBatchRenderer::Renderer::PreparedRenderBatch*) [clone .part.0] ()
    at /home/ubuntu/qt6/bin_host_x11/lib/libQt6Quick.so.6
#5  0x00007ffff7ad757e in QSGBatchRenderer::Renderer::prepareRenderPass(QSGBatchRenderer::Renderer::RenderPassContext*) () at /home/ubuntu/qt6/bin_host_x11/lib/libQt6Quick.so.6
#6  0x00007ffff7ad9787 in QSGBatchRenderer::Renderer::render() () at /home/ubuntu/qt6/bin_host_x11/lib/libQt6Quick.so.6
#7  0x00007ffff7af0126 in QSGRenderer::renderScene() () at /home/ubuntu/qt6/bin_host_x11/lib/libQt6Quick.so.6
#8  0x00007ffff7a9a0a3 in QQuickWindowPrivate::renderSceneGraph() () at /home/ubuntu/qt6/bin_host_x11/lib/libQt6Quick.so.6
#9  0x00007ffff7c6b26d in QSGRenderThread::syncAndRender() () at /home/ubuntu/qt6/bin_host_x11/lib/libQt6Quick.so.6
#10 0x00007ffff7c6c62f in QSGRenderThread::run() () at /home/ubuntu/qt6/bin_host_x11/lib/libQt6Quick.so.6
#11 0x00007ffff5ef1897 in QThreadPrivate::start(void*) () at /home/ubuntu/qt6/bin_host_x11/lib/libQt6Core.so.6
#12 0x00007ffff5494ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#13 0x00007ffff5526850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

If I delay the start of the 1st pipeline (snow) then I get the 2nd one (smpte) visible but as soon as the 1st pipeline starts the app crashes as well.
Still any help will be much appreciated...

[robgu]

I figured it out!
The problem comes from a dtor of GstQSGTexture (gstqsg6material.cc file) in the qt6 plugin of gstreamer. Insted of simply calling

delete m_texture;

one should call

m_texture->deleteLater();

as the texture may still be in use.
I'll file an issue with a patch request to gstreamer, hope it will be available soon in the next release.