GStreamer on iOS

FrancescoK · wrote on 18 May 2021, 08:52

Hello, I'm new to the forum, but I've been working with Qt for a while now,

I'm trying to develop a Remote Desktop application for iOS, in which the streaming parte of the desktop is done with GStreamer.

So far I've managed to correctly link the library and successfully run the project in the Simulator, but when I try to create GStreamer elements it fails at runtime.

Here it is my .pro

#iOS
INCLUDEPATH += /Users/user/Library/Developer/GStreamer/iPhone.sdk/GStreamer.framework/Headers
LIBS += -F /Users/user/Library/Developer/GStreamer/iPhone.sdk -framework GStreamer

LIBS += -L/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk/usr/lib -liconv

QMAKE_IOS_DEPLOYMENT_TARGET = 12.0

# Universal target (iPhone and iPad)
QMAKE_APPLE_TARGETED_DEVICE_FAMILY = 1,2

QMAKE_APPLE_DEVICE_ARCHS = arm64
QMAKE_APPLE_SIMULATOR_ARCHS = x86_64

And here it is where it fails:

data.pipeline = gst_pipeline_new ("pipeline");
        qDebug() << "pipeline" << data.pipeline;

        data.udpsrc = gst_element_factory_make("udpsrc", NULL);
        data.rtph264depay = gst_element_factory_make("rtph264depay", NULL);
        data.avdec_h264 = gst_element_factory_make("avdec_h264", NULL);
        data.videoconvert = gst_element_factory_make("videoconvert", NULL);
        data.videoflip = gst_element_factory_make ("videoflip", NULL);

        qDebug() << "udpsrc" << data.udpsrc;
        qDebug() << "rtph264depay" << data.rtph264depay;
        qDebug() << "avdec_h264" << data.avdec_h264;
        qDebug() << "videoconvert" << data.videoconvert;
        qDebug() << "videoflip" << data.videoflip;

All the calls to gst_element_factory_make return null.

Additional Info

Qt 5.15.2
Xcode 12.5
Develop on Big Sur

Any ideas?

Thanks

FrancescoK · wrote on 25 May 2021, 12:14

Finally managed to make it work.

Here it is my .pro:

CONFIG += c++11
CONFIG += objective_c

OBJECTIVE_HEADERS = gst_ios_init.h
OBJECTIVE_SOURCES = gst_ios_init.m
#''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
# macOS | Linux | Windows
# Remember to add  .../Versions/1.0/lib/pkgconfig/ to PKG_CONFIG_PATH
macx {
    QT_CONFIG -= no-pkg-config
   #PKG_CONFIG = /usr/local/bin/pkg-config
    PKG_CONFIG = /opt/homebrew/bin/pkg-config
}

macx|win32|linux {
CONFIG += link_pkgconfig
PKGCONFIG += gstreamer-1.0
PKGCONFIG += gstreamer-pbutils-1.0
#PKGCONFIG += gstreamer-video-1.0
#PKGCONFIG += gstreamer-audio-1.0
#PKGCONFIG += gstreamer-app-1.0
#PKGCONFIG += gstreamer-plugins-bad-1.0
#PKGCONFIG += gstreamer-plugins-base-1.0
#PKGCONFIG += gstreamer-plugins-good-1.0
}
#''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
#iOS
ios {
INCLUDEPATH += /Users/user/Library/Developer/GStreamer/iPhone.sdk/GStreamer.framework/Headers
LIBS += -F /Users/user/Library/Developer/GStreamer/iPhone.sdk -framework GStreamer

LIBS += -framework Foundation
LIBS += -framework CoreGraphics
LIBS += -framework UIKit

#LIBS += -L/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk/usr/lib -lresolv
#LIBS += -L/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk/usr/lib -liconv

LIBS += -L/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk/usr/lib -lresolv
LIBS += -L/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk/usr/lib -liconv

QMAKE_IOS_DEPLOYMENT_TARGET = 12.0

# Universal target (iPhone and iPad)
QMAKE_APPLE_TARGETED_DEVICE_FAMILY = 1,2

QMAKE_APPLE_DEVICE_ARCHS = arm64
QMAKE_APPLE_SIMULATOR_ARCHS = x86_64
}
#''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''

And then called gst_ios_init() (defined in gst_ios_init.h) in my main.cpp:

#ifdef Q_OS_IOS
    gst_ios_init();
#endif

The gst_ios_init.m was also missing some includes:

#include <gio/gio.h>
#include <UIKit/UIKit.h>
#include <Foundation/Foundation.h>

SGaist · wrote on 18 May 2021, 20:38

Hi and welcome to devnet,

Did you try to retrieve the error after calling that function ?

FrancescoK · wrote on 19 May 2021, 07:45

The error shown is:

0:00:12.281788000  5678 0x7fbc5cd1c200 WARN     GST_ELEMENT_FACTORY gstelementfactory.c:467:gst_element_factory_make: no such element factory "udpsrc"!
0:00:12.281795000  5678 0x7fbc5cd1c200 WARN     GST_ELEMENT_FACTORY gstelementfactory.c:467:gst_element_factory_make: no such element factory "rtph264depay"!
0:00:12.281801000  5678 0x7fbc5cd1c200 WARN     GST_ELEMENT_FACTORY gstelementfactory.c:467:gst_element_factory_make: no such element factory "avdec_h264"!
0:00:12.281805000  5678 0x7fbc5cd1c200 WARN     GST_ELEMENT_FACTORY gstelementfactory.c:467:gst_element_factory_make: no such element factory "videoconvert"!
0:00:12.281811000  5678 0x7fbc5cd1c200 WARN     GST_ELEMENT_FACTORY gstelementfactory.c:467:gst_element_factory_make: no such element factory "videoflip"!

so clearly it doesn't find the plugins.

I believe that the correct approach to solve the problem is to register such plugins, by writing something like this in main.cpp:

// Declar plugins
G_BEGIN_DECLS
GST_PLUGIN_STATIC_DECLARE(coreelements);
GST_PLUGIN_STATIC_DECLARE(videotestsrc);
GST_PLUGIN_STATIC_DECLARE(videoconvert);
GST_PLUGIN_STATIC_DECLARE(playback);
GST_PLUGIN_STATIC_DECLARE(x264);
GST_PLUGIN_STATIC_DECLARE(libav);
GST_PLUGIN_STATIC_DECLARE(encoding);
//...
G_END_DECLS

int main(int argc, char *argv[]) {

    // Initialize the GStreamer library
    if (gst_init_check(&argc, &argv, NULL)) {
        qDebug() << "This program is linked against" << gst_version_string();
    }

    // Register plugins
    GST_PLUGIN_STATIC_REGISTER(coreelements);
    GST_PLUGIN_STATIC_REGISTER(videotestsrc);
    GST_PLUGIN_STATIC_REGISTER(videoconvert);
    GST_PLUGIN_STATIC_REGISTER(playback);
    GST_PLUGIN_STATIC_REGISTER(x264);
    GST_PLUGIN_STATIC_REGISTER(libav);
    GST_PLUGIN_STATIC_REGISTER(encoding);
    //...
}

This does in fact give me some positive results, because now some elements are successfully created:

0:01:48.471557000  5973 0x7fa450c1ec00 INFO     GST_ELEMENT_FACTORY gstelementfactory.c:361:gst_element_factory_create: creating element "pipeline" named "pipeline"
pipeline 0x7fa452204160
0:01:48.472489000  5973 0x7fa450c1ec00 WARN     GST_ELEMENT_FACTORY gstelementfactory.c:467:gst_element_factory_make: no such element factory "udpsrc"!
0:01:48.472503000  5973 0x7fa450c1ec00 WARN     GST_ELEMENT_FACTORY gstelementfactory.c:467:gst_element_factory_make: no such element factory "rtph264depay"!
0:01:48.472513000  5973 0x7fa450c1ec00 INFO     GST_ELEMENT_FACTORY gstelementfactory.c:363:gst_element_factory_create: creating element "avdec_h264"
0:01:48.474720000  5973 0x7fa450c1ec00 INFO     GST_ELEMENT_FACTORY gstelementfactory.c:363:gst_element_factory_create: creating element "videoconvert"
0:01:48.474936000  5973 0x7fa450c1ec00 WARN     GST_ELEMENT_FACTORY gstelementfactory.c:467:gst_element_factory_make: no such element factory "videoflip"!
0:01:48.475071000  5973 0x7fa450c1ec00 WARN     GST_ELEMENT_FACTORY gstelementfactory.c:467:gst_element_factory_make: no such element factory "glimagesink"!

But if I try to register the others plugins, for example by adding

GST_PLUGIN_STATIC_DECLARE(udp);

and

GST_PLUGIN_STATIC_REGISTER(udp);

it gives me this error:

GStreamer(libgio-2_0_a-gresolver.c.o):-1: error: Undefined symbols for architecture x86_64:
  "_res_9_dn_expand", referenced from:
      _do_lookup_records in GStreamer(libgio-2_0_a-gthreadedresolver.c.o)
  "_res_9_ninit", referenced from:
      _do_lookup_records in GStreamer(libgio-2_0_a-gthreadedresolver.c.o)
  "_res_9_nquery", referenced from:
      _do_lookup_records in GStreamer(libgio-2_0_a-gthreadedresolver.c.o)
  "_res_9_ndestroy", referenced from:
      _do_lookup_records in GStreamer(libgio-2_0_a-gthreadedresolver.c.o)
  "_res_9_init", referenced from:
      _g_resolver_maybe_reload in GStreamer(libgio-2_0_a-gresolver.c.o)

I encountered a similar error before and I solved by adding the missing library (where the symbols were) in the .pro file:

LIBS += -L/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk/usr/lib -liconv

It seems that is missing the gio (or something like that) library, but I don't know where to find it.

[EDIT]

After a quick search

locate  libgio
         
/Library/Frameworks/GStreamer.framework/Versions/1.0/lib/gio/modules/libgioopenssl.a
/Library/Frameworks/GStreamer.framework/Versions/1.0/lib/gio/modules/libgioopenssl.so
/Library/Frameworks/GStreamer.framework/Versions/1.0/lib/libgio-2.0.0.dylib
/Library/Frameworks/GStreamer.framework/Versions/1.0/lib/libgio-2.0.a
/Library/Frameworks/GStreamer.framework/Versions/1.0/lib/libgio-2.0.dylib
/Library/Frameworks/GStreamer.framework/Versions/1.0/lib/libgio-2.0.la

I can locate the library for macOS, but not for iOS. In fact if I try to add it

LIBS += -L/Library/Frameworks/GStreamer.framework/Versions/1.0/lib/ -lgio-2.0

it (rightfully) complains about it:

:-1: error: building for iOS Simulator, but linking in dylib built for macOS, file '/Library/Frameworks/GStreamer.framework/Versions/1.0/lib/libgio-2.0.dylib' for architecture x86_64

I also tried to include the get_ios_init.h and get_ios_init.m configuration files of the GStreamer tutorials (https://gitlab.freedesktop.org/gstreamer/gst-docs/-/tree/master/examples/tutorials/xcode iOS/Tutorial 1) by copying them into the project by adding

CONFIG += c++11
CONFIG += objective_c

OBJECTIVE_HEADERS = gst_ios_init.h
OBJECTIVE_SOURCES = gst_ios_init.m

but it didn't help.

Any ideas?
Thanks

SGaist · 20 May 2021, 09:12

Did you build the GStreamer SDK yourself ?

FrancescoK · S SGaist 19 May 2021, 18:36

@SGaist Nope, I downloaded the GStreamer library from https://gstreamer.freedesktop.org/data/pkg/ios/, following the official instructions (https://gstreamer.freedesktop.org/documentation/installing/for-ios-development.html?gi-language=c)

SGaist · wrote on 20 May 2021, 20:03

Then it could be a glitch in the SDK. It's worth contacting them to see if it's something known.

FrancescoK · wrote on 25 May 2021, 12:14

Finally managed to make it work.

Here it is my .pro:

CONFIG += c++11
CONFIG += objective_c

OBJECTIVE_HEADERS = gst_ios_init.h
OBJECTIVE_SOURCES = gst_ios_init.m
#''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
# macOS | Linux | Windows
# Remember to add  .../Versions/1.0/lib/pkgconfig/ to PKG_CONFIG_PATH
macx {
    QT_CONFIG -= no-pkg-config
   #PKG_CONFIG = /usr/local/bin/pkg-config
    PKG_CONFIG = /opt/homebrew/bin/pkg-config
}

macx|win32|linux {
CONFIG += link_pkgconfig
PKGCONFIG += gstreamer-1.0
PKGCONFIG += gstreamer-pbutils-1.0
#PKGCONFIG += gstreamer-video-1.0
#PKGCONFIG += gstreamer-audio-1.0
#PKGCONFIG += gstreamer-app-1.0
#PKGCONFIG += gstreamer-plugins-bad-1.0
#PKGCONFIG += gstreamer-plugins-base-1.0
#PKGCONFIG += gstreamer-plugins-good-1.0
}
#''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
#iOS
ios {
INCLUDEPATH += /Users/user/Library/Developer/GStreamer/iPhone.sdk/GStreamer.framework/Headers
LIBS += -F /Users/user/Library/Developer/GStreamer/iPhone.sdk -framework GStreamer

LIBS += -framework Foundation
LIBS += -framework CoreGraphics
LIBS += -framework UIKit

#LIBS += -L/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk/usr/lib -lresolv
#LIBS += -L/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk/usr/lib -liconv

LIBS += -L/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk/usr/lib -lresolv
LIBS += -L/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk/usr/lib -liconv

QMAKE_IOS_DEPLOYMENT_TARGET = 12.0

# Universal target (iPhone and iPad)
QMAKE_APPLE_TARGETED_DEVICE_FAMILY = 1,2

QMAKE_APPLE_DEVICE_ARCHS = arm64
QMAKE_APPLE_SIMULATOR_ARCHS = x86_64
}
#''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''

And then called gst_ios_init() (defined in gst_ios_init.h) in my main.cpp:

#ifdef Q_OS_IOS
    gst_ios_init();
#endif

The gst_ios_init.m was also missing some includes:

#include <gio/gio.h>
#include <UIKit/UIKit.h>
#include <Foundation/Foundation.h>

SGaist · wrote on 25 May 2021, 18:31

Nice !

Thanks for the feedback :-)