Planned maintenance has been done but it did not solve the problem. So work will continue on this and a new time for trying updates will be announced asap.

Use gstreamer to record audio on android



  • Trying to record audio on android with gstreamer, I prefer gstreamer but not the java api because there are more functions need to be added in the future, like

    1. convert the audio to specific format(S16LE, channels=1, rate=16000 etc), encode as wav
    2. Separate video/audio

    The pipeline I am using are

    openslessrc ! audioconvert ! wavenc ! fakesink
    

    and

    openslessrc ! audioconvert ! wavenc ! openslessink location=\"%1\"
    

    But both of them give me error message

    E IAudioFlinger: createRecord returned error -22
    E AudioRecord: AudioFlinger could not create record track, status: -22
    E libOpenSLES: android_audioRecorder_realize(0xcea58a00) error creating AudioRecord object; status -22
    W libOpenSLES: Leaving Object::Realize (SL_RESULT_CONTENT_UNSUPPORTED)
    

    Part of my .pro file

    #plugins
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstcoreelements.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgsttypefindfunctions.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideoconvert.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideorate.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideoscale.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideofilter.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstautodetect.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideotestsrc.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstplayback.a
    #LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstsoup.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstx264.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideoparsersbad.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstandroidmedia.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstopengl.a
    
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstopensles.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstaudioconvert.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstaudiotestsrc.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstautodetect.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstautoconvert.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstaudioresample.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstplayback.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstwavenc.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvoaacenc.a
    
    LIBS += -lOpenSLES
    
    #LIBS += $${GSTREAMER_ROOT}/lib/libavcodec.a
    
    
    #plugin libs
    LIBS += $${GSTREAMER_ROOT}/lib/liborc-0.4.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstgl-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgraphene-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libpng16.a
    LIBS += $${GSTREAMER_ROOT}/lib/libjpeg.a
    LIBS += $${GSTREAMER_ROOT}/lib/libx264.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstvideo-1.0.a
    #LIBS += $${GSTREAMER_ROOT}/lib/libgstbadbase-1.0.a
    #LIBS += $${GSTREAMER_ROOT}/lib/libgstbadvideo-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstaudio-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstpbutils-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgsttag-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstphotography-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstcontroller-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstcodecparsers-1.0.a
    LIBS += -lEGL -lGLESv2
    
    #gstreamer libs
    LIBS += $${GSTREAMER_ROOT}/lib/libgstbase-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstreamer-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgmodule-2.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgobject-2.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libglib-2.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libffi.a
    #LIBS += $${GSTREAMER_ROOT}/lib/libpcre.a
    LIBS += $${GSTREAMER_ROOT}/lib/libiconv.a
    LIBS += $${GSTREAMER_ROOT}/lib/libintl.a
    LIBS += $${GSTREAMER_ROOT}/lib/libz.a
    

    Initialization codes of gstreamer, in main.cpp

    extern "C"
    {
    
    #include <gst/gst.h>
    
    #ifdef Q_OS_ANDROID
    GST_PLUGIN_STATIC_DECLARE(coreelements);
    GST_PLUGIN_STATIC_DECLARE(typefindfunctions);
    GST_PLUGIN_STATIC_DECLARE(videoconvert);
    GST_PLUGIN_STATIC_DECLARE(videorate);
    GST_PLUGIN_STATIC_DECLARE(videoscale);
    GST_PLUGIN_STATIC_DECLARE(videofilter);
    GST_PLUGIN_STATIC_DECLARE(autodetect);
    GST_PLUGIN_STATIC_DECLARE(playback);
    GST_PLUGIN_STATIC_DECLARE(videotestsrc);
    GST_PLUGIN_STATIC_DECLARE(playback);
    //GST_PLUGIN_STATIC_DECLARE(soup);
    GST_PLUGIN_STATIC_DECLARE(x264);
    GST_PLUGIN_STATIC_DECLARE(videoparsersbad);
    GST_PLUGIN_STATIC_DECLARE(androidmedia);
    GST_PLUGIN_STATIC_DECLARE(opengl);
    //GST_PLUGIN_STATIC_DECLARE(qmlgl);
    
    GST_PLUGIN_STATIC_DECLARE(opensles);
    GST_PLUGIN_STATIC_DECLARE(audioconvert);
    GST_PLUGIN_STATIC_DECLARE(audiotestsrc);
    GST_PLUGIN_STATIC_DECLARE(autodetect);
    GST_PLUGIN_STATIC_DECLARE(autoconvert);
    GST_PLUGIN_STATIC_DECLARE(audioresample);
    GST_PLUGIN_STATIC_DECLARE(playback);
    GST_PLUGIN_STATIC_DECLARE(wavenc);
    //GST_PLUGIN_STATIC_DECLARE(voaacenc);
    
    #endif
    
    }
    
    int main(int argc, char *argv[])
    {
        gst_init (&argc, &argv);
    
    #ifdef Q_OS_ANDROID
        GST_PLUGIN_STATIC_REGISTER(coreelements);
        GST_PLUGIN_STATIC_REGISTER(typefindfunctions);
        GST_PLUGIN_STATIC_REGISTER(videoconvert);
        GST_PLUGIN_STATIC_REGISTER(videorate);
        GST_PLUGIN_STATIC_REGISTER(videoscale);
        GST_PLUGIN_STATIC_REGISTER(videofilter);
        GST_PLUGIN_STATIC_REGISTER(autodetect);
        GST_PLUGIN_STATIC_REGISTER(playback);
        //GST_PLUGIN_STATIC_REGISTER(soup);
        GST_PLUGIN_STATIC_REGISTER(videotestsrc);
        GST_PLUGIN_STATIC_REGISTER(x264);
        GST_PLUGIN_STATIC_REGISTER(videoparsersbad);
        GST_PLUGIN_STATIC_REGISTER(androidmedia);
        GST_PLUGIN_STATIC_REGISTER(opengl);
        //GST_PLUGIN_STATIC_REGISTER(qmlgl);
    
        GST_PLUGIN_STATIC_REGISTER(opensles);
        GST_PLUGIN_STATIC_REGISTER(audioconvert);
        GST_PLUGIN_STATIC_REGISTER(audiotestsrc);
        GST_PLUGIN_STATIC_REGISTER(autodetect);
        GST_PLUGIN_STATIC_REGISTER(autoconvert);
        GST_PLUGIN_STATIC_REGISTER(audioresample);
        GST_PLUGIN_STATIC_REGISTER(playback);
        GST_PLUGIN_STATIC_REGISTER(wavenc);
        //GST_PLUGIN_STATIC_REGISTER(voaacenc);
    
        gst_debug_set_active(TRUE);
        gst_debug_set_default_threshold(GST_LEVEL_NONE);
    #endif
    

    Thanks

    Edit : Enable permission of audio record already



  • I solve the issue, share some steps, hope it could help someone(maybe I would become "someone" in the future). Before you start, make sure you build the .so file on the ubuntu(I am using 18.04), windows may work, but why find trouble for yourself?

    1. You need to build the libgstreamer_android.so file first
    2. The tutorials at here show you how to compile the projects. These tutorials and their examples are out of date already, you have a high probability to bump into holes
    3. Before you start, change the classpath of gst-docs\examples\tutorials\android\build.gradle to appropriate one, when I am writing this post, I am using classpath 'com.android.tools.build:gradle:3.4.2'
    4. Disable the projects you do not want to build in gst-docs\examples\tutorials\android\settings.gradle, in my case I disable android-tutorial-1~android-tutorial-4
    //include ':android-tutorial-1'
    //include ':android-tutorial-2'
    //include ':android-tutorial-3'
    //include ':android-tutorial-4'
    include ':android-tutorial-5'
    
    1. Open gst-docs\examples\tutorials\android\android-tutorial-5\build.gradle, update your settings

    Part of the change of my case

    android {
        compileSdkVersion 28
        buildToolsVersion '28.0.3'
    
        defaultConfig {
            applicationId "org.freedesktop.gstreamer.tutorials.tutorial_5"
            minSdkVersion 16
            targetSdkVersion 28
            versionCode 1
            versionName "1.0"
    ......
         }
    
    
    buildscript {
        repositories {
            jcenter()
            google()
        }
    
        dependencies {
            classpath 'com.android.tools.build:gradle:3.4.2'
        }
    }
    

    abiFilters 'x86', 'armabi-v7a'
    6. Open gst-docs\examples\tutorials\android\android-tutorial-5 by android studio(open an existing android studio project)
    7. Set up your ndk path and sdk path if you haven't(File->project structure->SDK location)
    8. Set gradle version(File->project strucutre->project)
    9. File->settings->Instant run->disable all
    10. Not sure this step is needed or not, I change my default gradle path to the location of gradle 5.1.1 locate(File->settings->gradle->use local gradle distribution)
    11. Change the arch of Application.mk to x86 and armabi-v7a
    12. sudo apt-get update
    13. sudo apt-get install pkg-config
    14. Set the GSTREAMER_ROOT_ANDROID to your gstreamer folder(ex : /home/yyyy/gstreamer/android, under the android folder got the arch folders like x86, armabi-v7a), I set the in gst-docs\examples\tutorials\android\android-tutorial-5\build.gradle, if you did not set it, the studio should give you warning later.

    externalNativeBuild {
                ndkBuild {
                    arguments "NDK_APPLICATION_MK=jni/Application.mk", "GSTREAMER_JAVA_SRC_DIR=src", "GSTREAMER_ROOT_ANDROID=/home/yyyy/Qt/gstreamer/android", "GSTREAMER_ASSETS_DIR=src/assets"
    
                    targets "tutorial-5"
    
                    // All archs except MIPS and MIPS64 are supported
                    abiFilters  'armeabi-v7a', 'x86'
                }
            }
    
    1. Set sdk.dir(ex : sdk.dir=/home/yyyy/Android/Sdk) in gst-docs\examples\tutorials\android\android-tutorial-5\local.properties and gst-docs\examples\tutorials\android\local.properties
    2. File->Sync project with gradle file
    3. If things error, delete the folders in android-tutorial-5 except of jni, res, src, remember to list the hidden by ls -la, rerun File->Sync project again
      17 comment out $(call assert-defined, SYSROOT_LINK) in the prebuild gstreamer folder(ex : x86\share\gst-android\ndk-build\gstreamer-1.0.mk)
    4. Build the apk(Build->make module android-tutorial-5)
    5. Add the libgstreamer_android.so as additional libraries
    6. Copy the file src/org/freedesktop/gstreamer/GStreamer.java to the android folder(with the same path)
    7. Copy folder src/assets and put it into the android folder
    8. Link and register the plugins you want to use in your main.cpp file, you can use the gstreamer_android.c as a reference to find out which plugins should be linked

    gstreamer api are well designed, easy to use(although resource managements are verbose, like any c libraries), much powerful multimedia capabilities compare with QtMultimedia, but there are too many hassles to use it on android(windows are fine), avoid it if you do not need more power.

    ps : Find out I able to record the voices and play it without libgstreamer_android.so after I written the steps, maybe It is because I did not register right plugin?



  • Hi, thanks for sharing this,

    also there is an attempt to enable GStreamer in QtMultimedia for Android:
    https://codereview.qt-project.org/c/qt/qtmultimedia/+/253411

    So it could be done using Qt Creator in normal way, just to rebuild QtMultimedia with GStreamer enabled.



  • @val-doroshchuk said in Use gstreamer to record audio on android:

    So it could be done using Qt Creator in normal way, just to rebuild QtMultimedia with GStreamer enabled.

    Thanks, trust me, figure out how to link with the plugins properly is not a pleasent job, since pkgconfig do not works out of the box, you need to open the files of pkgconfig + lots of trial and errors.

    I find out even without the .so, use the macro to register the plugins works for audio, but it do not works for video, maybe I did not register some plugins, fixing the bugs. After things done I will write down a small blog to record down how do I link, register those plugins.





  • @val-doroshchuk Thanks, I think it is not a problem of plugins after I look at the output of the adb.

    When I try to convert the video to audio, I saw following messages

    W RenderThread: type=1400 audit(0.0:1201381): avc: denied { read } for name="perf_ioctl" dev="proc" ino=4026533700 scontext=u:r:untrusted_app:s0:c512,c768 tcontext=u:object_r:proc:s0 tclass=file permissive=0
    

    However, it works perfectly well with voice recording and playing audio, I already add following permission into the manifest files and make sure they are checked when the app start.

    <uses-permission android:name=”android.permission.RECORD_AUDIO”/>
    <uses-permission android:name=”android.permission.READ_EXTERNAL_STORAGE”/>

    Do I miss any permission?Any suggestions except of hacking my machine in order to avoid this permission issue?Thanks


Log in to reply