Use gstreamer to record audio on android



  • Trying to record audio on android with gstreamer, I prefer gstreamer but not the java api because there are more functions need to be added in the future, like

    1. convert the audio to specific format(S16LE, channels=1, rate=16000 etc), encode as wav
    2. Separate video/audio

    The pipeline I am using are

    openslessrc ! audioconvert ! wavenc ! fakesink
    

    and

    openslessrc ! audioconvert ! wavenc ! openslessink location=\"%1\"
    

    But both of them give me error message

    E IAudioFlinger: createRecord returned error -22
    E AudioRecord: AudioFlinger could not create record track, status: -22
    E libOpenSLES: android_audioRecorder_realize(0xcea58a00) error creating AudioRecord object; status -22
    W libOpenSLES: Leaving Object::Realize (SL_RESULT_CONTENT_UNSUPPORTED)
    

    Part of my .pro file

    #plugins
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstcoreelements.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgsttypefindfunctions.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideoconvert.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideorate.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideoscale.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideofilter.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstautodetect.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideotestsrc.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstplayback.a
    #LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstsoup.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstx264.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvideoparsersbad.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstandroidmedia.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstopengl.a
    
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstopensles.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstaudioconvert.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstaudiotestsrc.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstautodetect.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstautoconvert.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstaudioresample.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstplayback.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstwavenc.a
    LIBS += $${GSTREAMER_ROOT}/lib/gstreamer-1.0/libgstvoaacenc.a
    
    LIBS += -lOpenSLES
    
    #LIBS += $${GSTREAMER_ROOT}/lib/libavcodec.a
    
    
    #plugin libs
    LIBS += $${GSTREAMER_ROOT}/lib/liborc-0.4.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstgl-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgraphene-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libpng16.a
    LIBS += $${GSTREAMER_ROOT}/lib/libjpeg.a
    LIBS += $${GSTREAMER_ROOT}/lib/libx264.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstvideo-1.0.a
    #LIBS += $${GSTREAMER_ROOT}/lib/libgstbadbase-1.0.a
    #LIBS += $${GSTREAMER_ROOT}/lib/libgstbadvideo-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstaudio-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstpbutils-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgsttag-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstphotography-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstcontroller-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstcodecparsers-1.0.a
    LIBS += -lEGL -lGLESv2
    
    #gstreamer libs
    LIBS += $${GSTREAMER_ROOT}/lib/libgstbase-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgstreamer-1.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgmodule-2.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libgobject-2.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libglib-2.0.a
    LIBS += $${GSTREAMER_ROOT}/lib/libffi.a
    #LIBS += $${GSTREAMER_ROOT}/lib/libpcre.a
    LIBS += $${GSTREAMER_ROOT}/lib/libiconv.a
    LIBS += $${GSTREAMER_ROOT}/lib/libintl.a
    LIBS += $${GSTREAMER_ROOT}/lib/libz.a
    

    Initialization codes of gstreamer, in main.cpp

    extern "C"
    {
    
    #include <gst/gst.h>
    
    #ifdef Q_OS_ANDROID
    GST_PLUGIN_STATIC_DECLARE(coreelements);
    GST_PLUGIN_STATIC_DECLARE(typefindfunctions);
    GST_PLUGIN_STATIC_DECLARE(videoconvert);
    GST_PLUGIN_STATIC_DECLARE(videorate);
    GST_PLUGIN_STATIC_DECLARE(videoscale);
    GST_PLUGIN_STATIC_DECLARE(videofilter);
    GST_PLUGIN_STATIC_DECLARE(autodetect);
    GST_PLUGIN_STATIC_DECLARE(playback);
    GST_PLUGIN_STATIC_DECLARE(videotestsrc);
    GST_PLUGIN_STATIC_DECLARE(playback);
    //GST_PLUGIN_STATIC_DECLARE(soup);
    GST_PLUGIN_STATIC_DECLARE(x264);
    GST_PLUGIN_STATIC_DECLARE(videoparsersbad);
    GST_PLUGIN_STATIC_DECLARE(androidmedia);
    GST_PLUGIN_STATIC_DECLARE(opengl);
    //GST_PLUGIN_STATIC_DECLARE(qmlgl);
    
    GST_PLUGIN_STATIC_DECLARE(opensles);
    GST_PLUGIN_STATIC_DECLARE(audioconvert);
    GST_PLUGIN_STATIC_DECLARE(audiotestsrc);
    GST_PLUGIN_STATIC_DECLARE(autodetect);
    GST_PLUGIN_STATIC_DECLARE(autoconvert);
    GST_PLUGIN_STATIC_DECLARE(audioresample);
    GST_PLUGIN_STATIC_DECLARE(playback);
    GST_PLUGIN_STATIC_DECLARE(wavenc);
    //GST_PLUGIN_STATIC_DECLARE(voaacenc);
    
    #endif
    
    }
    
    int main(int argc, char *argv[])
    {
        gst_init (&argc, &argv);
    
    #ifdef Q_OS_ANDROID
        GST_PLUGIN_STATIC_REGISTER(coreelements);
        GST_PLUGIN_STATIC_REGISTER(typefindfunctions);
        GST_PLUGIN_STATIC_REGISTER(videoconvert);
        GST_PLUGIN_STATIC_REGISTER(videorate);
        GST_PLUGIN_STATIC_REGISTER(videoscale);
        GST_PLUGIN_STATIC_REGISTER(videofilter);
        GST_PLUGIN_STATIC_REGISTER(autodetect);
        GST_PLUGIN_STATIC_REGISTER(playback);
        //GST_PLUGIN_STATIC_REGISTER(soup);
        GST_PLUGIN_STATIC_REGISTER(videotestsrc);
        GST_PLUGIN_STATIC_REGISTER(x264);
        GST_PLUGIN_STATIC_REGISTER(videoparsersbad);
        GST_PLUGIN_STATIC_REGISTER(androidmedia);
        GST_PLUGIN_STATIC_REGISTER(opengl);
        //GST_PLUGIN_STATIC_REGISTER(qmlgl);
    
        GST_PLUGIN_STATIC_REGISTER(opensles);
        GST_PLUGIN_STATIC_REGISTER(audioconvert);
        GST_PLUGIN_STATIC_REGISTER(audiotestsrc);
        GST_PLUGIN_STATIC_REGISTER(autodetect);
        GST_PLUGIN_STATIC_REGISTER(autoconvert);
        GST_PLUGIN_STATIC_REGISTER(audioresample);
        GST_PLUGIN_STATIC_REGISTER(playback);
        GST_PLUGIN_STATIC_REGISTER(wavenc);
        //GST_PLUGIN_STATIC_REGISTER(voaacenc);
    
        gst_debug_set_active(TRUE);
        gst_debug_set_default_threshold(GST_LEVEL_NONE);
    #endif
    

    Thanks

    Edit : Enable permission of audio record already



  • I solve the issue, share some steps, hope it could help someone(maybe I would become "someone" in the future). Before you start, make sure you build the .so file on the ubuntu(I am using 18.04), windows may work, but why find trouble for yourself?

    1. You need to build the libgstreamer_android.so file first
    2. The tutorials at here show you how to compile the projects. These tutorials and their examples are out of date already, you have a high probability to bump into holes
    3. Before you start, change the classpath of gst-docs\examples\tutorials\android\build.gradle to appropriate one, when I am writing this post, I am using classpath 'com.android.tools.build:gradle:3.4.2'
    4. Disable the projects you do not want to build in gst-docs\examples\tutorials\android\settings.gradle, in my case I disable android-tutorial-1~android-tutorial-4
    //include ':android-tutorial-1'
    //include ':android-tutorial-2'
    //include ':android-tutorial-3'
    //include ':android-tutorial-4'
    include ':android-tutorial-5'
    
    1. Open gst-docs\examples\tutorials\android\android-tutorial-5\build.gradle, update your settings

    Part of the change of my case

    android {
        compileSdkVersion 28
        buildToolsVersion '28.0.3'
    
        defaultConfig {
            applicationId "org.freedesktop.gstreamer.tutorials.tutorial_5"
            minSdkVersion 16
            targetSdkVersion 28
            versionCode 1
            versionName "1.0"
    ......
         }
    
    
    buildscript {
        repositories {
            jcenter()
            google()
        }
    
        dependencies {
            classpath 'com.android.tools.build:gradle:3.4.2'
        }
    }
    

    abiFilters 'x86', 'armabi-v7a'

    1. Open gst-docs\examples\tutorials\android\android-tutorial-5 by android studio(open an existing android studio project)
    2. Set up your ndk path and sdk path if you haven't(File->project structure->SDK location)
    3. Set gradle version(File->project strucutre->project)
    4. File->settings->Instant run->disable all
    5. Not sure this step is needed or not, I change my default gradle path to the location of gradle 5.1.1 locate(File->settings->gradle->use local gradle distribution)
    6. Change the arch of Application.mk to x86 and armabi-v7a
    7. sudo apt-get update
    8. sudo apt-get install pkg-config
    9. Set the GSTREAMER_ROOT_ANDROID to your gstreamer folder(ex : /home/yyyy/gstreamer/android, under the android folder got the arch folders like x86, armabi-v7a), I set the in gst-docs\examples\tutorials\android\android-tutorial-5\build.gradle, if you did not set it, the studio should give you warning later.
    externalNativeBuild {
                ndkBuild {
                    arguments "NDK_APPLICATION_MK=jni/Application.mk", "GSTREAMER_JAVA_SRC_DIR=src", "GSTREAMER_ROOT_ANDROID=/home/yyyy/Qt/gstreamer/android", "GSTREAMER_ASSETS_DIR=src/assets"
    
                    targets "tutorial-5"
    
                    // All archs except MIPS and MIPS64 are supported
                    abiFilters  'armeabi-v7a', 'x86'
                }
            }
    
    1. Set sdk.dir(ex : sdk.dir=/home/yyyy/Android/Sdk) in gst-docs\examples\tutorials\android\android-tutorial-5\local.properties and gst-docs\examples\tutorials\android\local.properties
    2. File->Sync project with gradle file
    3. If things error, delete the folders in android-tutorial-5 except of jni, res, src, remember to list the hidden by ls -la, rerun File->Sync project again
      17 comment out $(call assert-defined, SYSROOT_LINK) in the prebuild gstreamer folder(ex : x86\share\gst-android\ndk-build\gstreamer-1.0.mk)
    4. Build the apk(Build->make module android-tutorial-5)
    5. Add the libgstreamer_android.so as additional libraries
    6. Copy the file src/org/freedesktop/gstreamer/GStreamer.java to the android folder(with the same path)
    7. Copy folder src/assets and put it into the android folder
    8. Link and register the plugins you want to use in your main.cpp file, you can use the gstreamer_android.c as a reference to find out which plugins should be linked

    gstreamer api are well designed, easy to use(although resource managements are verbose, like any c libraries), much powerful multimedia capabilities compare with QtMultimedia, but there are too many hassles to use it on android(windows are fine), avoid it if you do not need more power.

    ps : Find out I able to record the voices and play it without libgstreamer_android.so after I written the steps, maybe It is because I did not register right plugin?


Log in to reply
 

Looks like your connection to Qt Forum was lost, please wait while we try to reconnect.