Important: Please read the Qt Code of Conduct - https://forum.qt.io/topic/113070/qt-code-of-conduct

Showing stream video



  • Hi!
    I'm receiving the raw h.264 video stream from QUdpSocket. So, i need to display that on widget.
    I know that QMediaPlayer can't play stream. After googling i found some solutions but all those need external libs. I would like to use only Qt components and decode h.264 using system codecs (QMediaPlayer nice plays h.264 from file using k-like codecs).

    So, I have QByteArray received from camera, I need to decode h.264 frame and show that on widget. How can i do that?
    Thanks)))



  • So, how to get QImage from stream:

    QUdpSocket *udpSocket;
    QByteArray	buffer;
    
    int read_packet(void *opaque, uint8_t *buf, int buf_size) {
    	//auto	fil = static_cast<QUdpSocket*>(opaque);
    	//QMutexLocker	l(&fil->mutex);
    	//qDebug() << buf_size << fil->buffer.length();
    
    	qDebug() << buf_size << buffer.length();
    
    	if (buffer.isEmpty() && !udpSocket->hasPendingDatagrams()) udpSocket->waitForReadyRead();
    	while (udpSocket->hasPendingDatagrams()) {
    		if (buffer.length() > 1024*500) {
    			buffer = buffer.right(1024*500);
    			buffer.clear();
    		}
    		buffer.append(udpSocket->receiveDatagram().data());
    	}
    
    	buf_size = buffer.length() > buf_size ? buf_size : buffer.length();
    	memcpy(buf, buffer.constData(), buf_size);
    	buffer.remove(0, buf_size);
    
    	return buf_size;
    };
    
    int MainWindow::start()
    {
    	QtConcurrent::run([this](){
    		udpSocket = new QUdpSocket;
    		udpSocket->bind(15004);
    
    		AVFormatContext *fmt_ctx = nullptr;
    		AVIOContext *avio_ctx = nullptr;
    		uint8_t *buffer = nullptr, *avio_ctx_buffer = nullptr;
    		size_t buffer_size = 0, avio_ctx_buffer_size = 4096;
    
    		int ret = 0;
    		fmt_ctx = avformat_alloc_context();
    		Q_ASSERT(fmt_ctx);
    
    		avio_ctx_buffer = reinterpret_cast<uint8_t*>(av_malloc(avio_ctx_buffer_size));
    		Q_ASSERT(avio_ctx_buffer);
    		avio_ctx = avio_alloc_context(avio_ctx_buffer, static_cast<int>(avio_ctx_buffer_size), 0, /*&bd*/udpSocket, &read_packet, nullptr, nullptr);
    		Q_ASSERT(avio_ctx);
    
    		fmt_ctx->pb = avio_ctx;
    		ret = avformat_open_input(&fmt_ctx, nullptr, nullptr, nullptr);
    		Q_ASSERT(ret >= 0);
    		ret = avformat_find_stream_info(fmt_ctx, nullptr);
    		Q_ASSERT(ret >= 0);
    
    		AVStream	*video_stream = nullptr;
    		for (uint i=0; i<fmt_ctx->nb_streams; ++i) {
    			auto	st = fmt_ctx->streams[i];
    			qDebug() << st->id << st->index << st->start_time << st->duration << st->codecpar->codec_type;
    			if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) video_stream = st;
    		}
    		Q_ASSERT(video_stream);
    
    
    		AVCodecParameters	*codecpar = video_stream->codecpar;
    		AVCodec* codec = avcodec_find_decoder(codecpar->codec_id);
    		AVCodecContext *codec_context = avcodec_alloc_context3(codec);
    		int err = avcodec_open2(codec_context, codec, nullptr);
    		Q_ASSERT(err>=0);
    		qDebug() << codec->name << codec->id;
    		qDebug() << codecpar->width << codecpar->height << codecpar->format << codec_context->pix_fmt;
    
    
    
    
    		AVFrame	*frameRGB = av_frame_alloc();
    		frameRGB->format = AV_PIX_FMT_RGB24;
    		frameRGB->width = codecpar->width;
    		frameRGB->height = codecpar->height;
    		err = av_frame_get_buffer(frameRGB, 0);
    		Q_ASSERT(err == 0);
    		///
    		SwsContext *imgConvertCtx = nullptr;
    
    		AVFrame* frame = av_frame_alloc();
    		AVPacket packet;
    
    		while (av_read_frame(fmt_ctx, &packet) >= 0) {
    			avcodec_send_packet(codec_context, &packet);
    			err = avcodec_receive_frame(codec_context, frame);
    			if (err == 0) {
    				//qDebug() << frame->height << frame->width << codec_context->pix_fmt;
    
    				imgConvertCtx = sws_getCachedContext(imgConvertCtx,
    													 codecpar->width, codecpar->height, static_cast<AVPixelFormat>(codecpar->format),
    													 frameRGB->width, frameRGB->height,
    													 AV_PIX_FMT_RGB24, SWS_BICUBIC, nullptr, nullptr, nullptr);
    				Q_ASSERT(imgConvertCtx);
    
    				//conversion frame to frameRGB
    				sws_scale(imgConvertCtx, frame->data, frame->linesize, 0, codec_context->height, frameRGB->data, frameRGB->linesize);
    
    				//setting QImage from frameRGB
    				QImage image(frameRGB->data[0],
    						frameRGB->width,
    						frameRGB->height,
    						frameRGB->linesize[0],
    						QImage::Format_RGB888);
    
    				emit imageUpdated(image, clock());
    			}
    
    			av_frame_unref(frame);
    			av_packet_unref(&packet);
    		}
    		if (imgConvertCtx) sws_freeContext(imgConvertCtx);
    
    
    
    
    
    		//end:
    		avformat_close_input(&fmt_ctx);
    		/* note: the internal buffer could have changed, and be != avio_ctx_buffer */
    		if (avio_ctx) {
    			av_freep(&avio_ctx->buffer);
    			av_freep(&avio_ctx);
    		}
    		av_file_unmap(buffer, buffer_size);
    		if (ret < 0) {
    			qWarning() << ret;
    			return 1;
    		}
    		return 0;
    	});
    }
    

  • Lifetime Qt Champion

    @BrMisha You could write your own QIODevice and use it with https://doc.qt.io/qt-5/qmediaplayer.html#setMedia



  • @jsulm I know but QMediaPlayer uses QIODevice::seek, which isn't available in stream


  • Lifetime Qt Champion

    Hi,

    What OS are you on ?



  • @SGaist Windows (10) and Linux (Ubuntu and other).


  • Lifetime Qt Champion

    In that case, I would rather consider using VLC and its Qt integration to handle the video stream.



  • @SGaist but why this solution better of using system codecs?


  • Lifetime Qt Champion

    Because VLC is the Swiss Army Knife of vidéo decoding and supports a larger palette of codecs than most OSs provide out of the box.



  • @SGaist I would't like to use VLC but you have convinced me)))
    Tomorrow i will try to build VLC for windows


  • Lifetime Qt Champion

    There's likely already a prebuilt package.

    See also VLC-Qt



  • @SGaist is VLC-Qt still actuality now? Last commit was in 2016 year and based on VLC 2.2.4


  • Lifetime Qt Champion

    It does not mean it does not work anymore. I'd say it's worth a try.



  • Unfortunately, VLC doesn't provide prebuilt packages for my compiler (MinGW32-64 7.3.0).
    After googling i decided to try ffmpeg. I build it following this manual https://ffmpeg.org/platform.html#Native-Windows-compilation-using-MinGW-or-MinGW_002dw64 , then add libs to *pro file:

    LIBS += -LF:/FFmpeg/tempinstall/usr/local/lib/ -llibavcodec.dll -llibavformat.dll
    INCLUDEPATH += F:/FFmpeg/tempinstall/usr/local/include
    DEPENDPATH += F:/FFmpeg/tempinstall/usr/local/include
    

    and i called avformat_open_input(some args). But got error^

    c:/qt/qt5.14.1/tools/mingw64/bin/../lib/gcc/x86_64-w64-mingw32/8.3.0/../../../../x86_64-w64-mingw32/bin/ld.exe: main.o: in function `qMain(int, char**)':
    D:\projects_test\qt\PlayerVideo\build-PlayerVideo-Desktop-Debug/../PlayerVideo/main.cpp:58: undefined reference to `avformat_open_input(AVFormatContext**, char const*, AVInputFormat*, AVDictionary**)'
    

    So, i'm searching my mistake... I compared compiler versions between msys2 and qt, booth have GNU Make 4.2.1, but msys2 has gcc 8.3.0 and qt has 7.3.0.
    That is problem?


  • Lifetime Qt Champion

    It's "-lavcodec"



  • @SGaist no, it isn't helpfull((


  • Lifetime Qt Champion

    Did you fix all libraries name ?



  • @SGaist this part of .pro

    LIBS += -LF:/Musor/FFmpeg_linbuild/ffmpeg/lib/ -lavformat
    
    INCLUDEPATH += F:/Musor/FFmpeg_linbuild/ffmpeg/include
    DEPENDPATH += F:/Musor/FFmpeg_linbuild/ffmpeg/include
    

    In code i'm calling only avformat_license(), which locates in avformat


  • Lifetime Qt Champion

    Do you have ffmpeg built for the same architecture as your Qt version ?



  • @SGaist yes, booth is x64



  • Men, I really don't understand what is happening.
    I installed msys2, and some packages for that, then i copied *c:\Qt\Qt5.14.2\Tools\mingw730_64* to *c:\msys64\mingw64*.
    Then built libav (it is "small" version of FFmpeg) and got this files:

    build\usr\local\bin\avcodec.lib
    build\usr\local\bin\avcodec-58.dll
    build\usr\local\bin\avconv.exe
    build\usr\local\bin\avdevice.lib
    build\usr\local\bin\avdevice-57.dll
    build\usr\local\bin\avfilter.lib
    build\usr\local\bin\avfilter-7.dll
    build\usr\local\bin\avformat.lib
    build\usr\local\bin\avformat-58.dll
    build\usr\local\bin\avprobe.exe
    build\usr\local\bin\avresample.lib
    build\usr\local\bin\avresample-4.dll
    build\usr\local\bin\avutil.lib
    build\usr\local\bin\avutil-56.dll
    build\usr\local\bin\swscale.lib
    build\usr\local\bin\swscale-5.dll
    
    build\usr\local\include\libavcodec\	<< here is *.h
    build\usr\local\include\libavdevice\	<< here is *.h
    build\usr\local\include\libavfilter\	<< here is *.h
    build\usr\local\include\libavformat\	<< here is *.h
    build\usr\local\include\libavresample\	<< here is *.h
    build\usr\local\include\libavutil\	<< here is *.h
    build\usr\local\include\libswscale\	<< here is *.h
    
    build\usr\local\lib\pkgconfig\
    build\usr\local\lib\avcodec-58.def
    build\usr\local\lib\avdevice-57.def
    build\usr\local\lib\avfilter-7.def
    build\usr\local\lib\avformat-58.def
    build\usr\local\lib\avresample-4.def
    build\usr\local\lib\avutil-56.def
    build\usr\local\lib\libavcodec.dll.a
    build\usr\local\lib\libavdevice.dll.a
    build\usr\local\lib\libavfilter.dll.a
    build\usr\local\lib\libavformat.dll.a
    build\usr\local\lib\libavresample.dll.a
    build\usr\local\lib\libavutil.dll.a
    build\usr\local\lib\libswscale.dll.a
    build\usr\local\lib\swscale-5.def
    

    *c:\Qt\Qt5.14.2\Tools\mingw730_64* was added to PATH. When I start, for example, avconv.exe, it works ok, so definitely libav was built with true mingw (which uses qt).
    Next, i added to *.pro (using "Add library...") this:

    win32: LIBS += -LF:/Musor/libav/build/usr/local/lib/ -llibavformat.dll
    INCLUDEPATH += F:/Musor/libav/build/usr/local/include
    DEPENDPATH += F:/Musor/libav/build/usr/local/include
    

    main.c:

    #include <libavformat/avformat.h>
    int main(int argc, char *argv[]) {
    	avformat_license();
    	return 0;
    }
    

    After compilation i got:

    C:/Qt/Qt5.14.2/Tools/mingw730_64/bin/mingw32-make -f Makefile.Debug
    mingw32-make[1]: Entering directory 'D:/projects_test/qt/qt_test/build-qt_test-Desktop_Qt_5_14_2_MinGW_64_bit-Debug'
    g++ -c -fno-keep-inline-dllexport -g -std=gnu++11 -Wall -Wextra -Wextra -fexceptions -mthreads -DUNICODE -D_UNICODE -DWIN32 -DMINGW_HAS_SECURE_API=1 -DQT_DEPRECATED_WARNINGS -DQT_QML_DEBUG -DQT_SERIALPORT_LIB -DQT_CORE_LIB -I..\qt_test -I. -IF:\Musor\libav\build\usr\local\include -IC:\Qt\Qt5.14.2\5.14.2\mingw73_64\include -IC:\Qt\Qt5.14.2\5.14.2\mingw73_64\include\QtSerialPort -IC:\Qt\Qt5.14.2\5.14.2\mingw73_64\include\QtCore -Idebug -IC:\Qt\Qt5.14.2\5.14.2\mingw73_64\mkspecs\win32-g++  -o debug\main.o ..\qt_test\main.cpp
    ..\qt_test\main.cpp: In function 'int main(int, char**)':
    ..\qt_test\main.cpp:4:14: warning: unused parameter 'argc' [-Wunused-parameter]
     int main(int argc, char *argv[])
                  ^~~~
    ..\qt_test\main.cpp:4:31: warning: unused parameter 'argv' [-Wunused-parameter]
     int main(int argc, char *argv[])
                                   ^
    g++ -Wl,-subsystem,console -mthreads -o debug\qt_test.exe debug/main.o  -LF:\Musor\libav\build\usr\local\lib F:\Musor\libav\build\usr\local\lib\libavformat.dll.a C:\Qt\Qt5.14.2\5.14.2\mingw73_64\lib\libQt5SerialPort.a C:\Qt\Qt5.14.2\5.14.2\mingw73_64\lib\libQt5Core.a   
    debug/main.o: In function `main':
    D:\projects_test\qt\qt_test\build-qt_test-Desktop_Qt_5_14_2_MinGW_64_bit-Debug/../qt_test/main.cpp:6: undefined reference to `avformat_license()'
    collect2.exe: error: ld returned 1 exit status
    mingw32-make[1]: *** [Makefile.Debug:68: debug/qt_test.exe] Error 1
    mingw32-make: *** [Makefile:45: debug] Error 2
    mingw32-make[1]: Leaving directory 'D:/projects_test/qt/qt_test/build-qt_test-Desktop_Qt_5_14_2_MinGW_64_bit-Debug'
    18:45:50: The process "C:\Qt\Qt5.14.2\Tools\mingw730_64\bin\mingw32-make.exe" exited with code 2.
    Error while building/deploying project qt_test (kit: Desktop Qt 5.14.2 MinGW 64-bit)
    When executing step "Make"
    

    Whyyyyyyy?????
    I have tried to change -llibavformat.dll to -lavformat, "-lavformat.dll" and "-lavformat" but i'm stil getting this error.


  • Lifetime Qt Champion

    Might be silly but try linking to all the ffmpeg libraries, just in case.



  • @BrMisha said in Showing stream video:

    build\usr\local\bin\avformat.lib
    build\usr\local\bin\avformat-58.dll

    not sure if that's the issue, but you're showing libraries built in build\usr\local\bin but you're using build\usr\local\lib in your .pro file:

    win32: LIBS += -LF:/Musor/libav/build/usr/local/lib/ -llibavformat.dll

    In addition, and not sure since it's a long time I don't use libraries in Windows, also keep in mind that you have different names between the .lib and .dll files (avformat.lib vs. avformat-58.dll)



  • @SGaist said in Showing stream video:

    Might be silly but try linking to all the ffmpeg libraries, just in case.

    I tried to add this LIBS += -llibavcodec.dll -llibavdevice.dll -llibavfilter.dll -llibavformat.dll -llibavresample.dll -llibavutil.dll -llibswscale.dll but i'm still getting errors.
    By the way, if in cpp i'm adding

    extern "C" {
    	#include <libavformat/avformat.h>
    }
    

    then compils but falls in runtime on calling avformat_license()



  • @Pablo-J-Rogina "bin" contains *.exe and *dll. Libraries locates in "lib" folder


  • Lifetime Qt Champion

    Because you are likely missing the path to the folder where the dlls are in the PATH environment variable.

    You can do that in the Run part of the Project panel.



  • But i did set *\libav\build\usr\local\bin\ to PATH before)))



  • Men, i have done it)))
    So, we need to add bin to PATH, in .pro LIBS += -L/libav/build/usr/local/lib/ -llibavformat.dll.
    I think, in a past it doesn't work because i didn't restart qt qreator after adding to PATH



  • @BrMisha said in Showing stream video:

    i have done it

    so please don't forget to mark your post as solved.



  • So, how to get QImage from stream:

    QUdpSocket *udpSocket;
    QByteArray	buffer;
    
    int read_packet(void *opaque, uint8_t *buf, int buf_size) {
    	//auto	fil = static_cast<QUdpSocket*>(opaque);
    	//QMutexLocker	l(&fil->mutex);
    	//qDebug() << buf_size << fil->buffer.length();
    
    	qDebug() << buf_size << buffer.length();
    
    	if (buffer.isEmpty() && !udpSocket->hasPendingDatagrams()) udpSocket->waitForReadyRead();
    	while (udpSocket->hasPendingDatagrams()) {
    		if (buffer.length() > 1024*500) {
    			buffer = buffer.right(1024*500);
    			buffer.clear();
    		}
    		buffer.append(udpSocket->receiveDatagram().data());
    	}
    
    	buf_size = buffer.length() > buf_size ? buf_size : buffer.length();
    	memcpy(buf, buffer.constData(), buf_size);
    	buffer.remove(0, buf_size);
    
    	return buf_size;
    };
    
    int MainWindow::start()
    {
    	QtConcurrent::run([this](){
    		udpSocket = new QUdpSocket;
    		udpSocket->bind(15004);
    
    		AVFormatContext *fmt_ctx = nullptr;
    		AVIOContext *avio_ctx = nullptr;
    		uint8_t *buffer = nullptr, *avio_ctx_buffer = nullptr;
    		size_t buffer_size = 0, avio_ctx_buffer_size = 4096;
    
    		int ret = 0;
    		fmt_ctx = avformat_alloc_context();
    		Q_ASSERT(fmt_ctx);
    
    		avio_ctx_buffer = reinterpret_cast<uint8_t*>(av_malloc(avio_ctx_buffer_size));
    		Q_ASSERT(avio_ctx_buffer);
    		avio_ctx = avio_alloc_context(avio_ctx_buffer, static_cast<int>(avio_ctx_buffer_size), 0, /*&bd*/udpSocket, &read_packet, nullptr, nullptr);
    		Q_ASSERT(avio_ctx);
    
    		fmt_ctx->pb = avio_ctx;
    		ret = avformat_open_input(&fmt_ctx, nullptr, nullptr, nullptr);
    		Q_ASSERT(ret >= 0);
    		ret = avformat_find_stream_info(fmt_ctx, nullptr);
    		Q_ASSERT(ret >= 0);
    
    		AVStream	*video_stream = nullptr;
    		for (uint i=0; i<fmt_ctx->nb_streams; ++i) {
    			auto	st = fmt_ctx->streams[i];
    			qDebug() << st->id << st->index << st->start_time << st->duration << st->codecpar->codec_type;
    			if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) video_stream = st;
    		}
    		Q_ASSERT(video_stream);
    
    
    		AVCodecParameters	*codecpar = video_stream->codecpar;
    		AVCodec* codec = avcodec_find_decoder(codecpar->codec_id);
    		AVCodecContext *codec_context = avcodec_alloc_context3(codec);
    		int err = avcodec_open2(codec_context, codec, nullptr);
    		Q_ASSERT(err>=0);
    		qDebug() << codec->name << codec->id;
    		qDebug() << codecpar->width << codecpar->height << codecpar->format << codec_context->pix_fmt;
    
    
    
    
    		AVFrame	*frameRGB = av_frame_alloc();
    		frameRGB->format = AV_PIX_FMT_RGB24;
    		frameRGB->width = codecpar->width;
    		frameRGB->height = codecpar->height;
    		err = av_frame_get_buffer(frameRGB, 0);
    		Q_ASSERT(err == 0);
    		///
    		SwsContext *imgConvertCtx = nullptr;
    
    		AVFrame* frame = av_frame_alloc();
    		AVPacket packet;
    
    		while (av_read_frame(fmt_ctx, &packet) >= 0) {
    			avcodec_send_packet(codec_context, &packet);
    			err = avcodec_receive_frame(codec_context, frame);
    			if (err == 0) {
    				//qDebug() << frame->height << frame->width << codec_context->pix_fmt;
    
    				imgConvertCtx = sws_getCachedContext(imgConvertCtx,
    													 codecpar->width, codecpar->height, static_cast<AVPixelFormat>(codecpar->format),
    													 frameRGB->width, frameRGB->height,
    													 AV_PIX_FMT_RGB24, SWS_BICUBIC, nullptr, nullptr, nullptr);
    				Q_ASSERT(imgConvertCtx);
    
    				//conversion frame to frameRGB
    				sws_scale(imgConvertCtx, frame->data, frame->linesize, 0, codec_context->height, frameRGB->data, frameRGB->linesize);
    
    				//setting QImage from frameRGB
    				QImage image(frameRGB->data[0],
    						frameRGB->width,
    						frameRGB->height,
    						frameRGB->linesize[0],
    						QImage::Format_RGB888);
    
    				emit imageUpdated(image, clock());
    			}
    
    			av_frame_unref(frame);
    			av_packet_unref(&packet);
    		}
    		if (imgConvertCtx) sws_freeContext(imgConvertCtx);
    
    
    
    
    
    		//end:
    		avformat_close_input(&fmt_ctx);
    		/* note: the internal buffer could have changed, and be != avio_ctx_buffer */
    		if (avio_ctx) {
    			av_freep(&avio_ctx->buffer);
    			av_freep(&avio_ctx);
    		}
    		av_file_unmap(buffer, buffer_size);
    		if (ret < 0) {
    			qWarning() << ret;
    			return 1;
    		}
    		return 0;
    	});
    }
    


  • @BrMisha said in Showing stream video:

    So, how to get QImage from stream:

    You've already marked the post as solved. Would you mind starting a new post then?
    Given that this is slightly different (although related) topic...


Log in to reply