Showing stream video

[BrMisha]

Hi!
I'm receiving the raw h.264 video stream from QUdpSocket. So, i need to display that on widget.
I know that QMediaPlayer can't play stream. After googling i found some solutions but all those need external libs. I would like to use only Qt components and decode h.264 using system codecs (QMediaPlayer nice plays h.264 from file using k-like codecs).

So, I have QByteArray received from camera, I need to decode h.264 frame and show that on widget. How can i do that?
Thanks)))

[BrMisha]

So, how to get QImage from stream:

QUdpSocket *udpSocket;
QByteArray	buffer;

int read_packet(void *opaque, uint8_t *buf, int buf_size) {
	//auto	fil = static_cast<QUdpSocket*>(opaque);
	//QMutexLocker	l(&fil->mutex);
	//qDebug() << buf_size << fil->buffer.length();

	qDebug() << buf_size << buffer.length();

	if (buffer.isEmpty() && !udpSocket->hasPendingDatagrams()) udpSocket->waitForReadyRead();
	while (udpSocket->hasPendingDatagrams()) {
		if (buffer.length() > 1024*500) {
			buffer = buffer.right(1024*500);
			buffer.clear();
		}
		buffer.append(udpSocket->receiveDatagram().data());
	}

	buf_size = buffer.length() > buf_size ? buf_size : buffer.length();
	memcpy(buf, buffer.constData(), buf_size);
	buffer.remove(0, buf_size);

	return buf_size;
};

int MainWindow::start()
{
	QtConcurrent::run([this](){
		udpSocket = new QUdpSocket;
		udpSocket->bind(15004);

		AVFormatContext *fmt_ctx = nullptr;
		AVIOContext *avio_ctx = nullptr;
		uint8_t *buffer = nullptr, *avio_ctx_buffer = nullptr;
		size_t buffer_size = 0, avio_ctx_buffer_size = 4096;

		int ret = 0;
		fmt_ctx = avformat_alloc_context();
		Q_ASSERT(fmt_ctx);

		avio_ctx_buffer = reinterpret_cast<uint8_t*>(av_malloc(avio_ctx_buffer_size));
		Q_ASSERT(avio_ctx_buffer);
		avio_ctx = avio_alloc_context(avio_ctx_buffer, static_cast<int>(avio_ctx_buffer_size), 0, /*&bd*/udpSocket, &read_packet, nullptr, nullptr);
		Q_ASSERT(avio_ctx);

		fmt_ctx->pb = avio_ctx;
		ret = avformat_open_input(&fmt_ctx, nullptr, nullptr, nullptr);
		Q_ASSERT(ret >= 0);
		ret = avformat_find_stream_info(fmt_ctx, nullptr);
		Q_ASSERT(ret >= 0);

		AVStream	*video_stream = nullptr;
		for (uint i=0; i<fmt_ctx->nb_streams; ++i) {
			auto	st = fmt_ctx->streams[i];
			qDebug() << st->id << st->index << st->start_time << st->duration << st->codecpar->codec_type;
			if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) video_stream = st;
		}
		Q_ASSERT(video_stream);


		AVCodecParameters	*codecpar = video_stream->codecpar;
		AVCodec* codec = avcodec_find_decoder(codecpar->codec_id);
		AVCodecContext *codec_context = avcodec_alloc_context3(codec);
		int err = avcodec_open2(codec_context, codec, nullptr);
		Q_ASSERT(err>=0);
		qDebug() << codec->name << codec->id;
		qDebug() << codecpar->width << codecpar->height << codecpar->format << codec_context->pix_fmt;




		AVFrame	*frameRGB = av_frame_alloc();
		frameRGB->format = AV_PIX_FMT_RGB24;
		frameRGB->width = codecpar->width;
		frameRGB->height = codecpar->height;
		err = av_frame_get_buffer(frameRGB, 0);
		Q_ASSERT(err == 0);
		///
		SwsContext *imgConvertCtx = nullptr;

		AVFrame* frame = av_frame_alloc();
		AVPacket packet;

		while (av_read_frame(fmt_ctx, &packet) >= 0) {
			avcodec_send_packet(codec_context, &packet);
			err = avcodec_receive_frame(codec_context, frame);
			if (err == 0) {
				//qDebug() << frame->height << frame->width << codec_context->pix_fmt;

				imgConvertCtx = sws_getCachedContext(imgConvertCtx,
													 codecpar->width, codecpar->height, static_cast<AVPixelFormat>(codecpar->format),
													 frameRGB->width, frameRGB->height,
													 AV_PIX_FMT_RGB24, SWS_BICUBIC, nullptr, nullptr, nullptr);
				Q_ASSERT(imgConvertCtx);

				//conversion frame to frameRGB
				sws_scale(imgConvertCtx, frame->data, frame->linesize, 0, codec_context->height, frameRGB->data, frameRGB->linesize);

				//setting QImage from frameRGB
				QImage image(frameRGB->data[0],
						frameRGB->width,
						frameRGB->height,
						frameRGB->linesize[0],
						QImage::Format_RGB888);

				emit imageUpdated(image, clock());
			}

			av_frame_unref(frame);
			av_packet_unref(&packet);
		}
		if (imgConvertCtx) sws_freeContext(imgConvertCtx);





		//end:
		avformat_close_input(&fmt_ctx);
		/* note: the internal buffer could have changed, and be != avio_ctx_buffer */
		if (avio_ctx) {
			av_freep(&avio_ctx->buffer);
			av_freep(&avio_ctx);
		}
		av_file_unmap(buffer, buffer_size);
		if (ret < 0) {
			qWarning() << ret;
			return 1;
		}
		return 0;
	});
}

[jsulm]

@BrMisha You could write your own QIODevice and use it with https://doc.qt.io/qt-5/qmediaplayer.html#setMedia

[BrMisha]

@jsulm I know but QMediaPlayer uses QIODevice::seek, which isn't available in stream

[SGaist]

Hi,

What OS are you on ?

[BrMisha]

@SGaist Windows (10) and Linux (Ubuntu and other).

[SGaist]

In that case, I would rather consider using VLC and its Qt integration to handle the video stream.

[BrMisha]

@SGaist but why this solution better of using system codecs?

[SGaist]

Because VLC is the Swiss Army Knife of vidéo decoding and supports a larger palette of codecs than most OSs provide out of the box.

[BrMisha]

@SGaist I would't like to use VLC but you have convinced me)))
Tomorrow i will try to build VLC for windows

[SGaist]

There's likely already a prebuilt package.

See also VLC-Qt

[BrMisha]

@SGaist is VLC-Qt still actuality now? Last commit was in 2016 year and based on VLC 2.2.4

[SGaist]

It does not mean it does not work anymore. I'd say it's worth a try.

[BrMisha]

Unfortunately, VLC doesn't provide prebuilt packages for my compiler (MinGW32-64 7.3.0).
After googling i decided to try ffmpeg. I build it following this manual https://ffmpeg.org/platform.html#Native-Windows-compilation-using-MinGW-or-MinGW_002dw64 , then add libs to *pro file:

LIBS += -LF:/FFmpeg/tempinstall/usr/local/lib/ -llibavcodec.dll -llibavformat.dll
INCLUDEPATH += F:/FFmpeg/tempinstall/usr/local/include
DEPENDPATH += F:/FFmpeg/tempinstall/usr/local/include

and i called avformat_open_input(some args). But got error^

c:/qt/qt5.14.1/tools/mingw64/bin/../lib/gcc/x86_64-w64-mingw32/8.3.0/../../../../x86_64-w64-mingw32/bin/ld.exe: main.o: in function `qMain(int, char**)':
D:\projects_test\qt\PlayerVideo\build-PlayerVideo-Desktop-Debug/../PlayerVideo/main.cpp:58: undefined reference to `avformat_open_input(AVFormatContext**, char const*, AVInputFormat*, AVDictionary**)'

So, i'm searching my mistake... I compared compiler versions between msys2 and qt, booth have GNU Make 4.2.1, but msys2 has gcc 8.3.0 and qt has 7.3.0.
That is problem?

[SGaist]

It's "-lavcodec"

[BrMisha]

@SGaist no, it isn't helpfull((

[SGaist]

Did you fix all libraries name ?

[BrMisha]

@SGaist this part of .pro

LIBS += -LF:/Musor/FFmpeg_linbuild/ffmpeg/lib/ -lavformat

INCLUDEPATH += F:/Musor/FFmpeg_linbuild/ffmpeg/include
DEPENDPATH += F:/Musor/FFmpeg_linbuild/ffmpeg/include

In code i'm calling only avformat_license(), which locates in avformat

[SGaist]

Do you have ffmpeg built for the same architecture as your Qt version ?

[BrMisha]

@SGaist yes, booth is x64

[BrMisha]

Men, I really don't understand what is happening.
I installed msys2, and some packages for that, then i copied *c:\Qt\Qt5.14.2\Tools\mingw730_64* to *c:\msys64\mingw64*.
Then built libav (it is "small" version of FFmpeg) and got this files:

build\usr\local\bin\avcodec.lib
build\usr\local\bin\avcodec-58.dll
build\usr\local\bin\avconv.exe
build\usr\local\bin\avdevice.lib
build\usr\local\bin\avdevice-57.dll
build\usr\local\bin\avfilter.lib
build\usr\local\bin\avfilter-7.dll
build\usr\local\bin\avformat.lib
build\usr\local\bin\avformat-58.dll
build\usr\local\bin\avprobe.exe
build\usr\local\bin\avresample.lib
build\usr\local\bin\avresample-4.dll
build\usr\local\bin\avutil.lib
build\usr\local\bin\avutil-56.dll
build\usr\local\bin\swscale.lib
build\usr\local\bin\swscale-5.dll

build\usr\local\include\libavcodec\	<< here is *.h
build\usr\local\include\libavdevice\	<< here is *.h
build\usr\local\include\libavfilter\	<< here is *.h
build\usr\local\include\libavformat\	<< here is *.h
build\usr\local\include\libavresample\	<< here is *.h
build\usr\local\include\libavutil\	<< here is *.h
build\usr\local\include\libswscale\	<< here is *.h

build\usr\local\lib\pkgconfig\
build\usr\local\lib\avcodec-58.def
build\usr\local\lib\avdevice-57.def
build\usr\local\lib\avfilter-7.def
build\usr\local\lib\avformat-58.def
build\usr\local\lib\avresample-4.def
build\usr\local\lib\avutil-56.def
build\usr\local\lib\libavcodec.dll.a
build\usr\local\lib\libavdevice.dll.a
build\usr\local\lib\libavfilter.dll.a
build\usr\local\lib\libavformat.dll.a
build\usr\local\lib\libavresample.dll.a
build\usr\local\lib\libavutil.dll.a
build\usr\local\lib\libswscale.dll.a
build\usr\local\lib\swscale-5.def

*c:\Qt\Qt5.14.2\Tools\mingw730_64* was added to PATH. When I start, for example, avconv.exe, it works ok, so definitely libav was built with true mingw (which uses qt).
Next, i added to *.pro (using "Add library...") this:

win32: LIBS += -LF:/Musor/libav/build/usr/local/lib/ -llibavformat.dll
INCLUDEPATH += F:/Musor/libav/build/usr/local/include
DEPENDPATH += F:/Musor/libav/build/usr/local/include

main.c:

#include <libavformat/avformat.h>
int main(int argc, char *argv[]) {
	avformat_license();
	return 0;
}

After compilation i got:

C:/Qt/Qt5.14.2/Tools/mingw730_64/bin/mingw32-make -f Makefile.Debug
mingw32-make[1]: Entering directory 'D:/projects_test/qt/qt_test/build-qt_test-Desktop_Qt_5_14_2_MinGW_64_bit-Debug'
g++ -c -fno-keep-inline-dllexport -g -std=gnu++11 -Wall -Wextra -Wextra -fexceptions -mthreads -DUNICODE -D_UNICODE -DWIN32 -DMINGW_HAS_SECURE_API=1 -DQT_DEPRECATED_WARNINGS -DQT_QML_DEBUG -DQT_SERIALPORT_LIB -DQT_CORE_LIB -I..\qt_test -I. -IF:\Musor\libav\build\usr\local\include -IC:\Qt\Qt5.14.2\5.14.2\mingw73_64\include -IC:\Qt\Qt5.14.2\5.14.2\mingw73_64\include\QtSerialPort -IC:\Qt\Qt5.14.2\5.14.2\mingw73_64\include\QtCore -Idebug -IC:\Qt\Qt5.14.2\5.14.2\mingw73_64\mkspecs\win32-g++  -o debug\main.o ..\qt_test\main.cpp
..\qt_test\main.cpp: In function 'int main(int, char**)':
..\qt_test\main.cpp:4:14: warning: unused parameter 'argc' [-Wunused-parameter]
 int main(int argc, char *argv[])
              ^~~~
..\qt_test\main.cpp:4:31: warning: unused parameter 'argv' [-Wunused-parameter]
 int main(int argc, char *argv[])
                               ^
g++ -Wl,-subsystem,console -mthreads -o debug\qt_test.exe debug/main.o  -LF:\Musor\libav\build\usr\local\lib F:\Musor\libav\build\usr\local\lib\libavformat.dll.a C:\Qt\Qt5.14.2\5.14.2\mingw73_64\lib\libQt5SerialPort.a C:\Qt\Qt5.14.2\5.14.2\mingw73_64\lib\libQt5Core.a   
debug/main.o: In function `main':
D:\projects_test\qt\qt_test\build-qt_test-Desktop_Qt_5_14_2_MinGW_64_bit-Debug/../qt_test/main.cpp:6: undefined reference to `avformat_license()'
collect2.exe: error: ld returned 1 exit status
mingw32-make[1]: *** [Makefile.Debug:68: debug/qt_test.exe] Error 1
mingw32-make: *** [Makefile:45: debug] Error 2
mingw32-make[1]: Leaving directory 'D:/projects_test/qt/qt_test/build-qt_test-Desktop_Qt_5_14_2_MinGW_64_bit-Debug'
18:45:50: The process "C:\Qt\Qt5.14.2\Tools\mingw730_64\bin\mingw32-make.exe" exited with code 2.
Error while building/deploying project qt_test (kit: Desktop Qt 5.14.2 MinGW 64-bit)
When executing step "Make"

Whyyyyyyy?????
I have tried to change -llibavformat.dll to -lavformat, "-lavformat.dll" and "-lavformat" but i'm stil getting this error.