QT application with OpenCV and multithreading dies, no errors given

bsmith · 9 Apr 2019, 20:05

Hello! I am working on building a simple QT interface that I later plan to add extra functionality, but for now I am simply trying to display 8 QImages in QLabels from cv::Mat images. The Mats are inside of a TBB concurrent_queue and are pushed from a cv::VideoCapture tbb::thread. This alone works outside of QT perfectly, but trying to integrate this with QT has been a struggle.

Here are my header files:

//camerastreamer.hpp
#pragma once
#include <iostream>
#include <string>
#include <thread>
#include <vector>
#include <concurrent_queue.h>
#include "opencv2/videoio.hpp"


using namespace std;
using namespace cv;
using namespace tbb;


class CameraStreamer{
    public:
    //this holds camera stream urls
    vector<string> camera_source;
    //this holds OpenCV VideoCapture pointers
    vector<VideoCapture*> camera_capture;
    //this holds queue(s) which hold images from each camera
    vector<concurrent_queue<Mat>*> frame_queue;
    //this holds thread(s) which run the camera capture process
    vector<thread*> camera_thread;
    //Constructor for IP Camera capture
    CameraStreamer(vector<string> source);
    //Destructor for releasing resource(s)
    ~CameraStreamer();

    private:
    int camera_count;
    //initialize and start the camera capturing process(es)
    void startMultiCapture();
    //release all camera capture resource(s)
    void stopMultiCapture();
    //main camera capturing process which will be done by the thread(s)
    void captureFrame(int index);
};

//mainwindow.h
#ifndef MAINWINDOW_H
#define MAINWINDOW_H

#include <QMainWindow>
#include <QThread>
#include <QVector>
#include <QtWidgets/QSlider>
#include <QtWidgets/QToolButton>
#include <QtWidgets/QToolButton>
#include <QtWidgets/QCheckBox>
#include <QtWidgets/QTextBrowser>


namespace Ui {
class MainWindow;
}

class MainWindow : public QMainWindow
{
    Q_OBJECT

public:
    explicit MainWindow(QWidget *parent = 0);
    ~MainWindow();

private:
    Ui::MainWindow *ui;
    QThread *thread;
    
    void setup();
   
signals:
    void sendSetup();
    void sendToggleStream();

private slots:
    void receiveFrame1(QImage img);
    void receiveFrame2(QImage img);
    void receiveFrame3(QImage img);
    void receiveFrame4(QImage img);
    void receiveFrame5(QImage img);
    void receiveFrame6(QImage img);
    void receiveFrame7(QImage img);
    void receiveFrame8(QImage img);
    void receiveToggleStream();
};

#endif // MAINWINDOW_H

//ocvworker.h
#ifndef OCVWORKER_H
#define OCVWORKER_H

#include <QObject>
#include <QImage>
#include <QtWidgets/QLabel>
#include <QVector>
#include <QtWidgets/QLabel>

#include "opencv2/highgui.hpp"
#include "opencv2/opencv.hpp"

#include "camerastreamer.hpp"

const string ip = "192.168.122.1";

class OCVworker : public QObject
{
    Q_OBJECT

private:
    vector<string> capture_source = {
            "http://"+ip+":8081/?action=stream",
            "http://"+ip+":8082/?action=stream",
            "http://"+ip+":8083/?action=stream",
            "http://"+ip+":8084/?action=stream",
            "http://"+ip+":8080/?action=stream",
            "http://"+ip+":8085/?action=stream",
            "http://"+ip+":8086/?action=stream"
        };
    CameraStreamer cam(capture_source);
    Mat frame,R,G,B,dst;
    vector<Mat> frames, col;
    Mat chans[3];
    Mat merged, edges1,refedges, throwaway1,throwaway2;
    double lutvals[7][2][3] = {{{63.,0.,125.},{252.,251.,253.}},{{8.,48.,107.},{247.,251.,255.}},{{0.,68.,27.},{247.,252.,245.}},{{103.,0.,12.},{152.,245.,228.}},{{102.,37.,102.},{255.,255.,255.}},{{-190.,-253.,255.},{922.,245.,63.5}},{{-191.,-94.,10.},{1016.,670.,665.}}};
    bool toggleStream, alignEnable;

public:
    explicit OCVworker(QObject *parent = nullptr);
    ~OCVworker();

signals:
    void sendFrame1(QImage img);
    void sendFrame2(QImage img);
    void sendFrame3(QImage img);
    void sendFrame4(QImage img);
    void sendFrame5(QImage img);
    void sendFrame6(QImage img);
    void sendFrame7(QImage img);
    void sendFrame8(QImage img);

public slots:
    void receiveGrabFrame();
    void receiveToggleStream();
    void receiveEnableAlign();
};

#endif // OCVWORKER_H

and my cpp files:

//camerastreamer.cpp
#include "camerastreamer.hpp"

CameraStreamer::CameraStreamer(vector<string> stream_source)
{
    camera_source = stream_source;
    camera_count = camera_source.size();
    startMultiCapture();
}

CameraStreamer::~CameraStreamer()
{
    //stopMultiCapture();
}

void CameraStreamer::captureFrame(int index)
{
    VideoCapture *capture = camera_capture[index];
    while (true)
    {
        Mat frame,frames[3];
        //Grab frame from camera capture
        (*capture) >> frame;
        cv::split(frame,frames);
        frame=frames[0];
        //Put frame to the queue
        frame_queue[index]->push(frame);
        //relase frame resource
        frame.release();
    }
}

void CameraStreamer::startMultiCapture()
{
    VideoCapture *capture;
    thread *t;
    concurrent_queue<Mat> *q;
    for (int i = 0; i < camera_count; i++)
    {   //Make VideoCapture instance
        string url = camera_source[i];
        capture = new VideoCapture(url);
        cout << "Camera Setup: " << url << endl;
        //Put VideoCapture to the vector
        camera_capture.push_back(capture);
        //Make thread instance
        t = new thread(&CameraStreamer::captureFrame, this, i);
        //Put thread to the vector
        camera_thread.push_back(t);
        //Make a queue instance
        q = new concurrent_queue<Mat>;
        //Put queue to the vector
        frame_queue.push_back(q);
    }
}

void CameraStreamer::stopMultiCapture()
{
    VideoCapture *cap;
    for (int i = 0; i < camera_count; i++) {
        cap = camera_capture[i];
        if (cap->isOpened())
        {   //Relase VideoCapture resource
            cap->release();
            cout << "Capture " << i << " released" << endl;
        }
    }
}

//mainwindow.cpp
#include "mainwindow.h"
#include "ui_mainwindow.h"
#include "ocvworker.h"
#include <QTimer>

MainWindow::MainWindow(QWidget *parent) :
    QMainWindow(parent),
    ui(new Ui::MainWindow)
{
    ui->setupUi(this);
    ui->frame1->setScaledContents(true);
    ui->frame2->setScaledContents(true);
    ui->frame3->setScaledContents(true);
    ui->frame4->setScaledContents(true);
    ui->frame5->setScaledContents(true);
    ui->frame6->setScaledContents(true);
    ui->frame7->setScaledContents(true);
    ui->frame8->setScaledContents(true);
    setup();
}

MainWindow::~MainWindow()
{
    thread->quit();
    while(!thread->isFinished());
    delete thread;
    delete ui;
}

void MainWindow::setup()
{
    thread = new QThread();
    OCVworker *worker = new OCVworker();

    QTimer *workerTrigger = new QTimer();
    workerTrigger->setInterval(1);

    connect(workerTrigger, SIGNAL(timeout()), worker, SLOT(receiveGrabFrame()));
    connect(thread, SIGNAL(finished()), worker, SLOT(deleteLater()));
    connect(thread, SIGNAL(finished()), workerTrigger, SLOT(deleteLater()));
    connect(this, SIGNAL(sendToggleStream()), worker, SLOT(receiveToggleStream()));
    connect(ui->playButton, SIGNAL(clicked(bool)), this, SLOT(receiveToggleStream()));
    connect(ui->checkBox, SIGNAL(toggled(bool)), worker, SLOT(receiveEnableAlign()));
    connect(worker, SIGNAL(sendFrame1(QImage)), this, SLOT(receiveFrame1(QImage)));
    connect(worker, SIGNAL(sendFrame2(QImage)), this, SLOT(receiveFrame2(QImage)));
    connect(worker, SIGNAL(sendFrame3(QImage)), this, SLOT(receiveFrame3(QImage)));
    connect(worker, SIGNAL(sendFrame4(QImage)), this, SLOT(receiveFrame4(QImage)));
    connect(worker, SIGNAL(sendFrame5(QImage)), this, SLOT(receiveFrame5(QImage)));
    connect(worker, SIGNAL(sendFrame6(QImage)), this, SLOT(receiveFrame6(QImage)));
    connect(worker, SIGNAL(sendFrame7(QImage)), this, SLOT(receiveFrame7(QImage)));
    connect(worker, SIGNAL(sendFrame8(QImage)), this, SLOT(receiveFrame8(QImage)));
    connect(thread, SIGNAL(finished()), worker, SLOT(deleteLater()));
    connect(thread, SIGNAL(finished()), workerTrigger, SLOT(deleteLater()));
    cout << "Got Here" <<endl;


    workerTrigger->start();
    worker->moveToThread(thread);
    workerTrigger->moveToThread(thread);

    thread->start();
}

void MainWindow::receiveFrame1(QImage img)
{
    ui->frame1->setPixmap(QPixmap::fromImage(img));
}
void MainWindow::receiveFrame2(QImage img)
{
    ui->frame2->setPixmap(QPixmap::fromImage(img));
}
void MainWindow::receiveFrame3(QImage img)
{
    ui->frame3->setPixmap(QPixmap::fromImage(img));
}
void MainWindow::receiveFrame4(QImage img)
{
    ui->frame4->setPixmap(QPixmap::fromImage(img));
}
void MainWindow::receiveFrame5(QImage img)
{
    ui->frame5->setPixmap(QPixmap::fromImage(img));
}
void MainWindow::receiveFrame6(QImage img)
{
    ui->frame6->setPixmap(QPixmap::fromImage(img));
}
void MainWindow::receiveFrame7(QImage img)
{
    ui->frame7->setPixmap(QPixmap::fromImage(img));
}
void MainWindow::receiveFrame8(QImage img)
{
    ui->frame8->setPixmap(QPixmap::fromImage(img));
}
void MainWindow::receiveToggleStream()
{
    if(!ui->playButton->text().compare(">")) ui->playButton->setText("||");
    else ui->playButton->setText(">");
    emit sendToggleStream();
}

//ocvworker.cpp
#include "ocvworker.h"

OCVworker::OCVworker(QObject *parent) :
    QObject(parent),
    toggleStream(false),
    alignEnable(false),
{

}

OCVworker::~OCVworker()
{
    /*cam->~CameraStreamer();
    delete cam;*/
}

void OCVworker::receiveGrabFrame()
{
    for (int i= 0; i < int(capture_source.size()); i++)
    {
        if (cam.frame_queue[i]->try_pop(frame))
        {
            frame.convertTo(R, CV_8UC1, lutvals[i][1][0]/255., lutvals[i][0][0]);
            frame.convertTo(G, CV_8UC1, lutvals[i][1][1]/255., lutvals[i][0][1]);
            frame.convertTo(B, CV_8UC1, lutvals[i][1][2]/255., lutvals[i][0][2]);
            col.push_back(B);
            col.push_back(G);
            col.push_back(R);
            merge(col,dst);
            QImage qimg((const unsigned char *) dst.data, dst.cols, dst.rows, QImage::Format_Indexed8);
            switch (i) {
            case 0: {
                emit sendFrame1(qimg);
                break;}
            case 1: {
                emit sendFrame2(qimg);
                break;}
            case 2: {
                emit sendFrame3(qimg);
                break;}
            case 3: {
                emit sendFrame4(qimg);
                break;}
            case 4: {
                applyColorMap(frame,dst,COLORMAP_PINK);
                QImage qimg2((const unsigned char *) frame.data, frame.cols, frame.rows, QImage::Format_Indexed8);
                emit sendFrame5(qimg2);
                break;}
            case 5: {
                applyColorMap(frame,dst,COLORMAP_MAGMA);
                QImage qimg3((const unsigned char *) frame.data, frame.cols, frame.rows, QImage::Format_Indexed8);
                emit sendFrame6(qimg3);
                break;}
            case 6: {
                applyColorMap(frame,dst,COLORMAP_INFERNO);
                QImage qimg4((const unsigned char *) frame.data, frame.cols, frame.rows, QImage::Format_Indexed8);
                emit sendFrame7(qimg4);
                break;}
            default:
                break;
            }
            frame.copyTo(frames[i]);
            if(frame.empty()){
                cout << "Frame "+to_string(i)+" Emtpy";
                return;
            }
            cout << "Got to Frame"+to_string(i);
        }
    }
    for (int i=3; i<6; i++)
    {
        frames[i].copyTo(chans[i-3]);blur(frames[i],throwaway1,Size(3,3));
        Canny(throwaway1,edges1,50,200,3);
        blur(frames[3],throwaway1,Size(3,3));
        Canny(throwaway1,refedges,50,200,3);
        edges1.convertTo(throwaway1,CV_32FC1);
        refedges.convertTo(throwaway2,CV_32FC1);
        Point2d offset = phaseCorrelate(throwaway1,throwaway2);
        Mat warpmat = (Mat_<double>(2,3) << 1, 0, offset.x,0,1,offset.y);
        warpAffine(chans[i-3],chans[i-3],warpmat,frames[3].size());
    }
    merge(chans,3,merged);
    QImage qmerge((const unsigned char *) merged.data, merged.cols, merged.rows, QImage::Format_Indexed8);
    emit sendFrame8(qmerge);
}

void OCVworker::receiveToggleStream()
{
    toggleStream = !toggleStream;
}
void OCVworker::receiveEnableAlign()
{
    alignEnable = !alignEnable;
}

//main.cpp
#include "mainwindow.h"
#include <QApplication>

int main(int argc, char *argv[])
{
    QApplication a(argc, argv);
    MainWindow w;
    w.show();

    return a.exec();
}

//.pro file
#-------------------------------------------------
#
# Project created by QtCreator 2019-07-15T13:48:32
#
#-------------------------------------------------

QT       += core gui

greaterThan(QT_MAJOR_VERSION, 4): QT += widgets

TARGET = TTCGUI
TEMPLATE = app

# The following define makes your compiler emit warnings if you use
# any feature of Qt which has been marked as deprecated (the exact warnings
# depend on your compiler). Please consult the documentation of the
# deprecated API in order to know how to port your code away from it.
DEFINES += QT_DEPRECATED_WARNINGS

# You can also make your code fail to compile if you use deprecated APIs.
# In order to do so, uncomment the following line.
# You can also select to disable deprecated APIs only up to a certain version of Qt.
#DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000    # disables all the APIs deprecated before Qt 6.0.0

QMAKE_CXX = g++
QMAKE_LINK = g++

CONFIG += c++11

SOURCES += \
        camerastreamer.cpp \
        main.cpp \
        mainwindow.cpp \
        ocvworker.cpp

HEADERS += \
        camerastreamer.hpp \
        mainwindow.h \
        ocvworker.h

FORMS += \
        mainwindow.ui

INCLUDEPATH += /usr/local/Cellar/opencv/4.1.0_2/include/opencv4/opencv \
/usr/local/Cellar/opencv/4.1.0_2/include/opencv4 \
/usr/local/Cellar/tbb/2019_U6/include/tbb

LIBS += `/opt/local/bin/pkg-config opencv4 --cflags --libs`
LIBS += -I/usr/local/Cellar/tbb/2019_U6/include/tbb
LIBS += -L/usr/local/Cellar/tbb/2019_U6/lib
LIBS+= -ltbb

# Default rules for deployment.
qnx: target.path = /tmp/$${TARGET}/bin
else: unix:!android: target.path = /opt/$${TARGET}/bin
!isEmpty(target.path): INSTALLS += target

My compile commands are:

/Applications/Xcode.app/Contents/Developer/usr/bin/g++ -std=c++11 camerastreamer.cpp `pkg-config --cflags --libs` -I/usr/local/Cellar/tbb/2019_U6/include/tbb -ltbb -c
/Applications/Xcode.app/Contents/Developer/usr/bin/g++ -std=c++11 ocvworker.cpp `pkg-config opencv4 --cflags --libs` -I/usr/local/Cellar/tbb/2019_U6/include/tbb -ltbb -I~/Qt/5.13.0/clang_64/lib/QtWidgets.framework/Versions/5/Headers -I~/Qt/5.13.0/clang_64/lib/QtCore.framework/Versions/5/Headers -I~/Qt/5.13.0/Src/qtbase/include -I~/Qt/5.13.0/clang_64/lib/QtGui.framework/Versions/5/Headers -c
qmake proj.pro -spec macx-g++ CONFIG+=debug CONFIG+=x86_64 CONFIG+=qml_debug
make -j36 in build-proj-Desktop_Qt_5_13_0_clang_64bit-Debug

The application build dies at runtime with no output. Debugging stops on line 36 of mainwindow.cpp, where a new OCVworker pointer is constructed (OCVworker *worker = new OCVWorker()).

Any wisdom on what might be causing this application to crash would be very appreciated, as I've been trying to figure this issue out for a couple of weeks now. Thank you!