VoIP QTcpSoket(audio streaming)



  • Hi there, I am trying to develop software that will send and receive audio through local network. TThat client's program starts to send data and unexpectedly stops.
    Client looks like:

    class AudioClient : public QObject
    {
        Q_OBJECT
    public:
        explicit AudioClient();
    
        ~AudioClient();
    
    signals:
        void error(int socketError,const QString &message);
    
    public slots:
    
        void StartStream();
    
        void StopStream();
    
    private:
        QString hostName;
        quint16 port;
        QTcpSocket *socket;
        QAudioInput* audio;
        QAudioFormat format;
        QAudioOutput *audio_out;
    
    };
    

    Constructor

    AudioClient::AudioClient ():
        hostName("127.0.0.1"),
        port(50306)
    {
        format.setSampleRate(48000);
        format.setChannelCount(2);
        format.setSampleSize(16);
        format.setCodec("audio/pcm");
        format.setByteOrder(QAudioFormat::LittleEndian);
        format.setSampleType(QAudioFormat::UnSignedInt);
    
        QAudioDeviceInfo info = QAudioDeviceInfo::defaultInputDevice();
    
        if (!info.isFormatSupported(format))
        {
            qWarning()<<"default format not supported try to use nearest";
            format = info.nearestFormat(format);
        }
        qDebug()<<"Created socket";
        socket = new QTcpSocket (this);
    
        qDebug()<<"Created audio input";
        audio = new QAudioInput(format, this);
        audio->setNotifyInterval(100);
    }
    

    Start stream method

    void AudioClient::StartStream()
    {
        qDebug()<<"Connecting to the host Audio";
        socket->connectToHost(hostName, port);
        if (!socket->waitForConnected(Timeout))
        {
            emit error(socket->error(), socket->errorString());
            return;
        }
        qDebug()<<"Start stream";
        audio -> start(socket);
        qDebug()<<" ";
        qDebug()<<"Audio stream from thread "<<QThread::currentThreadId();
        qDebug()<<" ";
    }
    

    Server cpp file looks like:

    MainDialog::MainDialog(QWidget *parent) :
        QDialog(parent),
        ui(new Ui::MainDialog),
        host_index(0),
        port(50306)
    {
        qDebug()<<"Init GUI";
        this -> SetGui();
        // setting server
        audioserver = new QTcpServer(this);
        
        qDebug()<<"Init audioserver";
        connect(audioserver, &QTcpServer::newConnection,
                this, &MainDialog::AudioStream);
         ...
        qDebug()<<"Init audio";
        /// audio
        // Set up the format, eg.
        format.setSampleRate(48000);
        format.setChannelCount(2);
        format.setSampleSize(16);
        format.setCodec("audio/pcm");
        format.setByteOrder(QAudioFormat::LittleEndian);
        format.setSampleType(QAudioFormat::UnSignedInt);
    
        QAudioDeviceInfo info(QAudioDeviceInfo::defaultOutputDevice());
        ...
        audio = new QAudioOutput(format, this);
        audio->setBufferSize(512000);
        audio->setNotifyInterval(100);
    }
    
    void MainDialog::AudioStream()
    {
        qDebug()<<"In audio stream";
        audioclientConnection = audioserver->nextPendingConnection();
        audio->start(audioclientConnection );
    }
    

    I will be glad to any help.


  • Moderators

    @Guess11
    start your application with a debugger attached to see where exactly it is crashing.
    E.g. QtCreator



  • @raven-worx Actually nothing is crashing, just output audio device do not produce any sound. But in a log I see that Client trying to send some data just after start streaming call.



  • does the output device ever produce any sound?

    I'm not sure QAudioOutput is as smart as you want it to be. When you call audio->start(audioclientConnection ); you would like the audio to respond to readyRead() signals and stream what it can read but I think everything start does is read everything that is already available and stream it out (Think as if the input was a file)



  • @VRonin I rewrite server and client sides. With readyRead approach using QBuffers and QByte Arrays The server looks as follows:

    MainDialog.h
    class MainDialog: public QWidget
    {
    ...
    private:
    ...
        QByteArray byte_buffer;
        QBuffer m_audioOutputIODevice;
    }
    
    MainDialog.cpp
    
    MainDialog::MainDialog(QWidget *parent) :
        QDialog(parent),
        ui(new Ui::MainDialog),
        host_index(0),
        port(50306)
    {
    ...
        audio->setBufferSize(512000);
        audio->setNotifyInterval(10);
    
        qDebug()<<"Real buffer size "<<audio->bufferSize();
        qDebug()<<"NotifyInterval "<<audio->notifyInterval();
        byte_buffer.clear();
    
        m_audioOutputIODevice.close();
        m_audioOutputIODevice.setBuffer(&byte_buffer);
        m_audioOutputIODevice.open(QIODevice::ReadOnly);
    }
    
    void MainDialog::AudioStream()
    {
        qDebug()<<"In audio stream";
        audioclientConnection = audioserver->nextPendingConnection();
    
        connect(audioclientConnection,SIGNAL(readyRead()),
                this,SLOT(ReadStream()));
        audio -> start(&m_audioOutputIODevice);
    }
    
    void MainDialog::ReadStream()
    {
        qDebug()<<"ReadAudio";
        byte_buffer.append(audioclientConnection->readAll());
    }
    
    
    

    Client side

    AudioClient.h
    class AudioClient : public QObject
    {
    ...
    private:
     ...
        QBuffer m_audioInputIODevice;
        QByteArray byte_buffer;
    }
    AudioClient.cpp
    AudioClient::AudioClient ():
        hostName("127.0.0.1"),
        port(50306)
    {
      ...
        qDebug()<<"Created audio input";
        audio = new QAudioInput(format, this);
        audio->setBufferSize(512000);
    
        byte_buffer.clear();
        m_audioInputIODevice.close();
        m_audioInputIODevice.setBuffer(&byte_buffer);
        m_audioInputIODevice.open(QIODevice::WriteOnly);
    
        connect(audio,SIGNAL(stateChanged(QAudio::State)),
                SLOT(handleStateChanged(QAudio::State)));
      
    }
    void AudioClient::StartStream()
    {
        qDebug()<<"Connecting to the host Audio";
        socket->connectToHost(hostName, port);
        if (!socket->waitForConnected(Timeout))
        {
            emit error(socket->error(), socket->errorString());
            return;
        }
        qDebug()<<" ";
        qDebug()<<"Audio stream from thread "<<QThread::currentThreadId();
        qDebug()<<" ";
    
        audio -> start(&m_audioInputIODevice);
    }
    
    void AudioClient::handleStateChanged(QAudio::State newState)
    {
        int bytes_ava = 0;
        switch (newState) {
        case QAudio::StoppedState:
                 ...
                break;
        case QAudio::ActiveState:
                qDebug()<<"Start recording ";
                bytes_ava = audio->bytesReady();
                qDebug()<<"Data ready"<<bytes_ava ;
                if(bytes_ava == 0)
                {
                    qDebug()<<"Nothing to send";
                }
                else
                {
                    qDebug()<<"Writing to socket";
                    socket->write(byte_buffer);
                }
                break;
        case QAudio::SuspendedState	:
                ...
                break;
        case QAudio::IdleState:
            qDebug()<<"IdleState streamin";
            ....
            break;
        }
    }
    
    

    I doubt that I understood it correctly. But now the problem is that on client side program goes to handleStateChanged, check that nothing to send and never sends anything to server.



  • handleStateChanged is called when the audio input device becomes available and then never more.
    this part:

    bytes_ava = audio->bytesReady();
                qDebug()<<"Data ready"<<bytes_ava ;
                if(bytes_ava == 0)
                {
                    qDebug()<<"Nothing to send";
                }
                else
                {
                    qDebug()<<"Writing to socket";
                    socket->write(byte_buffer);
                }
    

    should be put in a slot connected to QAudioInput::notify()



  • @VRonin Thank you for help. From qDebug I see that client write data to soket and send it. But on the server side still no sound, also buffer size is differ from the client side.

    void MainDialog::ReadStream()
    {
        qDebug()<<"Read Audio";
        byte_buffer.append(audioclientConnection->readAll());
        qDebug()<<"Size of Byte_buffer "<<byte_buffer.size();
        qDebug()<<"Size of Buffer "<<m_audioOutputIODevice.size();
    }
    

    I connect function to notify of AudioOutput and it seems that server never starts to process audio.

    void MainDialog::ReadNotify()
    {
       qDebug()<<"Here in Notify Output";
       qDebug()<<"Notify Byte_buffer "<<byte_buffer.size();
    }
    


  • The problem is in the client, not the server, the case QAudio::ActiveState: section should respond to the audio,&QAudioInput::notify signal

    also if you do not clean byte_buffer but keep appending you will send over all the audio from the start of the recording every time



  • I rewrote the client side as you said. And now it looks:

    
    void AudioClient::handleStateChanged(QAudio::State newState)
    {
    
        switch (newState) {
    ...
    
        case QAudio::ActiveState:
                // Started recording - read from IO device
                qDebug()<<"Device active ";
                connect(audio,SIGNAL(notify()),
                              SLOT(SendData()));
                break;
    
        case QAudio::SuspendedState	:
                qDebug()<<"SuspendedState";
                // ... other cases as appropriate;
                break;
    ...
        }
    }
    
    void AudioClient::SendData()
    {
        qDebug()<<"Send Data ";
        int bytes_ava =  byte_buffer.size();
        qDebug()<<"Data ready in notify "<<bytes_ava ;
        if(bytes_ava == 0)
        {
            qDebug()<<"Nothing to send";
            //audio->resume();
        }
        else
        {
            qDebug()<<"Writing to socket";
            socket -> write(byte_buffer);
            byte_buffer.clear();
        }
    }
    

    And on server side

    MainDialog.cpp
    
    MainDialog::MainDialog(QWidget *parent) :
        QDialog(parent),
        ui(new Ui::MainDialog),
        host_index(0),
        port(50306)
    {
    ...
    
       connect(audio,SIGNAL(notify()),
                     SLOT(ReadNotify()));
    }
    
    void MainDialog::AudioStream()
    {
        qDebug()<<"In audio stream";
        audioclientConnection = audioserver->nextPendingConnection();
        audio -> start(&m_audioOutputIODevice);
    }
    
    void MainDialog::ReadStream()
    {
        qDebug()<<"Read Audio";
        byte_buffer = audioclientConnection->readAll();
        qDebug()<<"Size of Byte_buffer "<<byte_buffer.size();
        qDebug()<<"Size of Buffer "<<m_audioOutputIODevice.size();
    }
    

    On the server it is never visit ReadNotify.



  • as i know, if you using TCP, there will have delay on network, so, your output buffer of audio will be drain to empty, so on ReadNotify(), you need check buffer of output audio, if it empty, you should generate some default data for it
    i think it better to using UDP, and using Opus codec to encode, decode & generate default data on case UDP lost package.
    i find an example about it, can you check
    https://github.com/antonypro/AudioStreaming



  • @kd_wala Thank you for example. I build it and run on my laptop, and audio start to repeat sounds after some time. Like playing from the begining on the background. Also on the first look example looks rather complicated. Is there way of doing it through AudioInput and AudioOutput through the start method?



  • "Is there way of doing it through AudioInput and AudioOutput through the start method?"
    I thinks , should not, because, when you have transfer over internet, you will have some delay, package lost(on udp), encode, decode ... so you must handle this before you put data to output ...
    I think this example not complicated,


Log in to reply
 

Looks like your connection to Qt Forum was lost, please wait while we try to reconnect.