From 6f5d29ac16ada8ba290f9278211ea9ab96850d36 Mon Sep 17 00:00:00 2001 From: Gonzalo Exequiel Pedone Date: Wed, 26 Nov 2014 19:44:50 -0300 Subject: Added Opus codec. --- src/base/QXmppRtpChannel.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'src/base/QXmppRtpChannel.cpp') diff --git a/src/base/QXmppRtpChannel.cpp b/src/base/QXmppRtpChannel.cpp index 078150cd..731772ae 100644 --- a/src/base/QXmppRtpChannel.cpp +++ b/src/base/QXmppRtpChannel.cpp @@ -300,6 +300,10 @@ QXmppCodec *QXmppRtpAudioChannelPrivate::codecForPayloadType(const QXmppJinglePa #ifdef QXMPP_USE_SPEEX else if (payloadType.name().toLower() == "speex") return new QXmppSpeexCodec(payloadType.clockrate()); +#endif +#ifdef QXMPP_USE_OPUS + else if (payloadType.name().toLower() == "opus") + return new QXmppOpusCodec(payloadType.clockrate(), payloadType.channels()); #endif return 0; } @@ -321,6 +325,14 @@ QXmppRtpAudioChannel::QXmppRtpAudioChannel(QObject *parent) // set supported codecs QXmppJinglePayloadType payload; +#ifdef QXMPP_USE_OPUS + payload.setId(100); + payload.setChannels(1); + payload.setName("opus"); + payload.setClockrate(8000); + m_outgoingPayloadTypes << payload; +#endif + #ifdef QXMPP_USE_SPEEX payload.setId(96); payload.setChannels(1); -- cgit v1.2.3 From 75fd085c60ca45a8fb838df88a34d944884bedba Mon Sep 17 00:00:00 2001 From: Gonzalo Exequiel Pedone Date: Wed, 26 Nov 2014 19:53:05 -0300 Subject: Added FEC for Vp8 codec. --- src/base/QXmppCodec.cpp | 83 ++++++++++++++++++++++++++++++++++---------- src/base/QXmppCodec_p.h | 2 +- src/base/QXmppRtpChannel.cpp | 4 +-- 3 files changed, 68 insertions(+), 21 deletions(-) (limited to 'src/base/QXmppRtpChannel.cpp') diff --git a/src/base/QXmppCodec.cpp b/src/base/QXmppCodec.cpp index 9cd520c6..ed346764 100644 --- a/src/base/QXmppCodec.cpp +++ b/src/base/QXmppCodec.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "QXmppCodec_p.h" #include "QXmppRtpChannel.h" @@ -64,6 +65,8 @@ #define SEG_SHIFT (4) /* Left shift for segment number. */ #define SEG_MASK (0x70) /* Segment field mask. */ +#define GOPSIZE 32 + enum FragmentType { NoFragment = 0, StartFragment, @@ -1078,7 +1081,11 @@ public: bool QXmppVpxDecoderPrivate::decodeFrame(const QByteArray &buffer, QXmppVideoFrame *frame) { - if (vpx_codec_decode(&codec, (const uint8_t*)buffer.constData(), buffer.size(), NULL, 0) != VPX_CODEC_OK) { + if (vpx_codec_decode(&codec, + (const uint8_t*)buffer.constData(), + buffer.size(), + NULL, + VPX_DL_REALTIME) != VPX_CODEC_OK) { qWarning("Vpx packet could not be decoded: %s", vpx_codec_error_detail(&codec)); return false; } @@ -1117,7 +1124,15 @@ bool QXmppVpxDecoderPrivate::decodeFrame(const QByteArray &buffer, QXmppVideoFra QXmppVpxDecoder::QXmppVpxDecoder() { d = new QXmppVpxDecoderPrivate; - if (vpx_codec_dec_init(&d->codec, vpx_codec_vp8_dx(), NULL, 0) != VPX_CODEC_OK) { + vpx_codec_flags_t flags = 0; + + if (vpx_codec_get_caps(vpx_codec_vp8_dx()) & VPX_CODEC_CAP_ERROR_CONCEALMENT) + flags |= VPX_CODEC_USE_ERROR_CONCEALMENT; + + if (vpx_codec_dec_init(&d->codec, + vpx_codec_vp8_dx(), + NULL, + flags) != VPX_CODEC_OK) { qWarning("Vpx decoder could not be initialised"); } } @@ -1159,31 +1174,46 @@ QList QXmppVpxDecoder::handlePacket(const QXmppRtpPacket &packe #endif QXmppVideoFrame frame; + static quint16 sequence = 0; if (frag_type == NoFragment) { // unfragmented packet - if (d->decodeFrame(packet.payload.mid(1), &frame)) - frames << frame; + if ((packet.payload[1] & 0x1) == 0 // is key frame + || packet.sequence == sequence) { + if (d->decodeFrame(packet.payload.mid(1), &frame)) + frames << frame; + + sequence = packet.sequence + 1; + } + d->packetBuffer.resize(0); } else { // fragments if (frag_type == StartFragment) { // start fragment - d->packetBuffer = packet.payload.mid(1); + if ((packet.payload[1] & 0x1) == 0 // is key frame + || packet.sequence == sequence) { + d->packetBuffer = packet.payload.mid(1); + sequence = packet.sequence + 1; + } } else { // continuation or end fragment - const int packetPos = d->packetBuffer.size(); - d->packetBuffer.resize(packetPos + packetLength); - stream.readRawData(d->packetBuffer.data() + packetPos, packetLength); - } + if (packet.sequence == sequence) { + const int packetPos = d->packetBuffer.size(); + d->packetBuffer.resize(packetPos + packetLength); + stream.readRawData(d->packetBuffer.data() + packetPos, packetLength); + + if (frag_type == EndFragment) { + // end fragment + if (d->decodeFrame(d->packetBuffer, &frame)) { + frames << frame; + d->packetBuffer.resize(0); + } + } - if (frag_type == EndFragment) { - // end fragment - if (d->decodeFrame(d->packetBuffer, &frame)) - frames << frame; - d->packetBuffer.resize(0); + sequence++; + } } - } return frames; @@ -1216,12 +1246,31 @@ void QXmppVpxEncoderPrivate::writeFragment(QDataStream &stream, FragmentType fra stream.writeRawData(data, length); } -QXmppVpxEncoder::QXmppVpxEncoder() +QXmppVpxEncoder::QXmppVpxEncoder(uint clockrate) { d = new QXmppVpxEncoderPrivate; d->frameCount = 0; d->imageBuffer = 0; vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &d->cfg, 0); + + // Set the encoding threads number to use + int nThreads = QThread::idealThreadCount(); + + if (nThreads > 0) + d->cfg.g_threads = nThreads - 1; + + // Make stream error resiliant + d->cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT + | VPX_ERROR_RESILIENT_PARTITIONS; + + d->cfg.g_pass = VPX_RC_ONE_PASS; + d->cfg.kf_mode = VPX_KF_AUTO; + + // Reduce GOP size + if (d->cfg.kf_max_dist > GOPSIZE) + d->cfg.kf_max_dist = GOPSIZE; + + d->cfg.rc_target_bitrate = clockrate / 1000; } QXmppVpxEncoder::~QXmppVpxEncoder() @@ -1239,8 +1288,6 @@ bool QXmppVpxEncoder::setFormat(const QXmppVideoFormat &format) qWarning("Vpx encoder does not support the given format"); return false; } - - d->cfg.rc_target_bitrate = format.frameSize().width() * format.frameSize().height() * d->cfg.rc_target_bitrate / d->cfg.g_w / d->cfg.g_h; d->cfg.g_w = format.frameSize().width(); d->cfg.g_h = format.frameSize().height(); if (vpx_codec_enc_init(&d->codec, vpx_codec_vp8_cx(), &d->cfg, 0) != VPX_CODEC_OK) { diff --git a/src/base/QXmppCodec_p.h b/src/base/QXmppCodec_p.h index 70f110c0..2b6c0371 100644 --- a/src/base/QXmppCodec_p.h +++ b/src/base/QXmppCodec_p.h @@ -228,7 +228,7 @@ private: class QXMPP_AUTOTEST_EXPORT QXmppVpxEncoder : public QXmppVideoEncoder { public: - QXmppVpxEncoder(); + QXmppVpxEncoder(uint clockrate=0); ~QXmppVpxEncoder(); bool setFormat(const QXmppVideoFormat &format); diff --git a/src/base/QXmppRtpChannel.cpp b/src/base/QXmppRtpChannel.cpp index 731772ae..7f8f49d2 100644 --- a/src/base/QXmppRtpChannel.cpp +++ b/src/base/QXmppRtpChannel.cpp @@ -890,7 +890,7 @@ QXmppRtpVideoChannel::QXmppRtpVideoChannel(QObject *parent) encoder->setFormat(d->outgoingFormat); payload.setId(96); payload.setName("vp8"); - payload.setClockrate(90000); + payload.setClockrate(256000); payload.setParameters(encoder->parameters()); m_outgoingPayloadTypes << payload; delete encoder; @@ -1022,7 +1022,7 @@ void QXmppRtpVideoChannel::payloadTypesChanged() #endif #ifdef QXMPP_USE_VPX else if (payload.name().toLower() == "vp8") { - encoder = new QXmppVpxEncoder; + encoder = new QXmppVpxEncoder(payload.clockrate()); } #endif if (encoder) { -- cgit v1.2.3 From c178db4e9ad78c9f8d434da9d49b0461123000c5 Mon Sep 17 00:00:00 2001 From: Gonzalo Exequiel Pedone Date: Wed, 26 Nov 2014 21:29:24 -0300 Subject: Added some comments for Opus codec. --- src/base/QXmppCodec.cpp | 30 ++++++++++++++++++++++++++---- src/base/QXmppRtpChannel.cpp | 2 +- 2 files changed, 27 insertions(+), 5 deletions(-) (limited to 'src/base/QXmppRtpChannel.cpp') diff --git a/src/base/QXmppCodec.cpp b/src/base/QXmppCodec.cpp index ed346764..0e47de63 100644 --- a/src/base/QXmppCodec.cpp +++ b/src/base/QXmppCodec.cpp @@ -398,12 +398,11 @@ QXmppOpusCodec::QXmppOpusCodec(int clockrate, int channels): sampleRate(clockrate), nChannels(channels) { - // https://tools.ietf.org/html/draft-ietf-payload-rtp-opus-04 - int error; encoder = opus_encoder_create(clockrate, channels, OPUS_APPLICATION_VOIP, &error); if (encoder || error == OPUS_OK) { + // Add some options for error correction. opus_encoder_ctl(encoder, OPUS_SET_INBAND_FEC(1)); opus_encoder_ctl(encoder, OPUS_SET_PACKET_LOSS_PERC(20)); opus_encoder_ctl(encoder, OPUS_SET_DTX(1)); @@ -417,11 +416,19 @@ QXmppOpusCodec::QXmppOpusCodec(int clockrate, int channels): if (!encoder || error != OPUS_OK) qCritical() << "Opus decoder initialization error:" << opus_strerror(error); + // Opus only supports fixed frame durations from 2.5ms to 60ms. + // + // NOTE: https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__encoder.html validFrameSize << 2.5e-3 << 5e-3 << 10e-3 << 20e-3 << 40e-3 << 60e-3; + // so now, calculate the equivalent number of samples to process in each + // frame. + // + // nSamples = t * frameRate for (int i = 0; i < validFrameSize.size(); i++) validFrameSize[i] *= clockrate; + // Maxmimum number of samples for the audio buffer. nSamples = validFrameSize.last(); } @@ -440,14 +447,21 @@ QXmppOpusCodec::~QXmppOpusCodec() qint64 QXmppOpusCodec::encode(QDataStream &input, QDataStream &output) { + // Read an audio frame. QByteArray pcm_buffer(input.device()->bytesAvailable(), 0); int length = input.readRawData(pcm_buffer.data(), pcm_buffer.size()); + + // and append it to the sample buffer. sampleBuffer.append(pcm_buffer.left(length)); + + // Get the maximum number of samples to encode. It must be a number + // accepted by the Opus encoder int samples = readWindow(sampleBuffer.size()); if (samples < 1) return 0; + // The encoded stream is supposed to be smaller than the raw stream, so QByteArray opus_buffer(sampleBuffer.size(), 0); length = opus_encode(encoder, @@ -459,8 +473,10 @@ qint64 QXmppOpusCodec::encode(QDataStream &input, QDataStream &output) if (length < 1) qWarning() << "Opus encoding error:" << opus_strerror(length); else + // Write the encoded stream to the output. output.writeRawData(opus_buffer.constData(), length); + // Remove the frame from the sample buffer. sampleBuffer.remove(0, samples * nChannels * 2); if (length < 1) @@ -477,8 +493,11 @@ qint64 QXmppOpusCodec::decode(QDataStream &input, QDataStream &output) if (length < 1) return 0; + // Audio frame is nSamples at maximum, so QByteArray pcm_buffer(nSamples * nChannels * 2, 0); + // The last argumment must be 1 to enable FEC, but I don't why it results + // in a SIGSEV. int samples = opus_decode(decoder, (uchar *) opus_buffer.constData(), length, @@ -492,6 +511,7 @@ qint64 QXmppOpusCodec::decode(QDataStream &input, QDataStream &output) return 0; } + // Write the audio frame to the output. output.writeRawData(pcm_buffer.constData(), samples * nChannels * 2); return samples; @@ -502,10 +522,12 @@ int QXmppOpusCodec::readWindow(int bufferSize) // WARNING: We are expecting 2 bytes signed samples, but this is wrong since // input stream can have a different sample formats. - int nFrames = bufferSize / nChannels / 2; + // Get the number of frames in the buffer. + int samples = bufferSize / nChannels / 2; + // Find an appropiate number of samples to read, according to Opus specs. for (int i = validFrameSize.size() - 1; i >= 0; i--) - if (validFrameSize[i] <= nFrames) + if (validFrameSize[i] <= samples) return validFrameSize[i]; return 0; diff --git a/src/base/QXmppRtpChannel.cpp b/src/base/QXmppRtpChannel.cpp index 7f8f49d2..ff478fd4 100644 --- a/src/base/QXmppRtpChannel.cpp +++ b/src/base/QXmppRtpChannel.cpp @@ -326,7 +326,7 @@ QXmppRtpAudioChannel::QXmppRtpAudioChannel(QObject *parent) QXmppJinglePayloadType payload; #ifdef QXMPP_USE_OPUS - payload.setId(100); + payload.setId(100); // NOTE: I don't know if this Id is ok for Opus. payload.setChannels(1); payload.setName("opus"); payload.setClockrate(8000); -- cgit v1.2.3