From f0a880c64d92db0f3aa97430749747aac4da7311 Mon Sep 17 00:00:00 2001 From: Dmytro Bogovych Date: Fri, 9 Oct 2026 11:01:20 +0200 Subject: [PATCH] - take the RTP packet duration from the payload (Opus ptime > 20 ms) AudioReceiver assumed every packet of a variable-size codec lasts frameTime(), i.e. the local 20 ms ptime. A 40 ms Opus packet then looked like 20 ms of audio followed by a 20 ms gap: decodePacketTo() emitted silence before every packet and the jitter buffer dropped about half of them (335 of 605 decoded on a 20/40 ms call), so vq-core scored clean audio at MOS ~1.3. Codec::packetTime() now returns the duration of a payload; Opus reads it with opus_packet_get_nb_samples(), fixed-frame codecs keep the old formula. With it the same call decodes 603 of 605. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LzU7iFc93X5jFATRKaoKiZ --- src/engine/media/MT_AudioCodec.cpp | 7 +++++++ src/engine/media/MT_AudioCodec.h | 2 ++ src/engine/media/MT_AudioReceiver.cpp | 11 ++++++----- src/engine/media/MT_Codec.h | 12 ++++++++++++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/engine/media/MT_AudioCodec.cpp b/src/engine/media/MT_AudioCodec.cpp index 5e7a2358..1cfa916f 100644 --- a/src/engine/media/MT_AudioCodec.cpp +++ b/src/engine/media/MT_AudioCodec.cpp @@ -434,6 +434,13 @@ Codec::Info OpusCodec::info() }; } +int OpusCodec::packetTime(std::span payload) +{ + int samples = payload.empty() ? 0 + : opus_packet_get_nb_samples(payload.data(), (opus_int32)payload.size(), mSamplerate); + return samples > 0 ? samples * 1000 / mSamplerate : frameTime(); +} + Codec::EncodeResult OpusCodec::encode(std::span input, std::span output) { // opus_encode() takes the frame size in samples per channel and the output diff --git a/src/engine/media/MT_AudioCodec.h b/src/engine/media/MT_AudioCodec.h index 9192df08..b01f2900 100644 --- a/src/engine/media/MT_AudioCodec.h +++ b/src/engine/media/MT_AudioCodec.h @@ -69,6 +69,8 @@ protected: int mDecoderChannels = 0; public: + int packetTime(std::span payload) override; + struct Params { bool mUseDtx = false, mUseInbandFec = false, mStereo = false; diff --git a/src/engine/media/MT_AudioReceiver.cpp b/src/engine/media/MT_AudioReceiver.cpp index 45776bd2..2d120a38 100644 --- a/src/engine/media/MT_AudioReceiver.cpp +++ b/src/engine/media/MT_AudioReceiver.cpp @@ -520,10 +520,8 @@ Codec* AudioReceiver::add(const std::shared_ptr& p) if (!codec) time_length = 10; - else if (!codec->rtpLength()) - time_length = codec->frameTime(); else - time_length = lround(double(payloadLength) / codec->rtpLength() * codec->frameTime()); + time_length = codec->packetTime(std::span{p->GetPayloadData(), (size_t)payloadLength}); if (codec) samplerate = codec->rtpClockRate(); // RTP timestamp units, not PCM rate (G.722) @@ -794,8 +792,11 @@ AudioReceiver::DecodeResult AudioReceiver::decodePacketTo(Audio::DataWindow& out mFrameCount = mCodec->rtpLength() ? rtp.GetPayloadLength() / mCodec->rtpLength() : 1; int frameLength = mCodec->rtpLength() ? mCodec->rtpLength() : (int)rtp.GetPayloadLength(); - // Save last packet time length - mLastPacketTimeLength = mFrameCount * mCodec->frameTime(); + // Save last packet time length. Taken from the payload, not mFrameCount * + // frameTime(): an Opus packet of 40 or 60 ms is one "frame" here, and counting + // it as frameTime() (20 ms) made the gap check above emit silence before every + // packet and pushed the jitter buffer into discarding about half of them. + mLastPacketTimeLength = mCodec->packetTime(std::span{rtp.GetPayloadData(), payload_length}); // Decode for (int i = 0; i < mFrameCount && !mCodecSettings.mSkipDecode; i++) diff --git a/src/engine/media/MT_Codec.h b/src/engine/media/MT_Codec.h index 799c0bff..034f42a3 100644 --- a/src/engine/media/MT_Codec.h +++ b/src/engine/media/MT_Codec.h @@ -6,6 +6,7 @@ #ifndef __MT_CODEC_H #define __MT_CODEC_H +#include #include #include @@ -73,6 +74,17 @@ public: Audio::Format getAudioFormat() { return Audio::Format(this->info().mSamplerate, this->info().mChannels); } + // Audio duration of one RTP payload, in milliseconds. The default suits fixed-size + // frames (rtpLength() bytes per frameTime()); a codec whose packets carry a variable + // amount of audio (Opus: 2.5-120 ms, whatever the sender's ptime) reads it from the + // payload instead, as frameTime() is only the local ptime. + virtual int packetTime(std::span payload) + { + if (!rtpLength()) + return frameTime(); + return static_cast(std::lround(double(payload.size()) / rtpLength() * frameTime())); + } + // Returns size of encoded data (RTP) in bytes struct EncodeResult {