Compare commits
5 Commits
7f638a5aef
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 23a28c293b | |||
| d22b8fe1ab | |||
| a19697f501 | |||
| 9f246e0d92 | |||
| bbad9d27b4 |
@@ -4,6 +4,7 @@
|
|||||||
#include <mutex>
|
#include <mutex>
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
|
#include <algorithm>
|
||||||
|
|
||||||
#include "../helper/HL_String.h"
|
#include "../helper/HL_String.h"
|
||||||
#include "../helper/HL_Time.h"
|
#include "../helper/HL_Time.h"
|
||||||
@@ -141,6 +142,9 @@ AndroidOutputDevice::AndroidOutputDevice(int devId)
|
|||||||
AndroidOutputDevice::~AndroidOutputDevice()
|
AndroidOutputDevice::~AndroidOutputDevice()
|
||||||
{
|
{
|
||||||
ICELogDebug(<< "Deleting AndroidOutputDevice.");
|
ICELogDebug(<< "Deleting AndroidOutputDevice.");
|
||||||
|
// Mark shutdown before closing so a disconnect callback racing with teardown
|
||||||
|
// does not resurrect the stream via onErrorAfterClose()'s restart.
|
||||||
|
mInShutdown = true;
|
||||||
close();
|
close();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -151,6 +155,7 @@ bool AndroidOutputDevice::open()
|
|||||||
if (mActive)
|
if (mActive)
|
||||||
return true;
|
return true;
|
||||||
|
|
||||||
|
mInShutdown = false;
|
||||||
mRequestedFrames = 0;
|
mRequestedFrames = 0;
|
||||||
mStartTime = 0.0;
|
mStartTime = 0.0;
|
||||||
mEndTime = 0.0;
|
mEndTime = 0.0;
|
||||||
@@ -161,6 +166,10 @@ bool AndroidOutputDevice::open()
|
|||||||
builder.setSharingMode(oboe::SharingMode::Exclusive);
|
builder.setSharingMode(oboe::SharingMode::Exclusive);
|
||||||
builder.setFormat(oboe::AudioFormat::I16);
|
builder.setFormat(oboe::AudioFormat::I16);
|
||||||
builder.setChannelCount(oboe::ChannelCount::Mono);
|
builder.setChannelCount(oboe::ChannelCount::Mono);
|
||||||
|
// Route through the platform voice-call path: correct device selection/volume
|
||||||
|
// and platform voice tuning for a softphone.
|
||||||
|
builder.setUsage(oboe::Usage::VoiceCommunication);
|
||||||
|
builder.setContentType(oboe::ContentType::Speech);
|
||||||
// builder.setDataCallback(this);
|
// builder.setDataCallback(this);
|
||||||
builder.setCallback(this);
|
builder.setCallback(this);
|
||||||
//builder.setErrorCallback(this)
|
//builder.setErrorCallback(this)
|
||||||
@@ -170,7 +179,7 @@ bool AndroidOutputDevice::open()
|
|||||||
return false;
|
return false;
|
||||||
|
|
||||||
mDeviceRate = mPlayingStream->getSampleRate();
|
mDeviceRate = mPlayingStream->getSampleRate();
|
||||||
ICELogInfo(<< "Input Opened with rate " << mDeviceRate);
|
ICELogInfo(<< "Output opened with rate " << mDeviceRate);
|
||||||
mActive = true;
|
mActive = true;
|
||||||
|
|
||||||
rescode = mPlayingStream->requestStart();
|
rescode = mPlayingStream->requestStart();
|
||||||
@@ -178,7 +187,18 @@ bool AndroidOutputDevice::open()
|
|||||||
{
|
{
|
||||||
close();
|
close();
|
||||||
mActive = false;
|
mActive = false;
|
||||||
|
return mActive;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Latch the burst size and start from a two-burst buffer. onAudioReady() grows
|
||||||
|
// this on XRuns (up to a cap) so we keep low latency when the device can sustain
|
||||||
|
// it and trade a little latency for glitch-free playback when it can't.
|
||||||
|
mBurstFrames = mPlayingStream->getFramesPerBurst();
|
||||||
|
mXRunLast = 0;
|
||||||
|
mHeartbeatLast = 0.0f;
|
||||||
|
if (mBurstFrames > 0)
|
||||||
|
mPlayingStream->setBufferSizeInFrames(mBurstFrames * 2);
|
||||||
|
|
||||||
return mActive;
|
return mActive;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -232,14 +252,50 @@ oboe::DataCallbackResult AndroidOutputDevice::onAudioReady(oboe::AudioStream *au
|
|||||||
}
|
}
|
||||||
mRequestedFrames += numFrames;
|
mRequestedFrames += numFrames;
|
||||||
|
|
||||||
|
// Adaptive buffer sizing: on new XRuns (device-side underruns, i.e. we missed a
|
||||||
|
// callback deadline), grow the buffer one burst at a time up to a cap. Both calls
|
||||||
|
// are documented as safe from within the data callback.
|
||||||
|
auto xrun = audioStream->getXRunCount();
|
||||||
|
if (xrun && xrun.value() > mXRunLast)
|
||||||
|
{
|
||||||
|
mXRunLast = xrun.value();
|
||||||
|
if (mBurstFrames > 0)
|
||||||
|
{
|
||||||
|
int32_t cap = mBurstFrames * 8;
|
||||||
|
int32_t cur = audioStream->getBufferSizeInFrames();
|
||||||
|
int32_t next = std::min(cur + mBurstFrames, cap);
|
||||||
|
if (next > cur)
|
||||||
|
audioStream->setBufferSizeInFrames(next);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Device-side heartbeat (Step 0): surfaces XRuns/buffer growth so device glitches
|
||||||
|
// can be told apart from network/jitter impairments.
|
||||||
|
float t = now_ms();
|
||||||
|
if (mHeartbeatLast == 0.0f)
|
||||||
|
mHeartbeatLast = t;
|
||||||
|
else if (t - mHeartbeatLast >= 5000.0f)
|
||||||
|
{
|
||||||
|
ICELogInfo(<< "[spk-heartbeat] xruns=" << mXRunLast
|
||||||
|
<< " bufFrames=" << audioStream->getBufferSizeInFrames()
|
||||||
|
<< " burst=" << mBurstFrames
|
||||||
|
<< " rate=" << mDeviceRate);
|
||||||
|
mHeartbeatLast = t;
|
||||||
|
}
|
||||||
|
|
||||||
return oboe::DataCallbackResult::Continue;
|
return oboe::DataCallbackResult::Continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO - special case https://github.com/google/oboe/blob/master/docs/notes/disconnect.md
|
// Disconnect recovery: on a route change (headset/BT plug/unplug) AAudio tears the
|
||||||
|
// stream down and calls this on its own thread after the stream is closed. Rebuild
|
||||||
|
// on the new default route so audio doesn't silently die mid-call.
|
||||||
|
// See https://github.com/google/oboe/blob/master/docs/notes/disconnect.md
|
||||||
void AndroidOutputDevice::onErrorAfterClose(oboe::AudioStream *stream, oboe::Result result) {
|
void AndroidOutputDevice::onErrorAfterClose(oboe::AudioStream *stream, oboe::Result result) {
|
||||||
if (result == oboe::Result::ErrorDisconnected) {
|
if (result == oboe::Result::ErrorDisconnected && !mInShutdown) {
|
||||||
// LOGI("Restarting AudioStream after disconnect");
|
ICELogInfo(<< "Output stream disconnected; restarting on the new route");
|
||||||
// soundEngine.restart(); // please check oboe samples for soundEngine.restart(); call
|
// close() and open() each take mMutex internally; this callback holds none.
|
||||||
|
close();
|
||||||
|
open();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif // TARGET_ANDROID
|
#endif // TARGET_ANDROID
|
||||||
|
|||||||
@@ -99,6 +99,11 @@ class AndroidOutputDevice: public OutputDevice, public oboe::AudioStreamCallback
|
|||||||
bool mInShutdown = false;
|
bool mInShutdown = false;
|
||||||
bool mActive = false;
|
bool mActive = false;
|
||||||
|
|
||||||
|
// Adaptive buffer sizing + device-side diagnostics (Step 0 / #5).
|
||||||
|
int32_t mBurstFrames = 0; // frames per burst, latched at open()
|
||||||
|
int32_t mXRunLast = 0; // last observed cumulative XRun count
|
||||||
|
float mHeartbeatLast = 0.0f; // now_ms() of last device heartbeat log
|
||||||
|
|
||||||
// Statistics
|
// Statistics
|
||||||
float mRequestedFrames = 0.0, mStartTime = 0.0, mEndTime = 0.0;
|
float mRequestedFrames = 0.0, mStartTime = 0.0, mEndTime = 0.0;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -27,8 +27,10 @@
|
|||||||
#define AUDIO_MIX_CHANNEL_COUNT 16
|
#define AUDIO_MIX_CHANNEL_COUNT 16
|
||||||
#define AUDIO_DEVICEPAIR_INPUTBUFFER 16384
|
#define AUDIO_DEVICEPAIR_INPUTBUFFER 16384
|
||||||
|
|
||||||
// Avoid too high resampler quality - it can take many CPU and cause gaps in playing
|
// Resampler quality on the Speex 0..10 scale. Quality 1 upsampling (e.g. G.711 8k -> 48k)
|
||||||
#define AUDIO_RESAMPLER_QUALITY 1
|
// adds audible aliasing and hurts PVQA/AQuA scores; 7 is a good fidelity/CPU balance for a
|
||||||
|
// single mono stream on modern ARM64. Raise toward 10 for max fidelity if CPU allows.
|
||||||
|
#define AUDIO_RESAMPLER_QUALITY 7
|
||||||
#define AEC_FRAME_TIME 10
|
#define AEC_FRAME_TIME 10
|
||||||
#define AEC_TAIL_TIME 160
|
#define AEC_TAIL_TIME 160
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
/// transparent passthrough to the global allocator (i.e. allocate_shared behaves like make_shared)
|
/// transparent passthrough to the global allocator (i.e. allocate_shared behaves like make_shared)
|
||||||
/// for A/B benchmarking without touching the call sites.
|
/// for A/B benchmarking without touching the call sites.
|
||||||
|
|
||||||
|
#include <atomic>
|
||||||
#include <cstddef>
|
#include <cstddef>
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
@@ -45,6 +46,7 @@ namespace hl
|
|||||||
{
|
{
|
||||||
uint8_t* raw = static_cast<uint8_t*>(::operator new(size + HeaderSize));
|
uint8_t* raw = static_cast<uint8_t*>(::operator new(size + HeaderSize));
|
||||||
tagOf(raw) = TagGlobal;
|
tagOf(raw) = TagGlobal;
|
||||||
|
s_activeGlobal.fetch_add(1, std::memory_order_relaxed);
|
||||||
return raw + HeaderSize;
|
return raw + HeaderSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,6 +56,7 @@ namespace hl
|
|||||||
|
|
||||||
uint8_t* block = static_cast<uint8_t*>(head);
|
uint8_t* block = static_cast<uint8_t*>(head);
|
||||||
head = nextOf(block);
|
head = nextOf(block);
|
||||||
|
s_activeBlocks.fetch_add(1, std::memory_order_relaxed);
|
||||||
return block + HeaderSize;
|
return block + HeaderSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,14 +71,49 @@ namespace hl
|
|||||||
void*& head = freeListHead();
|
void*& head = freeListHead();
|
||||||
nextOf(block) = head;
|
nextOf(block) = head;
|
||||||
head = block;
|
head = block;
|
||||||
|
s_activeBlocks.fetch_sub(1, std::memory_order_relaxed);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
::operator delete(static_cast<void*>(block));
|
::operator delete(static_cast<void*>(block));
|
||||||
|
s_activeGlobal.fetch_sub(1, std::memory_order_relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// @name Diagnostics (relaxed gauges — not synchronization)
|
||||||
|
/// @{
|
||||||
|
/// Blocks currently handed out from the pool = pooled allocate() minus
|
||||||
|
/// pooled deallocate(). Unlike capacityBlocks() (the chunk high-water,
|
||||||
|
/// which only ever grows) this DROPS when objects are freed, so a rising
|
||||||
|
/// activeBlocks() is a genuine leak of referenced objects rather than a
|
||||||
|
/// traffic peak that merely carved extra chunks. Covers every pooled
|
||||||
|
/// shared_ptr node (allocate_shared<RTPPacket> + jitter-buffer packets).
|
||||||
|
static std::int64_t activeBlocks() noexcept
|
||||||
|
{
|
||||||
|
return s_activeBlocks.load(std::memory_order_relaxed);
|
||||||
|
}
|
||||||
|
/// Live oversized allocations that overflowed to ::operator new.
|
||||||
|
static std::int64_t activeGlobalAllocations() noexcept
|
||||||
|
{
|
||||||
|
return s_activeGlobal.load(std::memory_order_relaxed);
|
||||||
|
}
|
||||||
|
/// Total blocks ever carved (chunks x BlocksPerChunk) = high-water capacity.
|
||||||
|
static std::int64_t capacityBlocks() noexcept
|
||||||
|
{
|
||||||
|
return s_chunks.load(std::memory_order_relaxed) * static_cast<std::int64_t>(BlocksPerChunk);
|
||||||
|
}
|
||||||
|
/// Approximate live pooled payload bytes (excludes per-block header).
|
||||||
|
static std::int64_t activeBytes() noexcept
|
||||||
|
{
|
||||||
|
return activeBlocks() * static_cast<std::int64_t>(PayloadSize);
|
||||||
|
}
|
||||||
|
/// @}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
inline static std::atomic<std::int64_t> s_activeBlocks{0};
|
||||||
|
inline static std::atomic<std::int64_t> s_activeGlobal{0};
|
||||||
|
inline static std::atomic<std::int64_t> s_chunks{0};
|
||||||
|
|
||||||
static constexpr std::size_t HeaderSize =
|
static constexpr std::size_t HeaderSize =
|
||||||
alignof(std::max_align_t) >= sizeof(uint64_t) ? alignof(std::max_align_t) : sizeof(uint64_t);
|
alignof(std::max_align_t) >= sizeof(uint64_t) ? alignof(std::max_align_t) : sizeof(uint64_t);
|
||||||
static constexpr std::size_t BlockSize = HeaderSize + PayloadSize;
|
static constexpr std::size_t BlockSize = HeaderSize + PayloadSize;
|
||||||
@@ -114,6 +152,7 @@ namespace hl
|
|||||||
{
|
{
|
||||||
const std::size_t chunkBytes = BlockSize * BlocksPerChunk;
|
const std::size_t chunkBytes = BlockSize * BlocksPerChunk;
|
||||||
uint8_t* chunk = static_cast<uint8_t*>(::operator new(chunkBytes));
|
uint8_t* chunk = static_cast<uint8_t*>(::operator new(chunkBytes));
|
||||||
|
s_chunks.fetch_add(1, std::memory_order_relaxed);
|
||||||
|
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(m_Mutex);
|
std::lock_guard<std::mutex> lock(m_Mutex);
|
||||||
@@ -144,6 +183,21 @@ namespace hl
|
|||||||
};
|
};
|
||||||
#endif // HL_RTP_POOL
|
#endif // HL_RTP_POOL
|
||||||
|
|
||||||
|
/// Mode-independent accessors for the pool diagnostics, so callers need not know
|
||||||
|
/// whether HL_RTP_POOL is compiled in. When pooling is disabled they return -1
|
||||||
|
/// ("not applicable") since allocate_shared then behaves like make_shared.
|
||||||
|
#if HL_RTP_POOL
|
||||||
|
inline std::int64_t poolActiveBlocks() noexcept { return FixedBlockPool::activeBlocks(); }
|
||||||
|
inline std::int64_t poolCapacityBlocks() noexcept { return FixedBlockPool::capacityBlocks(); }
|
||||||
|
inline std::int64_t poolActiveGlobal() noexcept { return FixedBlockPool::activeGlobalAllocations(); }
|
||||||
|
inline std::int64_t poolActiveBytes() noexcept { return FixedBlockPool::activeBytes(); }
|
||||||
|
#else
|
||||||
|
inline std::int64_t poolActiveBlocks() noexcept { return -1; }
|
||||||
|
inline std::int64_t poolCapacityBlocks() noexcept { return -1; }
|
||||||
|
inline std::int64_t poolActiveGlobal() noexcept { return -1; }
|
||||||
|
inline std::int64_t poolActiveBytes() noexcept { return -1; }
|
||||||
|
#endif
|
||||||
|
|
||||||
/// @class PoolAllocator
|
/// @class PoolAllocator
|
||||||
/// A stateless, std-conforming Allocator suitable for std::allocate_shared. When HL_RTP_POOL is
|
/// A stateless, std-conforming Allocator suitable for std::allocate_shared. When HL_RTP_POOL is
|
||||||
/// enabled it serves single-node allocations from FixedBlockPool; otherwise (and for any request
|
/// enabled it serves single-node allocations from FixedBlockPool; otherwise (and for any request
|
||||||
|
|||||||
@@ -571,6 +571,41 @@ size_t OpusCodec::plc(int lostPackets, std::span<uint8_t> output)
|
|||||||
return total;
|
return total;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t OpusCodec::fec(std::span<const uint8_t> nextPacket, std::span<uint8_t> output)
|
||||||
|
{
|
||||||
|
// Reconstruct the frame lost right before nextPacket from its LBRR data.
|
||||||
|
// Needs an initialized decoder; the very first gap of a call (no packet
|
||||||
|
// decoded yet) has none, so we bail and let the caller do silence/PLC.
|
||||||
|
if (nextPacket.empty() || !mDecoderCtx || output.empty())
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
// opus_decode(..., decode_fec=1) emits its audio at the decoder's channel
|
||||||
|
// count. Only the matched-channel case is handled here; on a mismatch we
|
||||||
|
// return 0 so decodeGapTo() falls back to plc() (which does convert).
|
||||||
|
if (mDecoderChannels != channels())
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
// The recovered frame is assumed to have the negotiated ptime - the same
|
||||||
|
// assumption plc() makes. frame_size is samples-per-channel.
|
||||||
|
int samples_per_channel = mPTime * mSamplerate / 1000;
|
||||||
|
if (samples_per_channel <= 0)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
size_t needed = (size_t)samples_per_channel * sizeof(opus_int16) * channels();
|
||||||
|
if (needed > output.size_bytes())
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
// decode_fec = 1: pull the redundant copy of the previous frame. If nextPacket
|
||||||
|
// carries no LBRR for it, Opus falls back to its own concealment internally,
|
||||||
|
// so the result is never worse than plc().
|
||||||
|
int decoded = opus_decode(mDecoderCtx, nextPacket.data(), nextPacket.size_bytes(),
|
||||||
|
(opus_int16*)output.data(), samples_per_channel, 1);
|
||||||
|
if (decoded <= 0)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
return (size_t)decoded * sizeof(opus_int16) * channels();
|
||||||
|
}
|
||||||
|
|
||||||
size_t OpusCodec::getNumberOfSamples(std::span<const uint8_t> payload)
|
size_t OpusCodec::getNumberOfSamples(std::span<const uint8_t> payload)
|
||||||
{
|
{
|
||||||
int r = opus_packet_get_nb_samples(payload.data(), payload.size(), mSamplerate);
|
int r = opus_packet_get_nb_samples(payload.data(), payload.size(), mSamplerate);
|
||||||
|
|||||||
@@ -113,6 +113,7 @@ public:
|
|||||||
EncodeResult encode(std::span<const uint8_t> input, std::span<uint8_t> output) override;
|
EncodeResult encode(std::span<const uint8_t> input, std::span<uint8_t> output) override;
|
||||||
DecodeResult decode(std::span<const uint8_t> input, std::span<uint8_t> output) override;
|
DecodeResult decode(std::span<const uint8_t> input, std::span<uint8_t> output) override;
|
||||||
size_t plc(int lostFrames, std::span<uint8_t> output) override;
|
size_t plc(int lostFrames, std::span<uint8_t> output) override;
|
||||||
|
size_t fec(std::span<const uint8_t> nextPacket, std::span<uint8_t> output) override;
|
||||||
|
|
||||||
size_t getNumberOfSamples(std::span<const uint8_t> payload);
|
size_t getNumberOfSamples(std::span<const uint8_t> payload);
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -348,6 +348,21 @@ std::chrono::milliseconds RtpBuffer::findTimelength()
|
|||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::shared_ptr<RtpBuffer::Packet> RtpBuffer::peekFront() const
|
||||||
|
{
|
||||||
|
Lock l(mGuard);
|
||||||
|
return mPacketList.empty() ? std::shared_ptr<Packet>() : mPacketList.front();
|
||||||
|
}
|
||||||
|
|
||||||
|
std::chrono::milliseconds RtpBuffer::bufferedTime() const
|
||||||
|
{
|
||||||
|
Lock l(mGuard);
|
||||||
|
std::chrono::milliseconds r = 0ms;
|
||||||
|
for (const auto& p: mPacketList)
|
||||||
|
r += p->timelength();
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
int RtpBuffer::getNumberOfReturnedPackets() const
|
int RtpBuffer::getNumberOfReturnedPackets() const
|
||||||
{
|
{
|
||||||
return mReturnedCounter;
|
return mReturnedCounter;
|
||||||
@@ -378,8 +393,23 @@ AudioReceiver::AudioReceiver(const CodecList::Settings& settings, MT::Statistics
|
|||||||
mDtmfBuffer.setLow(0ms);
|
mDtmfBuffer.setLow(0ms);
|
||||||
mDtmfBuffer.setHigh(1ms);
|
mDtmfBuffer.setHigh(1ms);
|
||||||
|
|
||||||
// Avoid collecting too much data
|
// Jitter-buffer target delay. Diagnostics on bursty networks showed the buffer level
|
||||||
mRtpBuffer.setHigh(240ms);
|
// swinging ~140ms peak-to-trough and hitting BOTH rails within one call - starving
|
||||||
|
// (underrun -> silence) and overflowing (trim -> drop) - with real packet loss near 1%.
|
||||||
|
// Widened the window to give burst jitter more room: prebuffer/low-water raised so a
|
||||||
|
// late clump arrives before starvation, and the high-water raised so the extra depth
|
||||||
|
// is not immediately trimmed back off.
|
||||||
|
mRtpBuffer.setHigh(360ms);
|
||||||
|
|
||||||
|
// Prebuffer: initial fill before the first packet is released (also the depth we aim to
|
||||||
|
// hold). Was the RTP_BUFFER_PREBUFFER default (100ms); set explicitly alongside low-water.
|
||||||
|
mRtpBuffer.setPrebuffer(120ms);
|
||||||
|
|
||||||
|
// Low-water cushion. With low-water at 0 the buffer drains to empty before reacting, so
|
||||||
|
// ordinary clock drift / jitter periodically starves the decoder and forces silence/PLC
|
||||||
|
// insertion (heard as periodic artifacts, uncounted by rtp_lost / rtp_dropped). Holding
|
||||||
|
// ~120ms lets a late packet clump arrive before the buffer underruns.
|
||||||
|
mRtpBuffer.setLow(120ms);
|
||||||
|
|
||||||
// Resamplers are lazy inside; there is no actual memory allocation
|
// Resamplers are lazy inside; there is no actual memory allocation
|
||||||
mResampler8.start(AUDIO_CHANNELS, 8000, AUDIO_SAMPLERATE);
|
mResampler8.start(AUDIO_CHANNELS, 8000, AUDIO_SAMPLERATE);
|
||||||
@@ -614,21 +644,60 @@ AudioReceiver::DecodeResult AudioReceiver::decodeGapTo(Audio::DataWindow& output
|
|||||||
else
|
else
|
||||||
if (mCodec && mFrameCount && !mCodecSettings.mSkipDecode)
|
if (mCodec && mFrameCount && !mCodecSettings.mSkipDecode)
|
||||||
{
|
{
|
||||||
// Do PLC to mDecodedFrame/mDecodedLength
|
// Conceal the gap into mDecodedFrame/mDecodedLength.
|
||||||
if (options.mSkipDecode)
|
if (options.mSkipDecode)
|
||||||
mDecodedLength = 0;
|
mDecodedLength = 0;
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
mDecodedLength = mCodec->plc(mFrameCount, {(uint8_t*)mDecodedFrame.data(), mDecodedFrame.size() * sizeof(int16_t)});
|
std::span<uint8_t> concealBuffer{(uint8_t*)mDecodedFrame.data(), mDecodedFrame.size() * sizeof(int16_t)};
|
||||||
|
mDecodedLength = 0;
|
||||||
|
|
||||||
|
// In-band FEC first: the packet sitting past the gap (still at the front
|
||||||
|
// of the jitter buffer - fetch() leaves it there on a Gap) may carry a
|
||||||
|
// redundant copy of the frame we just lost. Recover it exactly instead of
|
||||||
|
// synthesising it. Only attempt this when that packet decodes with the
|
||||||
|
// very same codec instance we have been using.
|
||||||
|
if (auto next = mRtpBuffer.peekFront(); next && next->rtp())
|
||||||
|
{
|
||||||
|
auto codecIter = mCodecMap.find(next->rtp()->GetPayloadType());
|
||||||
|
if (codecIter != mCodecMap.end() && codecIter->second == mCodec)
|
||||||
|
{
|
||||||
|
mDecodedLength = mCodec->fec({next->rtp()->GetPayloadData(), (size_t)next->rtp()->GetPayloadLength()},
|
||||||
|
concealBuffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// No FEC available (unsupported codec, no redundant data, or no next
|
||||||
|
// packet yet): fall back to the codec's own concealment.
|
||||||
|
if (!mDecodedLength)
|
||||||
|
mDecodedLength = mCodec->plc(mFrameCount, concealBuffer);
|
||||||
|
|
||||||
if (!mDecodedLength)
|
if (!mDecodedLength)
|
||||||
{
|
{
|
||||||
// PLC is not support or failed
|
// PLC unsupported or failed - substitute one frame of silence.
|
||||||
// So substitute the silence
|
size_t frameBytes = (size_t)mCodec->frameTime() * mCodec->samplerate() / 1000
|
||||||
size_t nr_of_samples = mCodec->frameTime() * mCodec->samplerate() / 1000 * sizeof(short);
|
* sizeof(short) * std::max(1, mCodec->channels());
|
||||||
mDecodedLength = nr_of_samples * sizeof(short);
|
frameBytes = std::min(frameBytes, concealBuffer.size_bytes());
|
||||||
|
mDecodedLength = frameBytes;
|
||||||
memset(mDecodedFrame.data(), 0, mDecodedLength);
|
memset(mDecodedFrame.data(), 0, mDecodedLength);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fix double-concealment: fetch() advanced mLastSeqno across the gap but not
|
||||||
|
// mLastPacketTimestamp, so without this the next real packet's timestamp-gap
|
||||||
|
// check in decodePacketTo() would pad the same span again with silence (gap
|
||||||
|
// concealed twice - once here as FEC/PLC, once there as silence). Advance the
|
||||||
|
// expected-timestamp cursor past the last real packet and record the audio we
|
||||||
|
// just produced as the new "last unit", so decodePacketTo() fills only whatever
|
||||||
|
// gap remains beyond it.
|
||||||
|
if (mDecodedLength && mLastPacketTimestamp && mCodec)
|
||||||
|
{
|
||||||
|
int sr = mCodec->samplerate();
|
||||||
|
int chans = std::max(1, mCodec->channels());
|
||||||
|
*mLastPacketTimestamp += (uint32_t)(mLastPacketTimeLength * (sr / 1000));
|
||||||
|
int concealedSamples = (int)(mDecodedLength / (sizeof(short) * chans));
|
||||||
|
mLastPacketTimeLength = concealedSamples * 1000 / sr;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (mDecodedLength)
|
if (mDecodedLength)
|
||||||
@@ -801,7 +870,14 @@ AudioReceiver::DecodeResult AudioReceiver::decodeEmptyTo(Audio::DataWindow& outp
|
|||||||
// Emit silence if codec information is available - it is to properly handle the gaps
|
// Emit silence if codec information is available - it is to properly handle the gaps
|
||||||
auto avail = output.getTimeLength(fmt);
|
auto avail = output.getTimeLength(fmt);
|
||||||
if (options.mElapsed > avail)
|
if (options.mElapsed > avail)
|
||||||
|
{
|
||||||
|
// Genuine decoder starvation: the buffer drained below low-water with no
|
||||||
|
// packet (and no CNG) to play, so we fill the timeline with silence. Count
|
||||||
|
// it for the heartbeat - this is the signature of jitter/clock-drift
|
||||||
|
// underruns that carry no packet-loss.
|
||||||
output.addZero(fmt.sizeFromTime(options.mElapsed - avail));
|
output.addZero(fmt.sizeFromTime(options.mElapsed - avail));
|
||||||
|
mUnderrunCount++;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -819,6 +895,42 @@ void MT::AudioReceiver::processDtmf()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void MT::AudioReceiver::logReceiveHeartbeat()
|
||||||
|
{
|
||||||
|
float t = now_ms();
|
||||||
|
if (mStatLogLast == 0.0f)
|
||||||
|
{
|
||||||
|
// First call - just latch the baselines, nothing to report yet.
|
||||||
|
mStatLogLast = t;
|
||||||
|
mStatLossLast = mStat.mPacketLoss;
|
||||||
|
mStatDropLast = mStat.mPacketDropped;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (t - mStatLogLast < 5000.0f)
|
||||||
|
return;
|
||||||
|
|
||||||
|
// Deltas over the elapsed window. Loss = missing sequence numbers seen on fetch;
|
||||||
|
// dropped = packets discarded by the high-water trim (late/overflow), which is
|
||||||
|
// NOT network loss; underruns = decoder starvation (buffer under low-water) that
|
||||||
|
// forced silence. These three separate the candidate impairment causes.
|
||||||
|
size_t lossDelta = mStat.mPacketLoss - mStatLossLast;
|
||||||
|
size_t dropDelta = mStat.mPacketDropped - mStatDropLast;
|
||||||
|
|
||||||
|
ICELogInfo(<< "[rx-heartbeat] buffered=" << mRtpBuffer.bufferedTime().count() << "ms"
|
||||||
|
<< " packets=" << mRtpBuffer.getCount()
|
||||||
|
<< " underruns/5s=" << mUnderrunCount
|
||||||
|
<< " gaps/5s=" << mGapCount
|
||||||
|
<< " loss/5s=" << lossDelta
|
||||||
|
<< " dropped/5s=" << dropDelta);
|
||||||
|
|
||||||
|
mStatLogLast = t;
|
||||||
|
mUnderrunCount = 0;
|
||||||
|
mGapCount = 0;
|
||||||
|
mStatLossLast = mStat.mPacketLoss;
|
||||||
|
mStatDropLast = mStat.mPacketDropped;
|
||||||
|
}
|
||||||
|
|
||||||
void MT::AudioReceiver::updateDecodingTimeStatistics()
|
void MT::AudioReceiver::updateDecodingTimeStatistics()
|
||||||
{
|
{
|
||||||
if (!mDecodeTimestamp)
|
if (!mDecodeTimestamp)
|
||||||
@@ -881,7 +993,7 @@ AudioReceiver::DecodeResult AudioReceiver::getAudioTo(Audio::DataWindow& output,
|
|||||||
// Decode to mAvailable buffer
|
// Decode to mAvailable buffer
|
||||||
switch (fr.mStatus)
|
switch (fr.mStatus)
|
||||||
{
|
{
|
||||||
case RtpBuffer::FetchResult::Status::Gap: result = decodeGapTo(mAvailable, options.decreaseElapsedBy(produced)); break;
|
case RtpBuffer::FetchResult::Status::Gap: mGapCount++; result = decodeGapTo(mAvailable, options.decreaseElapsedBy(produced)); break;
|
||||||
case RtpBuffer::FetchResult::Status::NoPacket: result = decodeEmptyTo(mAvailable, options.decreaseElapsedBy(produced)); break;
|
case RtpBuffer::FetchResult::Status::NoPacket: result = decodeEmptyTo(mAvailable, options.decreaseElapsedBy(produced)); break;
|
||||||
case RtpBuffer::FetchResult::Status::RegularPacket: result = decodePacketTo(mAvailable, options.decreaseElapsedBy(produced), fr.mPacket); updateDecodeIntervalStatistics(); break;
|
case RtpBuffer::FetchResult::Status::RegularPacket: result = decodePacketTo(mAvailable, options.decreaseElapsedBy(produced), fr.mPacket); updateDecodeIntervalStatistics(); break;
|
||||||
default:
|
default:
|
||||||
@@ -915,6 +1027,8 @@ AudioReceiver::DecodeResult AudioReceiver::getAudioTo(Audio::DataWindow& output,
|
|||||||
|
|
||||||
mProducedAudio += produced;
|
mProducedAudio += produced;
|
||||||
// ICELogDebug(<< "Requested " << options.mElapsed << ", produced " << produced << ", remains " << mAvailable.getTimeLength(fmt) << ", packets " << getRtpBuffer().getCount());
|
// ICELogDebug(<< "Requested " << options.mElapsed << ", produced " << produced << ", remains " << mAvailable.getTimeLength(fmt) << ", packets " << getRtpBuffer().getCount());
|
||||||
|
|
||||||
|
logReceiveHeartbeat();
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -106,6 +106,17 @@ public:
|
|||||||
|
|
||||||
FetchResult fetch();
|
FetchResult fetch();
|
||||||
|
|
||||||
|
// Return the oldest buffered packet without removing it, or nullptr when empty.
|
||||||
|
// After fetch() reports a Gap the post-gap packet stays at the front, so this
|
||||||
|
// lets the decoder peek at it (e.g. to pull Opus in-band FEC) before it is
|
||||||
|
// fetched and decoded normally on the next call.
|
||||||
|
std::shared_ptr<Packet> peekFront() const;
|
||||||
|
|
||||||
|
// Thread-safe snapshot of the currently buffered audio duration. Same value as
|
||||||
|
// findTimelength() but taken under the buffer lock, safe to call from the audio
|
||||||
|
// thread while the network thread adds packets (used by the diagnostic heartbeat).
|
||||||
|
std::chrono::milliseconds bufferedTime() const;
|
||||||
|
|
||||||
// Drop oldest packets so buffered audio stays within the high-water mark,
|
// Drop oldest packets so buffered audio stays within the high-water mark,
|
||||||
// recording packet-loss events for any sequence gaps crossed (the same
|
// recording packet-loss events for any sequence gaps crossed (the same
|
||||||
// accounting fetch() performs). Used to bound memory on streams that never
|
// accounting fetch() performs). Used to bound memory on streams that never
|
||||||
@@ -327,6 +338,16 @@ protected:
|
|||||||
size_t mDecodeCount = 0;
|
size_t mDecodeCount = 0;
|
||||||
void updateDecodeIntervalStatistics();
|
void updateDecodeIntervalStatistics();
|
||||||
|
|
||||||
|
// --- Step 0 diagnostics: periodic receive-path heartbeat ---
|
||||||
|
// Emits jitter-buffer fill level, starvation (underrun) count, gap/loss/drop
|
||||||
|
// deltas every ~5 s so we can tell which impairment mechanism dominates a call.
|
||||||
|
float mStatLogLast = 0.0f; // now_ms() of last heartbeat; 0 = not started
|
||||||
|
size_t mUnderrunCount = 0; // silence insertions on an empty/starved buffer since last log
|
||||||
|
size_t mGapCount = 0; // gap (loss) concealment events since last log
|
||||||
|
size_t mStatLossLast = 0; // mStat.mPacketLoss at last heartbeat
|
||||||
|
size_t mStatDropLast = 0; // mStat.mPacketDropped at last heartbeat
|
||||||
|
void logReceiveHeartbeat();
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,6 +88,13 @@ public:
|
|||||||
// Returns size of produced data (PCM signed short) in bytes
|
// Returns size of produced data (PCM signed short) in bytes
|
||||||
virtual size_t plc(int lostFrames, std::span<uint8_t> output) = 0;
|
virtual size_t plc(int lostFrames, std::span<uint8_t> output) = 0;
|
||||||
|
|
||||||
|
// In-band forward error correction. Reconstructs the single frame lost
|
||||||
|
// immediately before nextPacket, using the redundant (LBRR) copy that
|
||||||
|
// nextPacket may carry. Returns the number of PCM bytes written to output,
|
||||||
|
// or 0 when the codec has no in-band FEC or no redundant data was present
|
||||||
|
// (in which case the caller should fall back to plc()). Default: unsupported.
|
||||||
|
virtual size_t fec(std::span<const uint8_t> nextPacket, std::span<uint8_t> output) { return 0; }
|
||||||
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
Reference in New Issue
Block a user