public/rtphone
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

- ongoing work to fix issue with unstable audio

Browse Source
This commit is contained in:
Dmytro Bogovych
2021-08-06 18:31:42 +03:00
parent ed39725641
commit 74b5aa69cb
18 changed files with 220 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/engine/media/MT_AudioReceiver.cpp
View File

@@ -10,6 +10,7 @@
#include "MT_AudioCodec.h"
#include "MT_CngHelper.h"
#include "../helper/HL_Log.h"
#include "../helper/HL_Time.h"
#include "../audio/Audio_Interface.h"
#include "../audio/Audio_Resampler.h"
#include <cmath>
@@ -108,7 +109,16 @@ bool RtpBuffer::add(std::shared_ptr<jrtplib::RTPPacket> packet, int timelength,
Lock l(mGuard);
// Update statistics
if (mLastAddTime == 0.0)
mLastAddTime = now_ms();
else
{
float t = now_ms();
mStat.mPacketInterval.process(t - mLastAddTime);
mLastAddTime = t;
}
mStat.mSsrc = static_cast<uint16_t>(packet->GetSSRC());
// Update jitter
@@ -357,6 +367,7 @@ AudioReceiver::~AudioReceiver()
bool AudioReceiver::add(const std::shared_ptr<jrtplib::RTPPacket>& p, Codec** codec)
{
// ICELogInfo(<< "Adding packet No " << p->GetSequenceNumber());
// Increase codec counter
mStat.mCodecCount[p->GetPayloadType()]++;
@@ -435,7 +446,7 @@ void AudioReceiver::processDecoded(Audio::DataWindow& output, int options)
bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
{
bool result = false;
bool result = false, had_cng = false, had_decode = false;
// Get next packet from buffer
RtpBuffer::ResultList rl;
@@ -443,7 +454,7 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
switch (fr)
{
case RtpBuffer::FetchResult::Gap:
ICELogInfo(<< "Gap detected.");
ICELogDebug(<< "Gap detected.");
mDecodedLength = mResampledLength = 0;
if (mCngPacket && mCodec)
@@ -571,9 +582,13 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
mDecodedLength = 0;
else
{
// Trigger the statistics
had_decode = true;
// Decode frame by frame
mDecodedLength = mCodec->decode(p->GetPayloadData() + i*mCodec->rtpLength(),
mDecodedLength = mCodec->decode(p->GetPayloadData() + i * mCodec->rtpLength(),
frameLength, mDecodedFrame, sizeof mDecodedFrame);
// mDecodedLength = 3840; // Opus 20 ms stereo
if (mDecodedLength)
processDecoded(output, options);
}
@@ -594,6 +609,18 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
assert(0);
}
if (had_decode)
{
// mStat.mDecodeRequested++;
if (mLastDecodeTime == 0.0)
mLastDecodeTime = now_ms();
else
{
float t = now_ms();
mStat.mDecodingInterval.process(t - mLastDecodeTime);
mLastDecodeTime = t;
}
}
return result;
}
@@ -670,12 +697,12 @@ void AudioReceiver::updatePvqa(const void *data, int size)
mPvqaBuffer->addZero(size);
Audio::Format fmt;
int frames = (int)fmt.timeFromSize(mPvqaBuffer->filled()) / (PVQA_INTERVAL * 1000);
int frames = static_cast<int>(fmt.timeFromSize(mPvqaBuffer->filled())) / (PVQA_INTERVAL * 1000);
if (frames > 0)
{
int time4pvqa = (int)(frames * PVQA_INTERVAL * 1000);
int size4pvqa = (int)fmt.sizeFromTime(time4pvqa);
ICELogInfo(<< "PVQA buffer has " << time4pvqa << " milliseconds of audio.");
ICELogDebug(<< "PVQA buffer has " << time4pvqa << " milliseconds of audio.");
mPVQA->update(mPvqaBuffer->data(), size4pvqa);
mPvqaBuffer->erase(size4pvqa);
}

14
src/engine/media/MT_AudioReceiver.h
View File

@@ -60,16 +60,22 @@ namespace MT
unsigned ssrc();
void setSsrc(unsigned ssrc);
void setHigh(int milliseconds);
int high();
void setLow(int milliseconds);
int low();
void setPrebuffer(int milliseconds);
int prebuffer();
int getNumberOfReturnedPackets() const;
int getNumberOfAddPackets() const;
int findTimelength();
int getCount() const;
// Returns false if packet was not add - maybe too old or too new or duplicate
bool add(std::shared_ptr<RTPPacket> packet, int timelength, int rate);
@@ -89,6 +95,9 @@ namespace MT
bool mFirstPacketWillGo;
jrtplib::RTPSourceStats mRtpStats;
Packet mFetchedPacket;
// To calculate average interval between packet add. It is close to jitter but more useful in debugging.
float mLastAddTime = 0.0;
};
class Receiver
@@ -167,6 +176,11 @@ namespace MT
Audio::PWavFileWriter mDecodedDump;
float mLastDecodeTime = 0.0; // Time last call happened to codec->decode()
float mIntervalSum = 0.0;
int mIntervalCount = 0;
// Zero rate will make audio mono but resampling will be skipped
void makeMonoAndResample(int rate, int channels);

2
src/engine/media/MT_AudioStream.cpp
View File

@@ -97,7 +97,7 @@ AudioStream::~AudioStream()
if (mFinalStatistics)
*mFinalStatistics = mStat;
ICELogInfo(<< mStat.toShortString());
ICELogInfo(<< mStat.toString());
}
void AudioStream::setDestination(const RtpPair<InternetAddress>& dest)

4
src/engine/media/MT_CodecList.cpp
View File

@@ -126,14 +126,14 @@ CodecList::CodecList(const Settings& settings)
#endif
#endif
mFactoryList.push_back(new IsacCodec::IsacFactory16K(mSettings.mIsac16KPayloadType));
/*mFactoryList.push_back(new IsacCodec::IsacFactory16K(mSettings.mIsac16KPayloadType));
mFactoryList.push_back(new IlbcCodec::IlbcFactory(mSettings.mIlbc20PayloadType, mSettings.mIlbc30PayloadType));
mFactoryList.push_back(new G711Codec::AlawFactory());
mFactoryList.push_back(new G711Codec::UlawFactory());
mFactoryList.push_back(new GsmCodec::GsmFactory(mSettings.mGsmFrPayloadLength == 32 ? GsmCodec::Type::Bytes_32 : GsmCodec::Type::Bytes_33, mSettings.mGsmFrPayloadType));
mFactoryList.push_back(new G722Codec::G722Factory());
mFactoryList.push_back(new G729Codec::G729Factory());
mFactoryList.push_back(new G729Codec::G729Factory()); */
#ifndef TARGET_ANDROID
mFactoryList.push_back(new GsmHrCodec::GsmHrFactory(mSettings.mGsmHrPayloadType));
#endif

1
src/engine/media/MT_SingleAudioStream.cpp
View File

@@ -19,6 +19,7 @@ SingleAudioStream::SingleAudioStream(const CodecList::Settings& codecSettings, S
SingleAudioStream::~SingleAudioStream()
{
}
void SingleAudioStream::process(const std::shared_ptr<jrtplib::RTPPacket>& packet)

10
src/engine/media/MT_Statistics.cpp
View File

@@ -191,6 +191,9 @@ Statistics& Statistics::operator += (const Statistics& src)
mJitter = src.mJitter;
mRttDelay = src.mRttDelay;
mDecodingInterval = src.mDecodingInterval;
mDecodeRequested = src.mDecodeRequested;
if (!src.mCodecName.empty())
mCodecName = src.mCodecName;
@@ -239,13 +242,16 @@ Statistics& Statistics::operator -= (const Statistics& src)
}
std::string Statistics::toShortString() const
std::string Statistics::toString() const
{
std::ostringstream oss;
oss << "Received: " << mReceivedRtp
<< ", lost: " << mPacketLoss
<< ", dropped: " << mPacketDropped
<< ", sent: " << mSentRtp;
<< ", sent: " << mSentRtp
<< ", decoding interval: " << mDecodingInterval.average()
<< ", decode requested: " << mDecodeRequested.average()
<< ", packet interval: " << mPacketInterval.average();
return oss.str();
}

82
src/engine/media/MT_Statistics.h
View File

@@ -6,6 +6,8 @@
#include "audio/Audio_DataWindow.h"
#include "helper/HL_Optional.hpp"
#include "helper/HL_Statistics.h"
#include "jrtplib/src/rtptimeutilities.h"
#include "jrtplib/src/rtppacket.h"
@@ -13,78 +15,6 @@ using std::experimental::optional;
namespace MT
{
template<typename T>
struct Average
{
int mCount = 0;
T mSum = 0;
T average() const
{
if (!mCount)
return 0;
return mSum / mCount;
}
T value() const
{
return average();
}
void process(T value)
{
mCount++;
mSum += value;
}
};
template<typename T, int minimum = 100000, int maximum = 0, int default_value = 0>
struct TestResult
{
T mMin = minimum;
T mMax = maximum;
Average<T> mAverage;
T mCurrent = default_value;
void process(T value)
{
if (mMin > value)
mMin = value;
if (mMax < value)
mMax = value;
mCurrent = value;
mAverage.process(value);
}
bool is_initialized() const
{
return mAverage.mCount > 0;
}
T current() const
{
if (is_initialized())
return mCurrent;
else
return 0;
}
T value() const
{
return current();
}
TestResult<T>& operator = (T value)
{
process(value);
return *this;
}
operator T()
{
return mCurrent;
}
};
template<typename T>
struct StreamStats
@@ -130,9 +60,13 @@ public:
mDuplicatedRtp, // Number of received duplicated rtp packets
mOldRtp, // Number of late rtp packets
mPacketLoss, // Number of lost packets
mPacketDropped, // Number of dropped packets (due to time unsync when playing)
mPacketDropped, // Number of dropped packets (due to time unsync when playing)б
mIllegalRtp; // Number of rtp packets with bad payload type
TestResult<float> mDecodingInterval, // Average interval on call to packet decode
mDecodeRequested, // Average amount of requested audio frames to play
mPacketInterval; // Average interval between packet adding to jitter buffer
int mLoss[128]; // Every item is number of loss of corresping length
size_t mAudioTime; // Decoded/found time in milliseconds
uint16_t mSsrc; // Last known SSRC ID in a RTP stream
@@ -169,7 +103,7 @@ public:
std::string mPvqaReport;
#endif
std::string toShortString() const;
std::string toString() const;
};
} // end of namespace MT

2
src/engine/media/MT_Stream.cpp
View File

@@ -21,7 +21,7 @@ Stream::Stream()
Stream::~Stream()
{
ICELogInfo(<< mStat.toString());
}
void Stream::setDestination(const RtpPair<InternetAddress>& dest)