public
/
rtphone
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

- ongoing work to fix issue with unstable audio

This commit is contained in:
Dmytro Bogovych 2021-08-06 18:31:42 +03:00
parent ed39725641
commit 74b5aa69cb
18 changed files with 220 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/engine/audio/Audio_DataWindow.cpp
View File

@ -56,17 +56,25 @@ void DataWindow::add(const void* data, int length)
if (length > mCapacity)
{
// Use latest bytes from data buffer in this case.
data = (char*)data + length - mCapacity;
length = mCapacity;
}
// Check how much free space we have
int avail = mCapacity - mFilled;
if (avail < length)
{
memmove(mData, mData + length - avail, mFilled - (length - avail));
mFilled -= length - avail;
// Find the portion of data to move & save
int delta = length - avail;
// Move the data
if (mFilled - delta > 0)
memmove(mData, mData + delta, mFilled - delta);
mFilled -= delta;
}
memcpy(mData + mFilled, data, length);
mFilled += length;
}

2
src/engine/audio/Audio_DevicePair.cpp
View File

@ -197,7 +197,7 @@ void DevicePair::onSpkData(const Format& f, void* buffer, int length)
// Resample these 10 milliseconds it to native format
size_t wasProcessed = 0;
size_t wasProduced = mSpkResampler.resample(AUDIO_SAMPLERATE, mOutput10msBuffer.data(), mOutput10msBuffer.capacity(), wasProcessed, f.mRate,
size_t wasProduced = mSpkResampler.resample(nativeFormat.mRate, mOutput10msBuffer.data(), mOutput10msBuffer.capacity(), wasProcessed, f.mRate,
mOutputNativeData.mutableData() + mOutputNativeData.filled(), mOutputNativeData.capacity() - mOutputNativeData.filled());
mOutputNativeData.setFilled(mOutputNativeData.filled() + wasProduced);
#ifdef CONSOLE_LOGGING

46
src/engine/audio/Audio_Mixer.cpp
View File

@ -216,48 +216,52 @@ void Mixer::mix()
channelList[activeCounter++] = &mChannelList[i];
// No active channels - nothing to mix - exit
if (!activeCounter)
if (!activeCounter)
{
//ICELogDebug(<< "No active channel");
// ICELogDebug(<< "No active channel");
return;
}
// Optimized versions for 1& 2 active channels
if (activeCounter == 1)
{
// Copy much samples as we have
{
// Copy much samples as we have
Stream& audio = *channelList[0];
mOutput.add(audio.data().data(), audio.data().filled());
audio.data().erase(audio.data().filled());
// Copy the decoded data
mOutput.add(audio.data().data(), audio.data().filled());
// Erase copied audio samples
audio.data().erase(audio.data().filled());
//ICELogSpecial(<<"Length of mixer stream " << audio.data().filled());
}
}
else
if (activeCounter == 2)
{
if (activeCounter == 2)
{
Stream& audio1 = *channelList[0];
Stream& audio2 = *channelList[1];
int filled1 = audio1.data().filled() / 2, filled2 = audio2.data().filled() / 2;
Stream& audio2 = *channelList[1];
int filled1 = audio1.data().filled() / 2, filled2 = audio2.data().filled() / 2;
int available = filled1 > filled2 ? filled1 : filled2;
// Find how much samples can be mixed
int filled = mOutput.filled() / 2;
// Find how much samples can be mixed
int filled = mOutput.filled() / 2;
int maxsize = mOutput.capacity() / 2;
if (maxsize - filled < available)
available = maxsize - filled;
if (maxsize - filled < available)
available = maxsize - filled;
short sample = 0;
for (int i=0; i<available; i++)
{
short sample = 0;
for (int i=0; i<available; i++)
{
short sample1 = filled1 > i ? audio1.data().shortAt(i) : 0;
short sample2 = filled2 > i ? audio2.data().shortAt(i) : 0;
sample = (abs(sample1) > abs(sample2)) ? sample1 : sample2;
mOutput.add(sample);
}
audio1.data().erase(available*2);
audio2.data().erase(available*2);
}
audio1.data().erase(available*2);
audio2.data().erase(available*2);
}
else
{
do

14
src/engine/audio/Audio_Player.cpp
View File

@ -1,14 +1,18 @@
/* Copyright(C) 2007-2014 VoIP objects (voipobjects.com)
/* Copyright(C) 2007-2021 VoIP objects (voipobjects.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "Audio_Player.h"
#include "../helper/HL_Log.h"
#define LOG_SUBSYSTEM "Player"
using namespace Audio;
// -------------- Player -----------
Player::Player()
:mDelegate(NULL), mPlayedTime(0)
:mDelegate(nullptr), mPlayedTime(0)
{
}
@ -47,7 +51,7 @@ void Player::onMicData(const Format& f, const void* buffer, int length)
void Player::onSpkData(const Format& f, void* buffer, int length)
{
Lock l(mGuard);
// Fill buffer by zero if player owns dedicated device
if (mOutput)
memset(buffer, 0, length);
@ -99,7 +103,7 @@ void Player::onFilePlayed()
void Player::obtain(int usage)
{
Lock l(mGuard);
UsageMap::iterator usageIter = mUsage.find(usage);
auto usageIter = mUsage.find(usage);
if (usageIter == mUsage.end())
mUsage[usage] = 1;
else
@ -132,7 +136,7 @@ int Player::releasePlayed()
{
Lock l(mGuard);
int result = mFinishedUsages.size();
while (mFinishedUsages.size())
while (!mFinishedUsages.empty())
{
release(mFinishedUsages.front());
mFinishedUsages.erase(mFinishedUsages.begin());

8
src/engine/audio/Audio_Player.h
View File

@ -1,4 +1,4 @@
/* Copyright(C) 2007-2014 VoIP objects (voipobjects.com)
/* Copyright(C) 2007-2021 VoIP objects (voipobjects.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -8,6 +8,7 @@
#include "../helper/HL_Log.h"
#include "../helper/HL_Sync.h"
#include "../helper/HL_Statistics.h"
#include "Audio_Interface.h"
#include <deque>
#include <map>
@ -48,15 +49,18 @@ namespace Audio
void onMicData(const Format& f, const void* buffer, int length);
void onSpkData(const Format& f, void* buffer, int length);
void onFilePlayed();
void scheduleRelease();
void obtain(int usageId);
public:
Player();
~Player();
void setDelegate(EndOfAudioDelegate* d);
EndOfAudioDelegate* getDelegate() const;
void setOutput(POutputDevice output);
POutputDevice getOutput() const;
void add(int usageId, PWavFileReader file, bool loop, int timelength);
void release(int usageId);
void clear();

4
src/engine/config.h
View File

@ -17,7 +17,7 @@
#define AUDIO_CHANNELS 1
// Samplerate must be 8 / 16 / 24 / 32 / 48 KHz
#define AUDIO_SAMPLERATE 16000
#define AUDIO_SAMPLERATE 48000
#define AUDIO_MIC_BUFFER_COUNT 16
#define AUDIO_MIC_BUFFER_LENGTH 10
#define AUDIO_MIC_BUFFER_SIZE (AUDIO_MIC_BUFFER_LENGTH * AUDIO_SAMPLERATE / 1000 * 2 * AUDIO_CHANNELS)
@ -50,7 +50,7 @@
#define MT_MAXRTPPACKET 1500
#define MT_DTMF_END_PACKETS 3
#define RTP_BUFFER_HIGH 480
#define RTP_BUFFER_HIGH 24480
#define RTP_BUFFER_LOW 10
#define RTP_BUFFER_PREBUFFER 80
#define RTP_DECODED_CAPACITY 2048

1
src/engine/helper/HL_Statistics.cpp Normal file
View File

@ -0,0 +1 @@

82
src/engine/helper/HL_Statistics.h Normal file
View File

@ -0,0 +1,82 @@
#ifndef __HELPER_STATISTICS_H
#define __HELPER_STATISTICS_H
template<typename T>
struct Average
{
int mCount = 0;
T mSum = 0;
T average() const
{
if (!mCount)
return 0;
return mSum / mCount;
}
T value() const
{
return average();
}
void process(T value)
{
mCount++;
mSum += value;
}
};
template<typename T, int minimum = 100000, int maximum = 0, int default_value = 0>
struct TestResult
{
T mMin = minimum;
T mMax = maximum;
Average<T> mAverage;
T mCurrent = default_value;
void process(T value)
{
if (mMin > value)
mMin = value;
if (mMax < value)
mMax = value;
mCurrent = value;
mAverage.process(value);
}
bool is_initialized() const
{
return mAverage.mCount > 0;
}
T current() const
{
if (is_initialized())
return mCurrent;
else
return 0;
}
T value() const
{
return current();
}
T average() const
{
return mAverage.average();
}
TestResult<T>& operator = (T value)
{
process(value);
return *this;
}
operator T()
{
return mCurrent;
}
};
#endif

10
src/engine/helper/HL_Time.cpp Normal file
View File

@ -0,0 +1,10 @@
#include "HL_Time.h"
#include <time.h>
/* return current time in milliseconds */
double now_ms(void) {
struct timespec res;
clock_gettime(CLOCK_MONOTONIC, &res);
return 1000.0 * res.tv_sec + (double) res.tv_nsec / 1e6;
}

6
src/engine/helper/HL_Time.h Normal file
View File

@ -0,0 +1,6 @@
#ifndef __HELPER_TIME_H
#define __HELPER_TIME_H
extern double now_ms();
#endif

37
src/engine/media/MT_AudioReceiver.cpp
View File

@ -10,6 +10,7 @@
#include "MT_AudioCodec.h"
#include "MT_CngHelper.h"
#include "../helper/HL_Log.h"
#include "../helper/HL_Time.h"
#include "../audio/Audio_Interface.h"
#include "../audio/Audio_Resampler.h"
#include <cmath>
@ -108,7 +109,16 @@ bool RtpBuffer::add(std::shared_ptr<jrtplib::RTPPacket> packet, int timelength,
Lock l(mGuard);
// Update statistics
if (mLastAddTime == 0.0)
mLastAddTime = now_ms();
else
{
float t = now_ms();
mStat.mPacketInterval.process(t - mLastAddTime);
mLastAddTime = t;
}
mStat.mSsrc = static_cast<uint16_t>(packet->GetSSRC());
// Update jitter
@ -357,6 +367,7 @@ AudioReceiver::~AudioReceiver()
bool AudioReceiver::add(const std::shared_ptr<jrtplib::RTPPacket>& p, Codec** codec)
{
// ICELogInfo(<< "Adding packet No " << p->GetSequenceNumber());
// Increase codec counter
mStat.mCodecCount[p->GetPayloadType()]++;
@ -435,7 +446,7 @@ void AudioReceiver::processDecoded(Audio::DataWindow& output, int options)
bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
{
bool result = false;
bool result = false, had_cng = false, had_decode = false;
// Get next packet from buffer
RtpBuffer::ResultList rl;
@ -443,7 +454,7 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
switch (fr)
{
case RtpBuffer::FetchResult::Gap:
ICELogInfo(<< "Gap detected.");
ICELogDebug(<< "Gap detected.");
mDecodedLength = mResampledLength = 0;
if (mCngPacket && mCodec)
@ -571,9 +582,13 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
mDecodedLength = 0;
else
{
// Trigger the statistics
had_decode = true;
// Decode frame by frame
mDecodedLength = mCodec->decode(p->GetPayloadData() + i*mCodec->rtpLength(),
mDecodedLength = mCodec->decode(p->GetPayloadData() + i * mCodec->rtpLength(),
frameLength, mDecodedFrame, sizeof mDecodedFrame);
// mDecodedLength = 3840; // Opus 20 ms stereo
if (mDecodedLength)
processDecoded(output, options);
}
@ -594,6 +609,18 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
assert(0);
}
if (had_decode)
{
// mStat.mDecodeRequested++;
if (mLastDecodeTime == 0.0)
mLastDecodeTime = now_ms();
else
{
float t = now_ms();
mStat.mDecodingInterval.process(t - mLastDecodeTime);
mLastDecodeTime = t;
}
}
return result;
}
@ -670,12 +697,12 @@ void AudioReceiver::updatePvqa(const void *data, int size)
mPvqaBuffer->addZero(size);
Audio::Format fmt;
int frames = (int)fmt.timeFromSize(mPvqaBuffer->filled()) / (PVQA_INTERVAL * 1000);
int frames = static_cast<int>(fmt.timeFromSize(mPvqaBuffer->filled())) / (PVQA_INTERVAL * 1000);
if (frames > 0)
{
int time4pvqa = (int)(frames * PVQA_INTERVAL * 1000);
int size4pvqa = (int)fmt.sizeFromTime(time4pvqa);
ICELogInfo(<< "PVQA buffer has " << time4pvqa << " milliseconds of audio.");
ICELogDebug(<< "PVQA buffer has " << time4pvqa << " milliseconds of audio.");
mPVQA->update(mPvqaBuffer->data(), size4pvqa);
mPvqaBuffer->erase(size4pvqa);
}

14
src/engine/media/MT_AudioReceiver.h
View File

@ -60,16 +60,22 @@ namespace MT
unsigned ssrc();
void setSsrc(unsigned ssrc);
void setHigh(int milliseconds);
int high();
void setLow(int milliseconds);
int low();
void setPrebuffer(int milliseconds);
int prebuffer();
int getNumberOfReturnedPackets() const;
int getNumberOfAddPackets() const;
int findTimelength();
int getCount() const;
// Returns false if packet was not add - maybe too old or too new or duplicate
bool add(std::shared_ptr<RTPPacket> packet, int timelength, int rate);
@ -89,6 +95,9 @@ namespace MT
bool mFirstPacketWillGo;
jrtplib::RTPSourceStats mRtpStats;
Packet mFetchedPacket;
// To calculate average interval between packet add. It is close to jitter but more useful in debugging.
float mLastAddTime = 0.0;
};
class Receiver
@ -167,6 +176,11 @@ namespace MT
Audio::PWavFileWriter mDecodedDump;
float mLastDecodeTime = 0.0; // Time last call happened to codec->decode()
float mIntervalSum = 0.0;
int mIntervalCount = 0;
// Zero rate will make audio mono but resampling will be skipped
void makeMonoAndResample(int rate, int channels);

2
src/engine/media/MT_AudioStream.cpp
View File

@ -97,7 +97,7 @@ AudioStream::~AudioStream()
if (mFinalStatistics)
*mFinalStatistics = mStat;
ICELogInfo(<< mStat.toShortString());
ICELogInfo(<< mStat.toString());
}
void AudioStream::setDestination(const RtpPair<InternetAddress>& dest)

4
src/engine/media/MT_CodecList.cpp
View File

@ -126,14 +126,14 @@ CodecList::CodecList(const Settings& settings)
#endif
#endif
mFactoryList.push_back(new IsacCodec::IsacFactory16K(mSettings.mIsac16KPayloadType));
/*mFactoryList.push_back(new IsacCodec::IsacFactory16K(mSettings.mIsac16KPayloadType));
mFactoryList.push_back(new IlbcCodec::IlbcFactory(mSettings.mIlbc20PayloadType, mSettings.mIlbc30PayloadType));
mFactoryList.push_back(new G711Codec::AlawFactory());
mFactoryList.push_back(new G711Codec::UlawFactory());
mFactoryList.push_back(new GsmCodec::GsmFactory(mSettings.mGsmFrPayloadLength == 32 ? GsmCodec::Type::Bytes_32 : GsmCodec::Type::Bytes_33, mSettings.mGsmFrPayloadType));
mFactoryList.push_back(new G722Codec::G722Factory());
mFactoryList.push_back(new G729Codec::G729Factory());
mFactoryList.push_back(new G729Codec::G729Factory()); */
#ifndef TARGET_ANDROID
mFactoryList.push_back(new GsmHrCodec::GsmHrFactory(mSettings.mGsmHrPayloadType));
#endif

1
src/engine/media/MT_SingleAudioStream.cpp
View File

@ -19,6 +19,7 @@ SingleAudioStream::SingleAudioStream(const CodecList::Settings& codecSettings, S
SingleAudioStream::~SingleAudioStream()
{
}
void SingleAudioStream::process(const std::shared_ptr<jrtplib::RTPPacket>& packet)

10
src/engine/media/MT_Statistics.cpp
View File

@ -191,6 +191,9 @@ Statistics& Statistics::operator += (const Statistics& src)
mJitter = src.mJitter;
mRttDelay = src.mRttDelay;
mDecodingInterval = src.mDecodingInterval;
mDecodeRequested = src.mDecodeRequested;
if (!src.mCodecName.empty())
mCodecName = src.mCodecName;
@ -239,13 +242,16 @@ Statistics& Statistics::operator -= (const Statistics& src)
}
std::string Statistics::toShortString() const
std::string Statistics::toString() const
{
std::ostringstream oss;
oss << "Received: " << mReceivedRtp
<< ", lost: " << mPacketLoss
<< ", dropped: " << mPacketDropped
<< ", sent: " << mSentRtp;
<< ", sent: " << mSentRtp
<< ", decoding interval: " << mDecodingInterval.average()
<< ", decode requested: " << mDecodeRequested.average()
<< ", packet interval: " << mPacketInterval.average();
return oss.str();
}

82
src/engine/media/MT_Statistics.h
View File

@ -6,6 +6,8 @@
#include "audio/Audio_DataWindow.h"
#include "helper/HL_Optional.hpp"
#include "helper/HL_Statistics.h"
#include "jrtplib/src/rtptimeutilities.h"
#include "jrtplib/src/rtppacket.h"
@ -13,78 +15,6 @@ using std::experimental::optional;
namespace MT
{
template<typename T>
struct Average
{
int mCount = 0;
T mSum = 0;
T average() const
{
if (!mCount)
return 0;
return mSum / mCount;
}
T value() const
{
return average();
}
void process(T value)
{
mCount++;
mSum += value;
}
};
template<typename T, int minimum = 100000, int maximum = 0, int default_value = 0>
struct TestResult
{
T mMin = minimum;
T mMax = maximum;
Average<T> mAverage;
T mCurrent = default_value;
void process(T value)
{
if (mMin > value)
mMin = value;
if (mMax < value)
mMax = value;
mCurrent = value;
mAverage.process(value);
}
bool is_initialized() const
{
return mAverage.mCount > 0;
}
T current() const
{
if (is_initialized())
return mCurrent;
else
return 0;
}
T value() const
{
return current();
}
TestResult<T>& operator = (T value)
{
process(value);
return *this;
}
operator T()
{
return mCurrent;
}
};
template<typename T>
struct StreamStats
@ -130,9 +60,13 @@ public:
mDuplicatedRtp, // Number of received duplicated rtp packets
mOldRtp, // Number of late rtp packets
mPacketLoss, // Number of lost packets
mPacketDropped, // Number of dropped packets (due to time unsync when playing)
mPacketDropped, // Number of dropped packets (due to time unsync when playing)б
mIllegalRtp; // Number of rtp packets with bad payload type
TestResult<float> mDecodingInterval, // Average interval on call to packet decode
mDecodeRequested, // Average amount of requested audio frames to play
mPacketInterval; // Average interval between packet adding to jitter buffer
int mLoss[128]; // Every item is number of loss of corresping length
size_t mAudioTime; // Decoded/found time in milliseconds
uint16_t mSsrc; // Last known SSRC ID in a RTP stream
@ -169,7 +103,7 @@ public:
std::string mPvqaReport;
#endif
std::string toShortString() const;
std::string toString() const;
};
} // end of namespace MT

2
src/engine/media/MT_Stream.cpp
View File

@ -21,7 +21,7 @@ Stream::Stream()
Stream::~Stream()
{
ICELogInfo(<< mStat.toString());
}
void Stream::setDestination(const RtpPair<InternetAddress>& dest)