diff --git a/src/engine/audio/Audio_DataWindow.cpp b/src/engine/audio/Audio_DataWindow.cpp index 1b0c1f2a..6044ff78 100644 --- a/src/engine/audio/Audio_DataWindow.cpp +++ b/src/engine/audio/Audio_DataWindow.cpp @@ -56,17 +56,25 @@ void DataWindow::add(const void* data, int length) if (length > mCapacity) { + // Use latest bytes from data buffer in this case. data = (char*)data + length - mCapacity; length = mCapacity; } + // Check how much free space we have int avail = mCapacity - mFilled; - + if (avail < length) { - memmove(mData, mData + length - avail, mFilled - (length - avail)); - mFilled -= length - avail; + // Find the portion of data to move & save + int delta = length - avail; + + // Move the data + if (mFilled - delta > 0) + memmove(mData, mData + delta, mFilled - delta); + mFilled -= delta; } + memcpy(mData + mFilled, data, length); mFilled += length; } diff --git a/src/engine/audio/Audio_DevicePair.cpp b/src/engine/audio/Audio_DevicePair.cpp index 45c2bc65..2b46a4f2 100644 --- a/src/engine/audio/Audio_DevicePair.cpp +++ b/src/engine/audio/Audio_DevicePair.cpp @@ -197,7 +197,7 @@ void DevicePair::onSpkData(const Format& f, void* buffer, int length) // Resample these 10 milliseconds it to native format size_t wasProcessed = 0; - size_t wasProduced = mSpkResampler.resample(AUDIO_SAMPLERATE, mOutput10msBuffer.data(), mOutput10msBuffer.capacity(), wasProcessed, f.mRate, + size_t wasProduced = mSpkResampler.resample(nativeFormat.mRate, mOutput10msBuffer.data(), mOutput10msBuffer.capacity(), wasProcessed, f.mRate, mOutputNativeData.mutableData() + mOutputNativeData.filled(), mOutputNativeData.capacity() - mOutputNativeData.filled()); mOutputNativeData.setFilled(mOutputNativeData.filled() + wasProduced); #ifdef CONSOLE_LOGGING diff --git a/src/engine/audio/Audio_Mixer.cpp b/src/engine/audio/Audio_Mixer.cpp index 5e4aca7c..d6274206 100644 --- a/src/engine/audio/Audio_Mixer.cpp +++ b/src/engine/audio/Audio_Mixer.cpp @@ -216,48 +216,52 @@ void Mixer::mix() channelList[activeCounter++] = &mChannelList[i]; // No active channels - nothing to mix - exit - if (!activeCounter) + if (!activeCounter) { - //ICELogDebug(<< "No active channel"); + // ICELogDebug(<< "No active channel"); return; } // Optimized versions for 1& 2 active channels if (activeCounter == 1) - { - // Copy much samples as we have + { + // Copy much samples as we have Stream& audio = *channelList[0]; - mOutput.add(audio.data().data(), audio.data().filled()); - audio.data().erase(audio.data().filled()); + + // Copy the decoded data + mOutput.add(audio.data().data(), audio.data().filled()); + + // Erase copied audio samples + audio.data().erase(audio.data().filled()); //ICELogSpecial(<<"Length of mixer stream " << audio.data().filled()); - } + } else - if (activeCounter == 2) - { + if (activeCounter == 2) + { Stream& audio1 = *channelList[0]; - Stream& audio2 = *channelList[1]; - int filled1 = audio1.data().filled() / 2, filled2 = audio2.data().filled() / 2; + Stream& audio2 = *channelList[1]; + int filled1 = audio1.data().filled() / 2, filled2 = audio2.data().filled() / 2; int available = filled1 > filled2 ? filled1 : filled2; - // Find how much samples can be mixed - int filled = mOutput.filled() / 2; + // Find how much samples can be mixed + int filled = mOutput.filled() / 2; int maxsize = mOutput.capacity() / 2; - if (maxsize - filled < available) - available = maxsize - filled; + if (maxsize - filled < available) + available = maxsize - filled; - short sample = 0; - for (int i=0; i i ? audio1.data().shortAt(i) : 0; short sample2 = filled2 > i ? audio2.data().shortAt(i) : 0; sample = (abs(sample1) > abs(sample2)) ? sample1 : sample2; mOutput.add(sample); - } - audio1.data().erase(available*2); - audio2.data().erase(available*2); } + audio1.data().erase(available*2); + audio2.data().erase(available*2); + } else { do diff --git a/src/engine/audio/Audio_Player.cpp b/src/engine/audio/Audio_Player.cpp index cdb1952d..7aaa22ee 100644 --- a/src/engine/audio/Audio_Player.cpp +++ b/src/engine/audio/Audio_Player.cpp @@ -1,14 +1,18 @@ -/* Copyright(C) 2007-2014 VoIP objects (voipobjects.com) +/* Copyright(C) 2007-2021 VoIP objects (voipobjects.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "Audio_Player.h" +#include "../helper/HL_Log.h" + +#define LOG_SUBSYSTEM "Player" + using namespace Audio; // -------------- Player ----------- Player::Player() -:mDelegate(NULL), mPlayedTime(0) +:mDelegate(nullptr), mPlayedTime(0) { } @@ -47,7 +51,7 @@ void Player::onMicData(const Format& f, const void* buffer, int length) void Player::onSpkData(const Format& f, void* buffer, int length) { Lock l(mGuard); - + // Fill buffer by zero if player owns dedicated device if (mOutput) memset(buffer, 0, length); @@ -99,7 +103,7 @@ void Player::onFilePlayed() void Player::obtain(int usage) { Lock l(mGuard); - UsageMap::iterator usageIter = mUsage.find(usage); + auto usageIter = mUsage.find(usage); if (usageIter == mUsage.end()) mUsage[usage] = 1; else @@ -132,7 +136,7 @@ int Player::releasePlayed() { Lock l(mGuard); int result = mFinishedUsages.size(); - while (mFinishedUsages.size()) + while (!mFinishedUsages.empty()) { release(mFinishedUsages.front()); mFinishedUsages.erase(mFinishedUsages.begin()); diff --git a/src/engine/audio/Audio_Player.h b/src/engine/audio/Audio_Player.h index 614855af..82656d8c 100644 --- a/src/engine/audio/Audio_Player.h +++ b/src/engine/audio/Audio_Player.h @@ -1,4 +1,4 @@ -/* Copyright(C) 2007-2014 VoIP objects (voipobjects.com) +/* Copyright(C) 2007-2021 VoIP objects (voipobjects.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -8,6 +8,7 @@ #include "../helper/HL_Log.h" #include "../helper/HL_Sync.h" +#include "../helper/HL_Statistics.h" #include "Audio_Interface.h" #include #include @@ -48,15 +49,18 @@ namespace Audio void onMicData(const Format& f, const void* buffer, int length); void onSpkData(const Format& f, void* buffer, int length); void onFilePlayed(); - void scheduleRelease(); void obtain(int usageId); + public: Player(); ~Player(); + void setDelegate(EndOfAudioDelegate* d); EndOfAudioDelegate* getDelegate() const; + void setOutput(POutputDevice output); POutputDevice getOutput() const; + void add(int usageId, PWavFileReader file, bool loop, int timelength); void release(int usageId); void clear(); diff --git a/src/engine/config.h b/src/engine/config.h index b1390648..f6c4de0f 100644 --- a/src/engine/config.h +++ b/src/engine/config.h @@ -17,7 +17,7 @@ #define AUDIO_CHANNELS 1 // Samplerate must be 8 / 16 / 24 / 32 / 48 KHz -#define AUDIO_SAMPLERATE 16000 +#define AUDIO_SAMPLERATE 48000 #define AUDIO_MIC_BUFFER_COUNT 16 #define AUDIO_MIC_BUFFER_LENGTH 10 #define AUDIO_MIC_BUFFER_SIZE (AUDIO_MIC_BUFFER_LENGTH * AUDIO_SAMPLERATE / 1000 * 2 * AUDIO_CHANNELS) @@ -50,7 +50,7 @@ #define MT_MAXRTPPACKET 1500 #define MT_DTMF_END_PACKETS 3 -#define RTP_BUFFER_HIGH 480 +#define RTP_BUFFER_HIGH 24480 #define RTP_BUFFER_LOW 10 #define RTP_BUFFER_PREBUFFER 80 #define RTP_DECODED_CAPACITY 2048 diff --git a/src/engine/helper/HL_Statistics.cpp b/src/engine/helper/HL_Statistics.cpp new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/engine/helper/HL_Statistics.cpp @@ -0,0 +1 @@ + diff --git a/src/engine/helper/HL_Statistics.h b/src/engine/helper/HL_Statistics.h new file mode 100644 index 00000000..a9f29cef --- /dev/null +++ b/src/engine/helper/HL_Statistics.h @@ -0,0 +1,82 @@ +#ifndef __HELPER_STATISTICS_H +#define __HELPER_STATISTICS_H + +template +struct Average +{ + int mCount = 0; + T mSum = 0; + T average() const + { + if (!mCount) + return 0; + return mSum / mCount; + } + + T value() const + { + return average(); + } + + void process(T value) + { + mCount++; + mSum += value; + } +}; + +template +struct TestResult +{ + T mMin = minimum; + T mMax = maximum; + Average mAverage; + T mCurrent = default_value; + + void process(T value) + { + if (mMin > value) + mMin = value; + if (mMax < value) + mMax = value; + mCurrent = value; + mAverage.process(value); + } + + bool is_initialized() const + { + return mAverage.mCount > 0; + } + + T current() const + { + if (is_initialized()) + return mCurrent; + else + return 0; + } + + T value() const + { + return current(); + } + + T average() const + { + return mAverage.average(); + } + + TestResult& operator = (T value) + { + process(value); + return *this; + } + + operator T() + { + return mCurrent; + } +}; + + +#endif \ No newline at end of file diff --git a/src/engine/helper/HL_Time.cpp b/src/engine/helper/HL_Time.cpp new file mode 100644 index 00000000..9bfa995f --- /dev/null +++ b/src/engine/helper/HL_Time.cpp @@ -0,0 +1,10 @@ +#include "HL_Time.h" + +#include + +/* return current time in milliseconds */ +double now_ms(void) { + struct timespec res; + clock_gettime(CLOCK_MONOTONIC, &res); + return 1000.0 * res.tv_sec + (double) res.tv_nsec / 1e6; +} \ No newline at end of file diff --git a/src/engine/helper/HL_Time.h b/src/engine/helper/HL_Time.h new file mode 100644 index 00000000..639dd49f --- /dev/null +++ b/src/engine/helper/HL_Time.h @@ -0,0 +1,6 @@ +#ifndef __HELPER_TIME_H +#define __HELPER_TIME_H + +extern double now_ms(); + +#endif diff --git a/src/engine/media/MT_AudioReceiver.cpp b/src/engine/media/MT_AudioReceiver.cpp index a40c63df..7331a1c7 100644 --- a/src/engine/media/MT_AudioReceiver.cpp +++ b/src/engine/media/MT_AudioReceiver.cpp @@ -10,6 +10,7 @@ #include "MT_AudioCodec.h" #include "MT_CngHelper.h" #include "../helper/HL_Log.h" +#include "../helper/HL_Time.h" #include "../audio/Audio_Interface.h" #include "../audio/Audio_Resampler.h" #include @@ -108,7 +109,16 @@ bool RtpBuffer::add(std::shared_ptr packet, int timelength, Lock l(mGuard); + // Update statistics + if (mLastAddTime == 0.0) + mLastAddTime = now_ms(); + else + { + float t = now_ms(); + mStat.mPacketInterval.process(t - mLastAddTime); + mLastAddTime = t; + } mStat.mSsrc = static_cast(packet->GetSSRC()); // Update jitter @@ -357,6 +367,7 @@ AudioReceiver::~AudioReceiver() bool AudioReceiver::add(const std::shared_ptr& p, Codec** codec) { + // ICELogInfo(<< "Adding packet No " << p->GetSequenceNumber()); // Increase codec counter mStat.mCodecCount[p->GetPayloadType()]++; @@ -435,7 +446,7 @@ void AudioReceiver::processDecoded(Audio::DataWindow& output, int options) bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate) { - bool result = false; + bool result = false, had_cng = false, had_decode = false; // Get next packet from buffer RtpBuffer::ResultList rl; @@ -443,7 +454,7 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate) switch (fr) { case RtpBuffer::FetchResult::Gap: - ICELogInfo(<< "Gap detected."); + ICELogDebug(<< "Gap detected."); mDecodedLength = mResampledLength = 0; if (mCngPacket && mCodec) @@ -571,9 +582,13 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate) mDecodedLength = 0; else { + // Trigger the statistics + had_decode = true; + // Decode frame by frame - mDecodedLength = mCodec->decode(p->GetPayloadData() + i*mCodec->rtpLength(), + mDecodedLength = mCodec->decode(p->GetPayloadData() + i * mCodec->rtpLength(), frameLength, mDecodedFrame, sizeof mDecodedFrame); + // mDecodedLength = 3840; // Opus 20 ms stereo if (mDecodedLength) processDecoded(output, options); } @@ -594,6 +609,18 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate) assert(0); } + if (had_decode) + { + // mStat.mDecodeRequested++; + if (mLastDecodeTime == 0.0) + mLastDecodeTime = now_ms(); + else + { + float t = now_ms(); + mStat.mDecodingInterval.process(t - mLastDecodeTime); + mLastDecodeTime = t; + } + } return result; } @@ -670,12 +697,12 @@ void AudioReceiver::updatePvqa(const void *data, int size) mPvqaBuffer->addZero(size); Audio::Format fmt; - int frames = (int)fmt.timeFromSize(mPvqaBuffer->filled()) / (PVQA_INTERVAL * 1000); + int frames = static_cast(fmt.timeFromSize(mPvqaBuffer->filled())) / (PVQA_INTERVAL * 1000); if (frames > 0) { int time4pvqa = (int)(frames * PVQA_INTERVAL * 1000); int size4pvqa = (int)fmt.sizeFromTime(time4pvqa); - ICELogInfo(<< "PVQA buffer has " << time4pvqa << " milliseconds of audio."); + ICELogDebug(<< "PVQA buffer has " << time4pvqa << " milliseconds of audio."); mPVQA->update(mPvqaBuffer->data(), size4pvqa); mPvqaBuffer->erase(size4pvqa); } diff --git a/src/engine/media/MT_AudioReceiver.h b/src/engine/media/MT_AudioReceiver.h index 1f1cece3..f55e027c 100644 --- a/src/engine/media/MT_AudioReceiver.h +++ b/src/engine/media/MT_AudioReceiver.h @@ -60,16 +60,22 @@ namespace MT unsigned ssrc(); void setSsrc(unsigned ssrc); + void setHigh(int milliseconds); int high(); + void setLow(int milliseconds); int low(); + void setPrebuffer(int milliseconds); int prebuffer(); + int getNumberOfReturnedPackets() const; int getNumberOfAddPackets() const; + int findTimelength(); int getCount() const; + // Returns false if packet was not add - maybe too old or too new or duplicate bool add(std::shared_ptr packet, int timelength, int rate); @@ -89,6 +95,9 @@ namespace MT bool mFirstPacketWillGo; jrtplib::RTPSourceStats mRtpStats; Packet mFetchedPacket; + + // To calculate average interval between packet add. It is close to jitter but more useful in debugging. + float mLastAddTime = 0.0; }; class Receiver @@ -167,6 +176,11 @@ namespace MT Audio::PWavFileWriter mDecodedDump; + float mLastDecodeTime = 0.0; // Time last call happened to codec->decode() + + float mIntervalSum = 0.0; + int mIntervalCount = 0; + // Zero rate will make audio mono but resampling will be skipped void makeMonoAndResample(int rate, int channels); diff --git a/src/engine/media/MT_AudioStream.cpp b/src/engine/media/MT_AudioStream.cpp index 3f14e31a..32fa0d9d 100644 --- a/src/engine/media/MT_AudioStream.cpp +++ b/src/engine/media/MT_AudioStream.cpp @@ -97,7 +97,7 @@ AudioStream::~AudioStream() if (mFinalStatistics) *mFinalStatistics = mStat; - ICELogInfo(<< mStat.toShortString()); + ICELogInfo(<< mStat.toString()); } void AudioStream::setDestination(const RtpPair& dest) diff --git a/src/engine/media/MT_CodecList.cpp b/src/engine/media/MT_CodecList.cpp index e9adcc70..046e17d8 100644 --- a/src/engine/media/MT_CodecList.cpp +++ b/src/engine/media/MT_CodecList.cpp @@ -126,14 +126,14 @@ CodecList::CodecList(const Settings& settings) #endif #endif - mFactoryList.push_back(new IsacCodec::IsacFactory16K(mSettings.mIsac16KPayloadType)); + /*mFactoryList.push_back(new IsacCodec::IsacFactory16K(mSettings.mIsac16KPayloadType)); mFactoryList.push_back(new IlbcCodec::IlbcFactory(mSettings.mIlbc20PayloadType, mSettings.mIlbc30PayloadType)); mFactoryList.push_back(new G711Codec::AlawFactory()); mFactoryList.push_back(new G711Codec::UlawFactory()); mFactoryList.push_back(new GsmCodec::GsmFactory(mSettings.mGsmFrPayloadLength == 32 ? GsmCodec::Type::Bytes_32 : GsmCodec::Type::Bytes_33, mSettings.mGsmFrPayloadType)); mFactoryList.push_back(new G722Codec::G722Factory()); - mFactoryList.push_back(new G729Codec::G729Factory()); + mFactoryList.push_back(new G729Codec::G729Factory()); */ #ifndef TARGET_ANDROID mFactoryList.push_back(new GsmHrCodec::GsmHrFactory(mSettings.mGsmHrPayloadType)); #endif diff --git a/src/engine/media/MT_SingleAudioStream.cpp b/src/engine/media/MT_SingleAudioStream.cpp index 4b4273b0..6754b8cf 100644 --- a/src/engine/media/MT_SingleAudioStream.cpp +++ b/src/engine/media/MT_SingleAudioStream.cpp @@ -19,6 +19,7 @@ SingleAudioStream::SingleAudioStream(const CodecList::Settings& codecSettings, S SingleAudioStream::~SingleAudioStream() { + } void SingleAudioStream::process(const std::shared_ptr& packet) diff --git a/src/engine/media/MT_Statistics.cpp b/src/engine/media/MT_Statistics.cpp index aa0c3c9d..acad1e07 100644 --- a/src/engine/media/MT_Statistics.cpp +++ b/src/engine/media/MT_Statistics.cpp @@ -191,6 +191,9 @@ Statistics& Statistics::operator += (const Statistics& src) mJitter = src.mJitter; mRttDelay = src.mRttDelay; + mDecodingInterval = src.mDecodingInterval; + mDecodeRequested = src.mDecodeRequested; + if (!src.mCodecName.empty()) mCodecName = src.mCodecName; @@ -239,13 +242,16 @@ Statistics& Statistics::operator -= (const Statistics& src) } -std::string Statistics::toShortString() const +std::string Statistics::toString() const { std::ostringstream oss; oss << "Received: " << mReceivedRtp << ", lost: " << mPacketLoss << ", dropped: " << mPacketDropped - << ", sent: " << mSentRtp; + << ", sent: " << mSentRtp + << ", decoding interval: " << mDecodingInterval.average() + << ", decode requested: " << mDecodeRequested.average() + << ", packet interval: " << mPacketInterval.average(); return oss.str(); } diff --git a/src/engine/media/MT_Statistics.h b/src/engine/media/MT_Statistics.h index e7d10f08..b5c054ac 100644 --- a/src/engine/media/MT_Statistics.h +++ b/src/engine/media/MT_Statistics.h @@ -6,6 +6,8 @@ #include "audio/Audio_DataWindow.h" #include "helper/HL_Optional.hpp" +#include "helper/HL_Statistics.h" + #include "jrtplib/src/rtptimeutilities.h" #include "jrtplib/src/rtppacket.h" @@ -13,78 +15,6 @@ using std::experimental::optional; namespace MT { -template -struct Average -{ - int mCount = 0; - T mSum = 0; - T average() const - { - if (!mCount) - return 0; - return mSum / mCount; - } - - T value() const - { - return average(); - } - - void process(T value) - { - mCount++; - mSum += value; - } -}; - -template -struct TestResult -{ - T mMin = minimum; - T mMax = maximum; - Average mAverage; - T mCurrent = default_value; - - void process(T value) - { - if (mMin > value) - mMin = value; - if (mMax < value) - mMax = value; - mCurrent = value; - mAverage.process(value); - } - - bool is_initialized() const - { - return mAverage.mCount > 0; - } - - T current() const - { - if (is_initialized()) - return mCurrent; - else - return 0; - } - - T value() const - { - return current(); - } - - TestResult& operator = (T value) - { - process(value); - return *this; - } - - operator T() - { - return mCurrent; - } -}; - template struct StreamStats @@ -130,9 +60,13 @@ public: mDuplicatedRtp, // Number of received duplicated rtp packets mOldRtp, // Number of late rtp packets mPacketLoss, // Number of lost packets - mPacketDropped, // Number of dropped packets (due to time unsync when playing) + mPacketDropped, // Number of dropped packets (due to time unsync when playing)б mIllegalRtp; // Number of rtp packets with bad payload type + TestResult mDecodingInterval, // Average interval on call to packet decode + mDecodeRequested, // Average amount of requested audio frames to play + mPacketInterval; // Average interval between packet adding to jitter buffer + int mLoss[128]; // Every item is number of loss of corresping length size_t mAudioTime; // Decoded/found time in milliseconds uint16_t mSsrc; // Last known SSRC ID in a RTP stream @@ -169,7 +103,7 @@ public: std::string mPvqaReport; #endif - std::string toShortString() const; + std::string toString() const; }; } // end of namespace MT diff --git a/src/engine/media/MT_Stream.cpp b/src/engine/media/MT_Stream.cpp index 68fee660..05a5350c 100644 --- a/src/engine/media/MT_Stream.cpp +++ b/src/engine/media/MT_Stream.cpp @@ -21,7 +21,7 @@ Stream::Stream() Stream::~Stream() { - + ICELogInfo(<< mStat.toString()); } void Stream::setDestination(const RtpPair& dest)