- ongoing work to fix issue with unstable audio

src/engine/audio/Audio_DataWindow.cpp

@@ -56,17 +56,25 @@ void DataWindow::add(const void* data, int length)
   
   if (length > mCapacity)
   {
+    // Use latest bytes from data buffer in this case.
     data = (char*)data + length - mCapacity;
     length = mCapacity;
   }
 
+  // Check how much free space we have
   int avail = mCapacity - mFilled;
-  
+
   if (avail < length)
   {
-    memmove(mData, mData + length - avail, mFilled - (length - avail));
+    // Find the portion of data to move & save
-    mFilled -= length - avail;
+    int delta = length - avail;
+
+    // Move the data
+    if (mFilled - delta > 0)
+      memmove(mData, mData + delta, mFilled - delta);
+    mFilled -= delta;
   }
+
   memcpy(mData + mFilled, data, length);
   mFilled += length;
 }

src/engine/audio/Audio_DevicePair.cpp

@@ -197,7 +197,7 @@ void DevicePair::onSpkData(const Format& f, void* buffer, int length)
 
       // Resample these 10 milliseconds it to native format
       size_t wasProcessed = 0;
-      size_t wasProduced = mSpkResampler.resample(AUDIO_SAMPLERATE, mOutput10msBuffer.data(), mOutput10msBuffer.capacity(), wasProcessed, f.mRate,
+      size_t wasProduced = mSpkResampler.resample(nativeFormat.mRate, mOutput10msBuffer.data(), mOutput10msBuffer.capacity(), wasProcessed, f.mRate,
                                                mOutputNativeData.mutableData() + mOutputNativeData.filled(), mOutputNativeData.capacity() - mOutputNativeData.filled());
       mOutputNativeData.setFilled(mOutputNativeData.filled() + wasProduced);
 #ifdef CONSOLE_LOGGING

src/engine/audio/Audio_Mixer.cpp

@@ -216,48 +216,52 @@ void Mixer::mix()
 			channelList[activeCounter++] = &mChannelList[i];
 	
   // No active channels - nothing to mix - exit
-	if (!activeCounter)
+  if (!activeCounter)
   {
-    //ICELogDebug(<< "No active channel");
+    // ICELogDebug(<< "No active channel");
     return;
   }
 	
 	// Optimized versions for 1&  2  active channels
   if (activeCounter == 1)
-	{
+  {
-		// Copy much samples as we have
+    // Copy much samples as we have
     Stream& audio = *channelList[0];
-		mOutput.add(audio.data().data(), audio.data().filled());
+
-		audio.data().erase(audio.data().filled());
+    // Copy the decoded data
+    mOutput.add(audio.data().data(), audio.data().filled());
+
+	// Erase copied audio samples
+	audio.data().erase(audio.data().filled());
     //ICELogSpecial(<<"Length of mixer stream " << audio.data().filled());
-	}
+  }
   else	
-	if (activeCounter == 2)
+  if (activeCounter == 2)
-	{
+  {
     Stream& audio1 = *channelList[0];
-		Stream& audio2 = *channelList[1];
+	Stream& audio2 = *channelList[1];
-		int filled1 = audio1.data().filled() / 2, filled2 = audio2.data().filled() / 2;
+	int filled1 = audio1.data().filled() / 2, filled2 = audio2.data().filled() / 2;
     int available = filled1 > filled2 ? filled1 : filled2;
 		
-		// Find how much samples can be mixed
+	// Find how much samples can be mixed
-		int filled = mOutput.filled() / 2;
+	int filled = mOutput.filled() / 2;
 		
     int maxsize = mOutput.capacity() / 2;
-		if (maxsize - filled < available)
+	if (maxsize - filled < available)
-			available = maxsize - filled;
+      available = maxsize - filled;
 		
-		short sample = 0;
+	short sample = 0;
-		for (int i=0; i<available; i++)
+	for (int i=0; i<available; i++)
-		{
+	{
       short sample1 = filled1 > i ? audio1.data().shortAt(i) : 0;
       short sample2 = filled2 > i ? audio2.data().shortAt(i) : 0;
       sample = (abs(sample1) > abs(sample2)) ? sample1 : sample2;
 
       mOutput.add(sample);
-		}
-		audio1.data().erase(available*2);
-		audio2.data().erase(available*2);
 	}
+    audio1.data().erase(available*2);
+	audio2.data().erase(available*2);
+  }
   else
   {
     do

src/engine/audio/Audio_Player.cpp

@@ -1,14 +1,18 @@
-/* Copyright(C) 2007-2014 VoIP objects (voipobjects.com)
+/* Copyright(C) 2007-2021 VoIP objects (voipobjects.com)
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "Audio_Player.h"
 
+#include "../helper/HL_Log.h"
+
+#define LOG_SUBSYSTEM "Player"
+
 using namespace Audio;
 // -------------- Player -----------
 Player::Player()
-:mDelegate(NULL), mPlayedTime(0)
+:mDelegate(nullptr), mPlayedTime(0)
 {
 }
 
@@ -47,7 +51,7 @@ void Player::onMicData(const Format& f, const void* buffer, int length)
 void Player::onSpkData(const Format& f, void* buffer, int length)
 {
   Lock l(mGuard);
-  
+
   // Fill buffer by zero if player owns dedicated device
   if (mOutput)
     memset(buffer, 0, length);
@@ -99,7 +103,7 @@ void Player::onFilePlayed()
 void Player::obtain(int usage)
 {
   Lock l(mGuard);
-  UsageMap::iterator usageIter = mUsage.find(usage);
+  auto usageIter = mUsage.find(usage);
   if (usageIter == mUsage.end())
     mUsage[usage] = 1;
   else
@@ -132,7 +136,7 @@ int Player::releasePlayed()
 {
   Lock l(mGuard);
   int result = mFinishedUsages.size();
-  while (mFinishedUsages.size())
+  while (!mFinishedUsages.empty())
   {
     release(mFinishedUsages.front());
     mFinishedUsages.erase(mFinishedUsages.begin());

src/engine/audio/Audio_Player.h

@@ -1,4 +1,4 @@
-/* Copyright(C) 2007-2014 VoIP objects (voipobjects.com)
+/* Copyright(C) 2007-2021 VoIP objects (voipobjects.com)
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@@ -8,6 +8,7 @@
 
 #include "../helper/HL_Log.h"
 #include "../helper/HL_Sync.h"
+#include "../helper/HL_Statistics.h"
 #include "Audio_Interface.h"
 #include <deque>
 #include <map>
@@ -48,15 +49,18 @@ namespace Audio
     void onMicData(const Format& f, const void* buffer, int length);
     void onSpkData(const Format& f, void* buffer, int length);
     void onFilePlayed();
-    void scheduleRelease();
     void obtain(int usageId);
+
   public:
     Player();
     ~Player();
+
     void setDelegate(EndOfAudioDelegate* d);
     EndOfAudioDelegate* getDelegate() const;
+
     void setOutput(POutputDevice output);
     POutputDevice getOutput() const;
+
     void add(int usageId, PWavFileReader file, bool loop, int timelength);
     void release(int usageId);
     void clear();

src/engine/config.h

@@ -17,7 +17,7 @@
 #define AUDIO_CHANNELS 1
 
 // Samplerate must be 8 / 16 / 24 / 32 / 48 KHz
-#define AUDIO_SAMPLERATE 16000
+#define AUDIO_SAMPLERATE 48000
 #define AUDIO_MIC_BUFFER_COUNT 16
 #define AUDIO_MIC_BUFFER_LENGTH 10
 #define AUDIO_MIC_BUFFER_SIZE  (AUDIO_MIC_BUFFER_LENGTH * AUDIO_SAMPLERATE / 1000 * 2 * AUDIO_CHANNELS)
@@ -50,7 +50,7 @@
 #define MT_MAXRTPPACKET 1500
 #define MT_DTMF_END_PACKETS 3
 
-#define RTP_BUFFER_HIGH 480
+#define RTP_BUFFER_HIGH 24480
 #define RTP_BUFFER_LOW 10
 #define RTP_BUFFER_PREBUFFER 80
 #define RTP_DECODED_CAPACITY 2048

src/engine/helper/HL_Statistics.cpp

@@ -0,0 +1 @@
+

src/engine/helper/HL_Statistics.h

@@ -0,0 +1,82 @@
+#ifndef __HELPER_STATISTICS_H
+#define __HELPER_STATISTICS_H
+
+template<typename T>
+struct Average
+{
+    int mCount = 0;
+    T mSum = 0;
+    T average() const
+    {
+        if (!mCount)
+            return 0;
+        return mSum / mCount;
+    }
+
+    T value() const
+    {
+        return average();
+    }
+
+    void process(T value)
+    {
+        mCount++;
+        mSum += value;
+    }
+};
+
+template<typename T, int minimum = 100000, int maximum = 0, int default_value = 0>
+struct TestResult
+{
+    T mMin = minimum;
+    T mMax = maximum;
+    Average<T> mAverage;
+    T mCurrent = default_value;
+
+    void process(T value)
+    {
+        if (mMin > value)
+            mMin = value;
+        if (mMax < value)
+            mMax = value;
+        mCurrent = value;
+        mAverage.process(value);
+    }
+
+    bool is_initialized() const
+    {
+        return mAverage.mCount > 0;
+    }
+
+    T current() const
+    {
+        if (is_initialized())
+            return mCurrent;
+        else
+            return 0;
+    }
+
+    T value() const
+    {
+        return current();
+    }
+
+    T average() const
+    {
+        return mAverage.average();
+    }
+
+    TestResult<T>& operator = (T value)
+    {
+        process(value);
+        return *this;
+    }
+
+    operator T()
+    {
+        return mCurrent;
+    }
+};
+
+
+#endif

src/engine/helper/HL_Time.cpp

@@ -0,0 +1,10 @@
+#include "HL_Time.h"
+
+#include <time.h>
+
+/* return current time in milliseconds */
+double now_ms(void) {
+    struct timespec res;
+    clock_gettime(CLOCK_MONOTONIC, &res);
+    return 1000.0 * res.tv_sec + (double) res.tv_nsec / 1e6;
+}

src/engine/helper/HL_Time.h

@@ -0,0 +1,6 @@
+#ifndef __HELPER_TIME_H
+#define __HELPER_TIME_H
+
+extern double now_ms();
+
+#endif

src/engine/media/MT_AudioReceiver.cpp

@@ -10,6 +10,7 @@
 #include "MT_AudioCodec.h"
 #include "MT_CngHelper.h"
 #include "../helper/HL_Log.h"
+#include "../helper/HL_Time.h"
 #include "../audio/Audio_Interface.h"
 #include "../audio/Audio_Resampler.h"
 #include <cmath>
@@ -108,7 +109,16 @@ bool RtpBuffer::add(std::shared_ptr<jrtplib::RTPPacket> packet, int timelength,
 
     Lock l(mGuard);
 
+
     // Update statistics
+    if (mLastAddTime == 0.0)
+        mLastAddTime = now_ms();
+    else
+    {
+        float t = now_ms();
+        mStat.mPacketInterval.process(t - mLastAddTime);
+        mLastAddTime = t;
+    }
     mStat.mSsrc = static_cast<uint16_t>(packet->GetSSRC());
 
     // Update jitter
@@ -357,6 +367,7 @@ AudioReceiver::~AudioReceiver()
 
 bool AudioReceiver::add(const std::shared_ptr<jrtplib::RTPPacket>& p, Codec** codec)
 {
+    // ICELogInfo(<< "Adding packet No " << p->GetSequenceNumber());
     // Increase codec counter
     mStat.mCodecCount[p->GetPayloadType()]++;
 
@@ -435,7 +446,7 @@ void AudioReceiver::processDecoded(Audio::DataWindow& output, int options)
 
 bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
 {
-    bool result = false;
+    bool result = false, had_cng = false, had_decode = false;
 
     // Get next packet from buffer
     RtpBuffer::ResultList rl;
@@ -443,7 +454,7 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
     switch (fr)
     {
     case RtpBuffer::FetchResult::Gap:
-        ICELogInfo(<< "Gap detected.");
+        ICELogDebug(<< "Gap detected.");
 
         mDecodedLength = mResampledLength = 0;
         if (mCngPacket && mCodec)
@@ -571,9 +582,13 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
                                 mDecodedLength = 0;
                             else
                             {
+                                // Trigger the statistics
+                                had_decode = true;
+
                                 // Decode frame by frame
-                                mDecodedLength = mCodec->decode(p->GetPayloadData() + i*mCodec->rtpLength(),
+                                mDecodedLength = mCodec->decode(p->GetPayloadData() + i * mCodec->rtpLength(),
                                                                 frameLength, mDecodedFrame, sizeof mDecodedFrame);
+                                // mDecodedLength = 3840; // Opus 20 ms stereo
                                 if (mDecodedLength)
                                     processDecoded(output, options);
                             }
@@ -594,6 +609,18 @@ bool AudioReceiver::getAudio(Audio::DataWindow& output, int options, int* rate)
         assert(0);
     }
 
+    if (had_decode)
+    {
+        // mStat.mDecodeRequested++;
+        if (mLastDecodeTime == 0.0)
+            mLastDecodeTime = now_ms();
+        else
+        {
+            float t = now_ms();
+            mStat.mDecodingInterval.process(t - mLastDecodeTime);
+            mLastDecodeTime = t;
+        }
+    }
     return result;
 }
 
@@ -670,12 +697,12 @@ void AudioReceiver::updatePvqa(const void *data, int size)
             mPvqaBuffer->addZero(size);
 
         Audio::Format fmt;
-        int frames = (int)fmt.timeFromSize(mPvqaBuffer->filled()) / (PVQA_INTERVAL * 1000);
+        int frames = static_cast<int>(fmt.timeFromSize(mPvqaBuffer->filled())) / (PVQA_INTERVAL * 1000);
         if (frames > 0)
         {
             int time4pvqa = (int)(frames * PVQA_INTERVAL * 1000);
             int size4pvqa = (int)fmt.sizeFromTime(time4pvqa);
-            ICELogInfo(<< "PVQA buffer has " << time4pvqa << " milliseconds of audio.");
+            ICELogDebug(<< "PVQA buffer has " << time4pvqa << " milliseconds of audio.");
             mPVQA->update(mPvqaBuffer->data(), size4pvqa);
             mPvqaBuffer->erase(size4pvqa);
         }

src/engine/media/MT_AudioReceiver.h

@@ -60,16 +60,22 @@ namespace MT
 
     unsigned ssrc();
     void setSsrc(unsigned ssrc);
+
     void setHigh(int milliseconds);
     int high();
+
     void setLow(int milliseconds);
     int low();
+
     void setPrebuffer(int milliseconds);
     int prebuffer();
+
     int getNumberOfReturnedPackets() const;
     int getNumberOfAddPackets() const;
+
     int findTimelength();
     int getCount() const;
+
     // Returns false if packet was not add - maybe too old or too new or duplicate
     bool add(std::shared_ptr<RTPPacket> packet, int timelength, int rate);
 
@@ -89,6 +95,9 @@ namespace MT
     bool mFirstPacketWillGo;
     jrtplib::RTPSourceStats mRtpStats;
     Packet mFetchedPacket;
+
+    // To calculate average interval between packet add. It is close to jitter but more useful in debugging.
+    float mLastAddTime = 0.0;
   };
 
   class Receiver
@@ -167,6 +176,11 @@ namespace MT
 
     Audio::PWavFileWriter mDecodedDump;
 
+    float mLastDecodeTime = 0.0; // Time last call happened to codec->decode()
+
+    float mIntervalSum = 0.0;
+    int mIntervalCount = 0;
+
     // Zero rate will make audio mono but resampling will be skipped
     void makeMonoAndResample(int rate, int channels);
 

src/engine/media/MT_AudioStream.cpp

@@ -97,7 +97,7 @@ AudioStream::~AudioStream()
   if (mFinalStatistics)
     *mFinalStatistics = mStat;
 
-  ICELogInfo(<< mStat.toShortString());
+  ICELogInfo(<< mStat.toString());
 }
 
 void AudioStream::setDestination(const RtpPair<InternetAddress>& dest)

src/engine/media/MT_CodecList.cpp

@@ -126,14 +126,14 @@ CodecList::CodecList(const Settings& settings)
 #endif
 #endif
 
-  mFactoryList.push_back(new IsacCodec::IsacFactory16K(mSettings.mIsac16KPayloadType));
+  /*mFactoryList.push_back(new IsacCodec::IsacFactory16K(mSettings.mIsac16KPayloadType));
   mFactoryList.push_back(new IlbcCodec::IlbcFactory(mSettings.mIlbc20PayloadType, mSettings.mIlbc30PayloadType));
   mFactoryList.push_back(new G711Codec::AlawFactory());
   mFactoryList.push_back(new G711Codec::UlawFactory());
 
   mFactoryList.push_back(new GsmCodec::GsmFactory(mSettings.mGsmFrPayloadLength == 32 ? GsmCodec::Type::Bytes_32 : GsmCodec::Type::Bytes_33, mSettings.mGsmFrPayloadType));
   mFactoryList.push_back(new G722Codec::G722Factory());
-  mFactoryList.push_back(new G729Codec::G729Factory());
+  mFactoryList.push_back(new G729Codec::G729Factory()); */
 #ifndef TARGET_ANDROID
   mFactoryList.push_back(new GsmHrCodec::GsmHrFactory(mSettings.mGsmHrPayloadType));
 #endif

src/engine/media/MT_SingleAudioStream.cpp

@@ -19,6 +19,7 @@ SingleAudioStream::SingleAudioStream(const CodecList::Settings& codecSettings, S
 
 SingleAudioStream::~SingleAudioStream()
 {
+
 }
 
 void SingleAudioStream::process(const std::shared_ptr<jrtplib::RTPPacket>& packet)

src/engine/media/MT_Statistics.cpp

@@ -191,6 +191,9 @@ Statistics& Statistics::operator += (const Statistics& src)
 
     mJitter = src.mJitter;
     mRttDelay = src.mRttDelay;
+    mDecodingInterval = src.mDecodingInterval;
+    mDecodeRequested = src.mDecodeRequested;
+
     if (!src.mCodecName.empty())
         mCodecName = src.mCodecName;
 
@@ -239,13 +242,16 @@ Statistics& Statistics::operator -= (const Statistics& src)
 }
 
 
-std::string Statistics::toShortString() const
+std::string Statistics::toString() const
 {
     std::ostringstream oss;
     oss << "Received: "     << mReceivedRtp
         << ", lost: "       << mPacketLoss
         << ", dropped: "    << mPacketDropped
-        << ", sent: "       << mSentRtp;
+        << ", sent: "       << mSentRtp
+        << ", decoding interval: " << mDecodingInterval.average()
+        << ", decode requested: " << mDecodeRequested.average()
+        << ", packet interval: " << mPacketInterval.average();
 
     return oss.str();
 }

src/engine/media/MT_Statistics.h

@@ -6,6 +6,8 @@
 
 #include "audio/Audio_DataWindow.h"
 #include "helper/HL_Optional.hpp"
+#include "helper/HL_Statistics.h"
+
 #include "jrtplib/src/rtptimeutilities.h"
 #include "jrtplib/src/rtppacket.h"
 
@@ -13,78 +15,6 @@ using std::experimental::optional;
 
 namespace MT
 {
-template<typename T>
-struct Average
-{
-    int mCount = 0;
-    T mSum = 0;
-    T average() const
-    {
-        if (!mCount)
-            return 0;
-        return mSum / mCount;
-    }
-
-    T value() const
-    {
-        return average();
-    }
-
-    void process(T value)
-    {
-        mCount++;
-        mSum += value;
-    }
-};
-
-template<typename T, int minimum = 100000, int maximum = 0, int default_value = 0>
-struct TestResult
-{
-    T mMin = minimum;
-    T mMax = maximum;
-    Average<T> mAverage;
-    T mCurrent = default_value;
-
-    void process(T value)
-    {
-        if (mMin > value)
-            mMin = value;
-        if (mMax < value)
-            mMax = value;
-        mCurrent = value;
-        mAverage.process(value);
-    }
-
-    bool is_initialized() const
-    {
-        return mAverage.mCount > 0;
-    }
-
-    T current() const
-    {
-        if (is_initialized())
-            return mCurrent;
-        else
-            return 0;
-    }
-
-    T value() const
-    {
-        return current();
-    }
-
-    TestResult<T>& operator = (T value)
-    {
-        process(value);
-        return *this;
-    }
-
-    operator T()
-    {
-        return mCurrent;
-    }
-};
-
 
 template<typename T>
 struct StreamStats
@@ -130,9 +60,13 @@ public:
                 mDuplicatedRtp,   // Number of received duplicated rtp packets
                 mOldRtp,          // Number of late rtp packets
                 mPacketLoss,      // Number of lost packets
-                mPacketDropped,   // Number of dropped packets (due to time unsync when playing)
+                mPacketDropped,   // Number of dropped packets (due to time unsync when playing)б
                 mIllegalRtp;      // Number of rtp packets with bad payload type
 
+    TestResult<float> mDecodingInterval, // Average interval on call to packet decode
+                      mDecodeRequested,  // Average amount of requested audio frames to play
+                      mPacketInterval;   // Average interval between packet adding to jitter buffer
+
     int         mLoss[128];       // Every item is number of loss of corresping length
     size_t      mAudioTime;       // Decoded/found time in milliseconds
     uint16_t    mSsrc;            // Last known SSRC ID in a RTP stream
@@ -169,7 +103,7 @@ public:
     std::string mPvqaReport;
 #endif
 
-    std::string toShortString() const;
+    std::string toString() const;
 };
 
 } // end of namespace MT

src/engine/media/MT_Stream.cpp

@@ -21,7 +21,7 @@ Stream::Stream()
 
 Stream::~Stream()
 {
-
+  ICELogInfo(<< mStat.toString());
 }
 
 void Stream::setDestination(const RtpPair<InternetAddress>& dest)