diff --git a/src/engine/helper/HL_Rtp.cpp b/src/engine/helper/HL_Rtp.cpp index d244bba6..0a9cce6e 100644 --- a/src/engine/helper/HL_Rtp.cpp +++ b/src/engine/helper/HL_Rtp.cpp @@ -173,6 +173,11 @@ int RtpHelper::findPayloadLength(const void* buffer, size_t length) return static_cast(payloadLen); } +std::chrono::microseconds RtpHelper::toMicroseconds(const jrtplib::RTPTime& t) +{ + return std::chrono::microseconds(uint64_t(t.GetDouble() * 1000000)); +} + // --- RtpDump implementation --- std::shared_ptr RtpDump::parseRtpData(const uint8_t* data, size_t len) diff --git a/src/engine/helper/HL_Rtp.h b/src/engine/helper/HL_Rtp.h index a2c8798f..aeb13ae3 100644 --- a/src/engine/helper/HL_Rtp.h +++ b/src/engine/helper/HL_Rtp.h @@ -43,6 +43,8 @@ public: static unsigned findSsrc(const void* buffer, size_t length); static void setSsrc(void* buffer, size_t length, uint32_t ssrc); static int findPayloadLength(const void* buffer, size_t length); + + static std::chrono::microseconds toMicroseconds(const jrtplib::RTPTime& t); }; /** diff --git a/src/engine/media/MT_AmrCodec.cpp b/src/engine/media/MT_AmrCodec.cpp index 3c3fff27..735ff7a1 100644 --- a/src/engine/media/MT_AmrCodec.cpp +++ b/src/engine/media/MT_AmrCodec.cpp @@ -148,7 +148,7 @@ static AmrPayload parseAmrPayload(AmrPayloadInfo& input, size_t& cngCounter) // if (input.mWideband && f.mMode == 0xFF /* CNG */) // { // int a = 1; - // } + // }` if (input.mWideband && f.mFrameType == 15) { diff --git a/src/engine/media/MT_AudioReceiver.cpp b/src/engine/media/MT_AudioReceiver.cpp index 70f0bbd4..45becfce 100644 --- a/src/engine/media/MT_AudioReceiver.cpp +++ b/src/engine/media/MT_AudioReceiver.cpp @@ -11,6 +11,7 @@ #include "MT_AudioReceiver.h" #include "MT_AudioCodec.h" #include "MT_CngHelper.h" +#include "MT_Dtmf.h" #include "../helper/HL_Log.h" #include "../helper/HL_Time.h" #include "../audio/Audio_Interface.h" @@ -207,7 +208,8 @@ RtpBuffer::FetchResult RtpBuffer::fetch() // Save it as last packet however - to not confuse loss packet counter mFetchedPacket = mPacketList.front(); - mLastSeqno = mPacketList.front()->rtp()->GetExtendedSequenceNumber(); + mLastSeqno = mFetchedPacket->rtp()->GetExtendedSequenceNumber(); + mLastReceiveTime = mFetchedPacket->rtp()->GetReceiveTime(); // Erase from packet list mPacketList.erase(mPacketList.begin()); @@ -238,20 +240,27 @@ RtpBuffer::FetchResult RtpBuffer::fetch() // Gap between new packet and previous on int gap = (int64_t)seqno - (int64_t)*mLastSeqno - 1; - // gap = std::min(gap, 127); if (gap > 0) { // std::cout << "Increase the packet loss for SSRC " << std::hex << mSsrc << std::endl; mStat.mPacketLoss += gap; - auto currentTimestamp = std::chrono::microseconds(uint64_t(packet.rtp()->GetReceiveTime().GetDouble() * 1000000)); + // Report is the onetime; there is no many sequential 1-packet gap reports if (mStat.mPacketLossTimeline.empty() || (mStat.mPacketLossTimeline.back().mEndSeqno != seqno)) - mStat.mPacketLossTimeline.push_back({.mStartSeqno = *mLastSeqno, - .mEndSeqno = seqno, - .mGap = gap, - .mTimestamp = currentTimestamp}); + { + auto gapStart = RtpHelper::toMicroseconds(*mLastReceiveTime); + auto gapEnd = RtpHelper::toMicroseconds(packet.rtp()->GetReceiveTime()); + mStat.mPacketLossTimeline.emplace_back(PacketLossEvent{.mStartSeqno = *mLastSeqno, + .mEndSeqno = seqno, + .mGap = gap, + .mTimestampStart = gapStart, + .mTimestampEnd = gapEnd}); + } + // ToDo: here we should decide smth - 2-packet gap shoud report Status::Gap two times at least; but current implementation gives only one. + // It is not big problem - as gap is detected when we have smth to return usually mLastSeqno = seqno; + mLastReceiveTime = packet.rtp()->GetReceiveTime(); result = {FetchResult::Status::Gap}; } else @@ -261,6 +270,7 @@ RtpBuffer::FetchResult RtpBuffer::fetch() // Save last returned normal packet mFetchedPacket = result.mPacket; mLastSeqno = result.mPacket->rtp()->GetExtendedSequenceNumber(); + mLastReceiveTime = result.mPacket->rtp()->GetReceiveTime(); // Remove returned packet from the list mPacketList.erase(mPacketList.begin()); @@ -278,6 +288,7 @@ RtpBuffer::FetchResult RtpBuffer::fetch() // Remember returned packet mFetchedPacket = result.mPacket; mLastSeqno = result.mPacket->rtp()->GetExtendedSequenceNumber(); + mLastReceiveTime = result.mPacket->rtp()->GetReceiveTime(); // Remove returned packet from buffer list mPacketList.erase(mPacketList.begin()); @@ -955,5 +966,25 @@ DtmfReceiver::DtmfReceiver(Statistics& stat) DtmfReceiver::~DtmfReceiver() {} -void DtmfReceiver::add(std::shared_ptr /*p*/) -{} +void DtmfReceiver::add(const std::shared_ptr& p) +{ + // This receiver always work in context of single RTP stream; so there is no need to put SSRC map and so on + if (p->GetPayloadType() != 101) + return; + + auto ev = DtmfBuilder::parseRfc2833({p->GetPayloadData(), p->GetPayloadLength()}); + if (ev.mTone != mEvent || ev.mEnd != mEventEnded) + { + // New tone is here + if (mCallback) + mCallback(ev.mTone); + + // Queue statistics item + mStat.mDtmf2833Timeline.emplace_back(Dtmf2833Event{.mTone = ev.mTone, + .mTimestamp = RtpHelper::toMicroseconds(p->GetReceiveTime())}); + + // Store to avoid triggering on the packet + mEvent = ev.mTone; + mEventEnded = ev.mEnd; + } +} diff --git a/src/engine/media/MT_AudioReceiver.h b/src/engine/media/MT_AudioReceiver.h index ed76c310..3efb8b09 100644 --- a/src/engine/media/MT_AudioReceiver.h +++ b/src/engine/media/MT_AudioReceiver.h @@ -120,6 +120,7 @@ protected: jrtplib::RTPSourceStats mRtpStats; std::shared_ptr mFetchedPacket; std::optional mLastSeqno; + std::optional mLastReceiveTime; // To calculate average interval between packet add. It is close to jitter but more useful in debugging. float mLastAddTime = 0.0f; @@ -248,11 +249,18 @@ protected: class DtmfReceiver: public Receiver { +private: + char mEvent = 0; + bool mEventEnded = false; + std::chrono::milliseconds mEventStart = 0ms; + std::function mCallback; + public: DtmfReceiver(Statistics& stat); ~DtmfReceiver(); - void add(std::shared_ptr p); + void add(const std::shared_ptr& p); + void setCallback(std::function callback); }; } diff --git a/src/engine/media/MT_Dtmf.cpp b/src/engine/media/MT_Dtmf.cpp index 5ba3838e..173bea0e 100644 --- a/src/engine/media/MT_Dtmf.cpp +++ b/src/engine/media/MT_Dtmf.cpp @@ -16,38 +16,68 @@ using namespace MT; -void DtmfBuilder::buildRfc2833(int tone, int duration, int volume, bool endOfEvent, void* output) +void DtmfBuilder::buildRfc2833(const Rfc2833Event& ev, void* output) { - assert(duration); + assert(ev.mDuration != 0); assert(output); - assert(tone); + assert(ev.mTone != 0); unsigned char toneValue = 0; - if (tone >= '0' && tone <='9') - toneValue = tone - '0'; + if (ev.mTone >= '0' && ev.mTone <='9') + toneValue = ev.mTone - '0'; else - if (tone >= 'A' && tone <='D' ) - toneValue = tone - 'A' + 12; - else - if (tone == '*') - toneValue = 10; - else - if (tone == '#') - toneValue = 11; + if (ev.mTone >= 'A' && ev.mTone <='D' ) + toneValue = ev.mTone - 'A' + 12; + else + if (ev.mTone == '*') + toneValue = 10; + else + if (ev.mTone == '#') + toneValue = 11; char* packet = (char*)output; packet[0] = toneValue; - packet[1] = 1 | (volume << 2); - if (endOfEvent) + packet[1] = 1 | (ev.mVolume << 2); + if (ev.mEnd) packet[1] |= 128; else packet[1] &= 127; - unsigned short durationValue = htons(duration); + unsigned short durationValue = htons(ev.mDuration); memcpy(packet + 2, &durationValue, 2); } + +DtmfBuilder::Rfc2833Event DtmfBuilder::parseRfc2833(std::span payload) +{ + Rfc2833Event r; + if (payload.size_bytes() < 4) + return r; + + uint8_t b0 = payload[0]; + uint8_t b1 = payload[1]; + + if (b0 >=0 && b0 <= 9) + r.mTone = '0' + b0; + else + if (b0 >= 12 && b0 <= 17) + r.mTone = 'A' + b0; + else + if (b0 == 10) + r.mTone = '*'; + else + if (b0 == 11) + r.mTone = '#'; + + r.mEnd = (b1 & 128); + r.mVolume = (b1 & 127) >> 2; + r.mDuration = ntohs(*(uint16_t*)payload.data()+2); + + return r; +} + + #pragma region Inband DTMF support #include #ifndef TARGET_WIN @@ -302,24 +332,24 @@ bool DtmfContext::getRfc2833(int milliseconds, ByteBuffer& output, ByteBuffer& s { // Emit rfc2833 packet output.resize(4); - DtmfBuilder::buildRfc2833(d.mTone, milliseconds, d.mVolume, false, output.mutableData()); + DtmfBuilder::buildRfc2833({.mTone = (char)d.mTone, .mDuration = milliseconds, .mVolume = d.mVolume, .mEnd = false}, output.mutableData()); d.mDuration -= milliseconds; if(d.mDuration <= 0) d.mStopped = true; } else - if (!d.mStopped) - { - output.resize(4); - DtmfBuilder::buildRfc2833(d.mTone, milliseconds, d.mVolume, false, output.mutableData()); - } - else - output.clear(); + if (!d.mStopped) + { + output.resize(4); + DtmfBuilder::buildRfc2833({.mTone = (char)d.mTone, .mDuration = milliseconds, .mVolume = d.mVolume, .mEnd = false}, output.mutableData()); + } + else + output.clear(); if (d.mStopped) { stopPacket.resize(4); - DtmfBuilder::buildRfc2833(d.mTone, milliseconds, d.mVolume, true, stopPacket.mutableData()); + DtmfBuilder::buildRfc2833({.mTone = (char)d.mTone, .mDuration = milliseconds, .mVolume = d.mVolume, .mEnd = true}, stopPacket.mutableData()); } else stopPacket.clear(); @@ -375,7 +405,7 @@ void zap_dtmf_detect_init(dtmf_detect_state_t *s); int zap_dtmf_detect(dtmf_detect_state_t *s, int16_t amp[], int samples, int isradio); int zap_dtmf_get(dtmf_detect_state_t *s, char *buf, int max); -DTMFDetector::DTMFDetector() +InbandDtmfDetector::InbandDtmfDetector() :mState(NULL) { mState = malloc(sizeof(dtmf_detect_state_t)); @@ -384,13 +414,13 @@ DTMFDetector::DTMFDetector() zap_dtmf_detect_init((dtmf_detect_state_t*)mState); } -DTMFDetector::~DTMFDetector() +InbandDtmfDetector::~InbandDtmfDetector() { if (mState) free(mState); } -std::string DTMFDetector::streamPut(unsigned char* samples, unsigned int size) +std::string InbandDtmfDetector::streamPut(unsigned char* samples, unsigned int size) { char buf[16]; buf[0] = 0; if (zap_dtmf_detect((dtmf_detect_state_t*)mState, (int16_t*)samples, size/2, 0)) @@ -398,7 +428,7 @@ std::string DTMFDetector::streamPut(unsigned char* samples, unsigned int size) return buf; } -void DTMFDetector::resetState() +void InbandDtmfDetector::resetState() { zap_dtmf_detect_init((dtmf_detect_state_t*)mState); } diff --git a/src/engine/media/MT_Dtmf.h b/src/engine/media/MT_Dtmf.h index 998059e8..b0732377 100644 --- a/src/engine/media/MT_Dtmf.h +++ b/src/engine/media/MT_Dtmf.h @@ -15,38 +15,52 @@ namespace MT { - class DtmfBuilder - { - public: +class DtmfBuilder +{ +public: + struct Rfc2833Event + { + char mTone = 0; + int mDuration = 0; + int mVolume = 0; + bool mEnd = false; + + bool isValid() const { + return mTone != 0; + } + }; + // Output should be 4 bytes length - static void buildRfc2833(int tone, int duration, int volume, bool endOfEvent, void* output); - + static void buildRfc2833(const Rfc2833Event& ev, void* output); + + static Rfc2833Event parseRfc2833(std::span payload); + // Buf receives PCM audio static void buildInband(int tone, int startTime, int finishTime, int rate, short* buf); - }; +}; - struct Dtmf - { +struct Dtmf +{ Dtmf(): mTone(0), mDuration(0), mVolume(0), mFinishCount(3), mCurrentTime(0), mStopped(false) {} Dtmf(int tone, int volume, int duration): mTone(tone), mVolume(volume), mDuration(duration), mFinishCount(3), mCurrentTime(0), mStopped(false) {} - + int mStopped; int mTone; int mDuration; // It is zero for tones generated by startTone()..stopTone() calls. int mVolume; int mFinishCount; int mCurrentTime; - }; +}; - typedef std::vector DtmfQueue; +typedef std::vector DtmfQueue; - class DtmfContext - { - public: +class DtmfContext +{ +public: enum Type { - Dtmf_Inband, - Dtmf_Rfc2833 + Dtmf_Inband, + Dtmf_Rfc2833 }; DtmfContext(); @@ -65,33 +79,34 @@ namespace MT bool getInband(int milliseconds, int rate, ByteBuffer& output); bool getRfc2833(int milliseconds, ByteBuffer& output, ByteBuffer& stopPacket); - protected: +protected: Mutex mGuard; Type mType; DtmfQueue mQueue; - }; +}; -class DTMFDetector +class InbandDtmfDetector { public: - /*! The default constructor. Allocates space for detector context. */ - DTMFDetector(); - - /*! The destructor. Free the detector context's memory. */ - ~DTMFDetector(); + /*! The default constructor. Allocates space for detector context. */ + InbandDtmfDetector(); - /*! This method receives the input PCM 16-bit data and returns found DTMF event(s) in string representation. + /*! The destructor. Free the detector context's memory. */ + ~InbandDtmfDetector(); + + /*! This method receives the input PCM 16-bit data and returns found DTMF event(s) in string representation. * @param samples Input PCM buffer pointer. * @param size Size of input buffer in bytes * @return Found DTMF event(s) in string representation. The returned value has variable length. */ - std::string streamPut(unsigned char* samples, unsigned int size); + std::string streamPut(unsigned char* samples, unsigned int size); - void resetState(); + void resetState(); protected: - void* mState; /// DTMF detector context + void* mState; /// DTMF detector context }; + } #endif diff --git a/src/engine/media/MT_Statistics.cpp b/src/engine/media/MT_Statistics.cpp index a0a7931e..ac3583c7 100644 --- a/src/engine/media/MT_Statistics.cpp +++ b/src/engine/media/MT_Statistics.cpp @@ -74,13 +74,16 @@ Statistics::~Statistics() void Statistics::calculateBurstr(double* burstr, double* lossr) const { - int lost = 0; - int bursts = 0; - for (int i = 0; i < 128; i++) - { - lost += i * mLoss[i]; - bursts += mLoss[i]; - } + int lost = 0; // Total packet lost + for (const auto& item: mPacketLossTimeline) + lost += item.mGap; + int bursts = mPacketLossTimeline.size(); // number of events + + // for (const auto& entry: mLoss) + // { + // lost += entry.first * entry.second; + // bursts += entry.second; + // } if (lost < 5) { @@ -109,14 +112,14 @@ void Statistics::calculateBurstr(double* burstr, double* lossr) const double Statistics::calculateMos(double maximalMos) const { // calculate lossrate and burst rate - double burstr, lossr; + double burstr = 0, lossr = 0; calculateBurstr(&burstr, &lossr); - double r; + double r = 0.0; double bpl = 8.47627; //mos = -4.23836 + 0.29873 * r - 0.00416744 * r * r + 0.0000209855 * r * r * r; - double mos; + double mos = 0.0; - if (mReceivedRtp < 100) + if (mReceivedRtp < 10) return 0.0; if (lossr == 0.0 || burstr == 0.0) diff --git a/src/engine/media/MT_Statistics.h b/src/engine/media/MT_Statistics.h index d637f6e2..349cf03f 100644 --- a/src/engine/media/MT_Statistics.h +++ b/src/engine/media/MT_Statistics.h @@ -4,16 +4,15 @@ #include #include #include -#include -#include "audio/Audio_DataWindow.h" -#include "helper/HL_Optional.hpp" #include "helper/HL_Statistics.h" #include "helper/HL_Types.h" +#include "helper/HL_InternetAddress.h" #include "jrtplib/src/rtptimeutilities.h" #include "jrtplib/src/rtppacket.h" +using namespace std::chrono_literals; namespace MT { @@ -56,47 +55,55 @@ struct PacketLossEvent uint32_t mStartSeqno = 0, mEndSeqno = 0; int mGap = 0; + std::chrono::microseconds mTimestampStart = 0us, + mTimestampEnd = 0us; +}; + +struct Dtmf2833Event +{ + char mTone; std::chrono::microseconds mTimestamp; }; class Statistics { public: - size_t mReceived = 0, // Received traffic in bytes - mSent = 0, // Sent traffic in bytes - mReceivedRtp = 0, // Number of received rtp packets - mSentRtp = 0, // Number of sent rtp packets - mReceivedRtcp = 0, // Number of received rtcp packets - mSentRtcp = 0, // Number of sent rtcp packets - mDuplicatedRtp = 0, // Number of received duplicated rtp packets - mOldRtp = 0, // Number of late rtp packets - mPacketLoss = 0, // Number of lost packets - mPacketDropped = 0, // Number of dropped packets (due to time unsync when playing)б - mIllegalRtp = 0; // Number of rtp packets with bad payload type + size_t mReceived = 0, // Received traffic in bytes + mSent = 0, // Sent traffic in bytes + mReceivedRtp = 0, // Number of received rtp packets + mSentRtp = 0, // Number of sent rtp packets + mReceivedRtcp = 0, // Number of received rtcp packets + mSentRtcp = 0, // Number of sent rtcp packets + mDuplicatedRtp = 0, // Number of received duplicated rtp packets + mOldRtp = 0, // Number of late rtp packets + mPacketLoss = 0, // Number of lost packets + mPacketDropped = 0, // Number of dropped packets (due to time unsync when playing)б + mIllegalRtp = 0; // Number of rtp packets with bad payload type - TestResult mDecodingInterval, // Average interval on call to packet decode - mDecodeRequested, // Average amount of requested audio frames to play - mPacketInterval; // Average interval between packet adding to jitter buffer + TestResult mDecodingInterval, // Average interval on call to packet decode + mDecodeRequested, // Average amount of requested audio frames to play + mPacketInterval; // Average interval between packet adding to jitter buffer - std::array mLoss = {0}; // Every item is number of loss of corresping length - size_t mAudioTime = 0; // Decoded/found time in milliseconds - size_t mDecodedSize = 0; // Number of decoded bytes - uint16_t mSsrc = 0; // Last known SSRC ID in a RTP stream - ice::NetworkAddress mRemotePeer; // Last known remote RTP address + std::map mLoss; // Every item is number of loss of corresping length + size_t mAudioTime = 0; // Decoded/found time in milliseconds + size_t mDecodedSize = 0; // Number of decoded bytes + uint16_t mSsrc = 0; // Last known SSRC ID in a RTP stream + ice::NetworkAddress mRemotePeer; // Last known remote RTP address // AMR codec bitrate switch counter - int mBitrateSwitchCounter = 0; - int mCng = 0; - std::string mCodecName; - float mJitter = 0.0f; // Jitter - TestResult mRttDelay; // RTT delay + int mBitrateSwitchCounter = 0; + int mCng = 0; + std::string mCodecName; + float mJitter = 0.0f; // Jitter + TestResult mRttDelay; // RTT delay // Timestamp when first RTP packet has arrived - std::optional mFirstRtpTime; + std::optional mFirstRtpTime; - std::map mCodecCount; // Stats on used codecs + std::map mCodecCount; // Stats on used codecs - std::vector mPacketLossTimeline; // Packet loss timeline + std::vector mPacketLossTimeline; // Packet loss timeline + std::vector mDtmf2833Timeline; // It is to calculate network MOS void calculateBurstr(double* burstr, double* loss) const;