public
/
rtphone
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

- ongoing work

This commit is contained in:
Dmytro Bogovych 2023-05-28 15:42:32 +03:00
parent 7653fcf138
commit defe8531e7
14 changed files with 179 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/CMakeLists.txt
View File

@ -118,14 +118,23 @@ set (RTPHONE_SOURCES
${rtphone_engine}/media/MT_AudioCodec.cpp
${rtphone_engine}/media/MT_CngHelper.cpp
${rtphone_engine}/agent/Agent_Impl.cpp
${rtphone_engine}/agent/Agent_Impl.h
${rtphone_engine}/agent/Agent_AudioManager.cpp
${rtphone_engine}/agent/Agent_AudioManager.h
${rtphone_engine}/endpoint/EP_Account.cpp
${rtphone_engine}/endpoint/EP_Account.h
${rtphone_engine}/endpoint/EP_AudioProvider.cpp
${rtphone_engine}/endpoint/EP_AudioProvider.h
${rtphone_engine}/endpoint/EP_DataProvider.cpp
${rtphone_engine}/endpoint/EP_DataProvider.h
${rtphone_engine}/endpoint/EP_Engine.cpp
${rtphone_engine}/endpoint/EP_Engine.h
${rtphone_engine}/endpoint/EP_NetworkQueue.cpp
${rtphone_engine}/endpoint/EP_NetworkQueue.h
${rtphone_engine}/endpoint/EP_Observer.cpp
${rtphone_engine}/endpoint/EP_Observer.h
${rtphone_engine}/endpoint/EP_Session.cpp
${rtphone_engine}/endpoint/EP_Session.h
${rtphone_engine}/media/MT_Statistics.h
${rtphone_engine}/media/MT_WebRtc.h
@ -140,17 +149,7 @@ set (RTPHONE_SOURCES
${rtphone_engine}/media/MT_AudioStream.h
${rtphone_engine}/media/MT_AudioReceiver.h
${rtphone_engine}/media/MT_AudioCodec.h
${rtphone_engine}/media/MT_CngHelper.h
${rtphone_engine}/agent/Agent_Impl.h
${rtphone_engine}/agent/Agent_AudioManager.h
${rtphone_engine}/endpoint/EP_Account.h
${rtphone_engine}/endpoint/EP_AudioProvider.h
${rtphone_engine}/endpoint/EP_DataProvider.h
${rtphone_engine}/endpoint/EP_Engine.h
${rtphone_engine}/endpoint/EP_NetworkQueue.h
${rtphone_engine}/endpoint/EP_Observer.h
${rtphone_engine}/endpoint/EP_Session.h
)
if (USE_AMR_CODEC)

33
src/engine/agent/Agent_AudioManager.cpp
View File

@ -1,11 +1,10 @@
/* Copyright(C) 2007-2017 VoIP objects (voipobjects.com)
/* Copyright(C) 2007-2023 VoIP objects (voipobjects.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "Agent_AudioManager.h"
#include "../engine/audio/Audio_WavFile.h"
#include "../engine/helper/HL_String.h"
#include "../engine/audio/Audio_Null.h"
#if defined(TARGET_ANDROID)
@ -15,18 +14,15 @@
#define LOG_SUBSYSTEM "AudioManager"
// ---------------- AudioManager -------------
//static AudioManager GAudioManager;
AudioManager::AudioManager()
:mTerminal(nullptr)
:mTerminal(nullptr), mAudioMonitoring(nullptr)
{
mPlayer.setDelegate(this);
}
AudioManager::~AudioManager()
{
//stop();
// stop();
}
AudioManager& AudioManager::instance()
@ -49,6 +45,16 @@ MT::Terminal* AudioManager::terminal()
return mTerminal;
}
void AudioManager::setAudioMonitoring(Audio::DataConnection* monitoring)
{
mAudioMonitoring = monitoring;
}
Audio::DataConnection* AudioManager::audioMonitoring()
{
return mAudioMonitoring;
}
#define LOCK_MANAGER std::unique_lock<std::mutex> l(mGuard)
void AudioManager::start(int usageId)
{
@ -68,7 +74,14 @@ void AudioManager::start(int usageId)
// Disable AEC for now - because PVQA conflicts with speex AEC.
std::shared_ptr<Audio::Enumerator> enumerator(Audio::Enumerator::make(usageId == atNull));
if (!mTerminal->audio())
mTerminal->setAudio(std::make_shared<Audio::DevicePair>(false, true));
{
auto audio = std::make_shared<Audio::DevicePair>();
audio->setAgc(true);
audio->setAec(false);
audio->setMonitoring(mAudioMonitoring);
mTerminal->setAudio(audio);
}
if (!mAudioInput)
{
@ -96,8 +109,8 @@ void AudioManager::start(int usageId)
if (outputIndex >= enumerator->count())
outputIndex = 0;
mAudioOutput = Audio::POutputDevice(
Audio::OutputDevice::make(enumerator->idAt(outputIndex)));
mAudioOutput = Audio::POutputDevice(
Audio::OutputDevice::make(enumerator->idAt(outputIndex)));
}
else
mAudioOutput = Audio::POutputDevice(new Audio::NullOutputDevice());

8
src/engine/agent/Agent_AudioManager.h
View File

@ -8,10 +8,7 @@
#include "../engine/audio/Audio_Interface.h"
#include "../engine/audio/Audio_Player.h"
#include "../engine/endpoint/EP_Engine.h"
#include "../engine/media/MT_Box.h"
#include "../engine/helper/HL_Log.h"
#include "../engine/helper/HL_Sync.h"
@ -49,6 +46,9 @@ public:
void setTerminal(MT::Terminal* terminal);
MT::Terminal* terminal();
void setAudioMonitoring(Audio::DataConnection* monitoring);
Audio::DataConnection* audioMonitoring();
// Start/stop methods relies on usage counter; only first start and last stop opens/closes devices actually
void start(int usageId);
void stop(int usageId);
@ -69,6 +69,7 @@ public:
void startPlayFile(int usageId, const std::string& path, AudioTarget target, LoopMode lm, int timelimit = 0);
void stopPlayFile(int usageId);
void onFilePlayed(Audio::Player::PlaylistItem& item);
// Must be called from main loop to release used audio devices
@ -79,6 +80,7 @@ protected:
Audio::POutputDevice mAudioOutput;
Audio::Player mPlayer;
MT::Terminal* mTerminal;
Audio::DataConnection* mAudioMonitoring;
std::map<int, int> UsageMap;
UsageCounter mUsage;

22
src/engine/agent/Agent_Impl.cpp
View File

@ -32,6 +32,24 @@ AgentImpl::~AgentImpl()
stopAgentAndThread();
}
// Get access to internal audio manager. Value can be nullptr.
const std::shared_ptr<AudioManager>& AgentImpl::audioManager() const
{
return mAudioManager;
}
void AgentImpl::setAudioMonitoring(Audio::DataConnection* monitoring)
{
mAudioMonitoring = monitoring;
if (mAudioManager)
mAudioManager->setAudioMonitoring(monitoring);
}
Audio::DataConnection* AgentImpl::monitoring() const
{
return mAudioMonitoring;
}
void AgentImpl::run()
{
while (!mShutdown)
@ -212,8 +230,10 @@ void AgentImpl::processStart(JsonCpp::Value& request, JsonCpp::Value &answer)
// Enable audio
mAudioManager = std::make_shared<AudioManager>();
mAudioManager->setTerminal(mTerminal.get());
if (mAudioMonitoring)
mAudioManager->setAudioMonitoring(mAudioMonitoring);
// Do not start here. Start right before call.
// Do not start audio manager here. Start right before call.
// Initialize endpoint
start();

9
src/engine/agent/Agent_Impl.h
View File

@ -14,6 +14,7 @@
#include <mutex>
#include <condition_variable>
class AgentImpl: public UserAgent, public MT::Stream::MediaObserver
{
protected:
@ -34,7 +35,7 @@ protected:
volatile bool mShutdown;
std::shared_ptr<MT::Terminal> mTerminal;
std::shared_ptr<AudioManager> mAudioManager;
//Audio::PWavFileWriter mIncomingAudioDump, mOutgoingAudioDump;
Audio::DataConnection* mAudioMonitoring = nullptr;
void run();
void addEvent(const JsonCpp::Value& v);
@ -65,6 +66,12 @@ public:
bool waitForData(int milliseconds);
std::string read();
// Get access to internal audio manager. Value can be nullptr.
const std::shared_ptr<AudioManager>& audioManager() const;
void setAudioMonitoring(Audio::DataConnection* monitoring);
Audio::DataConnection* monitoring() const;
// UserAgent overrides
// Called on new incoming session; providers shoukld
PDataProvider onProviderNeeded(const std::string& name) override;

68
src/engine/audio/Audio_DevicePair.cpp
View File

@ -13,8 +13,9 @@
using namespace Audio;
// --- DevicePair ---
DevicePair::DevicePair(bool aec, bool agc)
:mConfig(NULL), mDelegate(NULL), mAec(aec), mAgc(agc), mAecFilter(AUDIO_MIC_BUFFER_LENGTH*10, AUDIO_MIC_BUFFER_LENGTH, AUDIO_SAMPLERATE), mAgcFilter(AUDIO_CHANNELS)
DevicePair::DevicePair()
:mConfig(nullptr), mDelegate(nullptr), mAec(false), mAgc(false), mAecFilter(AUDIO_MIC_BUFFER_LENGTH*10, AUDIO_MIC_BUFFER_LENGTH, AUDIO_SAMPLERATE), mAgcFilter(AUDIO_CHANNELS),
mMonitoring(nullptr)
{
mInputBuffer.setCapacity(AUDIO_MIC_BUFFER_SIZE * (AUDIO_MIC_BUFFER_COUNT + 1));
mOutputBuffer.setCapacity(AUDIO_SPK_BUFFER_SIZE * (AUDIO_SPK_BUFFER_COUNT + 1));
@ -28,26 +29,50 @@ DevicePair::~DevicePair()
if (mInput)
{
if (mInput->connection() == this)
mInput->setConnection(NULL);
mInput->setConnection(nullptr);
mInput.reset();
}
if (mOutput)
{
if (mOutput->connection() == this)
mOutput->setConnection(NULL);
mOutput->setConnection(nullptr);
mOutput.reset();
}
}
DevicePair& DevicePair::setAec(bool aec)
{
mAec = aec;
return *this;
}
bool DevicePair::aec()
{
return mAec;
}
DevicePair& DevicePair::setAgc(bool agc)
{
mAgc = agc;
return *this;
}
bool DevicePair::agc()
{
return mAgc;
}
VariantMap* DevicePair::config()
{
return mConfig;
}
void DevicePair::setConfig(VariantMap* config)
DevicePair& DevicePair::setConfig(VariantMap* config)
{
mConfig = config;
return *this;
}
PInputDevice DevicePair::input()
@ -55,15 +80,17 @@ PInputDevice DevicePair::input()
return mInput;
}
void DevicePair::setInput(PInputDevice input)
DevicePair& DevicePair::setInput(PInputDevice input)
{
if (mInput == input)
return;
return *this;
mInput = input;
mInput->setConnection(this);
if (mDelegate)
mDelegate->deviceChanged(this);
return *this;
}
POutputDevice DevicePair::output()
@ -71,14 +98,17 @@ POutputDevice DevicePair::output()
return mOutput;
}
void DevicePair::setOutput(POutputDevice output)
DevicePair& DevicePair::setOutput(POutputDevice output)
{
if (output == mOutput)
return;
return *this;
mOutput = output;
mOutput->setConnection(this);
if (mDelegate)
mDelegate->deviceChanged(this);
return *this;
}
bool DevicePair::start()
@ -88,6 +118,7 @@ bool DevicePair::start()
result = mInput->open();
if (mOutput && result)
result &= mOutput->open();
return result;
}
@ -99,9 +130,10 @@ void DevicePair::stop()
mOutput->close();
}
void DevicePair::setDelegate(Delegate* dc)
DevicePair& DevicePair::setDelegate(Delegate* dc)
{
mDelegate = dc;
return *this;
}
DevicePair::Delegate* DevicePair::delegate()
@ -109,6 +141,17 @@ DevicePair::Delegate* DevicePair::delegate()
return mDelegate;
}
DevicePair& DevicePair::setMonitoring(DataConnection* monitoring)
{
mMonitoring = monitoring;
return *this;
}
DataConnection* DevicePair::monitoring()
{
return mMonitoring;
}
Player& DevicePair::player()
{
return mPlayer;
@ -219,6 +262,10 @@ void DevicePair::onSpkData(const Format& f, void* buffer, int length)
mOutputNativeData.read(buffer, length);
// Send data to monitoring if needed
if (mMonitoring)
mMonitoring->onSpkData(f, buffer, length);
#define AEC_FRAME_SIZE (AUDIO_CHANNELS * (AUDIO_SAMPLERATE / 1000) * AEC_FRAME_TIME * sizeof(short))
// AEC filter wants frames.
@ -229,7 +276,6 @@ void DevicePair::onSpkData(const Format& f, void* buffer, int length)
mAecFilter.toSpeaker(mAecSpkBuffer.mutableData() + AEC_FRAME_SIZE * frameIndex);
mAecSpkBuffer.erase(nrOfFrames * AEC_FRAME_SIZE);
}
//ICELogMedia(<< "Audio::DevicePair::onSpkData() end")
}
void DevicePair::processMicData(const Format& f, void* buffer, int length)

18
src/engine/audio/Audio_DevicePair.h
View File

@ -26,29 +26,32 @@ namespace Audio
virtual void deviceChanged(DevicePair* dpair) = 0;
};
DevicePair(bool aec = true, bool agc = true);
DevicePair();
virtual ~DevicePair();
void setAec(bool aec);
DevicePair& setAec(bool aec);
bool aec();
void setAgc(bool agc);
DevicePair& setAgc(bool agc);
bool agc();
VariantMap* config();
void setConfig(VariantMap* config);
DevicePair& setConfig(VariantMap* config);
PInputDevice input();
void setInput(PInputDevice input);
DevicePair& setInput(PInputDevice input);
POutputDevice output();
void setOutput(POutputDevice output);
DevicePair& setOutput(POutputDevice output);
bool start();
void stop();
void setDelegate(Delegate* dc);
DevicePair& setDelegate(Delegate* dc);
Delegate* delegate();
DevicePair& setMonitoring(DataConnection* monitoring);
DataConnection* monitoring();
Player& player();
protected:
@ -63,6 +66,7 @@ namespace Audio
Player mPlayer;
UniversalResampler mMicResampler, mSpkResampler;
DataWindow mInputBuffer, mOutputBuffer, mAecSpkBuffer, mInputResampingData, mOutputNativeData, mOutput10msBuffer;
DataConnection* mMonitoring;
#ifdef DUMP_NATIVEOUTPUT
std::shared_ptr<WavFileWriter> mNativeOutputDump;

21
src/engine/audio/Audio_Interface.h
View File

@ -57,6 +57,27 @@ namespace Audio
sprintf(buffer, "%dHz %dch", mRate, mChannels);
return std::string(buffer);
}
bool operator == (const Format& rhs) const
{
return mRate == rhs.mRate && mChannels == rhs.mChannels;
}
bool operator != (const Format& rhs) const
{
return mRate != rhs.mRate || mChannels != rhs.mChannels;
}
int rate() const
{
return mRate;
}
int channels() const
{
return mChannels;
}
};
class DataConnection

6
src/engine/endpoint/EP_AudioProvider.cpp
View File

@ -64,7 +64,7 @@ void AudioProvider::configureMediaObserver(MT::Stream::MediaObserver *observer,
}
// Processes incoming data
void AudioProvider::processData(PDatagramSocket s, const void* dataBuffer, int dataSize, InternetAddress& source)
void AudioProvider::processData(const PDatagramSocket& s, const void* dataBuffer, int dataSize, InternetAddress& source)
{
if (!mActiveStream)
return;
@ -77,7 +77,7 @@ void AudioProvider::processData(PDatagramSocket s, const void* dataBuffer, int
}
// This method is called by user agent to send ICE packet from mediasocket
void AudioProvider::sendData(PDatagramSocket s, InternetAddress& destination, const void* buffer, unsigned int size)
void AudioProvider::sendData(const PDatagramSocket& s, InternetAddress& destination, const void* buffer, unsigned int size)
{
s->sendDatagram(destination, buffer, size);
}
@ -387,4 +387,4 @@ void AudioProvider::setStateImpl(unsigned int state) {
if (mActiveStream)
mActiveStream->setState(state);
}
}

8
src/engine/endpoint/EP_AudioProvider.h
View File

@ -1,4 +1,4 @@
/* Copyright(C) 2007-2014 VoIP objects (voipobjects.com)
/* Copyright(C) 2007-2023 VoIP objects (voipobjects.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -12,9 +12,7 @@
#include "../media/MT_Box.h"
#include "../media/MT_Stream.h"
#include "../media/MT_Codec.h"
#include "../audio/Audio_Interface.h"
#include "rutil/ThreadIf.hxx"
#include <vector>
#include <string>
@ -37,10 +35,10 @@ public:
void setDestinationAddress(const RtpPair<InternetAddress>& addr) override;
// Processes incoming data
void processData(PDatagramSocket s, const void* dataBuffer, int dataSize, InternetAddress& source) override;
void processData(const PDatagramSocket& s, const void* dataBuffer, int dataSize, InternetAddress& source) override;
// This method is called by user agent to send ICE packet from mediasocket
void sendData(PDatagramSocket s, InternetAddress& destination, const void* dataBuffer, unsigned int datasize) override;
void sendData(const PDatagramSocket& s, InternetAddress& destination, const void* dataBuffer, unsigned int datasize) override;
// Updates SDP offer
void updateSdpOffer(resip::SdpContents::Session::Medium& sdp, SdpDirection direction) override;

4
src/engine/endpoint/EP_DataProvider.h
View File

@ -45,10 +45,10 @@ public:
virtual void setDestinationAddress(const RtpPair<InternetAddress>& addr) = 0;
// Processes incoming data
virtual void processData(PDatagramSocket s, const void* dataBuffer, int dataSize, InternetAddress& address) = 0;
virtual void processData(const PDatagramSocket& s, const void* dataBuffer, int dataSize, InternetAddress& address) = 0;
// This method is called by user agent to send ICE packet from mediasocket
virtual void sendData(PDatagramSocket s, InternetAddress& destination, const void* dataBuffer, unsigned int datasize) = 0;
virtual void sendData(const PDatagramSocket& s, InternetAddress& destination, const void* dataBuffer, unsigned int datasize) = 0;
// Updates SDP offer
virtual void updateSdpOffer(resip::SdpContents::Session::Medium& sdp, SdpDirection direction) = 0;

30
src/engine/endpoint/EP_Session.cpp
View File

@ -1,4 +1,4 @@
/* Copyright(C) 2007-2017 VoIP objects (voipobjects.com)
/* Copyright(C) 2007-2023 VoIP objects (voipobjects.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -7,11 +7,7 @@
#include "EP_Engine.h"
#include "EP_AudioProvider.h"
#include "../media/MT_Stream.h"
#include "../media/MT_AudioStream.h"
#include "../media/MT_Dtmf.h"
#include "../helper/HL_Log.h"
#include "../helper/HL_Exception.h"
#include "../helper/HL_StreamState.h"
#include "../helper/HL_Sync.h"
#include "../helper/HL_String.h"
@ -20,7 +16,7 @@
typedef resip::SdpContents::Session::Medium Medium;
typedef resip::SdpContents::Session::MediumContainer MediumContainer;
#define DOMULTIPLEX() mUserAgent->mConfig[CONFIG_MULTIPLEXING].asBool() ? SocketHeap::DoMultiplexing : SocketHeap::DontMultiplexing
#define IS_MULTIPLEX() mUserAgent->mConfig[CONFIG_MULTIPLEXING].asBool() ? SocketHeap::DoMultiplexing : SocketHeap::DontMultiplexing
//------------ ResipSessionAppDialog ------------
@ -714,11 +710,11 @@ void Session::buildSdp(resip::SdpContents &sdp, SdpDirection sdpDirection)
if (mUserAgent->mConfig[CONFIG_MULTIPLEXING].asBool())
rtcpPort = rtpPort;
else
if (rtcpPort.isEmpty())
{
rtcpPort = rtpPort;
rtcpPort.setPort( rtpPort.port() + 1);
}
if (rtcpPort.isEmpty())
{
rtcpPort = rtpPort;
rtcpPort.setPort( rtpPort.port() + 1);
}
media.addAttribute("rtcp", resip::Data(rtcpPort.port()));
}
@ -792,8 +788,8 @@ void Session::addProvider(PDataProvider provider)
s.setProvider( provider );
// Allocate socket for provider
s.setSocket4( SocketHeap::instance().allocSocketPair(AF_INET, this, DOMULTIPLEX()) );
s.setSocket6( SocketHeap::instance().allocSocketPair(AF_INET6, this, DOMULTIPLEX()) );
s.setSocket4( SocketHeap::instance().allocSocketPair(AF_INET, this, IS_MULTIPLEX()) );
s.setSocket6( SocketHeap::instance().allocSocketPair(AF_INET6, this, IS_MULTIPLEX()) );
s.provider()->setSocket(s.socket4(), s.socket6());
// Create ICE stream/component
@ -896,8 +892,8 @@ void Session::refreshMediaPath()
SocketHeap::instance().freeSocketPair(p->socket(AF_INET));
// Bring new socket to provider and stream
RtpPair<PDatagramSocket> s4 = SocketHeap::instance().allocSocketPair(AF_INET, this, DOMULTIPLEX() ),
s6 = SocketHeap::instance().allocSocketPair(AF_INET, this, DOMULTIPLEX());
RtpPair<PDatagramSocket> s4 = SocketHeap::instance().allocSocketPair(AF_INET, this, IS_MULTIPLEX() ),
s6 = SocketHeap::instance().allocSocketPair(AF_INET, this, IS_MULTIPLEX());
p->setSocket(s4, s6);
s.setSocket4(s4);
@ -976,8 +972,8 @@ int Session::processSdp(uint64_t version, bool iceAvailable, std::string icePwd,
{
try
{
stream.setSocket4(SocketHeap::instance().allocSocketPair(AF_INET, this, DOMULTIPLEX()));
stream.setSocket6(SocketHeap::instance().allocSocketPair(AF_INET6, this, DOMULTIPLEX()));
stream.setSocket4(SocketHeap::instance().allocSocketPair(AF_INET, this, IS_MULTIPLEX()));
stream.setSocket6(SocketHeap::instance().allocSocketPair(AF_INET6, this, IS_MULTIPLEX()));
}
catch(...)
{

3
src/libs/ice/ICENetworkHelper.cpp
View File

@ -26,7 +26,8 @@
# include <linux/in6.h>
# if defined(TARGET_ANDROID)
# if __ANDROID_API__ < 24
# error not supported for ANDROID_API < 24
# include "android-ifaddrs/android-ifaddrs.h"
//# error not supported for ANDROID_API < 24
# else
# include <ifaddrs.h>
# endif

2
src/libs/oboe/CMakeLists.txt
View File

@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.4.1)
cmake_minimum_required(VERSION 3.18)
# Set the name of the project and store it in PROJECT_NAME. Also set the following variables:
# PROJECT_SOURCE_DIR (usually the root directory where Oboe has been cloned e.g.)