From ead9979db754111f2956adfc65f844b07ff589b1 Mon Sep 17 00:00:00 2001 From: Dmytro Bogovych Date: Fri, 3 Sep 2021 14:14:25 +0300 Subject: [PATCH] - more work on Android Oboe audio backend --- src/engine/audio/Audio_AndroidOboe.cpp | 245 +++++++++++++++++++++++ src/engine/audio/Audio_AndroidOboe.h | 109 ++++++++++ src/engine/endpoint/EP_AudioProvider.h | 10 +- src/engine/media/MT_CodecList.cpp | 3 +- src/libs/ice/ICELog.h | 2 +- src/libs/resiprocate/contrib/ares/ares.h | 20 +- 6 files changed, 376 insertions(+), 13 deletions(-) create mode 100644 src/engine/audio/Audio_AndroidOboe.cpp create mode 100644 src/engine/audio/Audio_AndroidOboe.h diff --git a/src/engine/audio/Audio_AndroidOboe.cpp b/src/engine/audio/Audio_AndroidOboe.cpp new file mode 100644 index 00000000..27896036 --- /dev/null +++ b/src/engine/audio/Audio_AndroidOboe.cpp @@ -0,0 +1,245 @@ +#include "Audio_AndroidOboe.h" +#include "../helper/HL_Sync.h" +#include "../helper/HL_Log.h" +#include +#include +#include + +#include "../helper/HL_String.h" +#include "../helper/HL_Time.h" + +#ifdef TARGET_ANDROID + +#define LOG_SUBSYSTEM "Audio" + +using namespace Audio; + + +// -------------------- AndroidEnumerator ----------------------------- + +AndroidEnumerator::AndroidEnumerator() +{} + +AndroidEnumerator::~AndroidEnumerator() +{} + +int AndroidEnumerator::indexOfDefaultDevice() +{ + return 0; +} + +int AndroidEnumerator::count() +{ + return 1; +} + +int AndroidEnumerator::idAt(int index) +{ + return 0; +} + +std::string AndroidEnumerator::nameAt(int index) +{ + return "Audio"; +} + +void AndroidEnumerator::open(int direction) +{} + +void AndroidEnumerator::close() +{} + +// --------------- Input implementation ---------------- +AndroidInputDevice::AndroidInputDevice(int devId) +{} + +AndroidInputDevice::~AndroidInputDevice() +{ + close(); +} + +bool AndroidInputDevice::open() +{ + if (active()) + return true; + + oboe::AudioStreamBuilder builder; + builder.setDirection(oboe::Direction::Input); + builder.setPerformanceMode(oboe::PerformanceMode::LowLatency); + builder.setSharingMode(oboe::SharingMode::Exclusive); + builder.setFormat(oboe::AudioFormat::I16); + builder.setChannelCount(oboe::ChannelCount::Mono); + builder.setCallback(this); + oboe::Result rescode = builder.openStream(&mRecordingStream); + if (rescode != oboe::Result::OK) + return false; + + mDeviceRate = mRecordingStream->getSampleRate(); + ICELogInfo(<< "Input Opened with rate " << mDeviceRate); + mActive = true; + + rescode = mRecordingStream->requestStart(); + if (rescode != oboe::Result::OK) + { + close(); + mActive = false; + } + return mActive; +} + +void AndroidInputDevice::close() +{ + // There is no check for active() value because close() can be called to cleanup after bad open() call. + if (mRecordingStream != nullptr) + { + mRecordingStream->close(); + delete mRecordingStream; mRecordingStream = nullptr; + } + mActive = false; +} + +oboe::DataCallbackResult +AndroidInputDevice::onAudioReady(oboe::AudioStream *audioStream, void *audioData, int32_t numFrames) +{ + std::unique_lock l(mMutex); + + // Send data to AudioPair + if (mConnection) + mConnection->onMicData(getFormat(), audioData, numFrames); + + return oboe::DataCallbackResult::Continue; +} + +Format AndroidInputDevice::getFormat() +{ + return Format(mDeviceRate, 1); +} + +bool AndroidInputDevice::active() const +{ + return mActive; +} + +bool AndroidInputDevice::fakeMode() +{ + return false; +} + +void AndroidInputDevice::setFakeMode(bool fakemode) +{} + +int AndroidInputDevice::readBuffer(void* buffer) +{ + throw std::runtime_error("AndroidInputDevice::readBuffer() is not implemented."); +} + +// ------------ AndroidOutputDevice ----------------- +AndroidOutputDevice::AndroidOutputDevice(int devId) +{ + ICELogDebug(<< "Creating AndroidOutputDevice. This is: " << StringHelper::toHex(this)); +} +AndroidOutputDevice::~AndroidOutputDevice() +{ + ICELogDebug(<< "Deleting AndroidOutputDevice."); + close(); +} + +bool AndroidOutputDevice::open() +{ + std::unique_lock l(mMutex); + + if (mActive) + return true; + + mRequestedFrames = 0; + mStartTime = 0.0; + mEndTime = 0.0; + + oboe::AudioStreamBuilder builder; + builder.setDirection(oboe::Direction::Output); + builder.setPerformanceMode(oboe::PerformanceMode::LowLatency); + builder.setSharingMode(oboe::SharingMode::Exclusive); + builder.setFormat(oboe::AudioFormat::I16); + builder.setChannelCount(oboe::ChannelCount::Mono); + // builder.setDataCallback(this); + builder.setCallback(this); + //builder.setErrorCallback(this) + + oboe::Result rescode = builder.openStream(&mPlayingStream); + if (rescode != oboe::Result::OK) + return false; + + mDeviceRate = mPlayingStream->getSampleRate(); + ICELogInfo(<< "Input Opened with rate " << mDeviceRate); + mActive = true; + + rescode = mPlayingStream->requestStart(); + if (rescode != oboe::Result::OK) + { + close(); + mActive = false; + } + return mActive; +} + +void AndroidOutputDevice::close() +{ + std::unique_lock l(mMutex); + if (!mActive) + return; + + if (mPlayingStream != nullptr) + { + mPlayingStream->close(); + delete mPlayingStream; mPlayingStream = nullptr; + } + mEndTime = now_ms(); + mActive = false; + + ICELogInfo(<< "For time " << mEndTime - mStartTime << " ms was requested " + << float(mRequestedFrames) / getFormat().mRate * 1000 << " ms"); +} + +Format AndroidOutputDevice::getFormat() +{ + return Format(mDeviceRate, 1); +} + +bool AndroidOutputDevice::fakeMode() +{ + return false; +} + +void AndroidOutputDevice::setFakeMode(bool /*fakemode*/) +{ +} + +oboe::DataCallbackResult AndroidOutputDevice::onAudioReady(oboe::AudioStream *audioStream, void *audioData, int32_t numFrames) +{ + if (mInShutdown) + return oboe::DataCallbackResult::Stop; + + if (mStartTime == 0.0) + mStartTime = now_ms(); + + // Ask producer about data + memset(audioData, 0, numFrames * 2); + if (mConnection) + { + Format f = getFormat(); + if (f.mRate != 0) + mConnection->onSpkData(f, audioData, numFrames * 2); + } + mRequestedFrames += numFrames; + + return oboe::DataCallbackResult::Continue; +} + +// TODO - special case https://github.com/google/oboe/blob/master/docs/notes/disconnect.md +void AndroidOutputDevice::onErrorAfterClose(oboe::AudioStream *stream, oboe::Result result) { + if (result == oboe::Result::ErrorDisconnected) { + // LOGI("Restarting AudioStream after disconnect"); + // soundEngine.restart(); // please check oboe samples for soundEngine.restart(); call + } +} +#endif // TARGET_ANDROID \ No newline at end of file diff --git a/src/engine/audio/Audio_AndroidOboe.h b/src/engine/audio/Audio_AndroidOboe.h new file mode 100644 index 00000000..b67e74f5 --- /dev/null +++ b/src/engine/audio/Audio_AndroidOboe.h @@ -0,0 +1,109 @@ +/* Copyright(C) 2007-2017 VoIP objects (voipobjects.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +#ifndef __AUDIO_ANDROID_OBOE_H +#define __AUDIO_ANDROID_OBOE_H + +#ifdef TARGET_ANDROID + +#include "Audio_Interface.h" +#include "Audio_Helper.h" +#include "Audio_Resampler.h" +#include "Audio_DataWindow.h" +#include "../helper/HL_Pointer.h" +#include "../helper/HL_ByteBuffer.h" +#include "../helper/HL_Exception.h" +#include "../helper/HL_Statistics.h" + +#include +#include + +#include "oboe/Oboe.h" + +namespace Audio +{ + class AndroidEnumerator: public Enumerator + { + public: + AndroidEnumerator(); + ~AndroidEnumerator(); + + void open(int direction); + void close(); + + int count(); + std::string nameAt(int index); + int idAt(int index); + int indexOfDefaultDevice(); + + protected: + }; + + class AndroidInputDevice: public InputDevice, public oboe::AudioStreamCallback + { + public: + AndroidInputDevice(int devId); + ~AndroidInputDevice(); + + bool open(); + void close(); + Format getFormat(); + + bool fakeMode(); + void setFakeMode(bool fakemode); + int readBuffer(void* buffer); + bool active() const; + + oboe::DataCallbackResult + onAudioReady(oboe::AudioStream *audioStream, void *audioData, int32_t numFrames); + + + protected: + bool mActive = false; + oboe::AudioStream* mRecordingStream = nullptr; + PResampler mResampler; + DataWindow mDeviceRateCache, mSdkRateCache; + int mDeviceRate; // Actual rate of opened recorder + int mBufferSize; // Size of buffer used for recording (at native sample rate) + DataWindow mRecorderBuffer; + std::condition_variable mDataCondVar; + int mRecorderBufferIndex; + std::mutex mMutex; + }; + +class AndroidOutputDevice: public OutputDevice, public oboe::AudioStreamCallback + { + public: + AndroidOutputDevice(int devId); + ~AndroidOutputDevice(); + + bool open(); + void close(); + Format getFormat(); + + bool fakeMode(); + void setFakeMode(bool fakemode); + + oboe::DataCallbackResult onAudioReady(oboe::AudioStream *audioStream, void *audioData, int32_t numFrames); + void onErrorAfterClose(oboe::AudioStream *stream, oboe::Result result); + + protected: + std::mutex mMutex; + int mDeviceRate = 0; + oboe::AudioStream* mPlayingStream = nullptr; + DataWindow mPlayBuffer; + int mBufferIndex = 0, mBufferSize = 0; + bool mInShutdown = false; + bool mActive = false; + + // Statistics + float mRequestedFrames = 0.0, mStartTime = 0.0, mEndTime = 0.0; + }; +} + +#endif // TARGET_ANDROID + +#endif // __AUDIO_ANDROID_H diff --git a/src/engine/endpoint/EP_AudioProvider.h b/src/engine/endpoint/EP_AudioProvider.h index b5968cbd..cb2388f6 100644 --- a/src/engine/endpoint/EP_AudioProvider.h +++ b/src/engine/endpoint/EP_AudioProvider.h @@ -28,19 +28,19 @@ public: virtual ~AudioProvider(); // Returns provider RTP name - std::string streamName(); + std::string streamName() override; // Returns provider RTP profile name - std::string streamProfile(); + std::string streamProfile() override; // Sets destination IP address - void setDestinationAddress(const RtpPair& addr); + void setDestinationAddress(const RtpPair& addr) override; // Processes incoming data - void processData(PDatagramSocket s, const void* dataBuffer, int dataSize, InternetAddress& source); + void processData(PDatagramSocket s, const void* dataBuffer, int dataSize, InternetAddress& source) override; // This method is called by user agent to send ICE packet from mediasocket - void sendData(PDatagramSocket s, InternetAddress& destination, const void* dataBuffer, unsigned int datasize); + void sendData(PDatagramSocket s, InternetAddress& destination, const void* dataBuffer, unsigned int datasize) override; // Updates SDP offer void updateSdpOffer(resip::SdpContents::Session::Medium& sdp, SdpDirection direction) override; diff --git a/src/engine/media/MT_CodecList.cpp b/src/engine/media/MT_CodecList.cpp index 046e17d8..66b5bbce 100644 --- a/src/engine/media/MT_CodecList.cpp +++ b/src/engine/media/MT_CodecList.cpp @@ -125,7 +125,6 @@ CodecList::CodecList(const Settings& settings) mFactoryList.push_back(new GsmEfrCodec::GsmEfrFactory(mSettings.mWrapIuUP, mSettings.mGsmEfrPayloadType)); #endif #endif - /*mFactoryList.push_back(new IsacCodec::IsacFactory16K(mSettings.mIsac16KPayloadType)); mFactoryList.push_back(new IlbcCodec::IlbcFactory(mSettings.mIlbc20PayloadType, mSettings.mIlbc30PayloadType)); mFactoryList.push_back(new G711Codec::AlawFactory()); @@ -133,7 +132,7 @@ CodecList::CodecList(const Settings& settings) mFactoryList.push_back(new GsmCodec::GsmFactory(mSettings.mGsmFrPayloadLength == 32 ? GsmCodec::Type::Bytes_32 : GsmCodec::Type::Bytes_33, mSettings.mGsmFrPayloadType)); mFactoryList.push_back(new G722Codec::G722Factory()); - mFactoryList.push_back(new G729Codec::G729Factory()); */ + mFactoryList.push_back(new G729Codec::G729Factory());*/ #ifndef TARGET_ANDROID mFactoryList.push_back(new GsmHrCodec::GsmHrFactory(mSettings.mGsmHrPayloadType)); #endif diff --git a/src/libs/ice/ICELog.h b/src/libs/ice/ICELog.h index 6e120fa4..105213a7 100644 --- a/src/libs/ice/ICELog.h +++ b/src/libs/ice/ICELog.h @@ -150,7 +150,7 @@ extern Logger GLogger; {\ if (GLogger.level() >= level_)\ {\ - LogLock l(GLogger.mutex());\ + LogLock log_lock(GLogger.mutex());\ GLogger.beginLine(level_, __FILE__, __LINE__, subsystem_);\ GLogger args_;\ GLogger.endLine();\ diff --git a/src/libs/resiprocate/contrib/ares/ares.h b/src/libs/resiprocate/contrib/ares/ares.h index c96bae88..4d5c60ba 100644 --- a/src/libs/resiprocate/contrib/ares/ares.h +++ b/src/libs/resiprocate/contrib/ares/ares.h @@ -215,7 +215,7 @@ extern const char *ares_strerror(int code); extern void ares_free_errmem(char *mem); -#if defined(WIN32) || defined (__CYGWIN__) || defined(__ANDROID_API__) +#if defined(WIN32) || defined (__CYGWIN__) //|| defined(__ANDROID_API__) #define T_A 1 /* host address */ #define T_NS 2 /* authoritative server */ @@ -257,11 +257,21 @@ extern void ares_free_errmem(char *mem); #define T_MAILA 254 /* transfer mail agent records */ #define T_ANY 255 /* wildcard match */ +#ifndef C_IN +# define C_IN 1 +#endif -#define C_IN 1 -#define C_CHAOS 3 -#define C_HS 4 -#define C_ANY 255 +#ifndef C_CHAOS +# define C_CHAOS 3 +#endif + +#ifndef C_HS +# define C_HS 4 +#endif + +#ifndef C_ANY +# define C_ANY 255 +#endif #define INDIR_MASK 0xc0 #define HFIXEDSZ 12