[WIP] feat: Try to enable software APM(aec,ns,agc) for rust-core.

.gitignore

@@ -1,2 +1,3 @@
 target
-/.idea
+/.idea
+.DS_Store

webrtc-sys/include/livekit/audio_device.h

@@ -20,6 +20,7 @@
 
 #include "api/task_queue/task_queue_factory.h"
 #include "modules/audio_device/include/audio_device.h"
+#include "modules/audio_device/audio_device_buffer.h"
 #include "rtc_base/synchronization/mutex.h"
 #include "rtc_base/task_queue.h"
 #include "rtc_base/task_utils/repeating_task.h"
@@ -123,6 +124,7 @@ class AudioDevice : public webrtc::AudioDeviceModule {
   webrtc::RepeatingTaskHandle audio_task_;
   webrtc::AudioTransport* audio_transport_;
   webrtc::TaskQueueFactory* task_queue_factory_;
+  webrtc::AudioDeviceBuffer audio_device_buffer_;
   bool playing_{false};
   bool initialized_{false};
 };

webrtc-sys/src/audio_device.cpp

@@ -25,7 +25,8 @@ namespace livekit {
 
 AudioDevice::AudioDevice(webrtc::TaskQueueFactory* task_queue_factory)
     : task_queue_factory_(task_queue_factory),
-      data_(kSamplesPer10Ms * kChannels) {}
+      data_(kSamplesPer10Ms * kChannels),
+      audio_device_buffer_(task_queue_factory) {}
 
 AudioDevice::~AudioDevice() {
   Terminate();
@@ -39,6 +40,7 @@ int32_t AudioDevice::ActiveAudioLayer(AudioLayer* audioLayer) const {
 int32_t AudioDevice::RegisterAudioCallback(webrtc::AudioTransport* transport) {
   webrtc::MutexLock lock(&mutex_);
   audio_transport_ = transport;
+  audio_device_buffer_.RegisterAudioCallback(transport);
   return 0;
 }
 
@@ -47,6 +49,11 @@ int32_t AudioDevice::Init() {
   if (initialized_)
     return 0;
 
+  audio_device_buffer_.SetRecordingSampleRate(kSampleRate);
+  audio_device_buffer_.SetPlayoutSampleRate(kSampleRate);
+  audio_device_buffer_.SetRecordingChannels(kChannels);
+  audio_device_buffer_.SetPlayoutChannels(kChannels);
+
   audio_queue_ =
       std::make_unique<rtc::TaskQueue>(task_queue_factory_->CreateTaskQueue(
           "AudioDevice", webrtc::TaskQueueFactory::Priority::NORMAL));
@@ -63,9 +70,13 @@ int32_t AudioDevice::Init() {
 
           // Request the AudioData, otherwise WebRTC will ignore the packets.
           // 10ms of audio data.
+          audio_device_buffer_.RequestPlayoutData(kSamplesPer10Ms);
+          audio_device_buffer_.GetPlayoutData(data);
+          /*
           audio_transport_->NeedMorePlayData(
               kSamplesPer10Ms, kBytesPerSample, kChannels, kSampleRate, data,
               n_samples_out, &elapsed_time_ms, &ntp_time_ms);
+          */
         }
 
         return webrtc::TimeDelta::Millis(10);

webrtc-sys/src/audio_track.cpp

@@ -163,15 +163,15 @@ AudioTrackSource::InternalSource::InternalSource(
 
         if (buffer_.size() >= samples10ms) {
           for (auto sink : sinks_)
-            sink->OnData(buffer_.data(), sizeof(int16_t), sample_rate_,
+            sink->OnData(buffer_.data(), sizeof(int16_t) * 8, sample_rate_,
                          num_channels_, samples10ms / num_channels_);
 
           buffer_.erase(buffer_.begin(), buffer_.begin() + samples10ms);
         } else {
           missed_frames_++;
           if (missed_frames_ >= silence_frames_threshold) {
             for (auto sink : sinks_)
-              sink->OnData(silence_buffer_, sizeof(int16_t), sample_rate_,
+              sink->OnData(silence_buffer_, sizeof(int16_t) * 8, sample_rate_,
                            num_channels_, samples10ms / num_channels_);
           }
         }

webrtc-sys/src/peer_connection_factory.cpp

@@ -21,6 +21,8 @@
 
 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "api/audio/echo_canceller3_factory.h"
+#include "api/audio/echo_canceller3_config.h"
 #include "api/peer_connection_interface.h"
 #include "api/rtc_error.h"
 #include "api/rtc_event_log/rtc_event_log_factory.h"
@@ -39,6 +41,7 @@
 #include "rtc_base/thread.h"
 #include "webrtc-sys/src/peer_connection.rs.h"
 #include "webrtc-sys/src/peer_connection_factory.rs.h"
+#include "modules/audio_mixer/audio_mixer_impl.h"
 
 namespace livekit {
 
@@ -76,7 +79,18 @@ PeerConnectionFactory::PeerConnectionFactory(
       std::move(std::make_unique<livekit::VideoDecoderFactory>());
   media_deps.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory();
   media_deps.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory();
-  media_deps.audio_processing = webrtc::AudioProcessingBuilder().Create();
+
+  auto apm = webrtc::AudioProcessingBuilder();
+  auto cfg = webrtc::EchoCanceller3Config();
+  auto echo_control = std::make_unique<webrtc::EchoCanceller3Factory>(cfg);
+
+  apm.SetEchoControlFactory(std::move(echo_control));
+  media_deps.audio_processing = apm.Create();
+
+  auto audio_mixer = webrtc::AudioMixerImpl::Create();
+
+  media_deps.audio_mixer = audio_mixer;
+
   media_deps.trials = dependencies.trials.get();
 
   dependencies.media_engine = cricket::CreateMediaEngine(std::move(media_deps));