Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2021-2024 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software; you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation; either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program; if not, write to the Free Software
      16             :  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
      17             :  */
      18             : 
      19             : #include "webrtc.h"
      20             : #include "logger.h"
      21             : 
      22             : #include <webrtc/modules/audio_processing/include/audio_processing.h>
      23             : 
      24             : namespace jami {
      25             : 
      26             : inline size_t
      27           0 : webrtcFrameSize(AudioFormat format)
      28             : {
      29           0 :     return (size_t) (webrtc::AudioProcessing::kChunkSizeMs * format.sample_rate / 1000);
      30             : }
      31             : 
      32             : constexpr int webrtcNoError = webrtc::AudioProcessing::kNoError;
      33             : 
      34           0 : WebRTCAudioProcessor::WebRTCAudioProcessor(AudioFormat format, unsigned /* frameSize */)
      35           0 :     : AudioProcessor(format.withSampleFormat(AV_SAMPLE_FMT_FLTP), webrtcFrameSize(format))
      36             : {
      37           0 :     JAMI_LOG("[webrtc-ap] WebRTCAudioProcessor, frame size = {:d} (={:d} ms), channels = {:d}",
      38             :              frameSize_,
      39             :              frameDurationMs_,
      40             :              format_.nb_channels);
      41           0 :     webrtc::Config config;
      42           0 :     config.Set<webrtc::ExtendedFilter>(new webrtc::ExtendedFilter(true));
      43           0 :     config.Set<webrtc::DelayAgnostic>(new webrtc::DelayAgnostic(true));
      44             : 
      45           0 :     apm.reset(webrtc::AudioProcessing::Create(config));
      46             : 
      47           0 :     webrtc::StreamConfig streamConfig((int) format_.sample_rate, (int) format_.nb_channels);
      48             :     webrtc::ProcessingConfig pconfig = {
      49             :         streamConfig, /* input stream */
      50             :         streamConfig, /* output stream */
      51             :         streamConfig, /* reverse input stream */
      52             :         streamConfig, /* reverse output stream */
      53           0 :     };
      54             : 
      55           0 :     if (apm->Initialize(pconfig) != webrtcNoError) {
      56           0 :         JAMI_ERROR("[webrtc-ap] Error initialising audio processing module");
      57             :     }
      58           0 : }
      59             : 
      60             : void
      61           0 : WebRTCAudioProcessor::enableNoiseSuppression(bool enabled)
      62             : {
      63           0 :     JAMI_LOG("[webrtc-ap] enableNoiseSuppression {}", enabled);
      64           0 :     if (apm->noise_suppression()->Enable(enabled) != webrtcNoError) {
      65           0 :         JAMI_ERROR("[webrtc-ap] Error enabling noise suppression");
      66             :     }
      67           0 :     if (apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kVeryHigh) != webrtcNoError) {
      68           0 :         JAMI_ERROR("[webrtc-ap] Error setting noise suppression level");
      69             :     }
      70           0 :     if (apm->high_pass_filter()->Enable(enabled) != webrtcNoError) {
      71           0 :         JAMI_ERROR("[webrtc-ap] Error enabling high pass filter");
      72             :     }
      73           0 : }
      74             : 
      75             : void
      76           0 : WebRTCAudioProcessor::enableAutomaticGainControl(bool enabled)
      77             : {
      78           0 :     JAMI_LOG("[webrtc-ap] enableAutomaticGainControl {}", enabled);
      79           0 :     if (apm->gain_control()->Enable(enabled) != webrtcNoError) {
      80           0 :         JAMI_ERROR("[webrtc-ap] Error enabling automatic gain control");
      81             :     }
      82           0 :     if (apm->gain_control()->set_analog_level_limits(0, 255) != webrtcNoError) {
      83           0 :         JAMI_ERROR("[webrtc-ap] Error setting automatic gain control analog level limits");
      84             :     }
      85           0 :     if (apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveAnalog) != webrtcNoError) {
      86           0 :         JAMI_ERROR("[webrtc-ap] Error setting automatic gain control mode");
      87             :     }
      88           0 : }
      89             : 
      90             : void
      91           0 : WebRTCAudioProcessor::enableEchoCancel(bool enabled)
      92             : {
      93           0 :     JAMI_LOG("[webrtc-ap] enableEchoCancel {}", enabled);
      94             : 
      95           0 :     if (apm->echo_cancellation()->Enable(enabled) != webrtcNoError) {
      96           0 :         JAMI_ERROR("[webrtc-ap] Error enabling echo cancellation");
      97             :     }
      98           0 :     if (apm->echo_cancellation()->set_suppression_level(
      99             :             webrtc::EchoCancellation::SuppressionLevel::kHighSuppression)
     100           0 :         != webrtcNoError) {
     101           0 :         JAMI_ERROR("[webrtc-ap] Error setting echo cancellation level");
     102             :     }
     103           0 :     if (apm->echo_cancellation()->enable_drift_compensation(true) != webrtcNoError) {
     104           0 :         JAMI_ERROR("[webrtc-ap] Error enabling echo cancellation drift compensation");
     105             :     }
     106           0 : }
     107             : 
     108             : void
     109           0 : WebRTCAudioProcessor::enableVoiceActivityDetection(bool enabled)
     110             : {
     111           0 :     JAMI_LOG("[webrtc-ap] enableVoiceActivityDetection {}", enabled);
     112           0 :     if (apm->voice_detection()->Enable(enabled) != webrtcNoError) {
     113           0 :         JAMI_ERROR("[webrtc-ap] Error enabling voice activation detection");
     114             :     }
     115           0 :     if (apm->voice_detection()->set_likelihood(webrtc::VoiceDetection::kVeryLowLikelihood)
     116           0 :         != webrtcNoError) {
     117           0 :         JAMI_ERROR("[webrtc-ap] Error setting voice detection likelihood");
     118             :     }
     119             :     // asserted to be 10 in voice_detection_impl.cc
     120           0 :     if (apm->voice_detection()->set_frame_size_ms(10) != webrtcNoError) {
     121           0 :         JAMI_ERROR("[webrtc-ap] Error setting voice detection frame size");
     122             :     }
     123           0 : }
     124             : 
     125             : std::shared_ptr<AudioFrame>
     126           0 : WebRTCAudioProcessor::getProcessed()
     127             : {
     128           0 :     if (tidyQueues()) {
     129           0 :         return {};
     130             :     }
     131             : 
     132           0 :     int driftSamples = playbackQueue_.samples() - recordQueue_.samples();
     133             : 
     134           0 :     auto playback = playbackQueue_.dequeue();
     135           0 :     auto record = recordQueue_.dequeue();
     136           0 :     if (!playback || !record) {
     137           0 :         return {};
     138             :     }
     139           0 :     webrtc::StreamConfig sc((int) format_.sample_rate, (int) format_.nb_channels);
     140             : 
     141             :     // process reverse in place
     142           0 :     float** playData = (float**) playback->pointer()->extended_data;
     143           0 :     if (apm->ProcessReverseStream(playData, sc, sc, playData) != webrtcNoError) {
     144           0 :         JAMI_ERR("[webrtc-ap] ProcessReverseStream failed");
     145             :     }
     146             : 
     147             :     // process deinterleaved float recorded data
     148             :     // TODO: maybe implement this to see if it's better than automatic drift compensation
     149             :     // (it MUST be called prior to ProcessStream)
     150             :     // delay = (t_render - t_analyze) + (t_process - t_capture)
     151           0 :     if (apm->set_stream_delay_ms(0) != webrtcNoError) {
     152           0 :         JAMI_ERR("[webrtc-ap] set_stream_delay_ms failed");
     153             :     }
     154             : 
     155           0 :     if (apm->gain_control()->set_stream_analog_level(analogLevel_) != webrtcNoError) {
     156           0 :         JAMI_ERR("[webrtc-ap] set_stream_analog_level failed");
     157             :     }
     158           0 :     apm->echo_cancellation()->set_stream_drift_samples(driftSamples);
     159             : 
     160             :     // process in place
     161           0 :     float** recData = (float**) record->pointer()->extended_data;
     162           0 :     if (apm->ProcessStream(recData, sc, sc, recData) != webrtcNoError) {
     163           0 :         JAMI_ERR("[webrtc-ap] ProcessStream failed");
     164             :     }
     165             : 
     166           0 :     analogLevel_ = apm->gain_control()->stream_analog_level();
     167           0 :     record->has_voice = apm->voice_detection()->is_enabled()
     168           0 :                            && getStabilizedVoiceActivity(apm->voice_detection()->stream_has_voice());
     169           0 :     return record;
     170           0 : }
     171             : 
     172             : } // namespace jami