Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2024 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  Author: Emmanuel Milou <emmanuel.milou@savoirfairelinux.com>
       5             :  *  Author: Alexandre Savard <alexandre.savard@savoirfairelinux.com>
       6             :  *
       7             :  *  This program is free software; you can redistribute it and/or modify
       8             :  *  it under the terms of the GNU General Public License as published by
       9             :  *  the Free Software Foundation; either version 3 of the License, or
      10             :  *  (at your option) any later version.
      11             :  *
      12             :  *  This program is distributed in the hope that it will be useful,
      13             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      14             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      15             :  *  GNU General Public License for more details.
      16             :  *
      17             :  *  You should have received a copy of the GNU General Public License
      18             :  *  along with this program; if not, write to the Free Software
      19             :  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
      20             :  */
      21             : 
      22             : #include "audiolayer.h"
      23             : #include "logger.h"
      24             : #include "manager.h"
      25             : #include "audio/ringbufferpool.h"
      26             : #include "audio/resampler.h"
      27             : #include "tonecontrol.h"
      28             : #include "client/ring_signal.h"
      29             : 
      30             : #include "audio-processing/null_audio_processor.h"
      31             : #include "tracepoint.h"
      32             : #if HAVE_WEBRTC_AP
      33             : #include "audio-processing/webrtc.h"
      34             : #endif
      35             : #if HAVE_SPEEXDSP
      36             : #include "audio-processing/speex.h"
      37             : #endif
      38             : 
      39             : #include <ctime>
      40             : #include <algorithm>
      41             : 
      42             : namespace jami {
      43             : 
      44          34 : AudioLayer::AudioLayer(const AudioPreference& pref)
      45          68 :     : isCaptureMuted_(pref.getCaptureMuted())
      46          34 :     , isPlaybackMuted_(pref.getPlaybackMuted())
      47          68 :     , captureGain_(pref.getVolumemic())
      48          34 :     , playbackGain_(pref.getVolumespkr())
      49          34 :     , pref_(pref)
      50          34 :     , mainRingBuffer_(
      51          34 :           Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID))
      52          34 :     , audioFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
      53          34 :     , audioInputFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
      54          34 :     , urgentRingBuffer_("urgentRingBuffer_id", SIZEBUF, audioFormat_)
      55          34 :     , resampler_(new Resampler)
      56         204 :     , lastNotificationTime_()
      57             : {
      58          34 :     urgentRingBuffer_.createReadOffset(RingBufferPool::DEFAULT_ID);
      59             : 
      60         102 :     JAMI_LOG("[audiolayer] AGC: {:d}, noiseReduce: {:s}, VAD: {:d}, echoCancel: {:s}, audioProcessor: {:s}",
      61             :               pref_.isAGCEnabled(),
      62             :               pref.getNoiseReduce(),
      63             :               pref.getVadEnabled(),
      64             :               pref.getEchoCanceller(),
      65             :               pref.getAudioProcessor());
      66          34 : }
      67             : 
      68          34 : AudioLayer::~AudioLayer() {}
      69             : 
      70             : void
      71         100 : AudioLayer::hardwareFormatAvailable(AudioFormat playback, size_t bufSize)
      72             : {
      73         300 :     JAMI_LOG("Hardware audio format available : {:s} {}", playback.toString(), bufSize);
      74         100 :     audioFormat_ = Manager::instance().hardwareAudioFormatChanged(playback);
      75         100 :     audioInputFormat_.sampleFormat = audioFormat_.sampleFormat;
      76         100 :     urgentRingBuffer_.setFormat(audioFormat_);
      77         100 :     nativeFrameSize_ = bufSize;
      78         100 : }
      79             : 
      80             : void
      81           0 : AudioLayer::hardwareInputFormatAvailable(AudioFormat capture)
      82             : {
      83           0 :     JAMI_LOG("Hardware input audio format available : {:s}", capture.toString());
      84           0 : }
      85             : 
      86             : void
      87           0 : AudioLayer::devicesChanged()
      88             : {
      89           0 :     emitSignal<libjami::AudioSignal::DeviceEvent>();
      90           0 : }
      91             : 
      92             : void
      93           0 : AudioLayer::flushMain()
      94             : {
      95           0 :     Manager::instance().getRingBufferPool().flushAllBuffers();
      96           0 : }
      97             : 
      98             : void
      99         386 : AudioLayer::flushUrgent()
     100             : {
     101         386 :     urgentRingBuffer_.flushAll();
     102         386 : }
     103             : 
     104             : void
     105           0 : AudioLayer::flush()
     106             : {
     107           0 :     Manager::instance().getRingBufferPool().flushAllBuffers();
     108           0 :     urgentRingBuffer_.flushAll();
     109           0 : }
     110             : 
     111             : void
     112         211 : AudioLayer::playbackChanged(bool started)
     113             : {
     114         211 :     playbackStarted_ = started;
     115         211 : }
     116             : 
     117             : void
     118         211 : AudioLayer::recordChanged(bool started)
     119             : {
     120         211 :     std::lock_guard lock(audioProcessorMutex);
     121         211 :     if (started) {
     122             :         // create audio processor
     123           0 :         createAudioProcessor();
     124             :     } else {
     125             :         // destroy audio processor
     126         211 :         destroyAudioProcessor();
     127             :     }
     128         211 :     recordStarted_ = started;
     129         211 : }
     130             : 
     131             : // helper function
     132             : static inline bool
     133           0 : shouldUseAudioProcessorEchoCancel(bool hasNativeAEC, const std::string& echoCancellerPref)
     134             : {
     135             :     return
     136             :         // user doesn't care which and there is not a system AEC
     137           0 :         (echoCancellerPref == "auto" && !hasNativeAEC)
     138             :         // user specifically wants audioProcessor
     139           0 :         or (echoCancellerPref == "audioProcessor");
     140             : }
     141             : 
     142             : // helper function
     143             : static inline bool
     144           0 : shouldUseAudioProcessorNoiseSuppression(bool hasNativeNS, const std::string& noiseSuppressionPref)
     145             : {
     146             :     return
     147             :         // user doesn't care which and there is no system noise suppression
     148           0 :         (noiseSuppressionPref == "auto" && !hasNativeNS)
     149             :         // user specifically wants audioProcessor
     150           0 :         or (noiseSuppressionPref == "audioProcessor");
     151             : }
     152             : 
     153             : void
     154          31 : AudioLayer::setHasNativeAEC(bool hasNativeAEC)
     155             : {
     156          31 :     JAMI_INFO("[audiolayer] setHasNativeAEC: %d", hasNativeAEC);
     157          31 :     std::lock_guard lock(audioProcessorMutex);
     158          31 :     hasNativeAEC_ = hasNativeAEC;
     159             :     // if we have a current audio processor, tell it to enable/disable its own AEC
     160          31 :     if (audioProcessor) {
     161           0 :         audioProcessor->enableEchoCancel(
     162           0 :             shouldUseAudioProcessorEchoCancel(hasNativeAEC, pref_.getEchoCanceller()));
     163             :     }
     164          31 : }
     165             : 
     166             : void
     167          34 : AudioLayer::setHasNativeNS(bool hasNativeNS)
     168             : {
     169          34 :     JAMI_INFO("[audiolayer] setHasNativeNS: %d", hasNativeNS);
     170          34 :     std::lock_guard lock(audioProcessorMutex);
     171          34 :     hasNativeNS_ = hasNativeNS;
     172             :     // if we have a current audio processor, tell it to enable/disable its own noise suppression
     173          34 :     if (audioProcessor) {
     174           0 :         audioProcessor->enableNoiseSuppression(
     175           0 :             shouldUseAudioProcessorNoiseSuppression(hasNativeNS, pref_.getNoiseReduce()));
     176             :     }
     177          34 : }
     178             : 
     179             : // must acquire lock beforehand
     180             : void
     181           0 : AudioLayer::createAudioProcessor()
     182             : {
     183           0 :     auto nb_channels = std::max(audioFormat_.nb_channels, audioInputFormat_.nb_channels);
     184           0 :     auto sample_rate = std::max(audioFormat_.sample_rate, audioInputFormat_.sample_rate);
     185             : 
     186           0 :     sample_rate = std::clamp(sample_rate, 16000u, 48000u);
     187             : 
     188           0 :     AudioFormat formatForProcessor {sample_rate, nb_channels};
     189             : 
     190             :     unsigned int frame_size;
     191           0 :     if (pref_.getAudioProcessor() == "speex") {
     192             :         // TODO: maybe force this to be equivalent to 20ms? as expected by speex
     193           0 :         frame_size = sample_rate / 50u;
     194             :     } else {
     195           0 :         frame_size = sample_rate / 100u;
     196             :     }
     197             : 
     198           0 :     JAMI_WARNING("Input {}", audioInputFormat_.toString());
     199           0 :     JAMI_WARNING("Output {}", audioFormat_.toString());
     200           0 :     JAMI_WARNING("Starting audio processor with: [{} Hz, {} channels, {} samples/frame]",
     201             :               sample_rate,
     202             :               nb_channels,
     203             :               frame_size);
     204             : 
     205           0 :     if (pref_.getAudioProcessor() == "webrtc") {
     206             : #if HAVE_WEBRTC_AP
     207           0 :         JAMI_WARN("[audiolayer] using WebRTCAudioProcessor");
     208           0 :         audioProcessor.reset(new WebRTCAudioProcessor(formatForProcessor, frame_size));
     209             : #else
     210             :         JAMI_ERR("[audiolayer] audioProcessor preference is webrtc, but library not linked! "
     211             :                  "using null AudioProcessor instead");
     212             :         audioProcessor.reset();
     213             : #endif
     214           0 :     } else if (pref_.getAudioProcessor() == "speex") {
     215             : #if HAVE_SPEEXDSP
     216           0 :         JAMI_WARN("[audiolayer] using SpeexAudioProcessor");
     217           0 :         audioProcessor.reset(new SpeexAudioProcessor(formatForProcessor, frame_size));
     218             : #else
     219             :         JAMI_ERR("[audiolayer] audioProcessor preference is speex, but library not linked! "
     220             :                  "using null AudioProcessor instead");
     221             :         audioProcessor.reset();
     222             : #endif
     223           0 :     } else if (pref_.getAudioProcessor() == "null") {
     224           0 :         JAMI_WARN("[audiolayer] using null AudioProcessor");
     225           0 :         audioProcessor.reset();
     226             :     } else {
     227           0 :         JAMI_ERR("[audiolayer] audioProcessor preference not recognized, using null AudioProcessor "
     228             :                  "instead");
     229           0 :         audioProcessor.reset();
     230             :     }
     231             : 
     232           0 :     if (audioProcessor) {
     233           0 :         audioProcessor->enableNoiseSuppression(
     234           0 :             shouldUseAudioProcessorNoiseSuppression(hasNativeNS_, pref_.getNoiseReduce()));
     235             : 
     236           0 :         audioProcessor->enableAutomaticGainControl(pref_.isAGCEnabled());
     237             : 
     238           0 :         audioProcessor->enableEchoCancel(
     239           0 :             shouldUseAudioProcessorEchoCancel(hasNativeAEC_, pref_.getEchoCanceller()));
     240             : 
     241           0 :         audioProcessor->enableVoiceActivityDetection(pref_.getVadEnabled());
     242             :     }
     243           0 : }
     244             : 
     245             : // must acquire lock beforehand
     246             : void
     247         211 : AudioLayer::destroyAudioProcessor()
     248             : {
     249             :     // delete it
     250         211 :     audioProcessor.reset();
     251         211 : }
     252             : 
     253             : void
     254           0 : AudioLayer::putUrgent(std::shared_ptr<AudioFrame> buffer)
     255             : {
     256           0 :     urgentRingBuffer_.put(std::move(buffer));
     257           0 : }
     258             : 
     259             : // Notify (with a beep) an incoming call when there is already a call in progress
     260             : void
     261           0 : AudioLayer::notifyIncomingCall()
     262             : {
     263           0 :     if (not playIncomingCallBeep_)
     264           0 :         return;
     265             : 
     266           0 :     auto now = std::chrono::system_clock::now();
     267             : 
     268             :     // Notify maximum once every 5 seconds
     269           0 :     if (now < lastNotificationTime_ + std::chrono::seconds(5))
     270           0 :         return;
     271             : 
     272           0 :     lastNotificationTime_ = now;
     273             : 
     274           0 :     Tone tone("440/160", getSampleRate(), audioFormat_.sampleFormat);
     275           0 :     size_t nbSample = tone.getSize();
     276             : 
     277             :     /* Put the data in the urgent ring buffer */
     278           0 :     urgentRingBuffer_.flushAll();
     279           0 :     urgentRingBuffer_.put(tone.getNext(nbSample));
     280           0 : }
     281             : 
     282             : std::shared_ptr<AudioFrame>
     283           0 : AudioLayer::getToRing(AudioFormat format, size_t writableSamples)
     284             : {
     285           0 :     if (auto fileToPlay = Manager::instance().getTelephoneFile()) {
     286           0 :         auto fileformat = fileToPlay->getFormat();
     287           0 :         bool resample = format != fileformat;
     288             : 
     289           0 :         size_t readableSamples = resample ? (rational<size_t>(writableSamples, format.sample_rate)
     290           0 :                                              * (size_t) fileformat.sample_rate)
     291           0 :                                                 .real<size_t>()
     292           0 :                                           : writableSamples;
     293             : 
     294           0 :         return resampler_->resample(fileToPlay->getNext(readableSamples, isRingtoneMuted_), format);
     295           0 :     }
     296           0 :     return {};
     297             : }
     298             : 
     299             : std::shared_ptr<AudioFrame>
     300           0 : AudioLayer::getToPlay(AudioFormat format, size_t writableSamples)
     301             : {
     302           0 :     notifyIncomingCall();
     303           0 :     auto& bufferPool = Manager::instance().getRingBufferPool();
     304             : 
     305           0 :     if (not playbackQueue_)
     306           0 :         playbackQueue_.reset(new AudioFrameResizer(format, writableSamples));
     307             :     else
     308           0 :         playbackQueue_->setFrameSize(writableSamples);
     309             : 
     310           0 :     std::shared_ptr<AudioFrame> playbackBuf {};
     311           0 :     while (!(playbackBuf = playbackQueue_->dequeue())) {
     312           0 :         std::shared_ptr<AudioFrame> resampled;
     313             : 
     314           0 :         if (auto urgentSamples = urgentRingBuffer_.get(RingBufferPool::DEFAULT_ID)) {
     315           0 :             bufferPool.discard(1, RingBufferPool::DEFAULT_ID);
     316           0 :             resampled = resampler_->resample(std::move(urgentSamples), format);
     317           0 :         } else if (auto toneToPlay = Manager::instance().getTelephoneTone()) {
     318           0 :             resampled = resampler_->resample(toneToPlay->getNext(), format);
     319           0 :         } else if (auto buf = bufferPool.getData(RingBufferPool::DEFAULT_ID)) {
     320           0 :             resampled = resampler_->resample(std::move(buf), format);
     321             :         } else {
     322           0 :             std::lock_guard lock(audioProcessorMutex);
     323           0 :             if (audioProcessor) {
     324           0 :                 auto silence = std::make_shared<AudioFrame>(format, writableSamples);
     325           0 :                 libav_utils::fillWithSilence(silence->pointer());
     326           0 :                 audioProcessor->putPlayback(silence);
     327           0 :             }
     328           0 :             break;
     329           0 :         }
     330             : 
     331           0 :         if (resampled) {
     332           0 :             std::lock_guard lock(audioProcessorMutex);
     333           0 :             if (audioProcessor) {
     334           0 :                 audioProcessor->putPlayback(resampled);
     335             :             }
     336           0 :             playbackQueue_->enqueue(std::move(resampled));
     337           0 :         } else
     338           0 :             break;
     339           0 :     }
     340             : 
     341             :     jami_tracepoint(audio_layer_get_to_play_end);
     342             : 
     343           0 :     return playbackBuf;
     344           0 : }
     345             : 
     346             : void
     347           0 : AudioLayer::putRecorded(std::shared_ptr<AudioFrame>&& frame)
     348             : {
     349           0 :     std::lock_guard lock(audioProcessorMutex);
     350           0 :     if (audioProcessor && playbackStarted_ && recordStarted_) {
     351           0 :         audioProcessor->putRecorded(std::move(frame));
     352           0 :         while (auto rec = audioProcessor->getProcessed()) {
     353           0 :             mainRingBuffer_->put(std::move(rec));
     354           0 :         }
     355             :     } else {
     356           0 :         mainRingBuffer_->put(std::move(frame));
     357             :     }
     358             : 
     359             :     jami_tracepoint(audio_layer_put_recorded_end, );
     360           0 : }
     361             : 
     362             : } // namespace jami