Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2026 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software: you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation, either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #include "audiolayer.h"
      19             : #include "audio/sound/tone.h"
      20             : #include "logger.h"
      21             : #include "manager.h"
      22             : #include "audio/ringbufferpool.h"
      23             : #include "audio/resampler.h"
      24             : #include "client/jami_signal.h"
      25             : 
      26             : #include "tracepoint.h"
      27             : #if HAVE_WEBRTC_AP
      28             : #include "audio-processing/webrtc.h"
      29             : #endif
      30             : #if HAVE_SPEEXDSP
      31             : #include "audio-processing/speex.h"
      32             : #endif
      33             : 
      34             : #include <ctime>
      35             : #include <algorithm>
      36             : 
      37             : namespace jami {
      38             : 
      39          34 : AudioLayer::AudioLayer(const AudioPreference& pref)
      40          68 :     : isCaptureMuted_(pref.getCaptureMuted())
      41          34 :     , isPlaybackMuted_(pref.getPlaybackMuted())
      42          68 :     , captureGain_(pref.getVolumemic())
      43          34 :     , playbackGain_(pref.getVolumespkr())
      44          34 :     , pref_(pref)
      45          34 :     , mainRingBuffer_(Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID))
      46          34 :     , audioFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
      47          34 :     , audioInputFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
      48          34 :     , urgentRingBuffer_("urgentRingBuffer_id", audioFormat_)
      49          34 :     , resampler_(new Resampler)
      50         204 :     , lastNotificationTime_()
      51             : {
      52          34 :     urgentRingBuffer_.createReadOffset(RingBufferPool::DEFAULT_ID);
      53             : 
      54         136 :     JAMI_LOG("[audiolayer] AGC: {:d}, noiseReduce: {:s}, VAD: {:d}, echoCancel: {:s}, audioProcessor: {:s}",
      55             :              pref_.isAGCEnabled(),
      56             :              pref.getNoiseReduce(),
      57             :              pref.getVadEnabled(),
      58             :              pref.getEchoCanceller(),
      59             :              pref.getAudioProcessor());
      60          34 : }
      61             : 
      62          34 : AudioLayer::~AudioLayer() {}
      63             : 
      64             : void
      65          84 : AudioLayer::hardwareFormatAvailable(AudioFormat playback, size_t bufSize)
      66             : {
      67         336 :     JAMI_LOG("Hardware audio format available: {:s} {}", playback.toString(), bufSize);
      68          84 :     audioFormat_ = Manager::instance().hardwareAudioFormatChanged(playback);
      69          84 :     audioInputFormat_.sampleFormat = audioFormat_.sampleFormat;
      70          84 :     urgentRingBuffer_.setFormat(audioFormat_);
      71          84 :     nativeFrameSize_ = bufSize;
      72          84 : }
      73             : 
      74             : void
      75           0 : AudioLayer::hardwareInputFormatAvailable(AudioFormat capture)
      76             : {
      77           0 :     JAMI_LOG("Hardware input audio format available: {:s}", capture.toString());
      78           0 : }
      79             : 
      80             : void
      81           0 : AudioLayer::devicesChanged()
      82             : {
      83           0 :     emitSignal<libjami::AudioSignal::DeviceEvent>();
      84           0 : }
      85             : 
      86             : void
      87           0 : AudioLayer::flushMain()
      88             : {
      89           0 :     Manager::instance().getRingBufferPool().flushAllBuffers();
      90           0 : }
      91             : 
      92             : void
      93         373 : AudioLayer::flushUrgent()
      94             : {
      95         373 :     urgentRingBuffer_.flushAll();
      96         373 : }
      97             : 
      98             : void
      99           0 : AudioLayer::flush()
     100             : {
     101           0 :     Manager::instance().getRingBufferPool().flushAllBuffers();
     102           0 :     urgentRingBuffer_.flushAll();
     103           0 : }
     104             : 
     105             : void
     106         188 : AudioLayer::playbackChanged(bool started)
     107             : {
     108         188 :     playbackStarted_ = started;
     109         188 : }
     110             : 
     111             : void
     112         188 : AudioLayer::recordChanged(bool started)
     113             : {
     114         188 :     std::lock_guard lock(audioProcessorMutex);
     115         188 :     if (started) {
     116             :         // create audio processor
     117           0 :         createAudioProcessor();
     118             :     } else {
     119             :         // destroy audio processor
     120         188 :         destroyAudioProcessor();
     121             :     }
     122         188 :     recordStarted_ = started;
     123         188 : }
     124             : 
     125             : // helper function
     126             : static inline bool
     127           0 : shouldUseAudioProcessorEchoCancel(bool hasNativeAEC, const std::string& echoCancellerPref)
     128             : {
     129             :     return
     130             :         // user doesn't care which and there is not a system AEC
     131           0 :         (echoCancellerPref == "auto" && !hasNativeAEC)
     132             :         // user specifically wants audioProcessor
     133           0 :         or (echoCancellerPref == "audioProcessor");
     134             : }
     135             : 
     136             : // helper function
     137             : static inline bool
     138           0 : shouldUseAudioProcessorNoiseSuppression(bool hasNativeNS, const std::string& noiseSuppressionPref)
     139             : {
     140             :     return
     141             :         // user doesn't care which and there is no system noise suppression
     142           0 :         (noiseSuppressionPref == "auto" && !hasNativeNS)
     143             :         // user specifically wants audioProcessor
     144           0 :         or (noiseSuppressionPref == "audioProcessor");
     145             : }
     146             : 
     147             : void
     148          31 : AudioLayer::setHasNativeAEC(bool hasNativeAEC)
     149             : {
     150          31 :     JAMI_INFO("[audiolayer] setHasNativeAEC: %d", hasNativeAEC);
     151          31 :     std::lock_guard lock(audioProcessorMutex);
     152          31 :     hasNativeAEC_ = hasNativeAEC;
     153             :     // if we have a current audio processor, tell it to enable/disable its own AEC
     154          31 :     if (audioProcessor) {
     155           0 :         audioProcessor->enableEchoCancel(shouldUseAudioProcessorEchoCancel(hasNativeAEC, pref_.getEchoCanceller()));
     156             :     }
     157          31 : }
     158             : 
     159             : void
     160          34 : AudioLayer::setHasNativeNS(bool hasNativeNS)
     161             : {
     162          34 :     JAMI_INFO("[audiolayer] setHasNativeNS: %d", hasNativeNS);
     163          34 :     std::lock_guard lock(audioProcessorMutex);
     164          34 :     hasNativeNS_ = hasNativeNS;
     165             :     // if we have a current audio processor, tell it to enable/disable its own noise suppression
     166          34 :     if (audioProcessor) {
     167           0 :         audioProcessor->enableNoiseSuppression(
     168           0 :             shouldUseAudioProcessorNoiseSuppression(hasNativeNS, pref_.getNoiseReduce()));
     169             :     }
     170          34 : }
     171             : 
     172             : // must acquire lock beforehand
     173             : void
     174           0 : AudioLayer::createAudioProcessor()
     175             : {
     176           0 :     auto nb_channels = std::max(audioFormat_.nb_channels, audioInputFormat_.nb_channels);
     177           0 :     auto sample_rate = std::max(audioFormat_.sample_rate, audioInputFormat_.sample_rate);
     178             : 
     179           0 :     sample_rate = std::clamp(sample_rate, 16000u, 48000u);
     180             : 
     181           0 :     AudioFormat formatForProcessor {sample_rate, nb_channels};
     182             : 
     183             :     unsigned int frame_size;
     184           0 :     if (pref_.getAudioProcessor() == "speex") {
     185             :         // TODO: maybe force this to be equivalent to 20ms? as expected by Speex
     186           0 :         frame_size = sample_rate / 50u;
     187             :     } else {
     188           0 :         frame_size = sample_rate / 100u;
     189             :     }
     190             : 
     191           0 :     JAMI_WARNING("Input {}", audioInputFormat_.toString());
     192           0 :     JAMI_WARNING("Output {}", audioFormat_.toString());
     193           0 :     JAMI_WARNING("Starting audio processor with: [{} Hz, {} channels, {} samples/frame]",
     194             :                  sample_rate,
     195             :                  nb_channels,
     196             :                  frame_size);
     197             : 
     198           0 :     if (pref_.getAudioProcessor() == "webrtc") {
     199             : #if HAVE_WEBRTC_AP
     200           0 :         JAMI_WARN("[audiolayer] using WebRTCAudioProcessor");
     201           0 :         audioProcessor.reset(new WebRTCAudioProcessor(formatForProcessor, frame_size));
     202             : #else
     203             :         JAMI_ERR("[audiolayer] audioProcessor preference is webrtc, but library not linked! "
     204             :                  "using null AudioProcessor instead");
     205             :         audioProcessor.reset();
     206             : #endif
     207           0 :     } else if (pref_.getAudioProcessor() == "speex") {
     208             : #if HAVE_SPEEXDSP
     209           0 :         JAMI_WARN("[audiolayer] using SpeexAudioProcessor");
     210           0 :         audioProcessor.reset(new SpeexAudioProcessor(formatForProcessor, frame_size));
     211             : #else
     212             :         JAMI_ERR("[audiolayer] audioProcessor preference is Speex, but library not linked! "
     213             :                  "using null AudioProcessor instead");
     214             :         audioProcessor.reset();
     215             : #endif
     216           0 :     } else if (pref_.getAudioProcessor() == "null") {
     217           0 :         JAMI_WARN("[audiolayer] using null AudioProcessor");
     218           0 :         audioProcessor.reset();
     219             :     } else {
     220           0 :         JAMI_ERR("[audiolayer] audioProcessor preference not recognized, using null AudioProcessor "
     221             :                  "instead");
     222           0 :         audioProcessor.reset();
     223             :     }
     224             : 
     225           0 :     if (audioProcessor) {
     226           0 :         audioProcessor->enableNoiseSuppression(
     227           0 :             shouldUseAudioProcessorNoiseSuppression(hasNativeNS_, pref_.getNoiseReduce()));
     228             : 
     229           0 :         audioProcessor->enableAutomaticGainControl(pref_.isAGCEnabled());
     230             : 
     231           0 :         audioProcessor->enableEchoCancel(shouldUseAudioProcessorEchoCancel(hasNativeAEC_, pref_.getEchoCanceller()));
     232             : 
     233           0 :         audioProcessor->enableVoiceActivityDetection(pref_.getVadEnabled());
     234             :     }
     235           0 : }
     236             : 
     237             : // must acquire lock beforehand
     238             : void
     239         188 : AudioLayer::destroyAudioProcessor()
     240             : {
     241             :     // delete it
     242         188 :     audioProcessor.reset();
     243         188 : }
     244             : 
     245             : void
     246           0 : AudioLayer::putUrgent(std::shared_ptr<AudioFrame> buffer)
     247             : {
     248           0 :     urgentRingBuffer_.put(std::move(buffer));
     249           0 : }
     250             : 
     251             : // Notify (with a beep) an incoming call when there is already a call in progress
     252             : void
     253           0 : AudioLayer::notifyIncomingCall()
     254             : {
     255           0 :     if (not playIncomingCallBeep_)
     256           0 :         return;
     257             : 
     258           0 :     auto now = std::chrono::system_clock::now();
     259             : 
     260             :     // Notify maximum once every 5 seconds
     261           0 :     if (now < lastNotificationTime_ + std::chrono::seconds(5))
     262           0 :         return;
     263             : 
     264           0 :     lastNotificationTime_ = now;
     265             : 
     266           0 :     Tone tone("440/160", getSampleRate(), audioFormat_.sampleFormat);
     267           0 :     size_t nbSample = tone.getSize();
     268             : 
     269             :     /* Put the data in the urgent ring buffer */
     270           0 :     urgentRingBuffer_.flushAll();
     271           0 :     urgentRingBuffer_.put(tone.getNext(nbSample));
     272           0 : }
     273             : 
     274             : std::shared_ptr<AudioFrame>
     275           0 : AudioLayer::getToRing(AudioFormat format, size_t writableSamples)
     276             : {
     277           0 :     if (auto fileToPlay = Manager::instance().getTelephoneFile()) {
     278           0 :         auto fileformat = fileToPlay->getFormat();
     279           0 :         bool resample = format != fileformat;
     280             : 
     281           0 :         size_t readableSamples = resample ? rational<size_t>(writableSamples * (size_t) fileformat.sample_rate,
     282           0 :                                                              format.sample_rate)
     283           0 :                                                 .real<size_t>()
     284           0 :                                           : writableSamples;
     285             : 
     286           0 :         return resampler_->resample(fileToPlay->getNext(readableSamples, isRingtoneMuted_), format);
     287           0 :     }
     288           0 :     return {};
     289             : }
     290             : 
     291             : std::shared_ptr<AudioFrame>
     292           0 : AudioLayer::getToPlay(AudioFormat format, size_t writableSamples)
     293             : {
     294           0 :     notifyIncomingCall();
     295           0 :     auto& bufferPool = Manager::instance().getRingBufferPool();
     296             : 
     297           0 :     if (not playbackQueue_)
     298           0 :         playbackQueue_.reset(new AudioFrameResizer(format, static_cast<int>(writableSamples)));
     299             :     else
     300           0 :         playbackQueue_->setFrameSize(static_cast<int>(writableSamples));
     301             : 
     302           0 :     std::shared_ptr<AudioFrame> playbackBuf {};
     303           0 :     while (!(playbackBuf = playbackQueue_->dequeue())) {
     304           0 :         std::shared_ptr<AudioFrame> resampled;
     305             : 
     306           0 :         if (auto urgentSamples = urgentRingBuffer_.get(RingBufferPool::DEFAULT_ID)) {
     307           0 :             bufferPool.discard(1, RingBufferPool::DEFAULT_ID);
     308           0 :             resampled = resampler_->resample(std::move(urgentSamples), format);
     309           0 :         } else if (auto toneToPlay = Manager::instance().getTelephoneTone()) {
     310           0 :             resampled = resampler_->resample(toneToPlay->getNext(), format);
     311           0 :         } else if (auto buf = bufferPool.getData(RingBufferPool::DEFAULT_ID)) {
     312           0 :             resampled = resampler_->resample(std::move(buf), format);
     313             :         } else {
     314           0 :             std::lock_guard lock(audioProcessorMutex);
     315           0 :             if (audioProcessor) {
     316           0 :                 auto silence = std::make_shared<AudioFrame>(format, writableSamples);
     317           0 :                 libav_utils::fillWithSilence(silence->pointer());
     318           0 :                 audioProcessor->putPlayback(silence);
     319           0 :             }
     320           0 :             break;
     321           0 :         }
     322             : 
     323           0 :         if (resampled) {
     324           0 :             std::lock_guard lock(audioProcessorMutex);
     325           0 :             if (audioProcessor) {
     326           0 :                 audioProcessor->putPlayback(resampled);
     327             :             }
     328           0 :             playbackQueue_->enqueue(std::move(resampled));
     329           0 :         } else
     330           0 :             break;
     331           0 :     }
     332             : 
     333             :     jami_tracepoint(audio_layer_get_to_play_end);
     334             : 
     335           0 :     return playbackBuf;
     336           0 : }
     337             : 
     338             : void
     339           0 : AudioLayer::putRecorded(std::shared_ptr<AudioFrame>&& frame)
     340             : {
     341           0 :     std::lock_guard lock(audioProcessorMutex);
     342           0 :     if (audioProcessor && playbackStarted_ && recordStarted_) {
     343           0 :         audioProcessor->putRecorded(std::move(frame));
     344           0 :         while (auto rec = audioProcessor->getProcessed()) {
     345           0 :             mainRingBuffer_->put(std::move(rec));
     346           0 :         }
     347             :     } else {
     348           0 :         mainRingBuffer_->put(std::move(frame));
     349             :     }
     350             : 
     351             :     jami_tracepoint(audio_layer_put_recorded_end, );
     352           0 : }
     353             : 
     354             : } // namespace jami