Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2025 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software: you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation, either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #include "audiolayer.h"
      19             : #include "logger.h"
      20             : #include "manager.h"
      21             : #include "audio/ringbufferpool.h"
      22             : #include "audio/resampler.h"
      23             : #include "tonecontrol.h"
      24             : #include "client/ring_signal.h"
      25             : 
      26             : #include "audio-processing/null_audio_processor.h"
      27             : #include "tracepoint.h"
      28             : #if HAVE_WEBRTC_AP
      29             : #include "audio-processing/webrtc.h"
      30             : #endif
      31             : #if HAVE_SPEEXDSP
      32             : #include "audio-processing/speex.h"
      33             : #endif
      34             : 
      35             : #include <ctime>
      36             : #include <algorithm>
      37             : 
      38             : namespace jami {
      39             : 
      40          35 : AudioLayer::AudioLayer(const AudioPreference& pref)
      41          70 :     : isCaptureMuted_(pref.getCaptureMuted())
      42          35 :     , isPlaybackMuted_(pref.getPlaybackMuted())
      43          70 :     , captureGain_(pref.getVolumemic())
      44          35 :     , playbackGain_(pref.getVolumespkr())
      45          35 :     , pref_(pref)
      46          35 :     , mainRingBuffer_(Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID))
      47          35 :     , audioFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
      48          35 :     , audioInputFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
      49          35 :     , urgentRingBuffer_("urgentRingBuffer_id", audioFormat_)
      50          35 :     , resampler_(new Resampler)
      51         210 :     , lastNotificationTime_()
      52             : {
      53          35 :     urgentRingBuffer_.createReadOffset(RingBufferPool::DEFAULT_ID);
      54             : 
      55         140 :     JAMI_LOG("[audiolayer] AGC: {:d}, noiseReduce: {:s}, VAD: {:d}, echoCancel: {:s}, audioProcessor: {:s}",
      56             :              pref_.isAGCEnabled(),
      57             :              pref.getNoiseReduce(),
      58             :              pref.getVadEnabled(),
      59             :              pref.getEchoCanceller(),
      60             :              pref.getAudioProcessor());
      61          35 : }
      62             : 
      63          35 : AudioLayer::~AudioLayer() {}
      64             : 
      65             : void
      66          99 : AudioLayer::hardwareFormatAvailable(AudioFormat playback, size_t bufSize)
      67             : {
      68         396 :     JAMI_LOG("Hardware audio format available: {:s} {}", playback.toString(), bufSize);
      69          99 :     audioFormat_ = Manager::instance().hardwareAudioFormatChanged(playback);
      70          99 :     audioInputFormat_.sampleFormat = audioFormat_.sampleFormat;
      71          99 :     urgentRingBuffer_.setFormat(audioFormat_);
      72          99 :     nativeFrameSize_ = bufSize;
      73          99 : }
      74             : 
      75             : void
      76           0 : AudioLayer::hardwareInputFormatAvailable(AudioFormat capture)
      77             : {
      78           0 :     JAMI_LOG("Hardware input audio format available: {:s}", capture.toString());
      79           0 : }
      80             : 
      81             : void
      82           0 : AudioLayer::devicesChanged()
      83             : {
      84           0 :     emitSignal<libjami::AudioSignal::DeviceEvent>();
      85           0 : }
      86             : 
      87             : void
      88           0 : AudioLayer::flushMain()
      89             : {
      90           0 :     Manager::instance().getRingBufferPool().flushAllBuffers();
      91           0 : }
      92             : 
      93             : void
      94         403 : AudioLayer::flushUrgent()
      95             : {
      96         403 :     urgentRingBuffer_.flushAll();
      97         403 : }
      98             : 
      99             : void
     100           0 : AudioLayer::flush()
     101             : {
     102           0 :     Manager::instance().getRingBufferPool().flushAllBuffers();
     103           0 :     urgentRingBuffer_.flushAll();
     104           0 : }
     105             : 
     106             : void
     107         210 : AudioLayer::playbackChanged(bool started)
     108             : {
     109         210 :     playbackStarted_ = started;
     110         210 : }
     111             : 
     112             : void
     113         210 : AudioLayer::recordChanged(bool started)
     114             : {
     115         210 :     std::lock_guard lock(audioProcessorMutex);
     116         210 :     if (started) {
     117             :         // create audio processor
     118           0 :         createAudioProcessor();
     119             :     } else {
     120             :         // destroy audio processor
     121         210 :         destroyAudioProcessor();
     122             :     }
     123         210 :     recordStarted_ = started;
     124         210 : }
     125             : 
     126             : // helper function
     127             : static inline bool
     128           0 : shouldUseAudioProcessorEchoCancel(bool hasNativeAEC, const std::string& echoCancellerPref)
     129             : {
     130             :     return
     131             :         // user doesn't care which and there is not a system AEC
     132           0 :         (echoCancellerPref == "auto" && !hasNativeAEC)
     133             :         // user specifically wants audioProcessor
     134           0 :         or (echoCancellerPref == "audioProcessor");
     135             : }
     136             : 
     137             : // helper function
     138             : static inline bool
     139           0 : shouldUseAudioProcessorNoiseSuppression(bool hasNativeNS, const std::string& noiseSuppressionPref)
     140             : {
     141             :     return
     142             :         // user doesn't care which and there is no system noise suppression
     143           0 :         (noiseSuppressionPref == "auto" && !hasNativeNS)
     144             :         // user specifically wants audioProcessor
     145           0 :         or (noiseSuppressionPref == "audioProcessor");
     146             : }
     147             : 
     148             : void
     149          32 : AudioLayer::setHasNativeAEC(bool hasNativeAEC)
     150             : {
     151          32 :     JAMI_INFO("[audiolayer] setHasNativeAEC: %d", hasNativeAEC);
     152          32 :     std::lock_guard lock(audioProcessorMutex);
     153          32 :     hasNativeAEC_ = hasNativeAEC;
     154             :     // if we have a current audio processor, tell it to enable/disable its own AEC
     155          32 :     if (audioProcessor) {
     156           0 :         audioProcessor->enableEchoCancel(shouldUseAudioProcessorEchoCancel(hasNativeAEC, pref_.getEchoCanceller()));
     157             :     }
     158          32 : }
     159             : 
     160             : void
     161          35 : AudioLayer::setHasNativeNS(bool hasNativeNS)
     162             : {
     163          35 :     JAMI_INFO("[audiolayer] setHasNativeNS: %d", hasNativeNS);
     164          35 :     std::lock_guard lock(audioProcessorMutex);
     165          35 :     hasNativeNS_ = hasNativeNS;
     166             :     // if we have a current audio processor, tell it to enable/disable its own noise suppression
     167          35 :     if (audioProcessor) {
     168           0 :         audioProcessor->enableNoiseSuppression(
     169           0 :             shouldUseAudioProcessorNoiseSuppression(hasNativeNS, pref_.getNoiseReduce()));
     170             :     }
     171          35 : }
     172             : 
     173             : // must acquire lock beforehand
     174             : void
     175           0 : AudioLayer::createAudioProcessor()
     176             : {
     177           0 :     auto nb_channels = std::max(audioFormat_.nb_channels, audioInputFormat_.nb_channels);
     178           0 :     auto sample_rate = std::max(audioFormat_.sample_rate, audioInputFormat_.sample_rate);
     179             : 
     180           0 :     sample_rate = std::clamp(sample_rate, 16000u, 48000u);
     181             : 
     182           0 :     AudioFormat formatForProcessor {sample_rate, nb_channels};
     183             : 
     184             :     unsigned int frame_size;
     185           0 :     if (pref_.getAudioProcessor() == "speex") {
     186             :         // TODO: maybe force this to be equivalent to 20ms? as expected by Speex
     187           0 :         frame_size = sample_rate / 50u;
     188             :     } else {
     189           0 :         frame_size = sample_rate / 100u;
     190             :     }
     191             : 
     192           0 :     JAMI_WARNING("Input {}", audioInputFormat_.toString());
     193           0 :     JAMI_WARNING("Output {}", audioFormat_.toString());
     194           0 :     JAMI_WARNING("Starting audio processor with: [{} Hz, {} channels, {} samples/frame]",
     195             :                  sample_rate,
     196             :                  nb_channels,
     197             :                  frame_size);
     198             : 
     199           0 :     if (pref_.getAudioProcessor() == "webrtc") {
     200             : #if HAVE_WEBRTC_AP
     201           0 :         JAMI_WARN("[audiolayer] using WebRTCAudioProcessor");
     202           0 :         audioProcessor.reset(new WebRTCAudioProcessor(formatForProcessor, frame_size));
     203             : #else
     204             :         JAMI_ERR("[audiolayer] audioProcessor preference is webrtc, but library not linked! "
     205             :                  "using null AudioProcessor instead");
     206             :         audioProcessor.reset();
     207             : #endif
     208           0 :     } else if (pref_.getAudioProcessor() == "speex") {
     209             : #if HAVE_SPEEXDSP
     210           0 :         JAMI_WARN("[audiolayer] using SpeexAudioProcessor");
     211           0 :         audioProcessor.reset(new SpeexAudioProcessor(formatForProcessor, frame_size));
     212             : #else
     213             :         JAMI_ERR("[audiolayer] audioProcessor preference is Speex, but library not linked! "
     214             :                  "using null AudioProcessor instead");
     215             :         audioProcessor.reset();
     216             : #endif
     217           0 :     } else if (pref_.getAudioProcessor() == "null") {
     218           0 :         JAMI_WARN("[audiolayer] using null AudioProcessor");
     219           0 :         audioProcessor.reset();
     220             :     } else {
     221           0 :         JAMI_ERR("[audiolayer] audioProcessor preference not recognized, using null AudioProcessor "
     222             :                  "instead");
     223           0 :         audioProcessor.reset();
     224             :     }
     225             : 
     226           0 :     if (audioProcessor) {
     227           0 :         audioProcessor->enableNoiseSuppression(
     228           0 :             shouldUseAudioProcessorNoiseSuppression(hasNativeNS_, pref_.getNoiseReduce()));
     229             : 
     230           0 :         audioProcessor->enableAutomaticGainControl(pref_.isAGCEnabled());
     231             : 
     232           0 :         audioProcessor->enableEchoCancel(shouldUseAudioProcessorEchoCancel(hasNativeAEC_, pref_.getEchoCanceller()));
     233             : 
     234           0 :         audioProcessor->enableVoiceActivityDetection(pref_.getVadEnabled());
     235             :     }
     236           0 : }
     237             : 
     238             : // must acquire lock beforehand
     239             : void
     240         210 : AudioLayer::destroyAudioProcessor()
     241             : {
     242             :     // delete it
     243         210 :     audioProcessor.reset();
     244         210 : }
     245             : 
     246             : void
     247           0 : AudioLayer::putUrgent(std::shared_ptr<AudioFrame> buffer)
     248             : {
     249           0 :     urgentRingBuffer_.put(std::move(buffer));
     250           0 : }
     251             : 
     252             : // Notify (with a beep) an incoming call when there is already a call in progress
     253             : void
     254           0 : AudioLayer::notifyIncomingCall()
     255             : {
     256           0 :     if (not playIncomingCallBeep_)
     257           0 :         return;
     258             : 
     259           0 :     auto now = std::chrono::system_clock::now();
     260             : 
     261             :     // Notify maximum once every 5 seconds
     262           0 :     if (now < lastNotificationTime_ + std::chrono::seconds(5))
     263           0 :         return;
     264             : 
     265           0 :     lastNotificationTime_ = now;
     266             : 
     267           0 :     Tone tone("440/160", getSampleRate(), audioFormat_.sampleFormat);
     268           0 :     size_t nbSample = tone.getSize();
     269             : 
     270             :     /* Put the data in the urgent ring buffer */
     271           0 :     urgentRingBuffer_.flushAll();
     272           0 :     urgentRingBuffer_.put(tone.getNext(nbSample));
     273           0 : }
     274             : 
     275             : std::shared_ptr<AudioFrame>
     276           0 : AudioLayer::getToRing(AudioFormat format, size_t writableSamples)
     277             : {
     278           0 :     if (auto fileToPlay = Manager::instance().getTelephoneFile()) {
     279           0 :         auto fileformat = fileToPlay->getFormat();
     280           0 :         bool resample = format != fileformat;
     281             : 
     282           0 :         size_t readableSamples = resample ? rational<size_t>(writableSamples * (size_t) fileformat.sample_rate,
     283           0 :                                                              format.sample_rate)
     284           0 :                                                 .real<size_t>()
     285           0 :                                           : writableSamples;
     286             : 
     287           0 :         return resampler_->resample(fileToPlay->getNext(readableSamples, isRingtoneMuted_), format);
     288           0 :     }
     289           0 :     return {};
     290             : }
     291             : 
     292             : std::shared_ptr<AudioFrame>
     293           0 : AudioLayer::getToPlay(AudioFormat format, size_t writableSamples)
     294             : {
     295           0 :     notifyIncomingCall();
     296           0 :     auto& bufferPool = Manager::instance().getRingBufferPool();
     297             : 
     298           0 :     if (not playbackQueue_)
     299           0 :         playbackQueue_.reset(new AudioFrameResizer(format, writableSamples));
     300             :     else
     301           0 :         playbackQueue_->setFrameSize(writableSamples);
     302             : 
     303           0 :     std::shared_ptr<AudioFrame> playbackBuf {};
     304           0 :     while (!(playbackBuf = playbackQueue_->dequeue())) {
     305           0 :         std::shared_ptr<AudioFrame> resampled;
     306             : 
     307           0 :         if (auto urgentSamples = urgentRingBuffer_.get(RingBufferPool::DEFAULT_ID)) {
     308           0 :             bufferPool.discard(1, RingBufferPool::DEFAULT_ID);
     309           0 :             resampled = resampler_->resample(std::move(urgentSamples), format);
     310           0 :         } else if (auto toneToPlay = Manager::instance().getTelephoneTone()) {
     311           0 :             resampled = resampler_->resample(toneToPlay->getNext(), format);
     312           0 :         } else if (auto buf = bufferPool.getData(RingBufferPool::DEFAULT_ID)) {
     313           0 :             resampled = resampler_->resample(std::move(buf), format);
     314             :         } else {
     315           0 :             std::lock_guard lock(audioProcessorMutex);
     316           0 :             if (audioProcessor) {
     317           0 :                 auto silence = std::make_shared<AudioFrame>(format, writableSamples);
     318           0 :                 libav_utils::fillWithSilence(silence->pointer());
     319           0 :                 audioProcessor->putPlayback(silence);
     320           0 :             }
     321           0 :             break;
     322           0 :         }
     323             : 
     324           0 :         if (resampled) {
     325           0 :             std::lock_guard lock(audioProcessorMutex);
     326           0 :             if (audioProcessor) {
     327           0 :                 audioProcessor->putPlayback(resampled);
     328             :             }
     329           0 :             playbackQueue_->enqueue(std::move(resampled));
     330           0 :         } else
     331           0 :             break;
     332           0 :     }
     333             : 
     334             :     jami_tracepoint(audio_layer_get_to_play_end);
     335             : 
     336           0 :     return playbackBuf;
     337           0 : }
     338             : 
     339             : void
     340           0 : AudioLayer::putRecorded(std::shared_ptr<AudioFrame>&& frame)
     341             : {
     342           0 :     std::lock_guard lock(audioProcessorMutex);
     343           0 :     if (audioProcessor && playbackStarted_ && recordStarted_) {
     344           0 :         audioProcessor->putRecorded(std::move(frame));
     345           0 :         while (auto rec = audioProcessor->getProcessed()) {
     346           0 :             mainRingBuffer_->put(std::move(rec));
     347           0 :         }
     348             :     } else {
     349           0 :         mainRingBuffer_->put(std::move(frame));
     350             :     }
     351             : 
     352             :     jami_tracepoint(audio_layer_put_recorded_end, );
     353           0 : }
     354             : 
     355             : } // namespace jami