Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2024 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software: you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation, either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #include "audiolayer.h"
      19             : #include "logger.h"
      20             : #include "manager.h"
      21             : #include "audio/ringbufferpool.h"
      22             : #include "audio/resampler.h"
      23             : #include "tonecontrol.h"
      24             : #include "client/ring_signal.h"
      25             : 
      26             : #include "audio-processing/null_audio_processor.h"
      27             : #include "tracepoint.h"
      28             : #if HAVE_WEBRTC_AP
      29             : #include "audio-processing/webrtc.h"
      30             : #endif
      31             : #if HAVE_SPEEXDSP
      32             : #include "audio-processing/speex.h"
      33             : #endif
      34             : 
      35             : #include <ctime>
      36             : #include <algorithm>
      37             : 
      38             : namespace jami {
      39             : 
      40          36 : AudioLayer::AudioLayer(const AudioPreference& pref)
      41          72 :     : isCaptureMuted_(pref.getCaptureMuted())
      42          36 :     , isPlaybackMuted_(pref.getPlaybackMuted())
      43          72 :     , captureGain_(pref.getVolumemic())
      44          36 :     , playbackGain_(pref.getVolumespkr())
      45          36 :     , pref_(pref)
      46          36 :     , mainRingBuffer_(
      47          36 :           Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID))
      48          36 :     , audioFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
      49          36 :     , audioInputFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
      50          36 :     , urgentRingBuffer_("urgentRingBuffer_id", SIZEBUF, audioFormat_)
      51          36 :     , resampler_(new Resampler)
      52         216 :     , lastNotificationTime_()
      53             : {
      54          36 :     urgentRingBuffer_.createReadOffset(RingBufferPool::DEFAULT_ID);
      55             : 
      56         108 :     JAMI_LOG("[audiolayer] AGC: {:d}, noiseReduce: {:s}, VAD: {:d}, echoCancel: {:s}, audioProcessor: {:s}",
      57             :               pref_.isAGCEnabled(),
      58             :               pref.getNoiseReduce(),
      59             :               pref.getVadEnabled(),
      60             :               pref.getEchoCanceller(),
      61             :               pref.getAudioProcessor());
      62          36 : }
      63             : 
      64          36 : AudioLayer::~AudioLayer() {}
      65             : 
      66             : void
      67         112 : AudioLayer::hardwareFormatAvailable(AudioFormat playback, size_t bufSize)
      68             : {
      69         336 :     JAMI_LOG("Hardware audio format available : {:s} {}", playback.toString(), bufSize);
      70         112 :     audioFormat_ = Manager::instance().hardwareAudioFormatChanged(playback);
      71         112 :     audioInputFormat_.sampleFormat = audioFormat_.sampleFormat;
      72         112 :     urgentRingBuffer_.setFormat(audioFormat_);
      73         112 :     nativeFrameSize_ = bufSize;
      74         112 : }
      75             : 
      76             : void
      77           0 : AudioLayer::hardwareInputFormatAvailable(AudioFormat capture)
      78             : {
      79           0 :     JAMI_LOG("Hardware input audio format available : {:s}", capture.toString());
      80           0 : }
      81             : 
      82             : void
      83           0 : AudioLayer::devicesChanged()
      84             : {
      85           0 :     emitSignal<libjami::AudioSignal::DeviceEvent>();
      86           0 : }
      87             : 
      88             : void
      89           0 : AudioLayer::flushMain()
      90             : {
      91           0 :     Manager::instance().getRingBufferPool().flushAllBuffers();
      92           0 : }
      93             : 
      94             : void
      95         437 : AudioLayer::flushUrgent()
      96             : {
      97         437 :     urgentRingBuffer_.flushAll();
      98         437 : }
      99             : 
     100             : void
     101           0 : AudioLayer::flush()
     102             : {
     103           0 :     Manager::instance().getRingBufferPool().flushAllBuffers();
     104           0 :     urgentRingBuffer_.flushAll();
     105           0 : }
     106             : 
     107             : void
     108         236 : AudioLayer::playbackChanged(bool started)
     109             : {
     110         236 :     playbackStarted_ = started;
     111         236 : }
     112             : 
     113             : void
     114         236 : AudioLayer::recordChanged(bool started)
     115             : {
     116         236 :     std::lock_guard lock(audioProcessorMutex);
     117         236 :     if (started) {
     118             :         // create audio processor
     119           0 :         createAudioProcessor();
     120             :     } else {
     121             :         // destroy audio processor
     122         236 :         destroyAudioProcessor();
     123             :     }
     124         236 :     recordStarted_ = started;
     125         236 : }
     126             : 
     127             : // helper function
     128             : static inline bool
     129           0 : shouldUseAudioProcessorEchoCancel(bool hasNativeAEC, const std::string& echoCancellerPref)
     130             : {
     131             :     return
     132             :         // user doesn't care which and there is not a system AEC
     133           0 :         (echoCancellerPref == "auto" && !hasNativeAEC)
     134             :         // user specifically wants audioProcessor
     135           0 :         or (echoCancellerPref == "audioProcessor");
     136             : }
     137             : 
     138             : // helper function
     139             : static inline bool
     140           0 : shouldUseAudioProcessorNoiseSuppression(bool hasNativeNS, const std::string& noiseSuppressionPref)
     141             : {
     142             :     return
     143             :         // user doesn't care which and there is no system noise suppression
     144           0 :         (noiseSuppressionPref == "auto" && !hasNativeNS)
     145             :         // user specifically wants audioProcessor
     146           0 :         or (noiseSuppressionPref == "audioProcessor");
     147             : }
     148             : 
     149             : void
     150          33 : AudioLayer::setHasNativeAEC(bool hasNativeAEC)
     151             : {
     152          33 :     JAMI_INFO("[audiolayer] setHasNativeAEC: %d", hasNativeAEC);
     153          33 :     std::lock_guard lock(audioProcessorMutex);
     154          33 :     hasNativeAEC_ = hasNativeAEC;
     155             :     // if we have a current audio processor, tell it to enable/disable its own AEC
     156          33 :     if (audioProcessor) {
     157           0 :         audioProcessor->enableEchoCancel(
     158           0 :             shouldUseAudioProcessorEchoCancel(hasNativeAEC, pref_.getEchoCanceller()));
     159             :     }
     160          33 : }
     161             : 
     162             : void
     163          36 : AudioLayer::setHasNativeNS(bool hasNativeNS)
     164             : {
     165          36 :     JAMI_INFO("[audiolayer] setHasNativeNS: %d", hasNativeNS);
     166          36 :     std::lock_guard lock(audioProcessorMutex);
     167          36 :     hasNativeNS_ = hasNativeNS;
     168             :     // if we have a current audio processor, tell it to enable/disable its own noise suppression
     169          36 :     if (audioProcessor) {
     170           0 :         audioProcessor->enableNoiseSuppression(
     171           0 :             shouldUseAudioProcessorNoiseSuppression(hasNativeNS, pref_.getNoiseReduce()));
     172             :     }
     173          36 : }
     174             : 
     175             : // must acquire lock beforehand
     176             : void
     177           0 : AudioLayer::createAudioProcessor()
     178             : {
     179           0 :     auto nb_channels = std::max(audioFormat_.nb_channels, audioInputFormat_.nb_channels);
     180           0 :     auto sample_rate = std::max(audioFormat_.sample_rate, audioInputFormat_.sample_rate);
     181             : 
     182           0 :     sample_rate = std::clamp(sample_rate, 16000u, 48000u);
     183             : 
     184           0 :     AudioFormat formatForProcessor {sample_rate, nb_channels};
     185             : 
     186             :     unsigned int frame_size;
     187           0 :     if (pref_.getAudioProcessor() == "speex") {
     188             :         // TODO: maybe force this to be equivalent to 20ms? as expected by speex
     189           0 :         frame_size = sample_rate / 50u;
     190             :     } else {
     191           0 :         frame_size = sample_rate / 100u;
     192             :     }
     193             : 
     194           0 :     JAMI_WARNING("Input {}", audioInputFormat_.toString());
     195           0 :     JAMI_WARNING("Output {}", audioFormat_.toString());
     196           0 :     JAMI_WARNING("Starting audio processor with: [{} Hz, {} channels, {} samples/frame]",
     197             :               sample_rate,
     198             :               nb_channels,
     199             :               frame_size);
     200             : 
     201           0 :     if (pref_.getAudioProcessor() == "webrtc") {
     202             : #if HAVE_WEBRTC_AP
     203           0 :         JAMI_WARN("[audiolayer] using WebRTCAudioProcessor");
     204           0 :         audioProcessor.reset(new WebRTCAudioProcessor(formatForProcessor, frame_size));
     205             : #else
     206             :         JAMI_ERR("[audiolayer] audioProcessor preference is webrtc, but library not linked! "
     207             :                  "using null AudioProcessor instead");
     208             :         audioProcessor.reset();
     209             : #endif
     210           0 :     } else if (pref_.getAudioProcessor() == "speex") {
     211             : #if HAVE_SPEEXDSP
     212           0 :         JAMI_WARN("[audiolayer] using SpeexAudioProcessor");
     213           0 :         audioProcessor.reset(new SpeexAudioProcessor(formatForProcessor, frame_size));
     214             : #else
     215             :         JAMI_ERR("[audiolayer] audioProcessor preference is speex, but library not linked! "
     216             :                  "using null AudioProcessor instead");
     217             :         audioProcessor.reset();
     218             : #endif
     219           0 :     } else if (pref_.getAudioProcessor() == "null") {
     220           0 :         JAMI_WARN("[audiolayer] using null AudioProcessor");
     221           0 :         audioProcessor.reset();
     222             :     } else {
     223           0 :         JAMI_ERR("[audiolayer] audioProcessor preference not recognized, using null AudioProcessor "
     224             :                  "instead");
     225           0 :         audioProcessor.reset();
     226             :     }
     227             : 
     228           0 :     if (audioProcessor) {
     229           0 :         audioProcessor->enableNoiseSuppression(
     230           0 :             shouldUseAudioProcessorNoiseSuppression(hasNativeNS_, pref_.getNoiseReduce()));
     231             : 
     232           0 :         audioProcessor->enableAutomaticGainControl(pref_.isAGCEnabled());
     233             : 
     234           0 :         audioProcessor->enableEchoCancel(
     235           0 :             shouldUseAudioProcessorEchoCancel(hasNativeAEC_, pref_.getEchoCanceller()));
     236             : 
     237           0 :         audioProcessor->enableVoiceActivityDetection(pref_.getVadEnabled());
     238             :     }
     239           0 : }
     240             : 
     241             : // must acquire lock beforehand
     242             : void
     243         236 : AudioLayer::destroyAudioProcessor()
     244             : {
     245             :     // delete it
     246         236 :     audioProcessor.reset();
     247         236 : }
     248             : 
     249             : void
     250           0 : AudioLayer::putUrgent(std::shared_ptr<AudioFrame> buffer)
     251             : {
     252           0 :     urgentRingBuffer_.put(std::move(buffer));
     253           0 : }
     254             : 
     255             : // Notify (with a beep) an incoming call when there is already a call in progress
     256             : void
     257           0 : AudioLayer::notifyIncomingCall()
     258             : {
     259           0 :     if (not playIncomingCallBeep_)
     260           0 :         return;
     261             : 
     262           0 :     auto now = std::chrono::system_clock::now();
     263             : 
     264             :     // Notify maximum once every 5 seconds
     265           0 :     if (now < lastNotificationTime_ + std::chrono::seconds(5))
     266           0 :         return;
     267             : 
     268           0 :     lastNotificationTime_ = now;
     269             : 
     270           0 :     Tone tone("440/160", getSampleRate(), audioFormat_.sampleFormat);
     271           0 :     size_t nbSample = tone.getSize();
     272             : 
     273             :     /* Put the data in the urgent ring buffer */
     274           0 :     urgentRingBuffer_.flushAll();
     275           0 :     urgentRingBuffer_.put(tone.getNext(nbSample));
     276           0 : }
     277             : 
     278             : std::shared_ptr<AudioFrame>
     279           0 : AudioLayer::getToRing(AudioFormat format, size_t writableSamples)
     280             : {
     281           0 :     if (auto fileToPlay = Manager::instance().getTelephoneFile()) {
     282           0 :         auto fileformat = fileToPlay->getFormat();
     283           0 :         bool resample = format != fileformat;
     284             : 
     285           0 :         size_t readableSamples = resample ? (rational<size_t>(writableSamples, format.sample_rate)
     286           0 :                                              * (size_t) fileformat.sample_rate)
     287           0 :                                                 .real<size_t>()
     288           0 :                                           : writableSamples;
     289             : 
     290           0 :         return resampler_->resample(fileToPlay->getNext(readableSamples, isRingtoneMuted_), format);
     291           0 :     }
     292           0 :     return {};
     293             : }
     294             : 
     295             : std::shared_ptr<AudioFrame>
     296           0 : AudioLayer::getToPlay(AudioFormat format, size_t writableSamples)
     297             : {
     298           0 :     notifyIncomingCall();
     299           0 :     auto& bufferPool = Manager::instance().getRingBufferPool();
     300             : 
     301           0 :     if (not playbackQueue_)
     302           0 :         playbackQueue_.reset(new AudioFrameResizer(format, writableSamples));
     303             :     else
     304           0 :         playbackQueue_->setFrameSize(writableSamples);
     305             : 
     306           0 :     std::shared_ptr<AudioFrame> playbackBuf {};
     307           0 :     while (!(playbackBuf = playbackQueue_->dequeue())) {
     308           0 :         std::shared_ptr<AudioFrame> resampled;
     309             : 
     310           0 :         if (auto urgentSamples = urgentRingBuffer_.get(RingBufferPool::DEFAULT_ID)) {
     311           0 :             bufferPool.discard(1, RingBufferPool::DEFAULT_ID);
     312           0 :             resampled = resampler_->resample(std::move(urgentSamples), format);
     313           0 :         } else if (auto toneToPlay = Manager::instance().getTelephoneTone()) {
     314           0 :             resampled = resampler_->resample(toneToPlay->getNext(), format);
     315           0 :         } else if (auto buf = bufferPool.getData(RingBufferPool::DEFAULT_ID)) {
     316           0 :             resampled = resampler_->resample(std::move(buf), format);
     317             :         } else {
     318           0 :             std::lock_guard lock(audioProcessorMutex);
     319           0 :             if (audioProcessor) {
     320           0 :                 auto silence = std::make_shared<AudioFrame>(format, writableSamples);
     321           0 :                 libav_utils::fillWithSilence(silence->pointer());
     322           0 :                 audioProcessor->putPlayback(silence);
     323           0 :             }
     324           0 :             break;
     325           0 :         }
     326             : 
     327           0 :         if (resampled) {
     328           0 :             std::lock_guard lock(audioProcessorMutex);
     329           0 :             if (audioProcessor) {
     330           0 :                 audioProcessor->putPlayback(resampled);
     331             :             }
     332           0 :             playbackQueue_->enqueue(std::move(resampled));
     333           0 :         } else
     334           0 :             break;
     335           0 :     }
     336             : 
     337             :     jami_tracepoint(audio_layer_get_to_play_end);
     338             : 
     339           0 :     return playbackBuf;
     340           0 : }
     341             : 
     342             : void
     343           0 : AudioLayer::putRecorded(std::shared_ptr<AudioFrame>&& frame)
     344             : {
     345           0 :     std::lock_guard lock(audioProcessorMutex);
     346           0 :     if (audioProcessor && playbackStarted_ && recordStarted_) {
     347           0 :         audioProcessor->putRecorded(std::move(frame));
     348           0 :         while (auto rec = audioProcessor->getProcessed()) {
     349           0 :             mainRingBuffer_->put(std::move(rec));
     350           0 :         }
     351             :     } else {
     352           0 :         mainRingBuffer_->put(std::move(frame));
     353             :     }
     354             : 
     355             :     jami_tracepoint(audio_layer_put_recorded_end, );
     356           0 : }
     357             : 
     358             : } // namespace jami