Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2025 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software: you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation, either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #include "audio_frame_resizer.h"
      19             : #include "audio_input.h"
      20             : #include "jami/media_const.h"
      21             : #include "fileutils.h" // access
      22             : #include "manager.h"
      23             : #include "media_decoder.h"
      24             : #include "resampler.h"
      25             : #include "ringbuffer.h"
      26             : #include "ringbufferpool.h"
      27             : #include "tracepoint.h"
      28             : 
      29             : #include <future>
      30             : #include <memory>
      31             : 
      32             : namespace jami {
      33             : 
      34             : static constexpr auto MS_PER_PACKET = std::chrono::milliseconds(20);
      35             : 
      36         250 : AudioInput::AudioInput(const std::string& id)
      37         250 :     : id_(id)
      38         250 :     , format_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
      39         250 :     , frameSize_(format_.sample_rate * MS_PER_PACKET.count() / 1000)
      40         250 :     , resampler_(new Resampler)
      41         500 :     , resizer_(new AudioFrameResizer(format_,
      42             :                                      frameSize_,
      43         250 :                                      [this](std::shared_ptr<AudioFrame>&& f) { frameResized(std::move(f)); }))
      44         250 :     , deviceGuard_()
      45       38947 :     , loop_([] { return true; }, [this] { process(); }, [] {})
      46             : {
      47        1000 :     JAMI_DEBUG("Creating audio input with id: {}", id_);
      48         250 :     ringBuf_ = Manager::instance().getRingBufferPool().createRingBuffer(id_);
      49         250 : }
      50             : 
      51           0 : AudioInput::AudioInput(const std::string& id, const std::string& resource)
      52           0 :     : AudioInput(id)
      53             : {
      54           0 :     switchInput(resource);
      55           0 : }
      56             : 
      57         250 : AudioInput::~AudioInput()
      58             : {
      59         250 :     if (playingFile_) {
      60           5 :         Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
      61           5 :         Manager::instance().getRingBufferPool().unBindHalfDuplexOut(id_, id_);
      62             :     }
      63         250 :     ringBuf_.reset();
      64         250 :     loop_.join();
      65             : 
      66         250 :     Manager::instance().getRingBufferPool().flush(id_);
      67         250 : }
      68             : 
      69             : void
      70       37523 : AudioInput::process()
      71             : {
      72       37523 :     readFromDevice();
      73       37523 : }
      74             : 
      75             : void
      76          18 : AudioInput::updateStartTime(int64_t start)
      77             : {
      78          18 :     if (decoder_) {
      79          18 :         decoder_->updateStartTime(start);
      80             :     }
      81          18 : }
      82             : 
      83             : void
      84           0 : AudioInput::frameResized(std::shared_ptr<AudioFrame>&& ptr)
      85             : {
      86           0 :     std::shared_ptr<AudioFrame> frame = std::move(ptr);
      87           0 :     frame->pointer()->pts = sent_samples;
      88           0 :     sent_samples += frame->pointer()->nb_samples;
      89             : 
      90           0 :     notify(std::static_pointer_cast<MediaFrame>(frame));
      91           0 : }
      92             : 
      93             : void
      94           9 : AudioInput::setSeekTime(int64_t time)
      95             : {
      96           9 :     if (decoder_) {
      97           9 :         decoder_->setSeekTime(time);
      98             :     }
      99           9 : }
     100             : 
     101             : void
     102       37523 : AudioInput::readFromDevice()
     103             : {
     104             :     {
     105       37523 :         std::lock_guard lk(resourceMutex_);
     106       37523 :         if (decodingFile_)
     107           0 :             while (ringBuf_ && ringBuf_->isEmpty())
     108           0 :                 readFromFile();
     109       37523 :         if (playingFile_) {
     110          15 :             while (ringBuf_ && ringBuf_->getLength(id_) == 0)
     111          10 :                 readFromQueue();
     112             :         }
     113       37523 :     }
     114             : 
     115             :     // Note: read for device is called in an audio thread and we don't
     116             :     // want to have a loop which takes 100% of the CPU.
     117             :     // Here, we basically want to mix available data without any glitch
     118             :     // and even if one buffer doesn't have audio data (call in hold,
     119             :     // connections issues, etc). So mix every MS_PER_PACKET
     120       37523 :     std::this_thread::sleep_until(wakeUp_);
     121       37523 :     wakeUp_ += MS_PER_PACKET;
     122             : 
     123       37523 :     auto& bufferPool = Manager::instance().getRingBufferPool();
     124       37523 :     auto audioFrame = bufferPool.getData(id_);
     125       37523 :     if (not audioFrame)
     126       37523 :         return;
     127             : 
     128           0 :     if (muteState_) {
     129           0 :         libav_utils::fillWithSilence(audioFrame->pointer());
     130           0 :         audioFrame->has_voice = false; // force no voice activity when muted
     131             :     }
     132             : 
     133           0 :     std::lock_guard lk(fmtMutex_);
     134           0 :     if (bufferPool.getInternalAudioFormat() != format_)
     135           0 :         audioFrame = resampler_->resample(std::move(audioFrame), format_);
     136           0 :     resizer_->enqueue(std::move(audioFrame));
     137             : 
     138           0 :     if (recorderCallback_ && settingMS_.exchange(false)) {
     139           0 :         recorderCallback_(MediaStream("a:local", format_, sent_samples));
     140             :     }
     141             : 
     142             :     jami_tracepoint(audio_input_read_from_device_end, id_.c_str());
     143       37523 : }
     144             : 
     145             : void
     146          10 : AudioInput::readFromQueue()
     147             : {
     148          10 :     if (!decoder_)
     149           0 :         return;
     150          10 :     if (paused_ || !decoder_->emitFrame(true)) {
     151          10 :         std::this_thread::sleep_for(MS_PER_PACKET);
     152             :     }
     153             : }
     154             : 
     155             : void
     156           0 : AudioInput::readFromFile()
     157             : {
     158           0 :     if (!decoder_)
     159           0 :         return;
     160           0 :     const auto ret = decoder_->decode();
     161           0 :     switch (ret) {
     162           0 :     case MediaDemuxer::Status::Success:
     163           0 :         break;
     164           0 :     case MediaDemuxer::Status::EndOfFile:
     165           0 :         createDecoder();
     166           0 :         break;
     167           0 :     case MediaDemuxer::Status::ReadError:
     168           0 :         JAMI_ERR() << "Failed to decode frame";
     169           0 :         break;
     170           0 :     case MediaDemuxer::Status::ReadBufferOverflow:
     171           0 :         JAMI_ERR() << "Read buffer overflow detected";
     172           0 :         break;
     173           0 :     case MediaDemuxer::Status::FallBack:
     174             :     case MediaDemuxer::Status::RestartRequired:
     175           0 :         break;
     176             :     }
     177             : }
     178             : 
     179             : bool
     180         211 : AudioInput::initDevice(const std::string& device)
     181             : {
     182         211 :     devOpts_ = {};
     183         211 :     devOpts_.input = device;
     184         211 :     devOpts_.channel = format_.nb_channels;
     185         211 :     devOpts_.framerate = format_.sample_rate;
     186         211 :     deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::CAPTURE);
     187         211 :     playingDevice_ = true;
     188         211 :     return true;
     189             : }
     190             : 
     191             : void
     192           5 : AudioInput::configureFilePlayback(const std::string& path, std::shared_ptr<MediaDemuxer>& demuxer, int index)
     193             : {
     194           5 :     decoder_.reset();
     195           5 :     devOpts_ = {};
     196           5 :     devOpts_.input = path;
     197           5 :     devOpts_.name = path;
     198           0 :     auto decoder = std::make_unique<MediaDecoder>(demuxer, index, [this](std::shared_ptr<MediaFrame>&& frame) {
     199           0 :         if (muteState_)
     200           0 :             libav_utils::fillWithSilence(frame->pointer());
     201           0 :         if (ringBuf_)
     202           0 :             ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
     203           5 :     });
     204           5 :     decoder->emulateRate();
     205           5 :     decoder->setInterruptCallback([](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); },
     206             :                                   this);
     207             : 
     208             :     // have file audio mixed into the local buffer so it gets played
     209           5 :     Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
     210             :     // Bind to itself to be able to read from the ringbuffer
     211           5 :     Manager::instance().getRingBufferPool().bindHalfDuplexOut(id_, id_);
     212             : 
     213           5 :     deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
     214             : 
     215           5 :     wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
     216           5 :     playingFile_ = true;
     217           5 :     decoder_ = std::move(decoder);
     218           5 :     resource_ = path;
     219           5 :     loop_.start();
     220           5 : }
     221             : 
     222             : void
     223          10 : AudioInput::setPaused(bool paused)
     224             : {
     225          10 :     if (paused) {
     226           8 :         Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
     227           8 :         deviceGuard_.reset();
     228             :     } else {
     229           2 :         Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
     230           2 :         deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
     231             :     }
     232          10 :     paused_ = paused;
     233          10 : }
     234             : 
     235             : void
     236           9 : AudioInput::flushBuffers()
     237             : {
     238           9 :     if (decoder_) {
     239           9 :         decoder_->flushBuffers();
     240             :     }
     241           9 : }
     242             : 
     243             : bool
     244           0 : AudioInput::initFile(const std::string& path)
     245             : {
     246           0 :     if (access(path.c_str(), R_OK) != 0) {
     247           0 :         JAMI_ERROR("File '{}' not available", path);
     248           0 :         return false;
     249             :     }
     250             : 
     251           0 :     devOpts_ = {};
     252           0 :     devOpts_.input = path;
     253           0 :     devOpts_.name = path;
     254           0 :     devOpts_.loop = "1";
     255             :     // sets devOpts_'s sample rate and number of channels
     256           0 :     if (!createDecoder()) {
     257           0 :         JAMI_WARN() << "Unable to decode audio from file, switching back to default device";
     258           0 :         return initDevice("");
     259             :     }
     260           0 :     wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
     261             : 
     262             :     // have file audio mixed into the local buffer so it gets played
     263           0 :     Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
     264           0 :     decodingFile_ = true;
     265           0 :     deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
     266           0 :     return true;
     267             : }
     268             : 
     269             : std::shared_future<DeviceParams>
     270         211 : AudioInput::switchInput(const std::string& resource)
     271             : {
     272             :     // Always switch inputs, even if it's the same resource, so audio will be in sync with video
     273         211 :     std::unique_lock lk(resourceMutex_);
     274             : 
     275         844 :     JAMI_DEBUG("Switching audio source from {} to {}", resource_, resource);
     276             : 
     277         211 :     auto oldGuard = std::move(deviceGuard_);
     278             : 
     279         211 :     decoder_.reset();
     280         211 :     if (decodingFile_) {
     281           0 :         decodingFile_ = false;
     282           0 :         Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
     283             :     }
     284             : 
     285         211 :     playingDevice_ = false;
     286         211 :     resource_ = resource;
     287         211 :     devOptsFound_ = false;
     288             : 
     289         211 :     std::promise<DeviceParams> p;
     290         211 :     foundDevOpts_.swap(p);
     291             : 
     292         211 :     if (resource_.empty()) {
     293         211 :         if (initDevice(""))
     294         211 :             foundDevOpts(devOpts_);
     295             :     } else {
     296           0 :         static const std::string& sep = libjami::Media::VideoProtocolPrefix::SEPARATOR;
     297           0 :         const auto pos = resource_.find(sep);
     298           0 :         if (pos == std::string::npos)
     299           0 :             return {};
     300             : 
     301           0 :         const auto prefix = resource_.substr(0, pos);
     302           0 :         if ((pos + sep.size()) >= resource_.size())
     303           0 :             return {};
     304             : 
     305           0 :         const auto suffix = resource_.substr(pos + sep.size());
     306           0 :         bool ready = false;
     307           0 :         if (prefix == libjami::Media::VideoProtocolPrefix::FILE)
     308           0 :             ready = initFile(suffix);
     309             :         else
     310           0 :             ready = initDevice(suffix);
     311             : 
     312           0 :         if (ready)
     313           0 :             foundDevOpts(devOpts_);
     314           0 :     }
     315             : 
     316         211 :     futureDevOpts_ = foundDevOpts_.get_future().share();
     317         211 :     wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
     318         211 :     lk.unlock();
     319         211 :     if (not loop_.isRunning())
     320         207 :         loop_.start();
     321         211 :     if (onSuccessfulSetup_)
     322         180 :         onSuccessfulSetup_(MEDIA_AUDIO, 0);
     323         211 :     return futureDevOpts_;
     324         211 : }
     325             : 
     326             : void
     327         211 : AudioInput::foundDevOpts(const DeviceParams& params)
     328             : {
     329         211 :     if (!devOptsFound_) {
     330         211 :         devOptsFound_ = true;
     331         211 :         foundDevOpts_.set_value(params);
     332             :     }
     333         211 : }
     334             : 
     335             : void
     336         181 : AudioInput::setRecorderCallback(const std::function<void(const MediaStream& ms)>& cb)
     337             : {
     338         181 :     settingMS_.exchange(true);
     339         181 :     recorderCallback_ = cb;
     340         181 :     if (decoder_)
     341           0 :         decoder_->setContextCallback([this]() {
     342           0 :             if (recorderCallback_)
     343           0 :                 recorderCallback_(getInfo());
     344           0 :         });
     345         181 : }
     346             : 
     347             : bool
     348           0 : AudioInput::createDecoder()
     349             : {
     350           0 :     decoder_.reset();
     351           0 :     if (devOpts_.input.empty()) {
     352           0 :         foundDevOpts(devOpts_);
     353           0 :         return false;
     354             :     }
     355             : 
     356           0 :     auto decoder = std::make_unique<MediaDecoder>([this](std::shared_ptr<MediaFrame>&& frame) {
     357           0 :         if (ringBuf_)
     358           0 :             ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
     359           0 :     });
     360             : 
     361             :     // NOTE don't emulate rate, file is read as frames are needed
     362             : 
     363           0 :     decoder->setInterruptCallback([](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); },
     364             :                                   this);
     365             : 
     366           0 :     if (decoder->openInput(devOpts_) < 0) {
     367           0 :         JAMI_ERR() << "Unable to open input '" << devOpts_.input << "'";
     368           0 :         foundDevOpts(devOpts_);
     369           0 :         return false;
     370             :     }
     371             : 
     372           0 :     if (decoder->setupAudio() < 0) {
     373           0 :         JAMI_ERR() << "Unable to setup decoder for '" << devOpts_.input << "'";
     374           0 :         foundDevOpts(devOpts_);
     375           0 :         return false;
     376             :     }
     377             : 
     378           0 :     auto ms = decoder->getStream(devOpts_.input);
     379           0 :     devOpts_.channel = ms.nbChannels;
     380           0 :     devOpts_.framerate = ms.sampleRate;
     381           0 :     JAMI_DBG() << "Created audio decoder: " << ms;
     382             : 
     383           0 :     decoder_ = std::move(decoder);
     384           0 :     foundDevOpts(devOpts_);
     385           0 :     decoder_->setContextCallback([this]() {
     386           0 :         if (recorderCallback_)
     387           0 :             recorderCallback_(getInfo());
     388           0 :     });
     389           0 :     return true;
     390           0 : }
     391             : 
     392             : void
     393         180 : AudioInput::setFormat(const AudioFormat& fmt)
     394             : {
     395         180 :     std::lock_guard lk(fmtMutex_);
     396         180 :     format_ = fmt;
     397         180 :     resizer_->setFormat(format_, format_.sample_rate * MS_PER_PACKET.count() / 1000);
     398         180 : }
     399             : 
     400             : void
     401         203 : AudioInput::setMuted(bool isMuted)
     402             : {
     403         203 :     JAMI_WARN("Audio Input muted [%s]", isMuted ? "YES" : "NO");
     404         203 :     muteState_ = isMuted;
     405         203 : }
     406             : 
     407             : MediaStream
     408           1 : AudioInput::getInfo() const
     409             : {
     410           1 :     std::lock_guard lk(fmtMutex_);
     411           2 :     return MediaStream("a:local", format_, sent_samples);
     412           1 : }
     413             : 
     414             : MediaStream
     415           1 : AudioInput::getInfo(const std::string& name) const
     416             : {
     417           1 :     std::lock_guard lk(fmtMutex_);
     418           1 :     auto ms = MediaStream(name, format_, sent_samples);
     419           2 :     return ms;
     420           1 : }
     421             : 
     422             : } // namespace jami