Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2024 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software: you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation, either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #include "audio_frame_resizer.h"
      19             : #include "audio_input.h"
      20             : #include "jami/media_const.h"
      21             : #include "fileutils.h" // access
      22             : #include "manager.h"
      23             : #include "media_decoder.h"
      24             : #include "resampler.h"
      25             : #include "ringbuffer.h"
      26             : #include "ringbufferpool.h"
      27             : #include "tracepoint.h"
      28             : 
      29             : #include <future>
      30             : #include <memory>
      31             : 
      32             : namespace jami {
      33             : 
      34             : static constexpr auto MS_PER_PACKET = std::chrono::milliseconds(20);
      35             : 
      36         244 : AudioInput::AudioInput(const std::string& id)
      37         244 :     : id_(id)
      38         244 :     , format_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
      39         244 :     , frameSize_(format_.sample_rate * MS_PER_PACKET.count() / 1000)
      40         244 :     , resampler_(new Resampler)
      41         488 :     , resizer_(new AudioFrameResizer(format_,
      42             :                                      frameSize_,
      43           0 :                                      [this](std::shared_ptr<AudioFrame>&& f) {
      44           0 :                                          frameResized(std::move(f));
      45         244 :                                      }))
      46         244 :     , deviceGuard_()
      47       45422 :     , loop_([] { return true; }, [this] { process(); }, [] {})
      48             : {
      49         732 :     JAMI_DEBUG("Creating audio input with id: {}", id_);
      50         244 :     ringBuf_ = Manager::instance().getRingBufferPool().createRingBuffer(id_);
      51         244 : }
      52             : 
      53           0 : AudioInput::AudioInput(const std::string& id, const std::string& resource)
      54           0 :     : AudioInput(id)
      55             : {
      56           0 :     switchInput(resource);
      57           0 : }
      58             : 
      59         244 : AudioInput::~AudioInput()
      60             : {
      61         244 :     if (playingFile_) {
      62          10 :         Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
      63          10 :         Manager::instance().getRingBufferPool().unBindHalfDuplexOut(id_, id_);
      64             :     }
      65         244 :     ringBuf_.reset();
      66         244 :     loop_.join();
      67             : 
      68         244 :     Manager::instance().getRingBufferPool().flush(id_);
      69         244 : }
      70             : 
      71             : void
      72       44026 : AudioInput::process()
      73             : {
      74       44026 :     readFromDevice();
      75       44026 : }
      76             : 
      77             : void
      78          28 : AudioInput::updateStartTime(int64_t start)
      79             : {
      80          28 :     if (decoder_) {
      81          28 :         decoder_->updateStartTime(start);
      82             :     }
      83          28 : }
      84             : 
      85             : void
      86           0 : AudioInput::frameResized(std::shared_ptr<AudioFrame>&& ptr)
      87             : {
      88           0 :     std::shared_ptr<AudioFrame> frame = std::move(ptr);
      89           0 :     frame->pointer()->pts = sent_samples;
      90           0 :     sent_samples += frame->pointer()->nb_samples;
      91             : 
      92           0 :     notify(std::static_pointer_cast<MediaFrame>(frame));
      93           0 : }
      94             : 
      95             : void
      96           9 : AudioInput::setSeekTime(int64_t time)
      97             : {
      98           9 :     if (decoder_) {
      99           9 :         decoder_->setSeekTime(time);
     100             :     }
     101           9 : }
     102             : 
     103             : void
     104       44026 : AudioInput::readFromDevice()
     105             : {
     106             :     {
     107       44026 :         std::lock_guard lk(resourceMutex_);
     108       44026 :         if (decodingFile_)
     109           0 :             while (ringBuf_ && ringBuf_->isEmpty())
     110           0 :                 readFromFile();
     111       44026 :         if (playingFile_) {
     112        5245 :             while (ringBuf_ && ringBuf_->getLength(id_) == 0)
     113        5235 :                 readFromQueue();
     114             :         }
     115       44026 :     }
     116             : 
     117             :     // Note: read for device is called in an audio thread and we don't
     118             :     // want to have a loop which takes 100% of the CPU.
     119             :     // Here, we basically want to mix available data without any glitch
     120             :     // and even if one buffer doesn't have audio data (call in hold,
     121             :     // connections issues, etc). So mix every MS_PER_PACKET
     122       44026 :     std::this_thread::sleep_until(wakeUp_);
     123       44026 :     wakeUp_ += MS_PER_PACKET;
     124             : 
     125       44026 :     auto& bufferPool = Manager::instance().getRingBufferPool();
     126       44026 :     auto audioFrame = bufferPool.getData(id_);
     127       44026 :     if (not audioFrame)
     128       44026 :         return;
     129             : 
     130           0 :     if (muteState_) {
     131           0 :         libav_utils::fillWithSilence(audioFrame->pointer());
     132           0 :         audioFrame->has_voice = false; // force no voice activity when muted
     133             :     }
     134             : 
     135           0 :     std::lock_guard lk(fmtMutex_);
     136           0 :     if (bufferPool.getInternalAudioFormat() != format_)
     137           0 :         audioFrame = resampler_->resample(std::move(audioFrame), format_);
     138           0 :     resizer_->enqueue(std::move(audioFrame));
     139             : 
     140           0 :     if (recorderCallback_ && settingMS_.exchange(false)) {
     141           0 :         recorderCallback_(MediaStream("a:local", format_, sent_samples));
     142             :     }
     143             : 
     144             :     jami_tracepoint(audio_input_read_from_device_end, id_.c_str());
     145       44026 : }
     146             : 
     147             : void
     148        5235 : AudioInput::readFromQueue()
     149             : {
     150        5235 :     if (!decoder_)
     151           0 :         return;
     152        5235 :     if (paused_ || !decoder_->emitFrame(true)) {
     153        5235 :         std::this_thread::sleep_for(MS_PER_PACKET);
     154             :     }
     155             : }
     156             : 
     157             : void
     158           0 : AudioInput::readFromFile()
     159             : {
     160           0 :     if (!decoder_)
     161           0 :         return;
     162           0 :     const auto ret = decoder_->decode();
     163           0 :     switch (ret) {
     164           0 :     case MediaDemuxer::Status::Success:
     165           0 :         break;
     166           0 :     case MediaDemuxer::Status::EndOfFile:
     167           0 :         createDecoder();
     168           0 :         break;
     169           0 :     case MediaDemuxer::Status::ReadError:
     170           0 :         JAMI_ERR() << "Failed to decode frame";
     171           0 :         break;
     172           0 :     case MediaDemuxer::Status::ReadBufferOverflow:
     173           0 :         JAMI_ERR() << "Read buffer overflow detected";
     174           0 :         break;
     175           0 :     case MediaDemuxer::Status::FallBack:
     176             :     case MediaDemuxer::Status::RestartRequired:
     177           0 :         break;
     178             :     }
     179             : }
     180             : 
     181             : bool
     182         204 : AudioInput::initDevice(const std::string& device)
     183             : {
     184         204 :     devOpts_ = {};
     185         204 :     devOpts_.input = device;
     186         204 :     devOpts_.channel = format_.nb_channels;
     187         204 :     devOpts_.framerate = format_.sample_rate;
     188         204 :     deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::CAPTURE);
     189         204 :     playingDevice_ = true;
     190         204 :     return true;
     191             : }
     192             : 
     193             : void
     194          10 : AudioInput::configureFilePlayback(const std::string& path,
     195             :                                   std::shared_ptr<MediaDemuxer>& demuxer,
     196             :                                   int index)
     197             : {
     198          10 :     decoder_.reset();
     199          10 :     devOpts_ = {};
     200          10 :     devOpts_.input = path;
     201          10 :     devOpts_.name = path;
     202             :     auto decoder
     203           0 :         = std::make_unique<MediaDecoder>(demuxer, index, [this](std::shared_ptr<MediaFrame>&& frame) {
     204           0 :               if (muteState_)
     205           0 :                   libav_utils::fillWithSilence(frame->pointer());
     206           0 :               if (ringBuf_)
     207           0 :                   ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
     208          10 :           });
     209          10 :     decoder->emulateRate();
     210          10 :     decoder->setInterruptCallback(
     211           0 :         [](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); }, this);
     212             : 
     213             :     // have file audio mixed into the local buffer so it gets played
     214          10 :     Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
     215             :     // Bind to itself to be able to read from the ringbuffer
     216          10 :     Manager::instance().getRingBufferPool().bindHalfDuplexOut(id_, id_);
     217             : 
     218          10 :     deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
     219             : 
     220          10 :     wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
     221          10 :     playingFile_ = true;
     222          10 :     decoder_ = std::move(decoder);
     223          10 :     resource_ = path;
     224          10 :     loop_.start();
     225          10 : }
     226             : 
     227             : void
     228          20 : AudioInput::setPaused(bool paused)
     229             : {
     230          20 :     if (paused) {
     231          18 :         Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
     232          18 :         deviceGuard_.reset();
     233             :     } else {
     234           2 :         Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
     235           2 :         deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
     236             :     }
     237          20 :     paused_ = paused;
     238          20 : }
     239             : 
     240             : void
     241           9 : AudioInput::flushBuffers()
     242             : {
     243           9 :     if (decoder_) {
     244           9 :         decoder_->flushBuffers();
     245             :     }
     246           9 : }
     247             : 
     248             : bool
     249           0 : AudioInput::initFile(const std::string& path)
     250             : {
     251           0 :     if (access(path.c_str(), R_OK) != 0) {
     252           0 :         JAMI_ERROR("File '{}' not available", path);
     253           0 :         return false;
     254             :     }
     255             : 
     256           0 :     devOpts_ = {};
     257           0 :     devOpts_.input = path;
     258           0 :     devOpts_.name = path;
     259           0 :     devOpts_.loop = "1";
     260             :     // sets devOpts_'s sample rate and number of channels
     261           0 :     if (!createDecoder()) {
     262           0 :         JAMI_WARN() << "Unable to decode audio from file, switching back to default device";
     263           0 :         return initDevice("");
     264             :     }
     265           0 :     wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
     266             : 
     267             :     // have file audio mixed into the local buffer so it gets played
     268           0 :     Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
     269           0 :     decodingFile_ = true;
     270           0 :     deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
     271           0 :     return true;
     272             : }
     273             : 
     274             : std::shared_future<DeviceParams>
     275         204 : AudioInput::switchInput(const std::string& resource)
     276             : {
     277             :     // Always switch inputs, even if it's the same resource, so audio will be in sync with video
     278         204 :     std::unique_lock lk(resourceMutex_);
     279             : 
     280         612 :     JAMI_DEBUG("Switching audio source from {} to {}", resource_, resource);
     281             : 
     282         204 :     auto oldGuard = std::move(deviceGuard_);
     283             : 
     284         204 :     decoder_.reset();
     285         204 :     if (decodingFile_) {
     286           0 :         decodingFile_ = false;
     287           0 :         Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID,
     288           0 :                                                                     id_);
     289             :     }
     290             : 
     291         204 :     playingDevice_ = false;
     292         204 :     resource_ = resource;
     293         204 :     devOptsFound_ = false;
     294             : 
     295         204 :     std::promise<DeviceParams> p;
     296         204 :     foundDevOpts_.swap(p);
     297             : 
     298         204 :     if (resource_.empty()) {
     299         204 :         if (initDevice(""))
     300         204 :             foundDevOpts(devOpts_);
     301             :     } else {
     302           0 :         static const std::string& sep = libjami::Media::VideoProtocolPrefix::SEPARATOR;
     303           0 :         const auto pos = resource_.find(sep);
     304           0 :         if (pos == std::string::npos)
     305           0 :             return {};
     306             : 
     307           0 :         const auto prefix = resource_.substr(0, pos);
     308           0 :         if ((pos + sep.size()) >= resource_.size())
     309           0 :             return {};
     310             : 
     311           0 :         const auto suffix = resource_.substr(pos + sep.size());
     312           0 :         bool ready = false;
     313           0 :         if (prefix == libjami::Media::VideoProtocolPrefix::FILE)
     314           0 :             ready = initFile(suffix);
     315             :         else
     316           0 :             ready = initDevice(suffix);
     317             : 
     318           0 :         if (ready)
     319           0 :             foundDevOpts(devOpts_);
     320           0 :     }
     321             : 
     322         204 :     futureDevOpts_ = foundDevOpts_.get_future().share();
     323         204 :     wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
     324         204 :     lk.unlock();
     325         204 :     loop_.start();
     326         204 :     if (onSuccessfulSetup_)
     327         190 :         onSuccessfulSetup_(MEDIA_AUDIO, 0);
     328         204 :     return futureDevOpts_;
     329         204 : }
     330             : 
     331             : void
     332         204 : AudioInput::foundDevOpts(const DeviceParams& params)
     333             : {
     334         204 :     if (!devOptsFound_) {
     335         204 :         devOptsFound_ = true;
     336         204 :         foundDevOpts_.set_value(params);
     337             :     }
     338         204 : }
     339             : 
     340             : void
     341         200 : AudioInput::setRecorderCallback(
     342             :     const std::function<void(const MediaStream& ms)>&
     343             :         cb)
     344             : {
     345         200 :     settingMS_.exchange(true);
     346         200 :     recorderCallback_ = cb;
     347         200 :     if (decoder_)
     348           0 :         decoder_->setContextCallback([this]() {
     349           0 :             if (recorderCallback_)
     350           0 :                 recorderCallback_(getInfo());
     351           0 :         });
     352         200 : }
     353             : 
     354             : 
     355             : bool
     356           0 : AudioInput::createDecoder()
     357             : {
     358           0 :     decoder_.reset();
     359           0 :     if (devOpts_.input.empty()) {
     360           0 :         foundDevOpts(devOpts_);
     361           0 :         return false;
     362             :     }
     363             : 
     364           0 :     auto decoder = std::make_unique<MediaDecoder>([this](std::shared_ptr<MediaFrame>&& frame) {
     365           0 :         if (ringBuf_)
     366           0 :             ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
     367           0 :     });
     368             : 
     369             :     // NOTE don't emulate rate, file is read as frames are needed
     370             : 
     371           0 :     decoder->setInterruptCallback(
     372           0 :         [](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); }, this);
     373             : 
     374           0 :     if (decoder->openInput(devOpts_) < 0) {
     375           0 :         JAMI_ERR() << "Unable to open input '" << devOpts_.input << "'";
     376           0 :         foundDevOpts(devOpts_);
     377           0 :         return false;
     378             :     }
     379             : 
     380           0 :     if (decoder->setupAudio() < 0) {
     381           0 :         JAMI_ERR() << "Unable to setup decoder for '" << devOpts_.input << "'";
     382           0 :         foundDevOpts(devOpts_);
     383           0 :         return false;
     384             :     }
     385             : 
     386           0 :     auto ms = decoder->getStream(devOpts_.input);
     387           0 :     devOpts_.channel = ms.nbChannels;
     388           0 :     devOpts_.framerate = ms.sampleRate;
     389           0 :     JAMI_DBG() << "Created audio decoder: " << ms;
     390             : 
     391           0 :     decoder_ = std::move(decoder);
     392           0 :     foundDevOpts(devOpts_);
     393           0 :     decoder_->setContextCallback([this]() {
     394           0 :         if (recorderCallback_)
     395           0 :             recorderCallback_(getInfo());
     396           0 :     });
     397           0 :     return true;
     398           0 : }
     399             : 
     400             : void
     401         190 : AudioInput::setFormat(const AudioFormat& fmt)
     402             : {
     403         190 :     std::lock_guard lk(fmtMutex_);
     404         190 :     format_ = fmt;
     405         190 :     resizer_->setFormat(format_, format_.sample_rate * MS_PER_PACKET.count() / 1000);
     406         190 : }
     407             : 
     408             : void
     409         212 : AudioInput::setMuted(bool isMuted)
     410             : {
     411         212 :     JAMI_WARN("Audio Input muted [%s]", isMuted ? "YES" : "NO");
     412         212 :     muteState_ = isMuted;
     413         212 : }
     414             : 
     415             : MediaStream
     416           5 : AudioInput::getInfo() const
     417             : {
     418           5 :     std::lock_guard lk(fmtMutex_);
     419          10 :     return MediaStream("a:local", format_, sent_samples);
     420           5 : }
     421             : 
     422             : MediaStream
     423           1 : AudioInput::getInfo(const std::string& name) const
     424             : {
     425           1 :     std::lock_guard lk(fmtMutex_);
     426           1 :     auto ms = MediaStream(name, format_, sent_samples);
     427           2 :     return ms;
     428           1 : }
     429             : 
     430             : } // namespace jami