Line data Source code
1 : /*
2 : * Copyright (C) 2004-2024 Savoir-faire Linux Inc.
3 : *
4 : * Author: Hugo Lefeuvre <hugo.lefeuvre@savoirfairelinux.com>
5 : * Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
6 : *
7 : * This program is free software; you can redistribute it and/or modify
8 : * it under the terms of the GNU General Public License as published by
9 : * the Free Software Foundation; either version 3 of the License, or
10 : * (at your option) any later version.
11 : *
12 : * This program is distributed in the hope that it will be useful,
13 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 : * GNU General Public License for more details.
16 : *
17 : * You should have received a copy of the GNU General Public License
18 : * along with this program; if not, write to the Free Software
19 : * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 : */
21 :
22 : #include "audio_frame_resizer.h"
23 : #include "audio_input.h"
24 : #include "jami/media_const.h"
25 : #include "fileutils.h" // access
26 : #include "manager.h"
27 : #include "media_decoder.h"
28 : #include "resampler.h"
29 : #include "ringbuffer.h"
30 : #include "ringbufferpool.h"
31 : #include "tracepoint.h"
32 :
33 : #include <future>
34 : #include <memory>
35 :
36 : namespace jami {
37 :
38 : static constexpr auto MS_PER_PACKET = std::chrono::milliseconds(20);
39 :
40 236 : AudioInput::AudioInput(const std::string& id)
41 236 : : id_(id)
42 236 : , format_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
43 236 : , frameSize_(format_.sample_rate * MS_PER_PACKET.count() / 1000)
44 236 : , resampler_(new Resampler)
45 472 : , resizer_(new AudioFrameResizer(format_,
46 : frameSize_,
47 0 : [this](std::shared_ptr<AudioFrame>&& f) {
48 0 : frameResized(std::move(f));
49 236 : }))
50 236 : , deviceGuard_()
51 45882 : , loop_([] { return true; }, [this] { process(); }, [] {})
52 : {
53 708 : JAMI_DEBUG("Creating audio input with id: {}", id_);
54 236 : ringBuf_ = Manager::instance().getRingBufferPool().createRingBuffer(id_);
55 236 : }
56 :
57 0 : AudioInput::AudioInput(const std::string& id, const std::string& resource)
58 0 : : AudioInput(id)
59 : {
60 0 : switchInput(resource);
61 0 : }
62 :
63 236 : AudioInput::~AudioInput()
64 : {
65 236 : if (playingFile_) {
66 10 : Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
67 10 : Manager::instance().getRingBufferPool().unBindHalfDuplexOut(id_, id_);
68 : }
69 236 : ringBuf_.reset();
70 236 : loop_.join();
71 :
72 236 : Manager::instance().getRingBufferPool().flush(id_);
73 236 : }
74 :
75 : void
76 44544 : AudioInput::process()
77 : {
78 44544 : readFromDevice();
79 44544 : }
80 :
81 : void
82 28 : AudioInput::updateStartTime(int64_t start)
83 : {
84 28 : if (decoder_) {
85 28 : decoder_->updateStartTime(start);
86 : }
87 28 : }
88 :
89 : void
90 0 : AudioInput::frameResized(std::shared_ptr<AudioFrame>&& ptr)
91 : {
92 0 : std::shared_ptr<AudioFrame> frame = std::move(ptr);
93 0 : frame->pointer()->pts = sent_samples;
94 0 : sent_samples += frame->pointer()->nb_samples;
95 :
96 0 : notify(std::static_pointer_cast<MediaFrame>(frame));
97 0 : }
98 :
99 : void
100 9 : AudioInput::setSeekTime(int64_t time)
101 : {
102 9 : if (decoder_) {
103 9 : decoder_->setSeekTime(time);
104 : }
105 9 : }
106 :
107 : void
108 44544 : AudioInput::readFromDevice()
109 : {
110 : {
111 44544 : std::lock_guard lk(resourceMutex_);
112 44544 : if (decodingFile_)
113 0 : while (ringBuf_ && ringBuf_->isEmpty())
114 0 : readFromFile();
115 44544 : if (playingFile_) {
116 5372 : while (ringBuf_ && ringBuf_->getLength(id_) == 0)
117 5362 : readFromQueue();
118 : }
119 44544 : }
120 :
121 : // Note: read for device is called in an audio thread and we don't
122 : // want to have a loop which takes 100% of the CPU.
123 : // Here, we basically want to mix available data without any glitch
124 : // and even if one buffer doesn't have audio data (call in hold,
125 : // connections issues, etc). So mix every MS_PER_PACKET
126 44544 : std::this_thread::sleep_until(wakeUp_);
127 44544 : wakeUp_ += MS_PER_PACKET;
128 :
129 44543 : auto& bufferPool = Manager::instance().getRingBufferPool();
130 44544 : auto audioFrame = bufferPool.getData(id_);
131 44544 : if (not audioFrame)
132 44544 : return;
133 :
134 0 : if (muteState_) {
135 0 : libav_utils::fillWithSilence(audioFrame->pointer());
136 0 : audioFrame->has_voice = false; // force no voice activity when muted
137 : }
138 :
139 0 : std::lock_guard lk(fmtMutex_);
140 0 : if (bufferPool.getInternalAudioFormat() != format_)
141 0 : audioFrame = resampler_->resample(std::move(audioFrame), format_);
142 0 : resizer_->enqueue(std::move(audioFrame));
143 :
144 0 : if (recorderCallback_ && settingMS_.exchange(false)) {
145 0 : recorderCallback_(MediaStream("a:local", format_, sent_samples));
146 : }
147 :
148 : jami_tracepoint(audio_input_read_from_device_end, id_.c_str());
149 44544 : }
150 :
151 : void
152 5362 : AudioInput::readFromQueue()
153 : {
154 5362 : if (!decoder_)
155 0 : return;
156 5362 : if (paused_ || !decoder_->emitFrame(true)) {
157 5362 : std::this_thread::sleep_for(MS_PER_PACKET);
158 : }
159 : }
160 :
161 : void
162 0 : AudioInput::readFromFile()
163 : {
164 0 : if (!decoder_)
165 0 : return;
166 0 : const auto ret = decoder_->decode();
167 0 : switch (ret) {
168 0 : case MediaDemuxer::Status::Success:
169 0 : break;
170 0 : case MediaDemuxer::Status::EndOfFile:
171 0 : createDecoder();
172 0 : break;
173 0 : case MediaDemuxer::Status::ReadError:
174 0 : JAMI_ERR() << "Failed to decode frame";
175 0 : break;
176 0 : case MediaDemuxer::Status::ReadBufferOverflow:
177 0 : JAMI_ERR() << "Read buffer overflow detected";
178 0 : break;
179 0 : case MediaDemuxer::Status::FallBack:
180 0 : break;
181 : }
182 : }
183 :
184 : bool
185 187 : AudioInput::initDevice(const std::string& device)
186 : {
187 187 : devOpts_ = {};
188 187 : devOpts_.input = device;
189 187 : devOpts_.channel = format_.nb_channels;
190 187 : devOpts_.framerate = format_.sample_rate;
191 187 : deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::CAPTURE);
192 187 : playingDevice_ = true;
193 187 : return true;
194 : }
195 :
196 : void
197 10 : AudioInput::configureFilePlayback(const std::string& path,
198 : std::shared_ptr<MediaDemuxer>& demuxer,
199 : int index)
200 : {
201 10 : decoder_.reset();
202 10 : devOpts_ = {};
203 10 : devOpts_.input = path;
204 10 : devOpts_.name = path;
205 : auto decoder
206 0 : = std::make_unique<MediaDecoder>(demuxer, index, [this](std::shared_ptr<MediaFrame>&& frame) {
207 0 : if (muteState_)
208 0 : libav_utils::fillWithSilence(frame->pointer());
209 0 : if (ringBuf_)
210 0 : ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
211 10 : });
212 10 : decoder->emulateRate();
213 10 : decoder->setInterruptCallback(
214 0 : [](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); }, this);
215 :
216 : // have file audio mixed into the local buffer so it gets played
217 10 : Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
218 : // Bind to itself to be able to read from the ringbuffer
219 10 : Manager::instance().getRingBufferPool().bindHalfDuplexOut(id_, id_);
220 :
221 10 : deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
222 :
223 10 : wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
224 10 : playingFile_ = true;
225 10 : decoder_ = std::move(decoder);
226 10 : resource_ = path;
227 10 : loop_.start();
228 10 : }
229 :
230 : void
231 20 : AudioInput::setPaused(bool paused)
232 : {
233 20 : if (paused) {
234 18 : Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
235 18 : deviceGuard_.reset();
236 : } else {
237 2 : Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
238 2 : deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
239 : }
240 20 : paused_ = paused;
241 20 : }
242 :
243 : void
244 9 : AudioInput::flushBuffers()
245 : {
246 9 : if (decoder_) {
247 9 : decoder_->flushBuffers();
248 : }
249 9 : }
250 :
251 : bool
252 0 : AudioInput::initFile(const std::string& path)
253 : {
254 0 : if (access(path.c_str(), R_OK) != 0) {
255 0 : JAMI_ERROR("File '{}' not available", path);
256 0 : return false;
257 : }
258 :
259 0 : devOpts_ = {};
260 0 : devOpts_.input = path;
261 0 : devOpts_.name = path;
262 0 : devOpts_.loop = "1";
263 : // sets devOpts_'s sample rate and number of channels
264 0 : if (!createDecoder()) {
265 0 : JAMI_WARN() << "Cannot decode audio from file, switching back to default device";
266 0 : return initDevice("");
267 : }
268 0 : wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
269 :
270 : // have file audio mixed into the local buffer so it gets played
271 0 : Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
272 0 : decodingFile_ = true;
273 0 : deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
274 0 : return true;
275 : }
276 :
277 : std::shared_future<DeviceParams>
278 187 : AudioInput::switchInput(const std::string& resource)
279 : {
280 : // Always switch inputs, even if it's the same resource, so audio will be in sync with video
281 187 : std::unique_lock lk(resourceMutex_);
282 :
283 561 : JAMI_DEBUG("Switching audio source from {} to {}", resource_, resource);
284 :
285 187 : auto oldGuard = std::move(deviceGuard_);
286 :
287 187 : decoder_.reset();
288 187 : if (decodingFile_) {
289 0 : decodingFile_ = false;
290 0 : Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID,
291 0 : id_);
292 : }
293 :
294 187 : playingDevice_ = false;
295 187 : resource_ = resource;
296 187 : devOptsFound_ = false;
297 :
298 187 : std::promise<DeviceParams> p;
299 187 : foundDevOpts_.swap(p);
300 :
301 187 : if (resource_.empty()) {
302 187 : if (initDevice(""))
303 187 : foundDevOpts(devOpts_);
304 : } else {
305 0 : static const std::string& sep = libjami::Media::VideoProtocolPrefix::SEPARATOR;
306 0 : const auto pos = resource_.find(sep);
307 0 : if (pos == std::string::npos)
308 0 : return {};
309 :
310 0 : const auto prefix = resource_.substr(0, pos);
311 0 : if ((pos + sep.size()) >= resource_.size())
312 0 : return {};
313 :
314 0 : const auto suffix = resource_.substr(pos + sep.size());
315 0 : bool ready = false;
316 0 : if (prefix == libjami::Media::VideoProtocolPrefix::FILE)
317 0 : ready = initFile(suffix);
318 : else
319 0 : ready = initDevice(suffix);
320 :
321 0 : if (ready)
322 0 : foundDevOpts(devOpts_);
323 0 : }
324 :
325 187 : futureDevOpts_ = foundDevOpts_.get_future().share();
326 187 : wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
327 187 : lk.unlock();
328 187 : loop_.start();
329 187 : if (onSuccessfulSetup_)
330 187 : onSuccessfulSetup_(MEDIA_AUDIO, 0);
331 187 : return futureDevOpts_;
332 187 : }
333 :
334 : void
335 187 : AudioInput::foundDevOpts(const DeviceParams& params)
336 : {
337 187 : if (!devOptsFound_) {
338 187 : devOptsFound_ = true;
339 187 : foundDevOpts_.set_value(params);
340 : }
341 187 : }
342 :
343 : void
344 197 : AudioInput::setRecorderCallback(
345 : const std::function<void(const MediaStream& ms)>&
346 : cb)
347 : {
348 197 : settingMS_.exchange(true);
349 197 : recorderCallback_ = cb;
350 197 : if (decoder_)
351 0 : decoder_->setContextCallback([this]() {
352 0 : if (recorderCallback_)
353 0 : recorderCallback_(getInfo());
354 0 : });
355 197 : }
356 :
357 :
358 : bool
359 0 : AudioInput::createDecoder()
360 : {
361 0 : decoder_.reset();
362 0 : if (devOpts_.input.empty()) {
363 0 : foundDevOpts(devOpts_);
364 0 : return false;
365 : }
366 :
367 0 : auto decoder = std::make_unique<MediaDecoder>([this](std::shared_ptr<MediaFrame>&& frame) {
368 0 : if (ringBuf_)
369 0 : ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
370 0 : });
371 :
372 : // NOTE don't emulate rate, file is read as frames are needed
373 :
374 0 : decoder->setInterruptCallback(
375 0 : [](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); }, this);
376 :
377 0 : if (decoder->openInput(devOpts_) < 0) {
378 0 : JAMI_ERR() << "Could not open input '" << devOpts_.input << "'";
379 0 : foundDevOpts(devOpts_);
380 0 : return false;
381 : }
382 :
383 0 : if (decoder->setupAudio() < 0) {
384 0 : JAMI_ERR() << "Could not setup decoder for '" << devOpts_.input << "'";
385 0 : foundDevOpts(devOpts_);
386 0 : return false;
387 : }
388 :
389 0 : auto ms = decoder->getStream(devOpts_.input);
390 0 : devOpts_.channel = ms.nbChannels;
391 0 : devOpts_.framerate = ms.sampleRate;
392 0 : JAMI_DBG() << "Created audio decoder: " << ms;
393 :
394 0 : decoder_ = std::move(decoder);
395 0 : foundDevOpts(devOpts_);
396 0 : decoder_->setContextCallback([this]() {
397 0 : if (recorderCallback_)
398 0 : recorderCallback_(getInfo());
399 0 : });
400 0 : return true;
401 0 : }
402 :
403 : void
404 187 : AudioInput::setFormat(const AudioFormat& fmt)
405 : {
406 187 : std::lock_guard lk(fmtMutex_);
407 187 : format_ = fmt;
408 187 : resizer_->setFormat(format_, format_.sample_rate * MS_PER_PACKET.count() / 1000);
409 187 : }
410 :
411 : void
412 211 : AudioInput::setMuted(bool isMuted)
413 : {
414 211 : JAMI_WARN("Audio Input muted [%s]", isMuted ? "YES" : "NO");
415 211 : muteState_ = isMuted;
416 211 : }
417 :
418 : MediaStream
419 5 : AudioInput::getInfo() const
420 : {
421 5 : std::lock_guard lk(fmtMutex_);
422 10 : return MediaStream("a:local", format_, sent_samples);
423 5 : }
424 :
425 : MediaStream
426 1 : AudioInput::getInfo(const std::string& name) const
427 : {
428 1 : std::lock_guard lk(fmtMutex_);
429 1 : auto ms = MediaStream(name, format_, sent_samples);
430 2 : return ms;
431 1 : }
432 :
433 : } // namespace jami
|
