Line data Source code
1 : /*
2 : * Copyright (C) 2004-2026 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "audio_frame_resizer.h"
19 : #include "audio_input.h"
20 : #include "jami/media_const.h"
21 : #include "manager.h"
22 : #include "media_decoder.h"
23 : #include "resampler.h"
24 : #include "logger.h"
25 : #include "ringbufferpool.h"
26 : #include "tracepoint.h"
27 : #include "video/video_device.h"
28 :
29 : #include <future>
30 : #include <memory>
31 : #include <stdexcept>
32 :
33 : namespace jami {
34 :
35 : static constexpr auto MS_PER_PACKET = std::chrono::milliseconds(20);
36 :
37 122 : AudioInput::AudioInput(const std::string& id)
38 122 : : id_(id)
39 122 : , format_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
40 122 : , frameSize_(static_cast<int>(format_.sample_rate * MS_PER_PACKET.count()) / 1000)
41 122 : , resampler_(new Resampler)
42 122 : , resizer_(new AudioFrameResizer(format_,
43 : frameSize_,
44 244 : [this](std::shared_ptr<AudioFrame>&& f) { frameResized(std::move(f)); }))
45 122 : , deviceGuard_()
46 24467 : , loop_([] { return true; }, [this] { process(); }, [] {})
47 : {
48 122 : JAMI_DEBUG("Creating audio input with id: {}", id_);
49 122 : ringBuf_ = Manager::instance().getRingBufferPool().createRingBuffer(id_);
50 122 : }
51 :
52 0 : AudioInput::AudioInput(const std::string& id, const std::string& resource)
53 0 : : AudioInput(id)
54 : {
55 0 : switchInput(resource);
56 0 : }
57 :
58 122 : AudioInput::~AudioInput()
59 : {
60 122 : if (playingFile_) {
61 20 : Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
62 10 : Manager::instance().getRingBufferPool().unBindHalfDuplexOut(id_, id_);
63 : }
64 122 : ringBuf_.reset();
65 122 : loop_.join();
66 :
67 122 : Manager::instance().getRingBufferPool().flush(id_);
68 122 : }
69 :
70 : void
71 23753 : AudioInput::process()
72 : {
73 23753 : readFromDevice();
74 23753 : }
75 :
76 : void
77 36 : AudioInput::updateStartTime(int64_t start)
78 : {
79 36 : if (decoder_) {
80 36 : decoder_->updateStartTime(start);
81 : }
82 36 : }
83 :
84 : void
85 0 : AudioInput::frameResized(std::shared_ptr<AudioFrame>&& ptr)
86 : {
87 0 : std::shared_ptr<AudioFrame> frame = std::move(ptr);
88 0 : frame->pointer()->pts = static_cast<int64_t>(sent_samples);
89 0 : sent_samples += frame->pointer()->nb_samples;
90 :
91 0 : notify(std::static_pointer_cast<MediaFrame>(std::move(frame)));
92 0 : }
93 :
94 : void
95 9 : AudioInput::setSeekTime(int64_t time)
96 : {
97 9 : if (decoder_) {
98 9 : decoder_->setSeekTime(time);
99 : }
100 9 : }
101 :
102 : void
103 23753 : AudioInput::readFromDevice()
104 : {
105 : {
106 23753 : std::lock_guard lk(resourceMutex_);
107 23753 : if (decodingFile_)
108 0 : while (ringBuf_ && ringBuf_->isEmpty())
109 0 : readFromFile();
110 23753 : if (playingFile_) {
111 5483 : while (ringBuf_ && ringBuf_->getLength(id_) == 0)
112 5473 : readFromQueue();
113 : }
114 23753 : }
115 :
116 23753 : auto& bufferPool = Manager::instance().getRingBufferPool();
117 23753 : if (not bufferPool.waitForDataAvailable(id_, wakeUp_))
118 23753 : std::this_thread::sleep_until(wakeUp_);
119 23753 : wakeUp_ += MS_PER_PACKET;
120 :
121 23753 : auto audioFrame = bufferPool.getData(id_);
122 23753 : if (not audioFrame)
123 23753 : return;
124 :
125 0 : if (muteState_) {
126 0 : libav_utils::fillWithSilence(audioFrame->pointer());
127 0 : audioFrame->has_voice = false; // force no voice activity when muted
128 : }
129 :
130 0 : std::lock_guard lk(fmtMutex_);
131 0 : if (bufferPool.getInternalAudioFormat() != format_)
132 0 : audioFrame = resampler_->resample(std::move(audioFrame), format_);
133 0 : resizer_->enqueue(std::move(audioFrame));
134 :
135 0 : if (recorderCallback_ && settingMS_.exchange(false)) {
136 0 : recorderCallback_(MediaStream("a:local", format_, static_cast<int64_t>(sent_samples)));
137 : }
138 :
139 : jami_tracepoint(audio_input_read_from_device_end, id_.c_str());
140 23753 : }
141 :
142 : void
143 5473 : AudioInput::readFromQueue()
144 : {
145 5473 : if (!decoder_)
146 0 : return;
147 5473 : if (paused_ || !decoder_->emitFrame(true)) {
148 5473 : std::this_thread::sleep_for(MS_PER_PACKET);
149 : }
150 : }
151 :
152 : void
153 0 : AudioInput::readFromFile()
154 : {
155 0 : if (!decoder_)
156 0 : return;
157 0 : const auto ret = decoder_->decode();
158 0 : switch (ret) {
159 0 : case MediaDemuxer::Status::Success:
160 0 : break;
161 0 : case MediaDemuxer::Status::EndOfFile:
162 0 : createDecoder();
163 0 : break;
164 0 : case MediaDemuxer::Status::ReadError:
165 0 : JAMI_ERROR("Failed to decode frame");
166 0 : break;
167 0 : case MediaDemuxer::Status::ReadBufferOverflow:
168 0 : JAMI_ERROR("Read buffer overflow detected");
169 0 : break;
170 0 : case MediaDemuxer::Status::FallBack:
171 : case MediaDemuxer::Status::RestartRequired:
172 0 : break;
173 : }
174 : }
175 :
176 : bool
177 0 : AudioInput::initCapture(const std::string& device)
178 : {
179 0 : std::string targetId = device;
180 : #if defined(_WIN32)
181 : // There are two possible formats for device:
182 : // 1. A string containing "window-id:hwnd=XXXX" where XXXX is the HWND of the window to capture
183 : // 2. A string that does not contain a window handle, in which case we capture desktop audio
184 : std::string pattern = "window-id:hwnd=";
185 : size_t winHandlePos = device.find(pattern);
186 :
187 : if (winHandlePos != std::string::npos) {
188 : // Get HWND from device URI
189 : size_t startPos = winHandlePos + pattern.size();
190 : size_t endPos = device.find(' ', startPos);
191 : if (endPos == std::string::npos) {
192 : endPos = device.size();
193 : }
194 : targetId = device.substr(startPos, endPos - startPos);
195 : } else {
196 : targetId = video::DEVICE_DESKTOP;
197 : }
198 : #elif defined(__linux__)
199 : // On Linux, we always capture desktop audio because window-specific audio capture is not yet implemented
200 : // Possible to implement window audio capture on X11 specifically in the future, but not Wayland as of Jan 2026
201 : // See https://github.com/flatpak/xdg-desktop-portal/issues/957
202 0 : targetId = video::DEVICE_DESKTOP;
203 : #elif defined(__APPLE__)
204 : // As of Jan 2026, audio capture has not been implemented for macOS (TODO)
205 : targetId = video::DEVICE_DESKTOP;
206 : #endif
207 :
208 0 : devOpts_ = {};
209 0 : devOpts_.input = targetId;
210 0 : devOpts_.channel = format_.nb_channels;
211 0 : devOpts_.framerate = format_.sample_rate;
212 :
213 : // This will cause the audio layer to create a ring buffer with id=targetId
214 : // The audio layer will then fill it with the audio from the captured window/desktop
215 0 : deviceGuard_ = Manager::instance().startCaptureStream(targetId);
216 0 : if (!deviceGuard_) {
217 0 : if (!targetId.empty())
218 0 : JAMI_ERROR("Failed to start capture stream for window-id: {}", targetId);
219 : else
220 0 : JAMI_ERROR("Failed to start capture stream for desktop audio");
221 0 : return false;
222 : }
223 :
224 : // We want the audio input's ring buffer to read the captured audio from the audio layer
225 : // Then the audio RTP session will handle sending the audio over the network
226 0 : Manager::instance().getRingBufferPool().bindHalfDuplexOut(id_, targetId);
227 :
228 0 : sourceRingBufferId_ = targetId;
229 0 : playingDevice_ = true;
230 0 : return true;
231 0 : }
232 :
233 : bool
234 111 : AudioInput::initDevice(const std::string& device)
235 : {
236 111 : devOpts_ = {};
237 111 : devOpts_.input = device;
238 111 : devOpts_.channel = format_.nb_channels;
239 111 : devOpts_.framerate = format_.sample_rate;
240 111 : deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::CAPTURE);
241 111 : playingDevice_ = true;
242 111 : return true;
243 111 : }
244 :
245 : void
246 10 : AudioInput::configureFilePlayback(const std::string& path, std::shared_ptr<MediaDemuxer>& demuxer, int index)
247 : {
248 10 : decoder_.reset();
249 10 : devOpts_ = {};
250 10 : devOpts_.input = path;
251 10 : devOpts_.name = path;
252 0 : auto decoder = std::make_unique<MediaDecoder>(demuxer, index, [this](std::shared_ptr<MediaFrame>&& frame) {
253 0 : if (muteState_)
254 0 : libav_utils::fillWithSilence(frame->pointer());
255 0 : if (ringBuf_)
256 0 : ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
257 10 : });
258 10 : if (!decoder->isReady()) {
259 0 : throw std::runtime_error("audio decoder setup failed for " + path);
260 : }
261 10 : decoder->emulateRate();
262 10 : decoder->setInterruptCallback([](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); },
263 : this);
264 :
265 : // have file audio mixed into the local buffer so it gets played
266 20 : Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
267 : // Bind to itself to be able to read from the ringbuffer
268 10 : Manager::instance().getRingBufferPool().bindHalfDuplexOut(id_, id_);
269 :
270 10 : sourceRingBufferId_ = id_;
271 10 : deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
272 :
273 10 : wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
274 10 : playingFile_ = true;
275 10 : decoder_ = std::move(decoder);
276 10 : resource_ = path;
277 10 : loop_.start();
278 20 : }
279 :
280 : void
281 28 : AudioInput::setPaused(bool paused)
282 : {
283 28 : if (paused) {
284 44 : Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
285 22 : deviceGuard_.reset();
286 : } else {
287 12 : Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
288 6 : deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
289 : }
290 28 : paused_ = paused;
291 28 : }
292 :
293 : void
294 9 : AudioInput::flushBuffers()
295 : {
296 9 : if (decoder_) {
297 9 : decoder_->flushBuffers();
298 : }
299 9 : }
300 :
301 : bool
302 0 : AudioInput::initFile(const std::string& path)
303 : {
304 0 : if (access(path.c_str(), R_OK) != 0) {
305 0 : JAMI_ERROR("File '{}' not available", path);
306 0 : return false;
307 : }
308 :
309 0 : devOpts_ = {};
310 0 : devOpts_.input = path;
311 0 : devOpts_.name = path;
312 0 : devOpts_.loop = "1";
313 : // sets devOpts_'s sample rate and number of channels
314 0 : if (!createDecoder()) {
315 0 : JAMI_WARNING("Unable to decode audio from file, switching back to default device");
316 0 : return initDevice("");
317 : }
318 0 : wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
319 :
320 : // have file audio mixed into the local buffer so it gets played
321 0 : Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
322 0 : sourceRingBufferId_ = id_;
323 0 : decodingFile_ = true;
324 0 : deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
325 0 : return true;
326 0 : }
327 :
328 : std::shared_future<DeviceParams>
329 111 : AudioInput::switchInput(const std::string& resource)
330 : {
331 : // Always switch inputs, even if it's the same resource, so audio will be in sync with video
332 111 : std::unique_lock lk(resourceMutex_);
333 :
334 111 : JAMI_DEBUG("Switching audio source from [{}] to [{}]", resource_, resource);
335 :
336 111 : auto oldGuard = std::move(deviceGuard_);
337 :
338 111 : decoder_.reset();
339 111 : if (decodingFile_) {
340 0 : decodingFile_ = false;
341 0 : Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
342 : }
343 :
344 111 : playingDevice_ = false;
345 111 : resource_ = resource;
346 111 : sourceRingBufferId_.clear();
347 111 : devOptsFound_ = false;
348 :
349 111 : std::promise<DeviceParams> p;
350 111 : foundDevOpts_.swap(p);
351 :
352 111 : if (resource_.empty()) {
353 222 : if (initDevice(""))
354 111 : foundDevOpts(devOpts_);
355 : } else {
356 : static const std::string& sep = libjami::Media::VideoProtocolPrefix::SEPARATOR;
357 0 : const auto pos = resource_.find(sep);
358 0 : if (pos == std::string::npos)
359 0 : return {};
360 :
361 0 : const auto prefix = resource_.substr(0, pos);
362 0 : if ((pos + sep.size()) >= resource_.size())
363 0 : return {};
364 :
365 0 : const auto suffix = resource_.substr(pos + sep.size());
366 :
367 0 : bool ready = false;
368 0 : if (prefix == libjami::Media::VideoProtocolPrefix::FILE)
369 0 : ready = initFile(suffix);
370 0 : else if (prefix == libjami::Media::VideoProtocolPrefix::DISPLAY)
371 0 : ready = initCapture(suffix);
372 : else
373 0 : ready = initDevice(suffix);
374 :
375 0 : if (ready)
376 0 : foundDevOpts(devOpts_);
377 0 : }
378 :
379 111 : futureDevOpts_ = foundDevOpts_.get_future().share();
380 111 : wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
381 111 : lk.unlock();
382 111 : if (not loop_.isRunning())
383 103 : loop_.start();
384 111 : if (onSuccessfulSetup_)
385 92 : onSuccessfulSetup_(MEDIA_AUDIO, 0);
386 111 : return futureDevOpts_;
387 111 : }
388 :
389 : void
390 111 : AudioInput::foundDevOpts(const DeviceParams& params)
391 : {
392 111 : if (!devOptsFound_) {
393 111 : devOptsFound_ = true;
394 111 : foundDevOpts_.set_value(params);
395 : }
396 111 : }
397 :
398 : void
399 100 : AudioInput::setRecorderCallback(const std::function<void(const MediaStream& ms)>& cb)
400 : {
401 100 : settingMS_.exchange(true);
402 100 : recorderCallback_ = cb;
403 100 : if (decoder_)
404 0 : decoder_->setContextCallback([this]() {
405 0 : if (recorderCallback_)
406 0 : recorderCallback_(getInfo());
407 0 : });
408 100 : }
409 :
410 : bool
411 0 : AudioInput::createDecoder()
412 : {
413 0 : decoder_.reset();
414 0 : if (devOpts_.input.empty()) {
415 0 : foundDevOpts(devOpts_);
416 0 : return false;
417 : }
418 :
419 0 : auto decoder = std::make_unique<MediaDecoder>([this](std::shared_ptr<MediaFrame>&& frame) {
420 0 : if (ringBuf_)
421 0 : ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
422 0 : });
423 :
424 : // NOTE don't emulate rate, file is read as frames are needed
425 :
426 0 : decoder->setInterruptCallback([](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); },
427 : this);
428 :
429 0 : if (decoder->openInput(devOpts_) < 0) {
430 0 : JAMI_ERROR("Unable to open input '{}'", devOpts_.input);
431 0 : foundDevOpts(devOpts_);
432 0 : return false;
433 : }
434 :
435 0 : if (decoder->setupAudio() < 0) {
436 0 : JAMI_ERROR("Unable to setup decoder for '{}'", devOpts_.input);
437 0 : foundDevOpts(devOpts_);
438 0 : return false;
439 : }
440 :
441 0 : auto ms = decoder->getStream(devOpts_.input);
442 0 : devOpts_.channel = ms.nbChannels;
443 0 : devOpts_.framerate = ms.sampleRate;
444 0 : JAMI_LOG("Created audio decoder: {}", ms);
445 :
446 0 : decoder_ = std::move(decoder);
447 0 : foundDevOpts(devOpts_);
448 0 : decoder_->setContextCallback([this]() {
449 0 : if (recorderCallback_)
450 0 : recorderCallback_(getInfo());
451 0 : });
452 0 : return true;
453 0 : }
454 :
455 : void
456 92 : AudioInput::setFormat(const AudioFormat& fmt)
457 : {
458 92 : std::lock_guard lk(fmtMutex_);
459 92 : format_ = fmt;
460 92 : resizer_->setFormat(format_, static_cast<int>(format_.sample_rate * MS_PER_PACKET.count()) / 1000);
461 92 : }
462 :
463 : void
464 184 : AudioInput::setMuted(bool isMuted)
465 : {
466 184 : JAMI_WARNING("Audio Input muted [{}]", isMuted ? "YES" : "NO");
467 184 : muteState_ = isMuted;
468 184 : }
469 :
470 : MediaStream
471 3 : AudioInput::getInfo() const
472 : {
473 3 : std::lock_guard lk(fmtMutex_);
474 9 : return MediaStream("a:local", format_, static_cast<int64_t>(sent_samples));
475 3 : }
476 :
477 : MediaStream
478 0 : AudioInput::getInfo(const std::string& name) const
479 : {
480 0 : std::lock_guard lk(fmtMutex_);
481 0 : auto ms = MediaStream(name, format_, static_cast<int64_t>(sent_samples));
482 0 : return ms;
483 0 : }
484 :
485 : } // namespace jami
|
