Line data Source code
1 : /*
2 : * Copyright (C) 2004-2026 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "audiolayer.h"
19 : #include "audio/sound/tone.h"
20 : #include "logger.h"
21 : #include "manager.h"
22 : #include "audio/ringbufferpool.h"
23 : #include "audio/resampler.h"
24 : #include "client/jami_signal.h"
25 :
26 : #include "tracepoint.h"
27 : #if HAVE_WEBRTC_AP
28 : #include "audio-processing/webrtc.h"
29 : #endif
30 : #if HAVE_SPEEXDSP
31 : #include "audio-processing/speex.h"
32 : #endif
33 :
34 : #include <ctime>
35 : #include <algorithm>
36 :
37 : namespace jami {
38 :
39 42 : AudioLayer::AudioLayer(const AudioPreference& pref)
40 84 : : isCaptureMuted_(pref.getCaptureMuted())
41 42 : , isPlaybackMuted_(pref.getPlaybackMuted())
42 84 : , captureGain_(pref.getVolumemic())
43 42 : , playbackGain_(pref.getVolumespkr())
44 42 : , pref_(pref)
45 84 : , mainRingBuffer_(Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID))
46 42 : , audioFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
47 42 : , audioInputFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
48 42 : , urgentRingBuffer_("urgentRingBuffer_id", audioFormat_)
49 42 : , resampler_(new Resampler)
50 252 : , lastNotificationTime_()
51 : {
52 84 : urgentRingBuffer_.createReadOffset(RingBufferPool::DEFAULT_ID);
53 :
54 42 : JAMI_LOG("[audiolayer] AGC: {:d}, noiseReduce: {:s}, VAD: {:d}, echoCancel: {:s}, audioProcessor: {:s}",
55 : pref_.isAGCEnabled(),
56 : pref.getNoiseReduce(),
57 : pref.getVadEnabled(),
58 : pref.getEchoCanceller(),
59 : pref.getAudioProcessor());
60 42 : }
61 :
62 42 : AudioLayer::~AudioLayer() {}
63 :
64 : void
65 105 : AudioLayer::hardwareFormatAvailable(AudioFormat playback, size_t bufSize)
66 : {
67 105 : JAMI_LOG("Hardware audio format available: {:s} {}", playback.toString(), bufSize);
68 105 : audioFormat_ = Manager::instance().hardwareAudioFormatChanged(playback);
69 105 : audioInputFormat_.sampleFormat = audioFormat_.sampleFormat;
70 105 : urgentRingBuffer_.setFormat(audioFormat_);
71 105 : nativeFrameSize_ = bufSize;
72 105 : }
73 :
74 : void
75 0 : AudioLayer::hardwareInputFormatAvailable(AudioFormat capture)
76 : {
77 0 : JAMI_LOG("Hardware input audio format available: {:s}", capture.toString());
78 :
79 : // Keep it: createAudioProcessor() sizes the echo canceller, the noise suppressor and the voice
80 : // activity detector from max(playback, capture), so a stale capture format silently sizes them
81 : // from the playback format alone.
82 0 : audioInputFormat_ = capture;
83 0 : }
84 :
85 : void
86 0 : AudioLayer::devicesChanged()
87 : {
88 0 : emitSignal<libjami::AudioSignal::DeviceEvent>();
89 0 : }
90 :
91 : void
92 0 : AudioLayer::flushMain()
93 : {
94 0 : Manager::instance().getRingBufferPool().flushAllBuffers();
95 0 : }
96 :
97 : void
98 447 : AudioLayer::flushUrgent()
99 : {
100 447 : urgentRingBuffer_.flushAll();
101 447 : }
102 :
103 : void
104 0 : AudioLayer::flush()
105 : {
106 0 : Manager::instance().getRingBufferPool().flushAllBuffers();
107 0 : urgentRingBuffer_.flushAll();
108 0 : }
109 :
110 : void
111 228 : AudioLayer::playbackChanged(bool started)
112 : {
113 228 : playbackStarted_ = started;
114 228 : }
115 :
116 : void
117 228 : AudioLayer::recordChanged(bool started)
118 : {
119 228 : std::lock_guard lock(audioProcessorMutex);
120 228 : if (started) {
121 : // create audio processor
122 0 : createAudioProcessor();
123 : } else {
124 : // destroy audio processor
125 228 : destroyAudioProcessor();
126 : }
127 228 : recordStarted_ = started;
128 228 : }
129 :
130 : // helper function
131 : static inline bool
132 0 : shouldUseAudioProcessorEchoCancel(bool hasNativeAEC, const std::string& echoCancellerPref)
133 : {
134 : return
135 : // user doesn't care which and there is not a system AEC
136 0 : (echoCancellerPref == "auto" && !hasNativeAEC)
137 : // user specifically wants audioProcessor
138 0 : or (echoCancellerPref == "audioProcessor");
139 : }
140 :
141 : // helper function
142 : static inline bool
143 0 : shouldUseAudioProcessorNoiseSuppression(bool hasNativeNS, const std::string& noiseSuppressionPref)
144 : {
145 : return
146 : // user doesn't care which and there is no system noise suppression
147 0 : (noiseSuppressionPref == "auto" && !hasNativeNS)
148 : // user specifically wants audioProcessor
149 0 : or (noiseSuppressionPref == "audioProcessor");
150 : }
151 :
152 : void
153 39 : AudioLayer::setHasNativeAEC(bool hasNativeAEC)
154 : {
155 39 : JAMI_LOG("[audiolayer] setHasNativeAEC: {}", hasNativeAEC);
156 39 : std::lock_guard lock(audioProcessorMutex);
157 39 : hasNativeAEC_ = hasNativeAEC;
158 : // if we have a current audio processor, tell it to enable/disable its own AEC
159 39 : if (audioProcessor) {
160 0 : audioProcessor->enableEchoCancel(shouldUseAudioProcessorEchoCancel(hasNativeAEC, pref_.getEchoCanceller()));
161 : }
162 39 : }
163 :
164 : void
165 42 : AudioLayer::setHasNativeNS(bool hasNativeNS)
166 : {
167 42 : JAMI_LOG("[audiolayer] setHasNativeNS: {}", hasNativeNS);
168 42 : std::lock_guard lock(audioProcessorMutex);
169 42 : hasNativeNS_ = hasNativeNS;
170 : // if we have a current audio processor, tell it to enable/disable its own noise suppression
171 42 : if (audioProcessor) {
172 0 : audioProcessor->enableNoiseSuppression(
173 0 : shouldUseAudioProcessorNoiseSuppression(hasNativeNS, pref_.getNoiseReduce()));
174 : }
175 42 : }
176 :
177 : // must acquire lock beforehand
178 : void
179 0 : AudioLayer::createAudioProcessor()
180 : {
181 0 : auto nb_channels = std::max(audioFormat_.nb_channels, audioInputFormat_.nb_channels);
182 0 : auto sample_rate = std::max(audioFormat_.sample_rate, audioInputFormat_.sample_rate);
183 :
184 0 : sample_rate = std::clamp(sample_rate, 16000u, 48000u);
185 :
186 0 : AudioFormat formatForProcessor {sample_rate, nb_channels};
187 :
188 : unsigned int frame_size;
189 0 : if (pref_.getAudioProcessor() == "speex") {
190 : // TODO: maybe force this to be equivalent to 20ms? as expected by Speex
191 0 : frame_size = sample_rate / 50u;
192 : } else {
193 0 : frame_size = sample_rate / 100u;
194 : }
195 :
196 0 : JAMI_WARNING("Input {}", audioInputFormat_.toString());
197 0 : JAMI_WARNING("Output {}", audioFormat_.toString());
198 0 : JAMI_WARNING("Starting audio processor with: [{} Hz, {} channels, {} samples/frame]",
199 : sample_rate,
200 : nb_channels,
201 : frame_size);
202 :
203 0 : if (pref_.getAudioProcessor() == "webrtc") {
204 : #if HAVE_WEBRTC_AP
205 0 : JAMI_WARNING("[audiolayer] using WebRTCAudioProcessor");
206 0 : audioProcessor.reset(new WebRTCAudioProcessor(formatForProcessor, frame_size));
207 : #else
208 : JAMI_ERROR("[audiolayer] audioProcessor preference is webrtc, but library not linked! using null "
209 : "AudioProcessor instead");
210 : audioProcessor.reset();
211 : #endif
212 0 : } else if (pref_.getAudioProcessor() == "speex") {
213 : #if HAVE_SPEEXDSP
214 0 : JAMI_WARNING("[audiolayer] using SpeexAudioProcessor");
215 0 : audioProcessor.reset(new SpeexAudioProcessor(formatForProcessor, frame_size));
216 : #else
217 : JAMI_ERROR("[audiolayer] audioProcessor preference is Speex, but library not linked! using null AudioProcessor "
218 : "instead");
219 : audioProcessor.reset();
220 : #endif
221 0 : } else if (pref_.getAudioProcessor() == "null") {
222 0 : JAMI_WARNING("[audiolayer] using null AudioProcessor");
223 0 : audioProcessor.reset();
224 : } else {
225 0 : JAMI_ERROR("[audiolayer] audioProcessor preference not recognized, using null AudioProcessor instead");
226 0 : audioProcessor.reset();
227 : }
228 :
229 0 : if (audioProcessor) {
230 0 : audioProcessor->enableNoiseSuppression(
231 0 : shouldUseAudioProcessorNoiseSuppression(hasNativeNS_, pref_.getNoiseReduce()));
232 :
233 0 : audioProcessor->enableAutomaticGainControl(pref_.isAGCEnabled());
234 :
235 0 : audioProcessor->enableEchoCancel(shouldUseAudioProcessorEchoCancel(hasNativeAEC_, pref_.getEchoCanceller()));
236 :
237 0 : audioProcessor->enableVoiceActivityDetection(pref_.getVadEnabled());
238 : }
239 0 : }
240 :
241 : // must acquire lock beforehand
242 : void
243 228 : AudioLayer::destroyAudioProcessor()
244 : {
245 : // delete it
246 228 : audioProcessor.reset();
247 228 : }
248 :
249 : void
250 0 : AudioLayer::putUrgent(std::shared_ptr<AudioFrame> buffer)
251 : {
252 0 : urgentRingBuffer_.put(std::move(buffer));
253 0 : }
254 :
255 : // Notify (with a beep) an incoming call when there is already a call in progress
256 : void
257 0 : AudioLayer::notifyIncomingCall()
258 : {
259 0 : if (not playIncomingCallBeep_)
260 0 : return;
261 :
262 0 : auto now = std::chrono::system_clock::now();
263 :
264 : // Notify maximum once every 5 seconds
265 0 : if (now < lastNotificationTime_ + std::chrono::seconds(5))
266 0 : return;
267 :
268 0 : lastNotificationTime_ = now;
269 :
270 0 : Tone tone("440/160", getSampleRate(), audioFormat_.sampleFormat);
271 0 : size_t nbSample = tone.getSize();
272 :
273 : /* Put the data in the urgent ring buffer */
274 0 : urgentRingBuffer_.flushAll();
275 0 : urgentRingBuffer_.put(tone.getNext(nbSample));
276 0 : }
277 :
278 : std::shared_ptr<AudioFrame>
279 0 : AudioLayer::getToRing(AudioFormat format, size_t writableSamples)
280 : {
281 0 : if (auto fileToPlay = Manager::instance().getTelephoneFile()) {
282 0 : auto fileformat = fileToPlay->getFormat();
283 0 : bool resample = format != fileformat;
284 :
285 0 : size_t readableSamples = resample ? rational<size_t>(writableSamples * (size_t) fileformat.sample_rate,
286 0 : format.sample_rate)
287 0 : .real<size_t>()
288 0 : : writableSamples;
289 :
290 0 : return resampler_->resample(fileToPlay->getNext(readableSamples, isRingtoneMuted_), format);
291 0 : }
292 0 : return {};
293 : }
294 :
295 : std::shared_ptr<AudioFrame>
296 0 : AudioLayer::getToPlay(AudioFormat format, size_t writableSamples)
297 : {
298 0 : notifyIncomingCall();
299 0 : auto& bufferPool = Manager::instance().getRingBufferPool();
300 :
301 0 : if (not playbackQueue_)
302 0 : playbackQueue_.reset(new AudioFrameResizer(format, static_cast<int>(writableSamples)));
303 : else
304 0 : playbackQueue_->setFrameSize(static_cast<int>(writableSamples));
305 :
306 0 : std::shared_ptr<AudioFrame> playbackBuf {};
307 0 : while (!(playbackBuf = playbackQueue_->dequeue())) {
308 0 : std::shared_ptr<AudioFrame> resampled;
309 :
310 0 : if (auto urgentSamples = urgentRingBuffer_.get(RingBufferPool::DEFAULT_ID)) {
311 0 : bufferPool.discard(1, RingBufferPool::DEFAULT_ID);
312 0 : resampled = resampler_->resample(std::move(urgentSamples), format);
313 0 : } else if (auto toneToPlay = Manager::instance().getTelephoneTone()) {
314 0 : resampled = resampler_->resample(toneToPlay->getNext(), format);
315 0 : } else if (auto buf = bufferPool.getData(RingBufferPool::DEFAULT_ID)) {
316 0 : resampled = resampler_->resample(std::move(buf), format);
317 : } else {
318 0 : std::lock_guard lock(audioProcessorMutex);
319 0 : if (audioProcessor) {
320 0 : auto silence = std::make_shared<AudioFrame>(format, writableSamples);
321 0 : libav_utils::fillWithSilence(silence->pointer());
322 0 : audioProcessor->putPlayback(silence);
323 0 : }
324 0 : break;
325 0 : }
326 :
327 0 : if (resampled) {
328 0 : std::lock_guard lock(audioProcessorMutex);
329 0 : if (audioProcessor) {
330 0 : audioProcessor->putPlayback(resampled);
331 : }
332 0 : playbackQueue_->enqueue(std::move(resampled));
333 0 : } else
334 0 : break;
335 0 : }
336 :
337 : jami_tracepoint(audio_layer_get_to_play_end);
338 :
339 0 : return playbackBuf;
340 0 : }
341 :
342 : void
343 0 : AudioLayer::putRecorded(std::shared_ptr<AudioFrame>&& frame)
344 : {
345 0 : std::lock_guard lock(audioProcessorMutex);
346 0 : if (audioProcessor && playbackStarted_ && recordStarted_) {
347 0 : audioProcessor->putRecorded(std::move(frame));
348 0 : while (auto rec = audioProcessor->getProcessed()) {
349 0 : mainRingBuffer_->put(std::move(rec));
350 0 : }
351 : } else {
352 0 : mainRingBuffer_->put(std::move(frame));
353 : }
354 :
355 : jami_tracepoint(audio_layer_put_recorded_end, );
356 0 : }
357 :
358 : } // namespace jami
|
