Line data Source code
1 : /*
2 : * Copyright (C) 2004-2026 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "audiolayer.h"
19 : #include "audio/sound/tone.h"
20 : #include "logger.h"
21 : #include "manager.h"
22 : #include "audio/ringbufferpool.h"
23 : #include "audio/resampler.h"
24 : #include "client/jami_signal.h"
25 :
26 : #include "tracepoint.h"
27 : #if HAVE_WEBRTC_AP
28 : #include "audio-processing/webrtc.h"
29 : #endif
30 : #if HAVE_SPEEXDSP
31 : #include "audio-processing/speex.h"
32 : #endif
33 :
34 : #include <ctime>
35 : #include <algorithm>
36 :
37 : namespace jami {
38 :
39 35 : AudioLayer::AudioLayer(const AudioPreference& pref)
40 70 : : isCaptureMuted_(pref.getCaptureMuted())
41 35 : , isPlaybackMuted_(pref.getPlaybackMuted())
42 70 : , captureGain_(pref.getVolumemic())
43 35 : , playbackGain_(pref.getVolumespkr())
44 35 : , pref_(pref)
45 70 : , mainRingBuffer_(Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID))
46 35 : , audioFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
47 35 : , audioInputFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
48 35 : , urgentRingBuffer_("urgentRingBuffer_id", audioFormat_)
49 35 : , resampler_(new Resampler)
50 210 : , lastNotificationTime_()
51 : {
52 70 : urgentRingBuffer_.createReadOffset(RingBufferPool::DEFAULT_ID);
53 :
54 140 : JAMI_LOG("[audiolayer] AGC: {:d}, noiseReduce: {:s}, VAD: {:d}, echoCancel: {:s}, audioProcessor: {:s}",
55 : pref_.isAGCEnabled(),
56 : pref.getNoiseReduce(),
57 : pref.getVadEnabled(),
58 : pref.getEchoCanceller(),
59 : pref.getAudioProcessor());
60 35 : }
61 :
62 35 : AudioLayer::~AudioLayer() {}
63 :
64 : void
65 101 : AudioLayer::hardwareFormatAvailable(AudioFormat playback, size_t bufSize)
66 : {
67 404 : JAMI_LOG("Hardware audio format available: {:s} {}", playback.toString(), bufSize);
68 101 : audioFormat_ = Manager::instance().hardwareAudioFormatChanged(playback);
69 101 : audioInputFormat_.sampleFormat = audioFormat_.sampleFormat;
70 101 : urgentRingBuffer_.setFormat(audioFormat_);
71 101 : nativeFrameSize_ = bufSize;
72 101 : }
73 :
74 : void
75 0 : AudioLayer::hardwareInputFormatAvailable(AudioFormat capture)
76 : {
77 0 : JAMI_LOG("Hardware input audio format available: {:s}", capture.toString());
78 0 : }
79 :
80 : void
81 0 : AudioLayer::devicesChanged()
82 : {
83 0 : emitSignal<libjami::AudioSignal::DeviceEvent>();
84 0 : }
85 :
86 : void
87 0 : AudioLayer::flushMain()
88 : {
89 0 : Manager::instance().getRingBufferPool().flushAllBuffers();
90 0 : }
91 :
92 : void
93 403 : AudioLayer::flushUrgent()
94 : {
95 403 : urgentRingBuffer_.flushAll();
96 403 : }
97 :
98 : void
99 0 : AudioLayer::flush()
100 : {
101 0 : Manager::instance().getRingBufferPool().flushAllBuffers();
102 0 : urgentRingBuffer_.flushAll();
103 0 : }
104 :
105 : void
106 215 : AudioLayer::playbackChanged(bool started)
107 : {
108 215 : playbackStarted_ = started;
109 215 : }
110 :
111 : void
112 215 : AudioLayer::recordChanged(bool started)
113 : {
114 215 : std::lock_guard lock(audioProcessorMutex);
115 215 : if (started) {
116 : // create audio processor
117 0 : createAudioProcessor();
118 : } else {
119 : // destroy audio processor
120 215 : destroyAudioProcessor();
121 : }
122 215 : recordStarted_ = started;
123 215 : }
124 :
125 : // helper function
126 : static inline bool
127 0 : shouldUseAudioProcessorEchoCancel(bool hasNativeAEC, const std::string& echoCancellerPref)
128 : {
129 : return
130 : // user doesn't care which and there is not a system AEC
131 0 : (echoCancellerPref == "auto" && !hasNativeAEC)
132 : // user specifically wants audioProcessor
133 0 : or (echoCancellerPref == "audioProcessor");
134 : }
135 :
136 : // helper function
137 : static inline bool
138 0 : shouldUseAudioProcessorNoiseSuppression(bool hasNativeNS, const std::string& noiseSuppressionPref)
139 : {
140 : return
141 : // user doesn't care which and there is no system noise suppression
142 0 : (noiseSuppressionPref == "auto" && !hasNativeNS)
143 : // user specifically wants audioProcessor
144 0 : or (noiseSuppressionPref == "audioProcessor");
145 : }
146 :
147 : void
148 32 : AudioLayer::setHasNativeAEC(bool hasNativeAEC)
149 : {
150 128 : JAMI_LOG("[audiolayer] setHasNativeAEC: {}", hasNativeAEC);
151 32 : std::lock_guard lock(audioProcessorMutex);
152 32 : hasNativeAEC_ = hasNativeAEC;
153 : // if we have a current audio processor, tell it to enable/disable its own AEC
154 32 : if (audioProcessor) {
155 0 : audioProcessor->enableEchoCancel(shouldUseAudioProcessorEchoCancel(hasNativeAEC, pref_.getEchoCanceller()));
156 : }
157 32 : }
158 :
159 : void
160 35 : AudioLayer::setHasNativeNS(bool hasNativeNS)
161 : {
162 140 : JAMI_LOG("[audiolayer] setHasNativeNS: {}", hasNativeNS);
163 35 : std::lock_guard lock(audioProcessorMutex);
164 35 : hasNativeNS_ = hasNativeNS;
165 : // if we have a current audio processor, tell it to enable/disable its own noise suppression
166 35 : if (audioProcessor) {
167 0 : audioProcessor->enableNoiseSuppression(
168 0 : shouldUseAudioProcessorNoiseSuppression(hasNativeNS, pref_.getNoiseReduce()));
169 : }
170 35 : }
171 :
172 : // must acquire lock beforehand
173 : void
174 0 : AudioLayer::createAudioProcessor()
175 : {
176 0 : auto nb_channels = std::max(audioFormat_.nb_channels, audioInputFormat_.nb_channels);
177 0 : auto sample_rate = std::max(audioFormat_.sample_rate, audioInputFormat_.sample_rate);
178 :
179 0 : sample_rate = std::clamp(sample_rate, 16000u, 48000u);
180 :
181 0 : AudioFormat formatForProcessor {sample_rate, nb_channels};
182 :
183 : unsigned int frame_size;
184 0 : if (pref_.getAudioProcessor() == "speex") {
185 : // TODO: maybe force this to be equivalent to 20ms? as expected by Speex
186 0 : frame_size = sample_rate / 50u;
187 : } else {
188 0 : frame_size = sample_rate / 100u;
189 : }
190 :
191 0 : JAMI_WARNING("Input {}", audioInputFormat_.toString());
192 0 : JAMI_WARNING("Output {}", audioFormat_.toString());
193 0 : JAMI_WARNING("Starting audio processor with: [{} Hz, {} channels, {} samples/frame]",
194 : sample_rate,
195 : nb_channels,
196 : frame_size);
197 :
198 0 : if (pref_.getAudioProcessor() == "webrtc") {
199 : #if HAVE_WEBRTC_AP
200 0 : JAMI_WARNING("[audiolayer] using WebRTCAudioProcessor");
201 0 : audioProcessor.reset(new WebRTCAudioProcessor(formatForProcessor, frame_size));
202 : #else
203 : JAMI_ERROR("[audiolayer] audioProcessor preference is webrtc, but library not linked! using null "
204 : "AudioProcessor instead");
205 : audioProcessor.reset();
206 : #endif
207 0 : } else if (pref_.getAudioProcessor() == "speex") {
208 : #if HAVE_SPEEXDSP
209 0 : JAMI_WARNING("[audiolayer] using SpeexAudioProcessor");
210 0 : audioProcessor.reset(new SpeexAudioProcessor(formatForProcessor, frame_size));
211 : #else
212 : JAMI_ERROR("[audiolayer] audioProcessor preference is Speex, but library not linked! using null AudioProcessor "
213 : "instead");
214 : audioProcessor.reset();
215 : #endif
216 0 : } else if (pref_.getAudioProcessor() == "null") {
217 0 : JAMI_WARNING("[audiolayer] using null AudioProcessor");
218 0 : audioProcessor.reset();
219 : } else {
220 0 : JAMI_ERROR("[audiolayer] audioProcessor preference not recognized, using null AudioProcessor instead");
221 0 : audioProcessor.reset();
222 : }
223 :
224 0 : if (audioProcessor) {
225 0 : audioProcessor->enableNoiseSuppression(
226 0 : shouldUseAudioProcessorNoiseSuppression(hasNativeNS_, pref_.getNoiseReduce()));
227 :
228 0 : audioProcessor->enableAutomaticGainControl(pref_.isAGCEnabled());
229 :
230 0 : audioProcessor->enableEchoCancel(shouldUseAudioProcessorEchoCancel(hasNativeAEC_, pref_.getEchoCanceller()));
231 :
232 0 : audioProcessor->enableVoiceActivityDetection(pref_.getVadEnabled());
233 : }
234 0 : }
235 :
236 : // must acquire lock beforehand
237 : void
238 215 : AudioLayer::destroyAudioProcessor()
239 : {
240 : // delete it
241 215 : audioProcessor.reset();
242 215 : }
243 :
244 : void
245 0 : AudioLayer::putUrgent(std::shared_ptr<AudioFrame> buffer)
246 : {
247 0 : urgentRingBuffer_.put(std::move(buffer));
248 0 : }
249 :
250 : // Notify (with a beep) an incoming call when there is already a call in progress
251 : void
252 0 : AudioLayer::notifyIncomingCall()
253 : {
254 0 : if (not playIncomingCallBeep_)
255 0 : return;
256 :
257 0 : auto now = std::chrono::system_clock::now();
258 :
259 : // Notify maximum once every 5 seconds
260 0 : if (now < lastNotificationTime_ + std::chrono::seconds(5))
261 0 : return;
262 :
263 0 : lastNotificationTime_ = now;
264 :
265 0 : Tone tone("440/160", getSampleRate(), audioFormat_.sampleFormat);
266 0 : size_t nbSample = tone.getSize();
267 :
268 : /* Put the data in the urgent ring buffer */
269 0 : urgentRingBuffer_.flushAll();
270 0 : urgentRingBuffer_.put(tone.getNext(nbSample));
271 0 : }
272 :
273 : std::shared_ptr<AudioFrame>
274 0 : AudioLayer::getToRing(AudioFormat format, size_t writableSamples)
275 : {
276 0 : if (auto fileToPlay = Manager::instance().getTelephoneFile()) {
277 0 : auto fileformat = fileToPlay->getFormat();
278 0 : bool resample = format != fileformat;
279 :
280 0 : size_t readableSamples = resample ? rational<size_t>(writableSamples * (size_t) fileformat.sample_rate,
281 0 : format.sample_rate)
282 0 : .real<size_t>()
283 0 : : writableSamples;
284 :
285 0 : return resampler_->resample(fileToPlay->getNext(readableSamples, isRingtoneMuted_), format);
286 0 : }
287 0 : return {};
288 : }
289 :
290 : std::shared_ptr<AudioFrame>
291 0 : AudioLayer::getToPlay(AudioFormat format, size_t writableSamples)
292 : {
293 0 : notifyIncomingCall();
294 0 : auto& bufferPool = Manager::instance().getRingBufferPool();
295 :
296 0 : if (not playbackQueue_)
297 0 : playbackQueue_.reset(new AudioFrameResizer(format, static_cast<int>(writableSamples)));
298 : else
299 0 : playbackQueue_->setFrameSize(static_cast<int>(writableSamples));
300 :
301 0 : std::shared_ptr<AudioFrame> playbackBuf {};
302 0 : while (!(playbackBuf = playbackQueue_->dequeue())) {
303 0 : std::shared_ptr<AudioFrame> resampled;
304 :
305 0 : if (auto urgentSamples = urgentRingBuffer_.get(RingBufferPool::DEFAULT_ID)) {
306 0 : bufferPool.discard(1, RingBufferPool::DEFAULT_ID);
307 0 : resampled = resampler_->resample(std::move(urgentSamples), format);
308 0 : } else if (auto toneToPlay = Manager::instance().getTelephoneTone()) {
309 0 : resampled = resampler_->resample(toneToPlay->getNext(), format);
310 0 : } else if (auto buf = bufferPool.getData(RingBufferPool::DEFAULT_ID)) {
311 0 : resampled = resampler_->resample(std::move(buf), format);
312 : } else {
313 0 : std::lock_guard lock(audioProcessorMutex);
314 0 : if (audioProcessor) {
315 0 : auto silence = std::make_shared<AudioFrame>(format, writableSamples);
316 0 : libav_utils::fillWithSilence(silence->pointer());
317 0 : audioProcessor->putPlayback(silence);
318 0 : }
319 0 : break;
320 0 : }
321 :
322 0 : if (resampled) {
323 0 : std::lock_guard lock(audioProcessorMutex);
324 0 : if (audioProcessor) {
325 0 : audioProcessor->putPlayback(resampled);
326 : }
327 0 : playbackQueue_->enqueue(std::move(resampled));
328 0 : } else
329 0 : break;
330 0 : }
331 :
332 : jami_tracepoint(audio_layer_get_to_play_end);
333 :
334 0 : return playbackBuf;
335 0 : }
336 :
337 : void
338 0 : AudioLayer::putRecorded(std::shared_ptr<AudioFrame>&& frame)
339 : {
340 0 : std::lock_guard lock(audioProcessorMutex);
341 0 : if (audioProcessor && playbackStarted_ && recordStarted_) {
342 0 : audioProcessor->putRecorded(std::move(frame));
343 0 : while (auto rec = audioProcessor->getProcessed()) {
344 0 : mainRingBuffer_->put(std::move(rec));
345 0 : }
346 : } else {
347 0 : mainRingBuffer_->put(std::move(frame));
348 : }
349 :
350 : jami_tracepoint(audio_layer_put_recorded_end, );
351 0 : }
352 :
353 : } // namespace jami
|
