Line data Source code
1 : /*
2 : * Copyright (C) 2004-2024 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "audiolayer.h"
19 : #include "logger.h"
20 : #include "manager.h"
21 : #include "audio/ringbufferpool.h"
22 : #include "audio/resampler.h"
23 : #include "tonecontrol.h"
24 : #include "client/ring_signal.h"
25 :
26 : #include "audio-processing/null_audio_processor.h"
27 : #include "tracepoint.h"
28 : #if HAVE_WEBRTC_AP
29 : #include "audio-processing/webrtc.h"
30 : #endif
31 : #if HAVE_SPEEXDSP
32 : #include "audio-processing/speex.h"
33 : #endif
34 :
35 : #include <ctime>
36 : #include <algorithm>
37 :
38 : namespace jami {
39 :
40 36 : AudioLayer::AudioLayer(const AudioPreference& pref)
41 72 : : isCaptureMuted_(pref.getCaptureMuted())
42 36 : , isPlaybackMuted_(pref.getPlaybackMuted())
43 72 : , captureGain_(pref.getVolumemic())
44 36 : , playbackGain_(pref.getVolumespkr())
45 36 : , pref_(pref)
46 36 : , mainRingBuffer_(
47 36 : Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID))
48 36 : , audioFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
49 36 : , audioInputFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
50 36 : , urgentRingBuffer_("urgentRingBuffer_id", SIZEBUF, audioFormat_)
51 36 : , resampler_(new Resampler)
52 216 : , lastNotificationTime_()
53 : {
54 36 : urgentRingBuffer_.createReadOffset(RingBufferPool::DEFAULT_ID);
55 :
56 108 : JAMI_LOG("[audiolayer] AGC: {:d}, noiseReduce: {:s}, VAD: {:d}, echoCancel: {:s}, audioProcessor: {:s}",
57 : pref_.isAGCEnabled(),
58 : pref.getNoiseReduce(),
59 : pref.getVadEnabled(),
60 : pref.getEchoCanceller(),
61 : pref.getAudioProcessor());
62 36 : }
63 :
64 36 : AudioLayer::~AudioLayer() {}
65 :
66 : void
67 115 : AudioLayer::hardwareFormatAvailable(AudioFormat playback, size_t bufSize)
68 : {
69 345 : JAMI_LOG("Hardware audio format available : {:s} {}", playback.toString(), bufSize);
70 115 : audioFormat_ = Manager::instance().hardwareAudioFormatChanged(playback);
71 115 : audioInputFormat_.sampleFormat = audioFormat_.sampleFormat;
72 115 : urgentRingBuffer_.setFormat(audioFormat_);
73 115 : nativeFrameSize_ = bufSize;
74 115 : }
75 :
76 : void
77 0 : AudioLayer::hardwareInputFormatAvailable(AudioFormat capture)
78 : {
79 0 : JAMI_LOG("Hardware input audio format available : {:s}", capture.toString());
80 0 : }
81 :
82 : void
83 0 : AudioLayer::devicesChanged()
84 : {
85 0 : emitSignal<libjami::AudioSignal::DeviceEvent>();
86 0 : }
87 :
88 : void
89 0 : AudioLayer::flushMain()
90 : {
91 0 : Manager::instance().getRingBufferPool().flushAllBuffers();
92 0 : }
93 :
94 : void
95 434 : AudioLayer::flushUrgent()
96 : {
97 434 : urgentRingBuffer_.flushAll();
98 434 : }
99 :
100 : void
101 0 : AudioLayer::flush()
102 : {
103 0 : Manager::instance().getRingBufferPool().flushAllBuffers();
104 0 : urgentRingBuffer_.flushAll();
105 0 : }
106 :
107 : void
108 237 : AudioLayer::playbackChanged(bool started)
109 : {
110 237 : playbackStarted_ = started;
111 237 : }
112 :
113 : void
114 237 : AudioLayer::recordChanged(bool started)
115 : {
116 237 : std::lock_guard lock(audioProcessorMutex);
117 237 : if (started) {
118 : // create audio processor
119 0 : createAudioProcessor();
120 : } else {
121 : // destroy audio processor
122 237 : destroyAudioProcessor();
123 : }
124 237 : recordStarted_ = started;
125 237 : }
126 :
127 : // helper function
128 : static inline bool
129 0 : shouldUseAudioProcessorEchoCancel(bool hasNativeAEC, const std::string& echoCancellerPref)
130 : {
131 : return
132 : // user doesn't care which and there is not a system AEC
133 0 : (echoCancellerPref == "auto" && !hasNativeAEC)
134 : // user specifically wants audioProcessor
135 0 : or (echoCancellerPref == "audioProcessor");
136 : }
137 :
138 : // helper function
139 : static inline bool
140 0 : shouldUseAudioProcessorNoiseSuppression(bool hasNativeNS, const std::string& noiseSuppressionPref)
141 : {
142 : return
143 : // user doesn't care which and there is no system noise suppression
144 0 : (noiseSuppressionPref == "auto" && !hasNativeNS)
145 : // user specifically wants audioProcessor
146 0 : or (noiseSuppressionPref == "audioProcessor");
147 : }
148 :
149 : void
150 33 : AudioLayer::setHasNativeAEC(bool hasNativeAEC)
151 : {
152 33 : JAMI_INFO("[audiolayer] setHasNativeAEC: %d", hasNativeAEC);
153 33 : std::lock_guard lock(audioProcessorMutex);
154 33 : hasNativeAEC_ = hasNativeAEC;
155 : // if we have a current audio processor, tell it to enable/disable its own AEC
156 33 : if (audioProcessor) {
157 0 : audioProcessor->enableEchoCancel(
158 0 : shouldUseAudioProcessorEchoCancel(hasNativeAEC, pref_.getEchoCanceller()));
159 : }
160 33 : }
161 :
162 : void
163 36 : AudioLayer::setHasNativeNS(bool hasNativeNS)
164 : {
165 36 : JAMI_INFO("[audiolayer] setHasNativeNS: %d", hasNativeNS);
166 36 : std::lock_guard lock(audioProcessorMutex);
167 36 : hasNativeNS_ = hasNativeNS;
168 : // if we have a current audio processor, tell it to enable/disable its own noise suppression
169 36 : if (audioProcessor) {
170 0 : audioProcessor->enableNoiseSuppression(
171 0 : shouldUseAudioProcessorNoiseSuppression(hasNativeNS, pref_.getNoiseReduce()));
172 : }
173 36 : }
174 :
175 : // must acquire lock beforehand
176 : void
177 0 : AudioLayer::createAudioProcessor()
178 : {
179 0 : auto nb_channels = std::max(audioFormat_.nb_channels, audioInputFormat_.nb_channels);
180 0 : auto sample_rate = std::max(audioFormat_.sample_rate, audioInputFormat_.sample_rate);
181 :
182 0 : sample_rate = std::clamp(sample_rate, 16000u, 48000u);
183 :
184 0 : AudioFormat formatForProcessor {sample_rate, nb_channels};
185 :
186 : unsigned int frame_size;
187 0 : if (pref_.getAudioProcessor() == "speex") {
188 : // TODO: maybe force this to be equivalent to 20ms? as expected by speex
189 0 : frame_size = sample_rate / 50u;
190 : } else {
191 0 : frame_size = sample_rate / 100u;
192 : }
193 :
194 0 : JAMI_WARNING("Input {}", audioInputFormat_.toString());
195 0 : JAMI_WARNING("Output {}", audioFormat_.toString());
196 0 : JAMI_WARNING("Starting audio processor with: [{} Hz, {} channels, {} samples/frame]",
197 : sample_rate,
198 : nb_channels,
199 : frame_size);
200 :
201 0 : if (pref_.getAudioProcessor() == "webrtc") {
202 : #if HAVE_WEBRTC_AP
203 0 : JAMI_WARN("[audiolayer] using WebRTCAudioProcessor");
204 0 : audioProcessor.reset(new WebRTCAudioProcessor(formatForProcessor, frame_size));
205 : #else
206 : JAMI_ERR("[audiolayer] audioProcessor preference is webrtc, but library not linked! "
207 : "using null AudioProcessor instead");
208 : audioProcessor.reset();
209 : #endif
210 0 : } else if (pref_.getAudioProcessor() == "speex") {
211 : #if HAVE_SPEEXDSP
212 0 : JAMI_WARN("[audiolayer] using SpeexAudioProcessor");
213 0 : audioProcessor.reset(new SpeexAudioProcessor(formatForProcessor, frame_size));
214 : #else
215 : JAMI_ERR("[audiolayer] audioProcessor preference is speex, but library not linked! "
216 : "using null AudioProcessor instead");
217 : audioProcessor.reset();
218 : #endif
219 0 : } else if (pref_.getAudioProcessor() == "null") {
220 0 : JAMI_WARN("[audiolayer] using null AudioProcessor");
221 0 : audioProcessor.reset();
222 : } else {
223 0 : JAMI_ERR("[audiolayer] audioProcessor preference not recognized, using null AudioProcessor "
224 : "instead");
225 0 : audioProcessor.reset();
226 : }
227 :
228 0 : if (audioProcessor) {
229 0 : audioProcessor->enableNoiseSuppression(
230 0 : shouldUseAudioProcessorNoiseSuppression(hasNativeNS_, pref_.getNoiseReduce()));
231 :
232 0 : audioProcessor->enableAutomaticGainControl(pref_.isAGCEnabled());
233 :
234 0 : audioProcessor->enableEchoCancel(
235 0 : shouldUseAudioProcessorEchoCancel(hasNativeAEC_, pref_.getEchoCanceller()));
236 :
237 0 : audioProcessor->enableVoiceActivityDetection(pref_.getVadEnabled());
238 : }
239 0 : }
240 :
241 : // must acquire lock beforehand
242 : void
243 237 : AudioLayer::destroyAudioProcessor()
244 : {
245 : // delete it
246 237 : audioProcessor.reset();
247 237 : }
248 :
249 : void
250 0 : AudioLayer::putUrgent(std::shared_ptr<AudioFrame> buffer)
251 : {
252 0 : urgentRingBuffer_.put(std::move(buffer));
253 0 : }
254 :
255 : // Notify (with a beep) an incoming call when there is already a call in progress
256 : void
257 0 : AudioLayer::notifyIncomingCall()
258 : {
259 0 : if (not playIncomingCallBeep_)
260 0 : return;
261 :
262 0 : auto now = std::chrono::system_clock::now();
263 :
264 : // Notify maximum once every 5 seconds
265 0 : if (now < lastNotificationTime_ + std::chrono::seconds(5))
266 0 : return;
267 :
268 0 : lastNotificationTime_ = now;
269 :
270 0 : Tone tone("440/160", getSampleRate(), audioFormat_.sampleFormat);
271 0 : size_t nbSample = tone.getSize();
272 :
273 : /* Put the data in the urgent ring buffer */
274 0 : urgentRingBuffer_.flushAll();
275 0 : urgentRingBuffer_.put(tone.getNext(nbSample));
276 0 : }
277 :
278 : std::shared_ptr<AudioFrame>
279 0 : AudioLayer::getToRing(AudioFormat format, size_t writableSamples)
280 : {
281 0 : if (auto fileToPlay = Manager::instance().getTelephoneFile()) {
282 0 : auto fileformat = fileToPlay->getFormat();
283 0 : bool resample = format != fileformat;
284 :
285 0 : size_t readableSamples = resample ? rational<size_t>(writableSamples * (size_t)fileformat.sample_rate, format.sample_rate)
286 0 : .real<size_t>()
287 0 : : writableSamples;
288 :
289 0 : return resampler_->resample(fileToPlay->getNext(readableSamples, isRingtoneMuted_), format);
290 0 : }
291 0 : return {};
292 : }
293 :
294 : std::shared_ptr<AudioFrame>
295 0 : AudioLayer::getToPlay(AudioFormat format, size_t writableSamples)
296 : {
297 0 : notifyIncomingCall();
298 0 : auto& bufferPool = Manager::instance().getRingBufferPool();
299 :
300 0 : if (not playbackQueue_)
301 0 : playbackQueue_.reset(new AudioFrameResizer(format, writableSamples));
302 : else
303 0 : playbackQueue_->setFrameSize(writableSamples);
304 :
305 0 : std::shared_ptr<AudioFrame> playbackBuf {};
306 0 : while (!(playbackBuf = playbackQueue_->dequeue())) {
307 0 : std::shared_ptr<AudioFrame> resampled;
308 :
309 0 : if (auto urgentSamples = urgentRingBuffer_.get(RingBufferPool::DEFAULT_ID)) {
310 0 : bufferPool.discard(1, RingBufferPool::DEFAULT_ID);
311 0 : resampled = resampler_->resample(std::move(urgentSamples), format);
312 0 : } else if (auto toneToPlay = Manager::instance().getTelephoneTone()) {
313 0 : resampled = resampler_->resample(toneToPlay->getNext(), format);
314 0 : } else if (auto buf = bufferPool.getData(RingBufferPool::DEFAULT_ID)) {
315 0 : resampled = resampler_->resample(std::move(buf), format);
316 : } else {
317 0 : std::lock_guard lock(audioProcessorMutex);
318 0 : if (audioProcessor) {
319 0 : auto silence = std::make_shared<AudioFrame>(format, writableSamples);
320 0 : libav_utils::fillWithSilence(silence->pointer());
321 0 : audioProcessor->putPlayback(silence);
322 0 : }
323 0 : break;
324 0 : }
325 :
326 0 : if (resampled) {
327 0 : std::lock_guard lock(audioProcessorMutex);
328 0 : if (audioProcessor) {
329 0 : audioProcessor->putPlayback(resampled);
330 : }
331 0 : playbackQueue_->enqueue(std::move(resampled));
332 0 : } else
333 0 : break;
334 0 : }
335 :
336 : jami_tracepoint(audio_layer_get_to_play_end);
337 :
338 0 : return playbackBuf;
339 0 : }
340 :
341 : void
342 0 : AudioLayer::putRecorded(std::shared_ptr<AudioFrame>&& frame)
343 : {
344 0 : std::lock_guard lock(audioProcessorMutex);
345 0 : if (audioProcessor && playbackStarted_ && recordStarted_) {
346 0 : audioProcessor->putRecorded(std::move(frame));
347 0 : while (auto rec = audioProcessor->getProcessed()) {
348 0 : mainRingBuffer_->put(std::move(rec));
349 0 : }
350 : } else {
351 0 : mainRingBuffer_->put(std::move(frame));
352 : }
353 :
354 : jami_tracepoint(audio_layer_put_recorded_end, );
355 0 : }
356 :
357 : } // namespace jami
|
