Line data Source code
1 : /*
2 : * Copyright (C) 2004-2025 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "speex.h"
19 :
20 : #include "audio/audiolayer.h"
21 :
22 : #ifndef _MSC_VER
23 : #if __has_include(<speex/speexdsp_config_types.h>)
24 : #include <speex/speexdsp_config_types.h>
25 : #else
26 : #include <speex/speex_config_types.h>
27 : #endif
28 : #endif
29 : extern "C" {
30 : #include <speex/speex_echo.h>
31 : #include <speex/speex_preprocess.h>
32 : }
33 :
34 : #include <cstdint>
35 : #include <memory>
36 : #include <vector>
37 :
38 : namespace jami {
39 :
40 : inline AudioFormat
41 : audioFormatToSampleFormat(AudioFormat format)
42 : {
43 : return {format.sample_rate, format.nb_channels, AV_SAMPLE_FMT_S16};
44 : }
45 :
46 0 : SpeexAudioProcessor::SpeexAudioProcessor(AudioFormat format, unsigned frameSize)
47 : : AudioProcessor(format.withSampleFormat(AV_SAMPLE_FMT_S16), frameSize)
48 0 : , echoState(speex_echo_state_init_mc((int) frameSize,
49 0 : (int) frameSize * 16,
50 0 : (int) format_.nb_channels,
51 0 : (int) format_.nb_channels),
52 0 : &speex_echo_state_destroy)
53 0 : , procBuffer(std::make_unique<AudioFrame>(format.withSampleFormat(AV_SAMPLE_FMT_S16P), frameSize_))
54 : {
55 0 : JAMI_DBG("[speex-dsp] SpeexAudioProcessor, frame size = %d (=%d ms), channels = %d",
56 : frameSize,
57 : frameDurationMs_,
58 : format_.nb_channels);
59 : // set up speex echo state
60 0 : speex_echo_ctl(echoState.get(), SPEEX_ECHO_SET_SAMPLING_RATE, &format_.sample_rate);
61 :
62 : // speex specific value to turn feature on (need to pass a pointer to it)
63 0 : spx_int32_t speexOn = 1;
64 :
65 : // probability integers, i.e. 50 means 50%
66 : // vad will be true if speex's raw probability calculation is higher than this in any case
67 0 : spx_int32_t probStart = 99;
68 :
69 : // vad will be true if voice was active last frame
70 : // AND speex's raw probability calculation is higher than this
71 0 : spx_int32_t probContinue = 90;
72 :
73 : // maximum noise suppression in dB (negative)
74 0 : spx_int32_t maxNoiseSuppress = -50;
75 :
76 : // set up speex preprocess states, one for each channel
77 : // note that they are not enabled here, but rather in the enable* functions
78 0 : for (unsigned int i = 0; i < format_.nb_channels; i++) {
79 : auto channelPreprocessorState = SpeexPreprocessStatePtr(speex_preprocess_state_init((int) frameSize,
80 0 : (int) format_.sample_rate),
81 0 : &speex_preprocess_state_destroy);
82 :
83 : // set max noise suppression level
84 0 : speex_preprocess_ctl(channelPreprocessorState.get(), SPEEX_PREPROCESS_SET_NOISE_SUPPRESS, &maxNoiseSuppress);
85 :
86 : // set up voice activity values
87 0 : speex_preprocess_ctl(channelPreprocessorState.get(), SPEEX_PREPROCESS_SET_VAD, &speexOn);
88 0 : speex_preprocess_ctl(channelPreprocessorState.get(), SPEEX_PREPROCESS_SET_PROB_START, &probStart);
89 0 : speex_preprocess_ctl(channelPreprocessorState.get(), SPEEX_PREPROCESS_SET_PROB_CONTINUE, &probContinue);
90 :
91 : // keep track of this channel's preprocessor state
92 0 : preprocessorStates.push_back(std::move(channelPreprocessorState));
93 0 : }
94 :
95 0 : JAMI_INFO("[speex-dsp] Done initializing");
96 0 : }
97 :
98 : void
99 0 : SpeexAudioProcessor::enableEchoCancel(bool enabled)
100 : {
101 0 : JAMI_DBG("[speex-dsp] enableEchoCancel %d", enabled);
102 : // need to set member variable so we know to do it in getProcessed
103 0 : shouldAEC = enabled;
104 :
105 0 : if (enabled) {
106 : // reset the echo canceller
107 0 : speex_echo_state_reset(echoState.get());
108 :
109 0 : for (auto& channelPreprocessorState : preprocessorStates) {
110 : // attach our already-created echo canceller
111 0 : speex_preprocess_ctl(channelPreprocessorState.get(), SPEEX_PREPROCESS_SET_ECHO_STATE, echoState.get());
112 : }
113 : } else {
114 0 : for (auto& channelPreprocessorState : preprocessorStates) {
115 : // detach echo canceller (set it to NULL)
116 : // don't destroy it though, we will reset it when necessary
117 0 : speex_preprocess_ctl(channelPreprocessorState.get(), SPEEX_PREPROCESS_SET_ECHO_STATE, NULL);
118 : }
119 : }
120 0 : }
121 :
122 : void
123 0 : SpeexAudioProcessor::enableNoiseSuppression(bool enabled)
124 : {
125 0 : JAMI_DBG("[speex-dsp] enableNoiseSuppression %d", enabled);
126 0 : spx_int32_t speexSetValue = (spx_int32_t) enabled;
127 :
128 : // for each preprocessor
129 0 : for (auto& channelPreprocessorState : preprocessorStates) {
130 : // set denoise status
131 0 : speex_preprocess_ctl(channelPreprocessorState.get(), SPEEX_PREPROCESS_SET_DENOISE, &speexSetValue);
132 : // set de-reverb status
133 0 : speex_preprocess_ctl(channelPreprocessorState.get(), SPEEX_PREPROCESS_SET_DEREVERB, &speexSetValue);
134 : }
135 0 : }
136 :
137 : void
138 0 : SpeexAudioProcessor::enableAutomaticGainControl(bool enabled)
139 : {
140 0 : JAMI_DBG("[speex-dsp] enableAutomaticGainControl %d", enabled);
141 0 : spx_int32_t speexSetValue = (spx_int32_t) enabled;
142 :
143 : // for each preprocessor
144 0 : for (auto& channelPreprocessorState : preprocessorStates) {
145 : // set AGC status
146 0 : speex_preprocess_ctl(channelPreprocessorState.get(), SPEEX_PREPROCESS_SET_AGC, &speexSetValue);
147 : }
148 0 : }
149 :
150 : void
151 0 : SpeexAudioProcessor::enableVoiceActivityDetection(bool enabled)
152 : {
153 0 : JAMI_DBG("[speex-dsp] enableVoiceActivityDetection %d", enabled);
154 :
155 0 : shouldDetectVoice = enabled;
156 :
157 0 : spx_int32_t speexSetValue = (spx_int32_t) enabled;
158 0 : for (auto& channelPreprocessorState : preprocessorStates) {
159 0 : speex_preprocess_ctl(channelPreprocessorState.get(), SPEEX_PREPROCESS_SET_VAD, &speexSetValue);
160 : }
161 0 : }
162 :
163 : std::shared_ptr<AudioFrame>
164 0 : SpeexAudioProcessor::getProcessed()
165 : {
166 0 : if (tidyQueues()) {
167 0 : return {};
168 : }
169 :
170 0 : auto playback = playbackQueue_.dequeue();
171 0 : auto record = recordQueue_.dequeue();
172 :
173 0 : if (!playback || !record) {
174 0 : return {};
175 : }
176 :
177 0 : std::shared_ptr<AudioFrame> processed;
178 0 : if (shouldAEC) {
179 : // we want to echo cancel
180 : // multichannel, output into processed
181 0 : processed = std::make_shared<AudioFrame>(record->getFormat(), record->getFrameSize());
182 0 : speex_echo_cancellation(echoState.get(),
183 0 : (int16_t*) record->pointer()->data[0],
184 0 : (int16_t*) playback->pointer()->data[0],
185 0 : (int16_t*) processed->pointer()->data[0]);
186 : } else {
187 : // don't want to echo cancel, so just use record frame instead
188 0 : processed = record;
189 : }
190 :
191 0 : deinterleaveResampler.resample(processed->pointer(), procBuffer->pointer());
192 :
193 : // overall voice activity
194 0 : bool overallVad = false;
195 : // current channel voice activity
196 : int channelVad;
197 :
198 : // run preprocess on each channel
199 0 : int channel = 0;
200 0 : for (auto& channelPreprocessorState : preprocessorStates) {
201 : // preprocesses in place, returns voice activity boolean
202 0 : channelVad = speex_preprocess_run(channelPreprocessorState.get(),
203 0 : (int16_t*) procBuffer->pointer()->data[channel]);
204 :
205 : // boolean OR
206 0 : overallVad |= channelVad;
207 :
208 0 : channel += 1;
209 : }
210 :
211 0 : interleaveResampler.resample(procBuffer->pointer(), processed->pointer());
212 :
213 : // add stabilized voice activity to the AudioFrame
214 0 : processed->has_voice = shouldDetectVoice && getStabilizedVoiceActivity(overallVad);
215 0 : return processed;
216 0 : }
217 :
218 : } // namespace jami
|
