Line data Source code
1 : /*
2 : * Copyright (C) 2004-2026 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "libav_deps.h"
19 : #include "logger.h"
20 : #include <libavutil/frame.h>
21 : #include <libavutil/mathematics.h>
22 : #include <libavutil/opt.h>
23 : #include "resampler.h"
24 : #include "libav_utils.h"
25 :
26 : extern "C" {
27 : #include <libswresample/swresample.h>
28 : }
29 :
30 : namespace jami {
31 :
32 855 : Resampler::Resampler()
33 855 : : swrCtx_(swr_alloc())
34 855 : , initCount_(0)
35 855 : {}
36 :
37 855 : Resampler::~Resampler()
38 : {
39 855 : swr_free(&swrCtx_);
40 855 : }
41 :
42 : void
43 3 : Resampler::reinit(const AVFrame* in, const AVFrame* out)
44 : {
45 : // NOTE swr_set_matrix should be called on an uninitialized context
46 3 : auto* swrCtx = swr_alloc();
47 3 : if (!swrCtx) {
48 0 : JAMI_ERROR("[{}] Unable to allocate resampler context", fmt::ptr(this));
49 0 : throw std::bad_alloc();
50 : }
51 :
52 3 : int ret = av_opt_set_chlayout(swrCtx, "ichl", &in->ch_layout, 0);
53 3 : if (ret < 0) {
54 0 : swr_free(&swrCtx);
55 : char layout_buf[64];
56 0 : av_channel_layout_describe(&in->ch_layout, layout_buf, sizeof(layout_buf));
57 0 : JAMI_ERROR("[{}] Failed to set input channel layout {}: {}",
58 : fmt::ptr(this),
59 : layout_buf,
60 : libav_utils::getError(ret));
61 0 : throw std::runtime_error("Failed to set input channel layout");
62 : }
63 3 : ret = av_opt_set_int(swrCtx, "isr", in->sample_rate, 0);
64 3 : if (ret < 0) {
65 0 : swr_free(&swrCtx);
66 0 : JAMI_ERROR("[{}] Failed to set input sample rate {}: {}",
67 : fmt::ptr(this),
68 : in->sample_rate,
69 : libav_utils::getError(ret));
70 0 : throw std::runtime_error("Failed to set input sample rate");
71 : }
72 3 : ret = av_opt_set_sample_fmt(swrCtx, "isf", static_cast<AVSampleFormat>(in->format), 0);
73 3 : if (ret < 0) {
74 0 : swr_free(&swrCtx);
75 0 : JAMI_ERROR("[{}] Failed to set input sample format {}: {}",
76 : fmt::ptr(this),
77 : av_get_sample_fmt_name(static_cast<AVSampleFormat>(in->format)),
78 : libav_utils::getError(ret));
79 0 : throw std::runtime_error("Failed to set input sample format");
80 : }
81 :
82 3 : ret = av_opt_set_chlayout(swrCtx, "ochl", &out->ch_layout, 0);
83 3 : if (ret < 0) {
84 0 : swr_free(&swrCtx);
85 : char layout_buf[64];
86 0 : av_channel_layout_describe(&out->ch_layout, layout_buf, sizeof(layout_buf));
87 0 : JAMI_ERROR("[{}] Failed to set output channel layout {}: {}",
88 : fmt::ptr(this),
89 : layout_buf,
90 : libav_utils::getError(ret));
91 0 : throw std::runtime_error("Failed to set output channel layout");
92 : }
93 3 : ret = av_opt_set_int(swrCtx, "osr", out->sample_rate, 0);
94 3 : if (ret < 0) {
95 0 : swr_free(&swrCtx);
96 0 : JAMI_ERROR("[{}] Failed to set output sample rate {}: {}",
97 : fmt::ptr(this),
98 : out->sample_rate,
99 : libav_utils::getError(ret));
100 0 : throw std::runtime_error("Failed to set output sample rate");
101 : }
102 3 : ret = av_opt_set_sample_fmt(swrCtx, "osf", static_cast<AVSampleFormat>(out->format), 0);
103 3 : if (ret < 0) {
104 0 : swr_free(&swrCtx);
105 0 : JAMI_ERROR("[{}] Failed to set output sample format {}: {}",
106 : fmt::ptr(this),
107 : av_get_sample_fmt_name(static_cast<AVSampleFormat>(out->format)),
108 : libav_utils::getError(ret));
109 0 : throw std::runtime_error("Failed to set output sample format");
110 : }
111 :
112 : /**
113 : * Downmixing from 5.1 requires extra setup, since libswresample is unable to do it
114 : * automatically (not yet implemented).
115 : *
116 : * Source: https://www.atsc.org/wp-content/uploads/2015/03/A52-201212-17.pdf
117 : * Section 7.8.2 for the algorithm
118 : * Tables 5.9 and 5.10 for the coefficients clev and slev
119 : *
120 : * LFE downmixing is optional, so any coefficient can be used, we use +6dB for mono and
121 : * +0dB in each channel for stereo.
122 : */
123 3 : if (in->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1 || in->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1_BACK) {
124 2 : int ret = 0;
125 : // NOTE: MSVC is unable to allocate dynamic size arrays on the stack
126 2 : if (out->ch_layout.nb_channels == 2) {
127 : double matrix[2][6];
128 : // L = 1.0*FL + 0.707*FC + 0.707*BL + 1.0*LFE
129 1 : matrix[0][0] = 1;
130 1 : matrix[0][1] = 0;
131 1 : matrix[0][2] = 0.707;
132 1 : matrix[0][3] = 1;
133 1 : matrix[0][4] = 0.707;
134 1 : matrix[0][5] = 0;
135 : // R = 1.0*FR + 0.707*FC + 0.707*BR + 1.0*LFE
136 1 : matrix[1][0] = 0;
137 1 : matrix[1][1] = 1;
138 1 : matrix[1][2] = 0.707;
139 1 : matrix[1][3] = 1;
140 1 : matrix[1][4] = 0;
141 1 : matrix[1][5] = 0.707;
142 1 : ret = swr_set_matrix(swrCtx, matrix[0], 6);
143 : } else {
144 : double matrix[1][6];
145 : // M = 1.0*FL + 1.414*FC + 1.0*FR + 0.707*BL + 0.707*BR + 2.0*LFE
146 1 : matrix[0][0] = 1;
147 1 : matrix[0][1] = 1;
148 1 : matrix[0][2] = 1.414;
149 1 : matrix[0][3] = 2;
150 1 : matrix[0][4] = 0.707;
151 1 : matrix[0][5] = 0.707;
152 1 : ret = swr_set_matrix(swrCtx, matrix[0], 6);
153 : }
154 2 : if (ret < 0) {
155 0 : swr_free(&swrCtx);
156 0 : JAMI_ERROR("[{}] Failed to set mixing matrix: {}", fmt::ptr(this), libav_utils::getError(ret));
157 0 : throw std::runtime_error("Failed to set mixing matrix");
158 : }
159 : }
160 :
161 3 : ret = swr_init(swrCtx);
162 3 : if (ret >= 0) {
163 3 : std::swap(swrCtx_, swrCtx);
164 3 : swr_free(&swrCtx);
165 12 : JAMI_DEBUG("[{}] Succesfully (re)initialized resampler context from {} to {}",
166 : fmt::ptr(this),
167 : libav_utils::getFormat(in).toString(),
168 : libav_utils::getFormat(out).toString());
169 3 : ++initCount_;
170 : } else {
171 0 : swr_free(&swrCtx);
172 0 : JAMI_ERROR("[{}] Runtime error: Failed to initialize resampler context: {}",
173 : fmt::ptr(this),
174 : libav_utils::getError(ret));
175 0 : throw std::runtime_error("Failed to initialize resampler context");
176 : }
177 3 : }
178 :
179 : int
180 4 : Resampler::resample(const AVFrame* input, AVFrame* output)
181 : {
182 4 : bool firstFrame = (initCount_ == 0);
183 4 : if (!initCount_)
184 2 : reinit(input, output);
185 :
186 4 : int ret = swr_convert_frame(swrCtx_, output, input);
187 4 : if (ret & AVERROR_INPUT_CHANGED || ret & AVERROR_OUTPUT_CHANGED) {
188 : // Under certain conditions, the resampler reinits itself in an infinite loop. This is
189 : // indicative of an underlying problem in the code. This check is so the backtrace
190 : // doesn't get mangled with a bunch of calls to Resampler::resample
191 1 : if (initCount_ > 1) {
192 : // JAMI_ERROR("Infinite loop detected in audio resampler, please open an issue on https://git.jami.net");
193 0 : JAMI_ERROR("[{}] Loop detected in audio resampler when resampling from {} to {}",
194 : fmt::ptr(this),
195 : libav_utils::getFormat(input).toString(),
196 : libav_utils::getFormat(output).toString());
197 0 : throw std::runtime_error("Infinite loop detected in audio resampler");
198 : }
199 1 : reinit(input, output);
200 1 : return resample(input, output);
201 : }
202 :
203 3 : if (ret < 0) {
204 0 : JAMI_ERROR("[{}] Failed to resample frame: {}", fmt::ptr(this), libav_utils::getError(ret));
205 0 : return -1;
206 : }
207 :
208 3 : if (firstFrame) {
209 : // we just resampled the first frame
210 4 : auto targetOutputLength = av_rescale_rnd(input->nb_samples,
211 2 : output->sample_rate,
212 2 : input->sample_rate,
213 : AV_ROUND_UP);
214 2 : if (output->nb_samples < targetOutputLength) {
215 : // create new frame with more samples, padded with silence
216 8 : JAMI_WARNING("[{}] Adding {} samples of silence at beginning of first frame to reach {} samples",
217 : fmt::ptr(this),
218 : targetOutputLength - output->nb_samples,
219 : targetOutputLength);
220 2 : auto* newOutput = av_frame_alloc();
221 2 : if (!newOutput) {
222 0 : JAMI_ERROR("[{}] Failed to clone output frame for resizing", fmt::ptr(this));
223 0 : return -1;
224 : }
225 2 : av_frame_copy_props(newOutput, output);
226 2 : newOutput->format = output->format;
227 2 : newOutput->nb_samples = static_cast<int>(targetOutputLength);
228 2 : newOutput->ch_layout = output->ch_layout;
229 2 : newOutput->channel_layout = output->channel_layout;
230 2 : newOutput->sample_rate = output->sample_rate;
231 2 : int bufferRet = av_frame_get_buffer(newOutput, 0);
232 2 : if (bufferRet < 0) {
233 0 : JAMI_ERROR("[{}] Failed to allocate new output frame buffer: {}",
234 : fmt::ptr(this),
235 : libav_utils::getError(bufferRet));
236 0 : av_frame_free(&newOutput);
237 0 : return -1;
238 : }
239 2 : auto sampleOffset = targetOutputLength - output->nb_samples;
240 4 : bufferRet = av_samples_set_silence(newOutput->data,
241 : 0,
242 : static_cast<int>(sampleOffset),
243 : output->ch_layout.nb_channels,
244 2 : static_cast<AVSampleFormat>(output->format));
245 2 : if (bufferRet < 0) {
246 0 : JAMI_ERROR("[{}] Failed to set silence on new output frame: {}",
247 : fmt::ptr(this),
248 : libav_utils::getError(bufferRet));
249 0 : av_frame_free(&newOutput);
250 0 : return -1;
251 : }
252 : // copy old data to new frame at offset sampleOffset
253 4 : bufferRet = av_samples_copy(newOutput->data,
254 2 : output->data,
255 : static_cast<int>(sampleOffset),
256 : 0,
257 : output->nb_samples,
258 : output->ch_layout.nb_channels,
259 2 : static_cast<AVSampleFormat>(output->format));
260 2 : if (bufferRet < 0) {
261 0 : JAMI_ERROR("[{}] Failed to copy data to new output frame: {}",
262 : fmt::ptr(this),
263 : libav_utils::getError(bufferRet));
264 0 : av_frame_free(&newOutput);
265 0 : return -1;
266 : }
267 8 : JAMI_DEBUG("[{}] Resampled first frame. Resized from {} to {} samples",
268 : fmt::ptr(this),
269 : output->nb_samples,
270 : newOutput->nb_samples);
271 : // replace output frame buffer
272 2 : av_frame_unref(output);
273 2 : av_frame_move_ref(output, newOutput);
274 2 : av_frame_free(&newOutput);
275 : }
276 : }
277 :
278 : // Resampling worked, reset count to 1 so reinit isn't called again
279 3 : initCount_ = 1;
280 3 : return 0;
281 : }
282 :
283 : std::unique_ptr<AudioFrame>
284 0 : Resampler::resample(std::unique_ptr<AudioFrame>&& in, const AudioFormat& format)
285 : {
286 0 : if (in->pointer()->sample_rate == (int) format.sample_rate
287 0 : && in->pointer()->ch_layout.nb_channels == (int) format.nb_channels
288 0 : && (AVSampleFormat) in->pointer()->format == format.sampleFormat) {
289 0 : return std::move(in);
290 : }
291 0 : auto output = std::make_unique<AudioFrame>(format);
292 0 : resample(in->pointer(), output->pointer());
293 0 : output->has_voice = in->has_voice;
294 0 : return output;
295 0 : }
296 :
297 : std::shared_ptr<AudioFrame>
298 0 : Resampler::resample(std::shared_ptr<AudioFrame>&& in, const AudioFormat& format)
299 : {
300 0 : if (not in) {
301 0 : return {};
302 : }
303 0 : auto* inPtr = in->pointer();
304 0 : if (inPtr == nullptr) {
305 0 : return {};
306 : }
307 :
308 0 : if (inPtr->sample_rate == (int) format.sample_rate && inPtr->ch_layout.nb_channels == (int) format.nb_channels
309 0 : && (AVSampleFormat) inPtr->format == format.sampleFormat) {
310 0 : return std::move(in);
311 : }
312 :
313 0 : auto output = std::make_shared<AudioFrame>(format);
314 0 : if (auto* outPtr = output->pointer()) {
315 0 : resample(inPtr, outPtr);
316 0 : output->has_voice = in->has_voice;
317 0 : return output;
318 : }
319 0 : return {};
320 0 : }
321 :
322 : } // namespace jami
|
