Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2025 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software: you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation, either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #include "libav_deps.h"
      19             : #include "logger.h"
      20             : #include <libavutil/frame.h>
      21             : #include <libavutil/mathematics.h>
      22             : #include "resampler.h"
      23             : 
      24             : extern "C" {
      25             : #include <libswresample/swresample.h>
      26             : }
      27             : 
      28             : namespace jami {
      29             : 
      30         859 : Resampler::Resampler()
      31         859 :     : swrCtx_(swr_alloc())
      32         859 :     , initCount_(0)
      33         859 : {}
      34             : 
      35         859 : Resampler::~Resampler()
      36             : {
      37         859 :     swr_free(&swrCtx_);
      38         859 : }
      39             : 
      40             : void
      41           3 : Resampler::reinit(const AVFrame* in, const AVFrame* out)
      42             : {
      43             :     // NOTE swr_set_matrix should be called on an uninitialized context
      44           3 :     auto swrCtx = swr_alloc();
      45           3 :     if (!swrCtx) {
      46           0 :         JAMI_ERR() << "Unable to allocate resampler context";
      47           0 :         throw std::bad_alloc();
      48             :     }
      49             : 
      50           3 :     av_opt_set_chlayout(swrCtx, "ichl", &in->ch_layout, 0);
      51           3 :     av_opt_set_int(swrCtx, "isr", in->sample_rate, 0);
      52           3 :     av_opt_set_sample_fmt(swrCtx, "isf", static_cast<AVSampleFormat>(in->format), 0);
      53             : 
      54           3 :     av_opt_set_chlayout(swrCtx, "ochl", &out->ch_layout, 0);
      55           3 :     av_opt_set_int(swrCtx, "osr", out->sample_rate, 0);
      56           3 :     av_opt_set_sample_fmt(swrCtx, "osf", static_cast<AVSampleFormat>(out->format), 0);
      57             : 
      58             :     /**
      59             :      * Downmixing from 5.1 requires extra setup, since libswresample is unable to do it
      60             :      * automatically (not yet implemented).
      61             :      *
      62             :      * Source: https://www.atsc.org/wp-content/uploads/2015/03/A52-201212-17.pdf
      63             :      * Section 7.8.2 for the algorithm
      64             :      * Tables 5.9 and 5.10 for the coefficients clev and slev
      65             :      *
      66             :      * LFE downmixing is optional, so any coefficient can be used, we use +6dB for mono and
      67             :      * +0dB in each channel for stereo.
      68             :      */
      69           3 :     if (in->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1 || in->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1_BACK) {
      70             :         // NOTE: MSVC is unable to allocate dynamic size arrays on the stack
      71           2 :         if (out->ch_layout.nb_channels == 2) {
      72             :             double matrix[2][6];
      73             :             // L = 1.0*FL + 0.707*FC + 0.707*BL + 1.0*LFE
      74           1 :             matrix[0][0] = 1;
      75           1 :             matrix[0][1] = 0;
      76           1 :             matrix[0][2] = 0.707;
      77           1 :             matrix[0][3] = 1;
      78           1 :             matrix[0][4] = 0.707;
      79           1 :             matrix[0][5] = 0;
      80             :             // R = 1.0*FR + 0.707*FC + 0.707*BR + 1.0*LFE
      81           1 :             matrix[1][0] = 0;
      82           1 :             matrix[1][1] = 1;
      83           1 :             matrix[1][2] = 0.707;
      84           1 :             matrix[1][3] = 1;
      85           1 :             matrix[1][4] = 0;
      86           1 :             matrix[1][5] = 0.707;
      87           1 :             swr_set_matrix(swrCtx, matrix[0], 6);
      88             :         } else {
      89             :             double matrix[1][6];
      90             :             // M = 1.0*FL + 1.414*FC + 1.0*FR + 0.707*BL + 0.707*BR + 2.0*LFE
      91           1 :             matrix[0][0] = 1;
      92           1 :             matrix[0][1] = 1;
      93           1 :             matrix[0][2] = 1.414;
      94           1 :             matrix[0][3] = 2;
      95           1 :             matrix[0][4] = 0.707;
      96           1 :             matrix[0][5] = 0.707;
      97           1 :             swr_set_matrix(swrCtx, matrix[0], 6);
      98             :         }
      99             :     }
     100             : 
     101           3 :     if (swr_init(swrCtx) >= 0) {
     102           3 :         std::swap(swrCtx_, swrCtx);
     103           3 :         swr_free(&swrCtx);
     104          12 :         JAMI_DEBUG("Succesfully (re)initialized resampler context");
     105           3 :         ++initCount_;
     106             :     } else {
     107           0 :         JAMI_ERROR("Runtime error: Failed to initialize resampler context");
     108           0 :         throw std::runtime_error("Failed to initialize resampler context");
     109             :     }
     110           3 : }
     111             : 
     112             : int
     113           4 : Resampler::resample(const AVFrame* input, AVFrame* output)
     114             : {
     115           4 :     bool firstFrame = !initCount_;
     116           4 :     if (!initCount_)
     117           2 :         reinit(input, output);
     118             : 
     119           4 :     int ret = swr_convert_frame(swrCtx_, output, input);
     120           4 :     if (ret & AVERROR_INPUT_CHANGED || ret & AVERROR_OUTPUT_CHANGED) {
     121             :         // Under certain conditions, the resampler reinits itself in an infinite loop. This is
     122             :         // indicative of an underlying problem in the code. This check is so the backtrace
     123             :         // doesn't get mangled with a bunch of calls to Resampler::resample
     124           1 :         if (initCount_ > 1) {
     125           0 :             JAMI_ERROR("Infinite loop detected in audio resampler, please open an issue on https://git.jami.net");
     126           0 :             throw std::runtime_error("Infinite loop detected in audio resampler");
     127             :         }
     128           1 :         reinit(input, output);
     129           1 :         return resample(input, output);
     130             :     }
     131             : 
     132           3 :     if (ret < 0) {
     133           0 :         JAMI_ERROR("Failed to resample frame");
     134           0 :         return -1;
     135             :     }
     136             : 
     137           3 :     if (firstFrame) {
     138             :         // we just resampled the first frame
     139           4 :         auto targetOutputLength = av_rescale_rnd(input->nb_samples,
     140           2 :                                                  output->sample_rate,
     141           2 :                                                  input->sample_rate,
     142             :                                                  AV_ROUND_UP);
     143           2 :         if (output->nb_samples < targetOutputLength) {
     144             :             // create new frame with more samples, padded with silence
     145           8 :             JAMI_WARNING(
     146             :                 "First resampled frame too small, adding {} samples of silence at beginning to reach {} samples",
     147             :                 targetOutputLength - output->nb_samples,
     148             :                 targetOutputLength);
     149           2 :             auto* newOutput = av_frame_alloc();
     150           2 :             if (!newOutput) {
     151           0 :                 JAMI_ERROR("Failed to clone output frame for resizing");
     152           0 :                 return -1;
     153             :             }
     154           2 :             av_frame_copy_props(newOutput, output);
     155           2 :             newOutput->format = output->format;
     156           2 :             newOutput->nb_samples = static_cast<int>(targetOutputLength);
     157           2 :             newOutput->ch_layout = output->ch_layout;
     158           2 :             newOutput->channel_layout = output->channel_layout;
     159           2 :             newOutput->sample_rate = output->sample_rate;
     160           2 :             if (av_frame_get_buffer(newOutput, 0) < 0) {
     161           0 :                 JAMI_ERROR("Failed to allocate new output frame buffer");
     162           0 :                 av_frame_free(&newOutput);
     163           0 :                 return -1;
     164             :             }
     165           2 :             auto sampleOffset = targetOutputLength - output->nb_samples;
     166           2 :             av_samples_set_silence(newOutput->data,
     167             :                                    0,
     168             :                                    static_cast<int>(sampleOffset),
     169             :                                    output->ch_layout.nb_channels,
     170           2 :                                    static_cast<AVSampleFormat>(output->format));
     171             :             // copy old data to new frame at offset sampleOffset
     172           2 :             av_samples_copy(newOutput->data,
     173           2 :                             output->data,
     174             :                             static_cast<int>(sampleOffset),
     175             :                             0,
     176             :                             output->nb_samples,
     177             :                             output->ch_layout.nb_channels,
     178           2 :                             static_cast<AVSampleFormat>(output->format));
     179           8 :             JAMI_DEBUG("Resampled first frame. Resized from {} to {} samples",
     180             :                        output->nb_samples,
     181             :                        newOutput->nb_samples);
     182             :             // replace output frame buffer
     183           2 :             av_frame_unref(output);
     184           2 :             av_frame_move_ref(output, newOutput);
     185           2 :             av_frame_free(&newOutput);
     186             :         }
     187             :     }
     188             : 
     189             :     // Resampling worked, reset count to 1 so reinit isn't called again
     190           3 :     initCount_ = 1;
     191           3 :     return 0;
     192             : }
     193             : 
     194             : std::unique_ptr<AudioFrame>
     195           0 : Resampler::resample(std::unique_ptr<AudioFrame>&& in, const AudioFormat& format)
     196             : {
     197           0 :     if (in->pointer()->sample_rate == (int) format.sample_rate
     198           0 :         && in->pointer()->ch_layout.nb_channels == (int) format.nb_channels
     199           0 :         && (AVSampleFormat) in->pointer()->format == format.sampleFormat) {
     200           0 :         return std::move(in);
     201             :     }
     202           0 :     auto output = std::make_unique<AudioFrame>(format);
     203           0 :     resample(in->pointer(), output->pointer());
     204           0 :     output->has_voice = in->has_voice;
     205           0 :     return output;
     206           0 : }
     207             : 
     208             : std::shared_ptr<AudioFrame>
     209           0 : Resampler::resample(std::shared_ptr<AudioFrame>&& in, const AudioFormat& format)
     210             : {
     211           0 :     if (not in) {
     212           0 :         return {};
     213             :     }
     214           0 :     auto inPtr = in->pointer();
     215           0 :     if (inPtr == nullptr) {
     216           0 :         return {};
     217             :     }
     218             : 
     219           0 :     if (inPtr->sample_rate == (int) format.sample_rate && inPtr->ch_layout.nb_channels == (int) format.nb_channels
     220           0 :         && (AVSampleFormat) inPtr->format == format.sampleFormat) {
     221           0 :         return std::move(in);
     222             :     }
     223             : 
     224           0 :     auto output = std::make_shared<AudioFrame>(format);
     225           0 :     if (auto outPtr = output->pointer()) {
     226           0 :         resample(inPtr, outPtr);
     227           0 :         output->has_voice = in->has_voice;
     228           0 :         return output;
     229             :     }
     230           0 :     return {};
     231           0 : }
     232             : 
     233             : } // namespace jami