Line data Source code
1 : /*
2 : * Copyright (C) 2004-2026 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "libav_deps.h" // MUST BE INCLUDED FIRST
19 : #include "logger.h"
20 : #include "media_filter.h"
21 : #include "media_buffer.h"
22 :
23 : extern "C" {
24 : #include <libavfilter/buffersink.h>
25 : #include <libavfilter/buffersrc.h>
26 : }
27 :
28 : #include <algorithm>
29 : #include <functional>
30 : #include <memory>
31 : #include <thread>
32 :
33 : namespace jami {
34 :
35 10 : MediaFilter::MediaFilter() {}
36 :
37 10 : MediaFilter::~MediaFilter()
38 : {
39 10 : clean();
40 10 : }
41 :
42 : std::string
43 0 : MediaFilter::getFilterDesc() const
44 : {
45 0 : return desc_;
46 : }
47 :
48 : int
49 11 : MediaFilter::initialize(const std::string& filterDesc, const std::vector<MediaStream>& msps)
50 : {
51 11 : int ret = 0;
52 11 : desc_ = filterDesc;
53 11 : graph_ = avfilter_graph_alloc();
54 :
55 11 : if (!graph_)
56 0 : return fail("Failed to allocate filter graph", AVERROR(ENOMEM));
57 :
58 11 : graph_->nb_threads = std::max(1, std::min(8, static_cast<int>(std::thread::hardware_concurrency()) / 2));
59 :
60 : AVFilterInOut* in;
61 : AVFilterInOut* out;
62 11 : if ((ret = avfilter_graph_parse2(graph_, desc_.c_str(), &in, &out)) < 0)
63 0 : return fail("Failed to parse filter graph", ret);
64 :
65 : using AVFilterInOutPtr = std::unique_ptr<AVFilterInOut, std::function<void(AVFilterInOut*)>>;
66 22 : AVFilterInOutPtr outputs(out, [](AVFilterInOut* f) { avfilter_inout_free(&f); });
67 22 : AVFilterInOutPtr inputs(in, [](AVFilterInOut* f) { avfilter_inout_free(&f); });
68 :
69 11 : if (outputs && outputs->next)
70 0 : return fail("Filters with multiple outputs are not supported", AVERROR(ENOTSUP));
71 :
72 11 : if ((ret = initOutputFilter(outputs.get())) < 0)
73 0 : return fail("Failed to create output for filter graph", ret);
74 :
75 : // make sure inputs linked list is the same size as msps
76 11 : size_t count = 0;
77 11 : AVFilterInOut* dummyInput = inputs.get();
78 26 : while (dummyInput && ++count) // increment count before evaluating its value
79 15 : dummyInput = dummyInput->next;
80 11 : if (count != msps.size())
81 0 : return fail("Size mismatch between number of inputs in filter graph and input parameter array", AVERROR(EINVAL));
82 :
83 26 : for (AVFilterInOut* current = inputs.get(); current; current = current->next) {
84 15 : if (!current->name)
85 0 : return fail("Filters require non empty names", AVERROR(EINVAL));
86 15 : std::string_view name = current->name;
87 35 : auto it = std::find_if(msps.begin(), msps.end(), [name](const MediaStream& msp) { return msp.name == name; });
88 15 : if (it != msps.end()) {
89 15 : if ((ret = initInputFilter(current, *it)) < 0) {
90 0 : return fail(fmt::format("Failed to initialize input: {}", name), ret);
91 : }
92 : } else {
93 0 : return fail(fmt::format("Failed to find matching parameters for: {}", name), ret);
94 : }
95 : }
96 :
97 11 : if ((ret = avfilter_graph_config(graph_, nullptr)) < 0)
98 0 : return fail("Failed to configure filter graph", ret);
99 :
100 11 : JAMI_DBG() << "Filter graph initialized with: " << desc_;
101 11 : initialized_ = true;
102 11 : return 0;
103 11 : }
104 :
105 : bool
106 0 : MediaFilter::needsReinitForNewStream(const std::string& name) const
107 : {
108 0 : for (const auto& ms : inputParams_)
109 0 : if (ms.name == name)
110 0 : return false;
111 0 : return true;
112 : }
113 :
114 : const MediaStream&
115 2 : MediaFilter::getInputParams(const std::string& inputName) const
116 : {
117 3 : for (const auto& ms : inputParams_)
118 3 : if (ms.name == inputName)
119 2 : return ms;
120 0 : throw std::out_of_range("Input '" + inputName + "' not found");
121 : }
122 :
123 : MediaStream
124 5 : MediaFilter::getOutputParams() const
125 : {
126 5 : MediaStream output;
127 5 : if (!output_ || !initialized_) {
128 1 : fail("Filter not initialized", -1);
129 1 : return output;
130 : }
131 :
132 4 : switch (av_buffersink_get_type(output_)) {
133 3 : case AVMEDIA_TYPE_VIDEO:
134 3 : output.name = "videoOutput";
135 3 : output.format = av_buffersink_get_format(output_);
136 3 : output.isVideo = true;
137 3 : output.timeBase = av_buffersink_get_time_base(output_);
138 3 : output.width = av_buffersink_get_w(output_);
139 3 : output.height = av_buffersink_get_h(output_);
140 3 : output.bitrate = 0;
141 3 : output.frameRate = av_buffersink_get_frame_rate(output_);
142 3 : break;
143 1 : case AVMEDIA_TYPE_AUDIO:
144 1 : output.name = "audioOutput";
145 1 : output.format = av_buffersink_get_format(output_);
146 1 : output.isVideo = false;
147 1 : output.timeBase = av_buffersink_get_time_base(output_);
148 1 : output.sampleRate = av_buffersink_get_sample_rate(output_);
149 1 : output.nbChannels = av_buffersink_get_channels(output_);
150 1 : break;
151 0 : default:
152 0 : output.format = -1;
153 0 : break;
154 : }
155 4 : return output;
156 0 : }
157 :
158 : int
159 305 : MediaFilter::feedInput(AVFrame* frame, const std::string& inputName)
160 : {
161 305 : int ret = 0;
162 305 : if (!initialized_)
163 0 : return fail("Filter not initialized", -1);
164 :
165 305 : if (!frame)
166 0 : return 0;
167 :
168 606 : for (size_t i = 0; i < inputs_.size(); ++i) {
169 606 : auto& ms = inputParams_[i];
170 606 : if (ms.name != inputName)
171 301 : continue;
172 :
173 305 : bool formatChanged = ms.format != frame->format;
174 305 : bool videoParamsChanged = ms.isVideo && (ms.width != frame->width || ms.height != frame->height);
175 610 : bool audioParamsChanged = !ms.isVideo
176 606 : && (ms.sampleRate != frame->sample_rate
177 301 : || ms.nbChannels != frame->ch_layout.nb_channels);
178 :
179 305 : if (formatChanged || videoParamsChanged || audioParamsChanged) {
180 1 : ms.update(frame);
181 1 : if ((ret = reinitialize()) < 0)
182 0 : return fail("Failed to reinitialize filter with new input parameters", ret);
183 : }
184 :
185 305 : int flags = AV_BUFFERSRC_FLAG_KEEP_REF;
186 305 : if ((ret = av_buffersrc_add_frame_flags(inputs_[i], frame, flags)) < 0)
187 0 : return fail("Unable to pass frame to filters", ret);
188 : else
189 305 : return 0;
190 : }
191 0 : return fail(fmt::format("Specified filter '{}' not found", inputName), AVERROR(EINVAL));
192 : }
193 :
194 : std::unique_ptr<MediaFrame>
195 103 : MediaFilter::readOutput()
196 : {
197 103 : if (!initialized_) {
198 0 : fail("Not properly initialized", -1);
199 0 : return {};
200 : }
201 :
202 103 : std::unique_ptr<MediaFrame> frame;
203 103 : switch (av_buffersink_get_type(output_)) {
204 : #ifdef ENABLE_VIDEO
205 2 : case AVMEDIA_TYPE_VIDEO:
206 2 : frame = std::make_unique<libjami::VideoFrame>();
207 2 : break;
208 : #endif
209 101 : case AVMEDIA_TYPE_AUDIO:
210 101 : frame = std::make_unique<AudioFrame>();
211 101 : break;
212 0 : default:
213 0 : return {};
214 : }
215 103 : auto err = av_buffersink_get_frame(output_, frame->pointer());
216 103 : if (err >= 0) {
217 102 : return frame;
218 1 : } else if (err == AVERROR(EAGAIN)) {
219 : // no data available right now, try again
220 0 : } else if (err == AVERROR_EOF) {
221 0 : JAMI_WARN() << "Filters have reached EOF, no more frames will be output";
222 : } else {
223 0 : fail("Error occurred while pulling from filter graph", err);
224 : }
225 1 : return {};
226 103 : }
227 :
228 : void
229 5 : MediaFilter::flush()
230 : {
231 10 : for (size_t i = 0; i < inputs_.size(); ++i) {
232 5 : int ret = av_buffersrc_add_frame_flags(inputs_[i], nullptr, 0);
233 5 : if (ret < 0) {
234 0 : JAMI_ERR() << "Failed to flush filter '" << inputParams_[i].name << "': " << libav_utils::getError(ret);
235 : }
236 : }
237 5 : }
238 :
239 : int
240 11 : MediaFilter::initOutputFilter(AVFilterInOut* out)
241 : {
242 11 : int ret = 0;
243 : const AVFilter* buffersink;
244 11 : AVFilterContext* buffersinkCtx = nullptr;
245 11 : AVMediaType mediaType = avfilter_pad_get_type(out->filter_ctx->input_pads, out->pad_idx);
246 :
247 11 : if (mediaType == AVMEDIA_TYPE_VIDEO)
248 6 : buffersink = avfilter_get_by_name("buffersink");
249 : else
250 5 : buffersink = avfilter_get_by_name("abuffersink");
251 :
252 11 : if ((ret = avfilter_graph_create_filter(&buffersinkCtx, buffersink, "out", nullptr, nullptr, graph_)) < 0) {
253 0 : avfilter_free(buffersinkCtx);
254 0 : return fail("Failed to create buffer sink", ret);
255 : }
256 :
257 11 : if ((ret = avfilter_link(out->filter_ctx, out->pad_idx, buffersinkCtx, 0)) < 0) {
258 0 : avfilter_free(buffersinkCtx);
259 0 : return fail("Unable to link buffer sink to graph", ret);
260 : }
261 :
262 11 : output_ = buffersinkCtx;
263 11 : return ret;
264 : }
265 :
266 : int
267 15 : MediaFilter::initInputFilter(AVFilterInOut* in, const MediaStream& msp)
268 : {
269 15 : int ret = 0;
270 15 : AVBufferSrcParameters* params = av_buffersrc_parameters_alloc();
271 15 : if (!params)
272 0 : return -1;
273 :
274 : const AVFilter* buffersrc;
275 15 : AVMediaType mediaType = avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx);
276 15 : params->format = msp.format;
277 15 : params->time_base = msp.timeBase;
278 15 : if (mediaType == AVMEDIA_TYPE_VIDEO) {
279 8 : params->width = msp.width;
280 8 : params->height = msp.height;
281 8 : params->frame_rate = msp.frameRate;
282 8 : buffersrc = avfilter_get_by_name("buffer");
283 : } else {
284 7 : params->sample_rate = msp.sampleRate;
285 7 : av_channel_layout_default(¶ms->ch_layout, msp.nbChannels);
286 7 : buffersrc = avfilter_get_by_name("abuffer");
287 : }
288 :
289 15 : AVFilterContext* buffersrcCtx = nullptr;
290 15 : if (buffersrc) {
291 : char name[128];
292 15 : snprintf(name, sizeof(name), "buffersrc_%s_%d", in->name, in->pad_idx);
293 15 : buffersrcCtx = avfilter_graph_alloc_filter(graph_, buffersrc, name);
294 : }
295 15 : if (!buffersrcCtx) {
296 0 : av_free(params);
297 0 : return fail("Failed to allocate filter graph input", AVERROR(ENOMEM));
298 : }
299 15 : ret = av_buffersrc_parameters_set(buffersrcCtx, params);
300 15 : av_free(params);
301 15 : if (ret < 0)
302 0 : return fail("Failed to set filter graph input parameters", ret);
303 :
304 15 : if ((ret = avfilter_init_str(buffersrcCtx, nullptr)) < 0)
305 0 : return fail("Failed to initialize buffer source", ret);
306 :
307 15 : if ((ret = avfilter_link(buffersrcCtx, 0, in->filter_ctx, in->pad_idx)) < 0)
308 0 : return fail("Failed to link buffer source to graph", ret);
309 :
310 15 : inputs_.push_back(buffersrcCtx);
311 15 : inputParams_.emplace_back(msp);
312 15 : inputParams_.back().name = in->name;
313 15 : return ret;
314 : }
315 :
316 : int
317 1 : MediaFilter::reinitialize()
318 : {
319 : // keep parameters needed for initialization before clearing filter
320 1 : auto params = std::move(inputParams_);
321 1 : auto desc = std::move(desc_);
322 1 : clean();
323 1 : auto ret = initialize(desc, params);
324 1 : if (ret >= 0)
325 1 : JAMI_DBG() << "Filter graph reinitialized";
326 1 : return ret;
327 1 : }
328 :
329 : int
330 1 : MediaFilter::fail(std::string_view msg, int err) const
331 : {
332 1 : if (!msg.empty())
333 1 : JAMI_ERR() << msg << ": " << libav_utils::getError(err);
334 1 : return err;
335 : }
336 :
337 : void
338 11 : MediaFilter::clean()
339 : {
340 11 : initialized_ = false;
341 11 : avfilter_graph_free(&graph_); // frees inputs_ and output_
342 11 : desc_.clear();
343 11 : inputs_.clear(); // don't point to freed memory
344 11 : output_ = nullptr; // don't point to freed memory
345 11 : inputParams_.clear();
346 11 : }
347 :
348 : } // namespace jami
|
