Line data Source code
1 : /*
2 : * Copyright (C) 2004-2025 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "libav_deps.h" // MUST BE INCLUDED FIRST
19 : #include "logger.h"
20 : #include "media_filter.h"
21 : #include "media_buffer.h"
22 :
23 : extern "C" {
24 : #include <libavfilter/buffersink.h>
25 : #include <libavfilter/buffersrc.h>
26 : }
27 :
28 : #include <algorithm>
29 : #include <functional>
30 : #include <memory>
31 : #include <sstream>
32 : #include <thread>
33 :
34 : namespace jami {
35 :
36 10 : MediaFilter::MediaFilter() {}
37 :
38 10 : MediaFilter::~MediaFilter()
39 : {
40 10 : clean();
41 10 : }
42 :
43 : std::string
44 0 : MediaFilter::getFilterDesc() const
45 : {
46 0 : return desc_;
47 : }
48 :
49 : int
50 11 : MediaFilter::initialize(const std::string& filterDesc, const std::vector<MediaStream>& msps)
51 : {
52 11 : int ret = 0;
53 11 : desc_ = filterDesc;
54 11 : graph_ = avfilter_graph_alloc();
55 :
56 11 : if (!graph_)
57 0 : return fail("Failed to allocate filter graph", AVERROR(ENOMEM));
58 :
59 11 : graph_->nb_threads = std::max(1u, std::min(8u, std::thread::hardware_concurrency() / 2));
60 :
61 : AVFilterInOut* in;
62 : AVFilterInOut* out;
63 11 : if ((ret = avfilter_graph_parse2(graph_, desc_.c_str(), &in, &out)) < 0)
64 0 : return fail("Failed to parse filter graph", ret);
65 :
66 : using AVFilterInOutPtr = std::unique_ptr<AVFilterInOut, std::function<void(AVFilterInOut*)>>;
67 22 : AVFilterInOutPtr outputs(out, [](AVFilterInOut* f) { avfilter_inout_free(&f); });
68 22 : AVFilterInOutPtr inputs(in, [](AVFilterInOut* f) { avfilter_inout_free(&f); });
69 :
70 11 : if (outputs && outputs->next)
71 0 : return fail("Filters with multiple outputs are not supported", AVERROR(ENOTSUP));
72 :
73 11 : if ((ret = initOutputFilter(outputs.get())) < 0)
74 0 : return fail("Failed to create output for filter graph", ret);
75 :
76 : // make sure inputs linked list is the same size as msps
77 11 : size_t count = 0;
78 11 : AVFilterInOut* dummyInput = inputs.get();
79 26 : while (dummyInput && ++count) // increment count before evaluating its value
80 15 : dummyInput = dummyInput->next;
81 11 : if (count != msps.size())
82 0 : return fail("Size mismatch between number of inputs in filter graph and input parameter array", AVERROR(EINVAL));
83 :
84 26 : for (AVFilterInOut* current = inputs.get(); current; current = current->next) {
85 15 : if (!current->name)
86 0 : return fail("Filters require non empty names", AVERROR(EINVAL));
87 15 : std::string_view name = current->name;
88 35 : auto it = std::find_if(msps.begin(), msps.end(), [name](const MediaStream& msp) { return msp.name == name; });
89 15 : if (it != msps.end()) {
90 15 : if ((ret = initInputFilter(current, *it)) < 0) {
91 0 : return fail(fmt::format("Failed to initialize input: {}", name), ret);
92 : }
93 : } else {
94 0 : return fail(fmt::format("Failed to find matching parameters for: {}", name), ret);
95 : }
96 : }
97 :
98 11 : if ((ret = avfilter_graph_config(graph_, nullptr)) < 0)
99 0 : return fail("Failed to configure filter graph", ret);
100 :
101 11 : JAMI_DBG() << "Filter graph initialized with: " << desc_;
102 11 : initialized_ = true;
103 11 : return 0;
104 11 : }
105 :
106 : bool
107 0 : MediaFilter::needsReinitForNewStream(const std::string& name) const
108 : {
109 0 : for (const auto& ms : inputParams_)
110 0 : if (ms.name == name)
111 0 : return false;
112 0 : return true;
113 : }
114 :
115 : const MediaStream&
116 2 : MediaFilter::getInputParams(const std::string& inputName) const
117 : {
118 3 : for (const auto& ms : inputParams_)
119 3 : if (ms.name == inputName)
120 2 : return ms;
121 0 : throw std::out_of_range("Input '" + inputName + "' not found");
122 : }
123 :
124 : MediaStream
125 5 : MediaFilter::getOutputParams() const
126 : {
127 5 : MediaStream output;
128 5 : if (!output_ || !initialized_) {
129 1 : fail("Filter not initialized", -1);
130 1 : return output;
131 : }
132 :
133 4 : switch (av_buffersink_get_type(output_)) {
134 3 : case AVMEDIA_TYPE_VIDEO:
135 3 : output.name = "videoOutput";
136 3 : output.format = av_buffersink_get_format(output_);
137 3 : output.isVideo = true;
138 3 : output.timeBase = av_buffersink_get_time_base(output_);
139 3 : output.width = av_buffersink_get_w(output_);
140 3 : output.height = av_buffersink_get_h(output_);
141 3 : output.bitrate = 0;
142 3 : output.frameRate = av_buffersink_get_frame_rate(output_);
143 3 : break;
144 1 : case AVMEDIA_TYPE_AUDIO:
145 1 : output.name = "audioOutput";
146 1 : output.format = av_buffersink_get_format(output_);
147 1 : output.isVideo = false;
148 1 : output.timeBase = av_buffersink_get_time_base(output_);
149 1 : output.sampleRate = av_buffersink_get_sample_rate(output_);
150 1 : output.nbChannels = av_buffersink_get_channels(output_);
151 1 : break;
152 0 : default:
153 0 : output.format = -1;
154 0 : break;
155 : }
156 4 : return output;
157 0 : }
158 :
159 : int
160 304 : MediaFilter::feedInput(AVFrame* frame, const std::string& inputName)
161 : {
162 304 : int ret = 0;
163 304 : if (!initialized_)
164 0 : return fail("Filter not initialized", -1);
165 :
166 304 : if (!frame)
167 0 : return 0;
168 :
169 605 : for (size_t i = 0; i < inputs_.size(); ++i) {
170 605 : auto& ms = inputParams_[i];
171 605 : if (ms.name != inputName)
172 301 : continue;
173 :
174 304 : bool formatChanged = ms.format != frame->format;
175 304 : bool videoParamsChanged = ms.isVideo && (ms.width != frame->width || ms.height != frame->height);
176 608 : bool audioParamsChanged = !ms.isVideo
177 605 : && (ms.sampleRate != frame->sample_rate
178 301 : || ms.nbChannels != frame->ch_layout.nb_channels);
179 :
180 304 : if (formatChanged || videoParamsChanged || audioParamsChanged) {
181 1 : ms.update(frame);
182 1 : if ((ret = reinitialize()) < 0)
183 0 : return fail("Failed to reinitialize filter with new input parameters", ret);
184 : }
185 :
186 304 : int flags = AV_BUFFERSRC_FLAG_KEEP_REF;
187 304 : if ((ret = av_buffersrc_add_frame_flags(inputs_[i], frame, flags)) < 0)
188 0 : return fail("Unable to pass frame to filters", ret);
189 : else
190 304 : return 0;
191 : }
192 0 : return fail(fmt::format("Specified filter '{}' not found", inputName), AVERROR(EINVAL));
193 : }
194 :
195 : std::unique_ptr<MediaFrame>
196 102 : MediaFilter::readOutput()
197 : {
198 102 : if (!initialized_) {
199 0 : fail("Not properly initialized", -1);
200 0 : return {};
201 : }
202 :
203 102 : std::unique_ptr<MediaFrame> frame;
204 102 : switch (av_buffersink_get_type(output_)) {
205 : #ifdef ENABLE_VIDEO
206 1 : case AVMEDIA_TYPE_VIDEO:
207 1 : frame = std::make_unique<libjami::VideoFrame>();
208 1 : break;
209 : #endif
210 101 : case AVMEDIA_TYPE_AUDIO:
211 101 : frame = std::make_unique<AudioFrame>();
212 101 : break;
213 0 : default:
214 0 : return {};
215 : }
216 102 : auto err = av_buffersink_get_frame(output_, frame->pointer());
217 102 : if (err >= 0) {
218 102 : return frame;
219 0 : } else if (err == AVERROR(EAGAIN)) {
220 : // no data available right now, try again
221 0 : } else if (err == AVERROR_EOF) {
222 0 : JAMI_WARN() << "Filters have reached EOF, no more frames will be output";
223 : } else {
224 0 : fail("Error occurred while pulling from filter graph", err);
225 : }
226 0 : return {};
227 102 : }
228 :
229 : void
230 5 : MediaFilter::flush()
231 : {
232 10 : for (size_t i = 0; i < inputs_.size(); ++i) {
233 5 : int ret = av_buffersrc_add_frame_flags(inputs_[i], nullptr, 0);
234 5 : if (ret < 0) {
235 0 : JAMI_ERR() << "Failed to flush filter '" << inputParams_[i].name << "': " << libav_utils::getError(ret);
236 : }
237 : }
238 5 : }
239 :
240 : int
241 11 : MediaFilter::initOutputFilter(AVFilterInOut* out)
242 : {
243 11 : int ret = 0;
244 : const AVFilter* buffersink;
245 11 : AVFilterContext* buffersinkCtx = nullptr;
246 11 : AVMediaType mediaType = avfilter_pad_get_type(out->filter_ctx->input_pads, out->pad_idx);
247 :
248 11 : if (mediaType == AVMEDIA_TYPE_VIDEO)
249 6 : buffersink = avfilter_get_by_name("buffersink");
250 : else
251 5 : buffersink = avfilter_get_by_name("abuffersink");
252 :
253 11 : if ((ret = avfilter_graph_create_filter(&buffersinkCtx, buffersink, "out", nullptr, nullptr, graph_)) < 0) {
254 0 : avfilter_free(buffersinkCtx);
255 0 : return fail("Failed to create buffer sink", ret);
256 : }
257 :
258 11 : if ((ret = avfilter_link(out->filter_ctx, out->pad_idx, buffersinkCtx, 0)) < 0) {
259 0 : avfilter_free(buffersinkCtx);
260 0 : return fail("Unable to link buffer sink to graph", ret);
261 : }
262 :
263 11 : output_ = buffersinkCtx;
264 11 : return ret;
265 : }
266 :
267 : int
268 15 : MediaFilter::initInputFilter(AVFilterInOut* in, const MediaStream& msp)
269 : {
270 15 : int ret = 0;
271 15 : AVBufferSrcParameters* params = av_buffersrc_parameters_alloc();
272 15 : if (!params)
273 0 : return -1;
274 :
275 : const AVFilter* buffersrc;
276 15 : AVMediaType mediaType = avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx);
277 15 : params->format = msp.format;
278 15 : params->time_base = msp.timeBase;
279 15 : if (mediaType == AVMEDIA_TYPE_VIDEO) {
280 8 : params->width = msp.width;
281 8 : params->height = msp.height;
282 8 : params->frame_rate = msp.frameRate;
283 8 : buffersrc = avfilter_get_by_name("buffer");
284 : } else {
285 7 : params->sample_rate = msp.sampleRate;
286 7 : av_channel_layout_default(¶ms->ch_layout, msp.nbChannels);
287 7 : buffersrc = avfilter_get_by_name("abuffer");
288 : }
289 :
290 15 : AVFilterContext* buffersrcCtx = nullptr;
291 15 : if (buffersrc) {
292 : char name[128];
293 15 : snprintf(name, sizeof(name), "buffersrc_%s_%d", in->name, in->pad_idx);
294 15 : buffersrcCtx = avfilter_graph_alloc_filter(graph_, buffersrc, name);
295 : }
296 15 : if (!buffersrcCtx) {
297 0 : av_free(params);
298 0 : return fail("Failed to allocate filter graph input", AVERROR(ENOMEM));
299 : }
300 15 : ret = av_buffersrc_parameters_set(buffersrcCtx, params);
301 15 : av_free(params);
302 15 : if (ret < 0)
303 0 : return fail("Failed to set filter graph input parameters", ret);
304 :
305 15 : if ((ret = avfilter_init_str(buffersrcCtx, nullptr)) < 0)
306 0 : return fail("Failed to initialize buffer source", ret);
307 :
308 15 : if ((ret = avfilter_link(buffersrcCtx, 0, in->filter_ctx, in->pad_idx)) < 0)
309 0 : return fail("Failed to link buffer source to graph", ret);
310 :
311 15 : inputs_.push_back(buffersrcCtx);
312 15 : inputParams_.emplace_back(msp);
313 15 : inputParams_.back().name = in->name;
314 15 : return ret;
315 : }
316 :
317 : int
318 1 : MediaFilter::reinitialize()
319 : {
320 : // keep parameters needed for initialization before clearing filter
321 1 : auto params = std::move(inputParams_);
322 1 : auto desc = std::move(desc_);
323 1 : clean();
324 1 : auto ret = initialize(desc, params);
325 1 : if (ret >= 0)
326 1 : JAMI_DBG() << "Filter graph reinitialized";
327 1 : return ret;
328 1 : }
329 :
330 : int
331 1 : MediaFilter::fail(std::string_view msg, int err) const
332 : {
333 1 : if (!msg.empty())
334 1 : JAMI_ERR() << msg << ": " << libav_utils::getError(err);
335 1 : return err;
336 : }
337 :
338 : void
339 11 : MediaFilter::clean()
340 : {
341 11 : initialized_ = false;
342 11 : avfilter_graph_free(&graph_); // frees inputs_ and output_
343 11 : desc_.clear();
344 11 : inputs_.clear(); // don't point to freed memory
345 11 : output_ = nullptr; // don't point to freed memory
346 11 : inputParams_.clear();
347 11 : }
348 :
349 : } // namespace jami
|
