Line data Source code
1 : /*
2 : * Copyright (C) 2004-2024 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "libav_deps.h" // MUST BE INCLUDED FIRST
19 : #include "logger.h"
20 : #include "media_filter.h"
21 : #include "media_buffer.h"
22 :
23 : extern "C" {
24 : #include <libavfilter/buffersink.h>
25 : #include <libavfilter/buffersrc.h>
26 : }
27 :
28 : #include <algorithm>
29 : #include <functional>
30 : #include <memory>
31 : #include <sstream>
32 : #include <thread>
33 :
34 : namespace jami {
35 :
36 41 : MediaFilter::MediaFilter() {}
37 :
38 41 : MediaFilter::~MediaFilter()
39 : {
40 41 : clean();
41 41 : }
42 :
43 : std::string
44 0 : MediaFilter::getFilterDesc() const
45 : {
46 0 : return desc_;
47 : }
48 :
49 : int
50 12 : MediaFilter::initialize(const std::string& filterDesc, const std::vector<MediaStream>& msps)
51 : {
52 12 : int ret = 0;
53 12 : desc_ = filterDesc;
54 12 : graph_ = avfilter_graph_alloc();
55 :
56 12 : if (!graph_)
57 0 : return fail("Failed to allocate filter graph", AVERROR(ENOMEM));
58 :
59 12 : graph_->nb_threads = std::max(1u, std::min(8u, std::thread::hardware_concurrency() / 2));
60 :
61 : AVFilterInOut* in;
62 : AVFilterInOut* out;
63 12 : if ((ret = avfilter_graph_parse2(graph_, desc_.c_str(), &in, &out)) < 0)
64 0 : return fail("Failed to parse filter graph", ret);
65 :
66 : using AVFilterInOutPtr = std::unique_ptr<AVFilterInOut, std::function<void(AVFilterInOut*)>>;
67 24 : AVFilterInOutPtr outputs(out, [](AVFilterInOut* f) { avfilter_inout_free(&f); });
68 24 : AVFilterInOutPtr inputs(in, [](AVFilterInOut* f) { avfilter_inout_free(&f); });
69 :
70 12 : if (outputs && outputs->next)
71 0 : return fail("Filters with multiple outputs are not supported", AVERROR(ENOTSUP));
72 :
73 12 : if ((ret = initOutputFilter(outputs.get())) < 0)
74 0 : return fail("Failed to create output for filter graph", ret);
75 :
76 : // make sure inputs linked list is the same size as msps
77 12 : size_t count = 0;
78 12 : AVFilterInOut* dummyInput = inputs.get();
79 28 : while (dummyInput && ++count) // increment count before evaluating its value
80 16 : dummyInput = dummyInput->next;
81 12 : if (count != msps.size())
82 0 : return fail(
83 : "Size mismatch between number of inputs in filter graph and input parameter array",
84 0 : AVERROR(EINVAL));
85 :
86 28 : for (AVFilterInOut* current = inputs.get(); current; current = current->next) {
87 16 : if (!current->name)
88 0 : return fail("Filters require non empty names", AVERROR(EINVAL));
89 16 : std::string_view name = current->name;
90 16 : auto it = std::find_if(msps.begin(), msps.end(), [name](const MediaStream& msp) {
91 21 : return msp.name == name;
92 : });
93 16 : if (it != msps.end()) {
94 16 : if ((ret = initInputFilter(current, *it)) < 0) {
95 0 : return fail(fmt::format("Failed to initialize input: {}", name), ret);
96 : }
97 : } else {
98 0 : return fail(fmt::format("Failed to find matching parameters for: {}", name), ret);
99 : }
100 : }
101 :
102 12 : if ((ret = avfilter_graph_config(graph_, nullptr)) < 0)
103 0 : return fail("Failed to configure filter graph", ret);
104 :
105 12 : JAMI_DBG() << "Filter graph initialized with: " << desc_;
106 12 : initialized_ = true;
107 12 : return 0;
108 12 : }
109 :
110 : const MediaStream&
111 2 : MediaFilter::getInputParams(const std::string& inputName) const
112 : {
113 3 : for (const auto& ms : inputParams_)
114 3 : if (ms.name == inputName)
115 2 : return ms;
116 0 : throw std::out_of_range("Input '" + inputName + "' not found");
117 : }
118 :
119 : MediaStream
120 5 : MediaFilter::getOutputParams() const
121 : {
122 5 : MediaStream output;
123 5 : if (!output_ || !initialized_) {
124 1 : fail("Filter not initialized", -1);
125 1 : return output;
126 : }
127 :
128 4 : switch (av_buffersink_get_type(output_)) {
129 3 : case AVMEDIA_TYPE_VIDEO:
130 3 : output.name = "videoOutput";
131 3 : output.format = av_buffersink_get_format(output_);
132 3 : output.isVideo = true;
133 3 : output.timeBase = av_buffersink_get_time_base(output_);
134 3 : output.width = av_buffersink_get_w(output_);
135 3 : output.height = av_buffersink_get_h(output_);
136 3 : output.bitrate = 0;
137 3 : output.frameRate = av_buffersink_get_frame_rate(output_);
138 3 : break;
139 1 : case AVMEDIA_TYPE_AUDIO:
140 1 : output.name = "audioOutput";
141 1 : output.format = av_buffersink_get_format(output_);
142 1 : output.isVideo = false;
143 1 : output.timeBase = av_buffersink_get_time_base(output_);
144 1 : output.sampleRate = av_buffersink_get_sample_rate(output_);
145 1 : output.nbChannels = av_buffersink_get_channels(output_);
146 1 : break;
147 0 : default:
148 0 : output.format = -1;
149 0 : break;
150 : }
151 4 : return output;
152 0 : }
153 :
154 : int
155 305 : MediaFilter::feedInput(AVFrame* frame, const std::string& inputName)
156 : {
157 305 : int ret = 0;
158 305 : if (!initialized_)
159 0 : return fail("Filter not initialized", -1);
160 :
161 305 : if (!frame)
162 0 : return 0;
163 :
164 606 : for (size_t i = 0; i < inputs_.size(); ++i) {
165 606 : auto& ms = inputParams_[i];
166 606 : if (ms.name != inputName)
167 301 : continue;
168 :
169 305 : if (ms.format != frame->format
170 305 : || (ms.isVideo && (ms.width != frame->width || ms.height != frame->height))
171 305 : || (!ms.isVideo
172 302 : && (ms.sampleRate != frame->sample_rate || ms.nbChannels != frame->ch_layout.nb_channels))) {
173 1 : ms.update(frame);
174 1 : if ((ret = reinitialize()) < 0)
175 0 : return fail("Failed to reinitialize filter with new input parameters", ret);
176 : }
177 :
178 305 : int flags = AV_BUFFERSRC_FLAG_KEEP_REF;
179 305 : if ((ret = av_buffersrc_add_frame_flags(inputs_[i], frame, flags)) < 0)
180 0 : return fail("Unable to pass frame to filters", ret);
181 : else
182 305 : return 0;
183 : }
184 0 : return fail(fmt::format("Specified filter '{}' not found", inputName), AVERROR(EINVAL));
185 : }
186 :
187 : std::unique_ptr<MediaFrame>
188 103 : MediaFilter::readOutput()
189 : {
190 103 : if (!initialized_) {
191 0 : fail("Not properly initialized", -1);
192 0 : return {};
193 : }
194 :
195 103 : std::unique_ptr<MediaFrame> frame;
196 103 : switch (av_buffersink_get_type(output_)) {
197 : #ifdef ENABLE_VIDEO
198 2 : case AVMEDIA_TYPE_VIDEO:
199 2 : frame = std::make_unique<libjami::VideoFrame>();
200 2 : break;
201 : #endif
202 101 : case AVMEDIA_TYPE_AUDIO:
203 101 : frame = std::make_unique<AudioFrame>();
204 101 : break;
205 0 : default:
206 0 : return {};
207 : }
208 103 : auto err = av_buffersink_get_frame(output_, frame->pointer());
209 103 : if (err >= 0) {
210 102 : return frame;
211 1 : } else if (err == AVERROR(EAGAIN)) {
212 : // no data available right now, try again
213 0 : } else if (err == AVERROR_EOF) {
214 0 : JAMI_WARN() << "Filters have reached EOF, no more frames will be output";
215 : } else {
216 0 : fail("Error occurred while pulling from filter graph", err);
217 : }
218 1 : return {};
219 103 : }
220 :
221 : void
222 36 : MediaFilter::flush()
223 : {
224 42 : for (size_t i = 0; i < inputs_.size(); ++i) {
225 6 : int ret = av_buffersrc_add_frame_flags(inputs_[i], nullptr, 0);
226 6 : if (ret < 0) {
227 0 : JAMI_ERR() << "Failed to flush filter '" << inputParams_[i].name
228 0 : << "': " << libav_utils::getError(ret);
229 : }
230 : }
231 36 : }
232 :
233 : int
234 12 : MediaFilter::initOutputFilter(AVFilterInOut* out)
235 : {
236 12 : int ret = 0;
237 : const AVFilter* buffersink;
238 12 : AVFilterContext* buffersinkCtx = nullptr;
239 12 : AVMediaType mediaType = avfilter_pad_get_type(out->filter_ctx->input_pads, out->pad_idx);
240 :
241 12 : if (mediaType == AVMEDIA_TYPE_VIDEO)
242 6 : buffersink = avfilter_get_by_name("buffersink");
243 : else
244 6 : buffersink = avfilter_get_by_name("abuffersink");
245 :
246 12 : if ((ret
247 12 : = avfilter_graph_create_filter(&buffersinkCtx, buffersink, "out", nullptr, nullptr, graph_))
248 12 : < 0) {
249 0 : avfilter_free(buffersinkCtx);
250 0 : return fail("Failed to create buffer sink", ret);
251 : }
252 :
253 12 : if ((ret = avfilter_link(out->filter_ctx, out->pad_idx, buffersinkCtx, 0)) < 0) {
254 0 : avfilter_free(buffersinkCtx);
255 0 : return fail("Unable to link buffer sink to graph", ret);
256 : }
257 :
258 12 : output_ = buffersinkCtx;
259 12 : return ret;
260 : }
261 :
262 : int
263 16 : MediaFilter::initInputFilter(AVFilterInOut* in, const MediaStream& msp)
264 : {
265 16 : int ret = 0;
266 16 : AVBufferSrcParameters* params = av_buffersrc_parameters_alloc();
267 16 : if (!params)
268 0 : return -1;
269 :
270 : const AVFilter* buffersrc;
271 16 : AVMediaType mediaType = avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx);
272 16 : params->format = msp.format;
273 16 : params->time_base = msp.timeBase;
274 16 : if (mediaType == AVMEDIA_TYPE_VIDEO) {
275 8 : params->width = msp.width;
276 8 : params->height = msp.height;
277 8 : params->frame_rate = msp.frameRate;
278 8 : buffersrc = avfilter_get_by_name("buffer");
279 : } else {
280 8 : params->sample_rate = msp.sampleRate;
281 8 : av_channel_layout_default(¶ms->ch_layout, msp.nbChannels);
282 8 : buffersrc = avfilter_get_by_name("abuffer");
283 : }
284 :
285 16 : AVFilterContext* buffersrcCtx = nullptr;
286 16 : if (buffersrc) {
287 : char name[128];
288 16 : snprintf(name, sizeof(name), "buffersrc_%s_%d", in->name, in->pad_idx);
289 16 : buffersrcCtx = avfilter_graph_alloc_filter(graph_, buffersrc, name);
290 : }
291 16 : if (!buffersrcCtx) {
292 0 : av_free(params);
293 0 : return fail("Failed to allocate filter graph input", AVERROR(ENOMEM));
294 : }
295 16 : ret = av_buffersrc_parameters_set(buffersrcCtx, params);
296 16 : av_free(params);
297 16 : if (ret < 0)
298 0 : return fail("Failed to set filter graph input parameters", ret);
299 :
300 16 : if ((ret = avfilter_init_str(buffersrcCtx, nullptr)) < 0)
301 0 : return fail("Failed to initialize buffer source", ret);
302 :
303 16 : if ((ret = avfilter_link(buffersrcCtx, 0, in->filter_ctx, in->pad_idx)) < 0)
304 0 : return fail("Failed to link buffer source to graph", ret);
305 :
306 16 : inputs_.push_back(buffersrcCtx);
307 16 : inputParams_.emplace_back(msp);
308 16 : inputParams_.back().name = in->name;
309 16 : return ret;
310 : }
311 :
312 : int
313 1 : MediaFilter::reinitialize()
314 : {
315 : // keep parameters needed for initialization before clearing filter
316 1 : auto params = std::move(inputParams_);
317 1 : auto desc = std::move(desc_);
318 1 : clean();
319 1 : auto ret = initialize(desc, params);
320 1 : if (ret >= 0)
321 1 : JAMI_DBG() << "Filter graph reinitialized";
322 1 : return ret;
323 1 : }
324 :
325 : int
326 1 : MediaFilter::fail(std::string_view msg, int err) const
327 : {
328 1 : if (!msg.empty())
329 1 : JAMI_ERR() << msg << ": " << libav_utils::getError(err);
330 1 : return err;
331 : }
332 :
333 : void
334 42 : MediaFilter::clean()
335 : {
336 42 : initialized_ = false;
337 42 : avfilter_graph_free(&graph_); // frees inputs_ and output_
338 42 : desc_.clear();
339 42 : inputs_.clear(); // don't point to freed memory
340 42 : output_ = nullptr; // don't point to freed memory
341 42 : inputParams_.clear();
342 42 : }
343 :
344 : } // namespace jami
|
