Line data Source code
1 : /*
2 : * Copyright (C) 2004-2024 Savoir-faire Linux Inc.
3 : *
4 : * Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
5 : *
6 : * This program is free software; you can redistribute it and/or modify
7 : * it under the terms of the GNU General Public License as published by
8 : * the Free Software Foundation; either version 3 of the License, or
9 : * (at your option) any later version.
10 : *
11 : * This program is distributed in the hope that it will be useful,
12 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : * GNU General Public License for more details.
15 : *
16 : * You should have received a copy of the GNU General Public License
17 : * along with this program; if not, write to the Free Software
18 : * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19 : */
20 :
21 : #include "libav_deps.h" // MUST BE INCLUDED FIRST
22 : #include "logger.h"
23 : #include "media_filter.h"
24 : #include "media_buffer.h"
25 :
26 : extern "C" {
27 : #include <libavfilter/buffersink.h>
28 : #include <libavfilter/buffersrc.h>
29 : }
30 :
31 : #include <algorithm>
32 : #include <functional>
33 : #include <memory>
34 : #include <sstream>
35 : #include <thread>
36 :
37 : namespace jami {
38 :
39 42 : MediaFilter::MediaFilter() {}
40 :
41 42 : MediaFilter::~MediaFilter()
42 : {
43 42 : clean();
44 42 : }
45 :
46 : std::string
47 0 : MediaFilter::getFilterDesc() const
48 : {
49 0 : return desc_;
50 : }
51 :
52 : int
53 12 : MediaFilter::initialize(const std::string& filterDesc, const std::vector<MediaStream>& msps)
54 : {
55 12 : int ret = 0;
56 12 : desc_ = filterDesc;
57 12 : graph_ = avfilter_graph_alloc();
58 :
59 12 : if (!graph_)
60 0 : return fail("Failed to allocate filter graph", AVERROR(ENOMEM));
61 :
62 12 : graph_->nb_threads = std::max(1u, std::min(8u, std::thread::hardware_concurrency() / 2));
63 :
64 : AVFilterInOut* in;
65 : AVFilterInOut* out;
66 12 : if ((ret = avfilter_graph_parse2(graph_, desc_.c_str(), &in, &out)) < 0)
67 0 : return fail("Failed to parse filter graph", ret);
68 :
69 : using AVFilterInOutPtr = std::unique_ptr<AVFilterInOut, std::function<void(AVFilterInOut*)>>;
70 24 : AVFilterInOutPtr outputs(out, [](AVFilterInOut* f) { avfilter_inout_free(&f); });
71 24 : AVFilterInOutPtr inputs(in, [](AVFilterInOut* f) { avfilter_inout_free(&f); });
72 :
73 12 : if (outputs && outputs->next)
74 0 : return fail("Filters with multiple outputs are not supported", AVERROR(ENOTSUP));
75 :
76 12 : if ((ret = initOutputFilter(outputs.get())) < 0)
77 0 : return fail("Failed to create output for filter graph", ret);
78 :
79 : // make sure inputs linked list is the same size as msps
80 12 : size_t count = 0;
81 12 : AVFilterInOut* dummyInput = inputs.get();
82 28 : while (dummyInput && ++count) // increment count before evaluating its value
83 16 : dummyInput = dummyInput->next;
84 12 : if (count != msps.size())
85 0 : return fail(
86 : "Size mismatch between number of inputs in filter graph and input parameter array",
87 0 : AVERROR(EINVAL));
88 :
89 28 : for (AVFilterInOut* current = inputs.get(); current; current = current->next) {
90 16 : if (!current->name)
91 0 : return fail("Filters require non empty names", AVERROR(EINVAL));
92 16 : std::string_view name = current->name;
93 16 : auto it = std::find_if(msps.begin(), msps.end(), [name](const MediaStream& msp) {
94 21 : return msp.name == name;
95 : });
96 16 : if (it != msps.end()) {
97 16 : if ((ret = initInputFilter(current, *it)) < 0) {
98 0 : return fail(fmt::format("Failed to initialize input: {}", name), ret);
99 : }
100 : } else {
101 0 : return fail(fmt::format("Failed to find matching parameters for: {}", name), ret);
102 : }
103 : }
104 :
105 12 : if ((ret = avfilter_graph_config(graph_, nullptr)) < 0)
106 0 : return fail("Failed to configure filter graph", ret);
107 :
108 12 : JAMI_DBG() << "Filter graph initialized with: " << desc_;
109 12 : initialized_ = true;
110 12 : return 0;
111 12 : }
112 :
113 : const MediaStream&
114 2 : MediaFilter::getInputParams(const std::string& inputName) const
115 : {
116 3 : for (const auto& ms : inputParams_)
117 3 : if (ms.name == inputName)
118 2 : return ms;
119 0 : return {};
120 : }
121 :
122 : MediaStream
123 5 : MediaFilter::getOutputParams() const
124 : {
125 5 : MediaStream output;
126 5 : if (!output_ || !initialized_) {
127 1 : fail("Filter not initialized", -1);
128 1 : return output;
129 : }
130 :
131 4 : switch (av_buffersink_get_type(output_)) {
132 3 : case AVMEDIA_TYPE_VIDEO:
133 3 : output.name = "videoOutput";
134 3 : output.format = av_buffersink_get_format(output_);
135 3 : output.isVideo = true;
136 3 : output.timeBase = av_buffersink_get_time_base(output_);
137 3 : output.width = av_buffersink_get_w(output_);
138 3 : output.height = av_buffersink_get_h(output_);
139 3 : output.bitrate = 0;
140 3 : output.frameRate = av_buffersink_get_frame_rate(output_);
141 3 : break;
142 1 : case AVMEDIA_TYPE_AUDIO:
143 1 : output.name = "audioOutput";
144 1 : output.format = av_buffersink_get_format(output_);
145 1 : output.isVideo = false;
146 1 : output.timeBase = av_buffersink_get_time_base(output_);
147 1 : output.sampleRate = av_buffersink_get_sample_rate(output_);
148 1 : output.nbChannels = av_buffersink_get_channels(output_);
149 1 : break;
150 0 : default:
151 0 : output.format = -1;
152 0 : break;
153 : }
154 4 : return output;
155 0 : }
156 :
157 : int
158 305 : MediaFilter::feedInput(AVFrame* frame, const std::string& inputName)
159 : {
160 305 : int ret = 0;
161 305 : if (!initialized_)
162 0 : return fail("Filter not initialized", -1);
163 :
164 305 : if (!frame)
165 0 : return 0;
166 :
167 606 : for (size_t i = 0; i < inputs_.size(); ++i) {
168 606 : auto& ms = inputParams_[i];
169 606 : if (ms.name != inputName)
170 301 : continue;
171 :
172 305 : if (ms.format != frame->format
173 305 : || (ms.isVideo && (ms.width != frame->width || ms.height != frame->height))
174 305 : || (!ms.isVideo
175 302 : && (ms.sampleRate != frame->sample_rate || ms.nbChannels != frame->ch_layout.nb_channels))) {
176 1 : ms.update(frame);
177 1 : if ((ret = reinitialize()) < 0)
178 0 : return fail("Failed to reinitialize filter with new input parameters", ret);
179 : }
180 :
181 305 : int flags = AV_BUFFERSRC_FLAG_KEEP_REF;
182 305 : if ((ret = av_buffersrc_add_frame_flags(inputs_[i], frame, flags)) < 0)
183 0 : return fail("Could not pass frame to filters", ret);
184 : else
185 305 : return 0;
186 : }
187 0 : return fail(fmt::format("Specified filter '{}' not found", inputName), AVERROR(EINVAL));
188 : }
189 :
190 : std::unique_ptr<MediaFrame>
191 103 : MediaFilter::readOutput()
192 : {
193 103 : if (!initialized_) {
194 0 : fail("Not properly initialized", -1);
195 0 : return {};
196 : }
197 :
198 103 : std::unique_ptr<MediaFrame> frame;
199 103 : switch (av_buffersink_get_type(output_)) {
200 : #ifdef ENABLE_VIDEO
201 2 : case AVMEDIA_TYPE_VIDEO:
202 2 : frame = std::make_unique<libjami::VideoFrame>();
203 2 : break;
204 : #endif
205 101 : case AVMEDIA_TYPE_AUDIO:
206 101 : frame = std::make_unique<AudioFrame>();
207 101 : break;
208 0 : default:
209 0 : return {};
210 : }
211 103 : auto err = av_buffersink_get_frame(output_, frame->pointer());
212 103 : if (err >= 0) {
213 102 : return frame;
214 1 : } else if (err == AVERROR(EAGAIN)) {
215 : // no data available right now, try again
216 0 : } else if (err == AVERROR_EOF) {
217 0 : JAMI_WARN() << "Filters have reached EOF, no more frames will be output";
218 : } else {
219 0 : fail("Error occurred while pulling from filter graph", err);
220 : }
221 1 : return {};
222 103 : }
223 :
224 : void
225 36 : MediaFilter::flush()
226 : {
227 42 : for (size_t i = 0; i < inputs_.size(); ++i) {
228 6 : int ret = av_buffersrc_add_frame_flags(inputs_[i], nullptr, 0);
229 6 : if (ret < 0) {
230 0 : JAMI_ERR() << "Failed to flush filter '" << inputParams_[i].name
231 0 : << "': " << libav_utils::getError(ret);
232 : }
233 : }
234 36 : }
235 :
236 : int
237 12 : MediaFilter::initOutputFilter(AVFilterInOut* out)
238 : {
239 12 : int ret = 0;
240 : const AVFilter* buffersink;
241 12 : AVFilterContext* buffersinkCtx = nullptr;
242 12 : AVMediaType mediaType = avfilter_pad_get_type(out->filter_ctx->input_pads, out->pad_idx);
243 :
244 12 : if (mediaType == AVMEDIA_TYPE_VIDEO)
245 6 : buffersink = avfilter_get_by_name("buffersink");
246 : else
247 6 : buffersink = avfilter_get_by_name("abuffersink");
248 :
249 12 : if ((ret
250 12 : = avfilter_graph_create_filter(&buffersinkCtx, buffersink, "out", nullptr, nullptr, graph_))
251 12 : < 0) {
252 0 : avfilter_free(buffersinkCtx);
253 0 : return fail("Failed to create buffer sink", ret);
254 : }
255 :
256 12 : if ((ret = avfilter_link(out->filter_ctx, out->pad_idx, buffersinkCtx, 0)) < 0) {
257 0 : avfilter_free(buffersinkCtx);
258 0 : return fail("Could not link buffer sink to graph", ret);
259 : }
260 :
261 12 : output_ = buffersinkCtx;
262 12 : return ret;
263 : }
264 :
265 : int
266 16 : MediaFilter::initInputFilter(AVFilterInOut* in, const MediaStream& msp)
267 : {
268 16 : int ret = 0;
269 16 : AVBufferSrcParameters* params = av_buffersrc_parameters_alloc();
270 16 : if (!params)
271 0 : return -1;
272 :
273 : const AVFilter* buffersrc;
274 16 : AVMediaType mediaType = avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx);
275 16 : params->format = msp.format;
276 16 : params->time_base = msp.timeBase;
277 16 : if (mediaType == AVMEDIA_TYPE_VIDEO) {
278 8 : params->width = msp.width;
279 8 : params->height = msp.height;
280 8 : params->frame_rate = msp.frameRate;
281 8 : buffersrc = avfilter_get_by_name("buffer");
282 : } else {
283 8 : params->sample_rate = msp.sampleRate;
284 8 : av_channel_layout_default(¶ms->ch_layout, msp.nbChannels);
285 8 : buffersrc = avfilter_get_by_name("abuffer");
286 : }
287 :
288 16 : AVFilterContext* buffersrcCtx = nullptr;
289 16 : if (buffersrc) {
290 : char name[128];
291 16 : snprintf(name, sizeof(name), "buffersrc_%s_%d", in->name, in->pad_idx);
292 16 : buffersrcCtx = avfilter_graph_alloc_filter(graph_, buffersrc, name);
293 : }
294 16 : if (!buffersrcCtx) {
295 0 : av_free(params);
296 0 : return fail("Failed to allocate filter graph input", AVERROR(ENOMEM));
297 : }
298 16 : ret = av_buffersrc_parameters_set(buffersrcCtx, params);
299 16 : av_free(params);
300 16 : if (ret < 0)
301 0 : return fail("Failed to set filter graph input parameters", ret);
302 :
303 16 : if ((ret = avfilter_init_str(buffersrcCtx, nullptr)) < 0)
304 0 : return fail("Failed to initialize buffer source", ret);
305 :
306 16 : if ((ret = avfilter_link(buffersrcCtx, 0, in->filter_ctx, in->pad_idx)) < 0)
307 0 : return fail("Failed to link buffer source to graph", ret);
308 :
309 16 : inputs_.push_back(buffersrcCtx);
310 16 : inputParams_.emplace_back(msp);
311 16 : inputParams_.back().name = in->name;
312 16 : return ret;
313 : }
314 :
315 : int
316 1 : MediaFilter::reinitialize()
317 : {
318 : // keep parameters needed for initialization before clearing filter
319 1 : auto params = std::move(inputParams_);
320 1 : auto desc = std::move(desc_);
321 1 : clean();
322 1 : auto ret = initialize(desc, params);
323 1 : if (ret >= 0)
324 1 : JAMI_DBG() << "Filter graph reinitialized";
325 1 : return ret;
326 1 : }
327 :
328 : int
329 1 : MediaFilter::fail(std::string_view msg, int err) const
330 : {
331 1 : if (!msg.empty())
332 1 : JAMI_ERR() << msg << ": " << libav_utils::getError(err);
333 1 : return err;
334 : }
335 :
336 : void
337 43 : MediaFilter::clean()
338 : {
339 43 : initialized_ = false;
340 43 : avfilter_graph_free(&graph_); // frees inputs_ and output_
341 43 : desc_.clear();
342 43 : inputs_.clear(); // don't point to freed memory
343 43 : output_ = nullptr; // don't point to freed memory
344 43 : inputParams_.clear();
345 43 : }
346 :
347 : } // namespace jami
|
