Line data Source code
1 : /*
2 : * Copyright (C) 2004-2026 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include <algorithm>
19 :
20 : #ifdef HAVE_CONFIG_H
21 : #include "config.h"
22 : #endif
23 :
24 : #include "media_buffer.h"
25 : #include "logger.h"
26 : #include "accel.h"
27 :
28 : namespace jami {
29 : namespace video {
30 :
31 : struct HardwareAPI
32 : {
33 : std::string name;
34 : AVHWDeviceType hwType;
35 : AVPixelFormat format;
36 : AVPixelFormat swFormat;
37 : std::vector<AVCodecID> supportedCodecs;
38 : std::list<std::pair<std::string, DeviceState>> possible_devices;
39 : bool dynBitrate;
40 : };
41 :
42 : static std::list<HardwareAPI> apiListDec = {
43 : {"nvdec",
44 : AV_HWDEVICE_TYPE_CUDA,
45 : AV_PIX_FMT_CUDA,
46 : AV_PIX_FMT_NV12,
47 : {AV_CODEC_ID_H264, AV_CODEC_ID_HEVC, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG},
48 : {{"default", DeviceState::NOT_TESTED}, {"1", DeviceState::NOT_TESTED}, {"2", DeviceState::NOT_TESTED}},
49 : false},
50 : {"vaapi",
51 : AV_HWDEVICE_TYPE_VAAPI,
52 : AV_PIX_FMT_VAAPI,
53 : AV_PIX_FMT_NV12,
54 : {AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_VP8},
55 : {{"default", DeviceState::NOT_TESTED},
56 : {"/dev/dri/renderD128", DeviceState::NOT_TESTED},
57 : {"/dev/dri/renderD129", DeviceState::NOT_TESTED},
58 : {":0", DeviceState::NOT_TESTED}},
59 : false},
60 : {"videotoolbox",
61 : AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
62 : AV_PIX_FMT_VIDEOTOOLBOX,
63 : AV_PIX_FMT_NV12,
64 : {AV_CODEC_ID_H264, AV_CODEC_ID_HEVC, AV_CODEC_ID_MPEG4},
65 : {{"default", DeviceState::NOT_TESTED}},
66 : false},
67 : {"qsv",
68 : AV_HWDEVICE_TYPE_QSV,
69 : AV_PIX_FMT_QSV,
70 : AV_PIX_FMT_NV12,
71 : {AV_CODEC_ID_H264, AV_CODEC_ID_HEVC, AV_CODEC_ID_MJPEG, AV_CODEC_ID_VP8, AV_CODEC_ID_VP9},
72 : {{"default", DeviceState::NOT_TESTED}},
73 : false},
74 : };
75 :
76 : static std::list<HardwareAPI> apiListEnc = {
77 : {"nvenc",
78 : AV_HWDEVICE_TYPE_CUDA,
79 : AV_PIX_FMT_CUDA,
80 : AV_PIX_FMT_NV12,
81 : {AV_CODEC_ID_H264, AV_CODEC_ID_HEVC},
82 : {{"default", DeviceState::NOT_TESTED}, {"1", DeviceState::NOT_TESTED}, {"2", DeviceState::NOT_TESTED}},
83 : true},
84 : {"vaapi",
85 : AV_HWDEVICE_TYPE_VAAPI,
86 : AV_PIX_FMT_VAAPI,
87 : AV_PIX_FMT_NV12,
88 : {AV_CODEC_ID_H264, AV_CODEC_ID_HEVC, AV_CODEC_ID_VP8},
89 : {{"default", DeviceState::NOT_TESTED},
90 : {"/dev/dri/renderD128", DeviceState::NOT_TESTED},
91 : {"/dev/dri/renderD129", DeviceState::NOT_TESTED},
92 : {":0", DeviceState::NOT_TESTED}},
93 : false},
94 : {"videotoolbox",
95 : AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
96 : AV_PIX_FMT_VIDEOTOOLBOX,
97 : AV_PIX_FMT_NV12,
98 : {AV_CODEC_ID_H264, AV_CODEC_ID_HEVC},
99 : {{"default", DeviceState::NOT_TESTED}},
100 : false},
101 : // Disable temporarily QSVENC
102 : // {"qsv",
103 : // AV_HWDEVICE_TYPE_QSV,
104 : // AV_PIX_FMT_QSV,
105 : // AV_PIX_FMT_NV12,
106 : // {AV_CODEC_ID_H264, AV_CODEC_ID_HEVC, AV_CODEC_ID_MJPEG, AV_CODEC_ID_VP8},
107 : // {{"default", DeviceState::NOT_TESTED}},
108 : // false},
109 : };
110 :
111 348 : HardwareAccel::HardwareAccel(AVCodecID id,
112 : const std::string& name,
113 : AVHWDeviceType hwType,
114 : AVPixelFormat format,
115 : AVPixelFormat swFormat,
116 : CodecType type,
117 348 : bool dynBitrate)
118 348 : : id_(id)
119 348 : , name_(name)
120 348 : , hwType_(hwType)
121 348 : , format_(format)
122 348 : , swFormat_(swFormat)
123 348 : , type_(type)
124 348 : , dynBitrate_(dynBitrate)
125 348 : {}
126 :
127 1044 : HardwareAccel::~HardwareAccel()
128 : {
129 1044 : if (deviceCtx_)
130 0 : av_buffer_unref(&deviceCtx_);
131 1044 : if (framesCtx_)
132 0 : av_buffer_unref(&framesCtx_);
133 1044 : }
134 :
135 : static AVPixelFormat
136 0 : getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats)
137 : {
138 0 : auto* accel = static_cast<HardwareAccel*>(codecCtx->opaque);
139 :
140 0 : for (int i = 0; formats[i] != AV_PIX_FMT_NONE; ++i) {
141 0 : if (accel && formats[i] == accel->getFormat()) {
142 : // found hardware format for codec with api
143 0 : JAMI_LOG("Found compatible hardware format for {} decoder with {}",
144 : avcodec_get_name(static_cast<AVCodecID>(accel->getCodecId())),
145 : accel->getName());
146 : // hardware tends to under-report supported levels
147 0 : codecCtx->hwaccel_flags |= AV_HWACCEL_FLAG_IGNORE_LEVEL;
148 0 : return formats[i];
149 : }
150 : }
151 0 : return AV_PIX_FMT_NONE;
152 : }
153 :
154 : int
155 80 : HardwareAccel::init_device(const char* name, const char* device, int flags)
156 : {
157 80 : const AVHWDeviceContext* dev = nullptr;
158 :
159 : // Create device ctx
160 : int err;
161 80 : err = av_hwdevice_ctx_create(&deviceCtx_, hwType_, device, NULL, flags);
162 80 : if (err < 0) {
163 80 : JAMI_LOG("Failed to create {} device: {}.", name, err);
164 80 : return 1;
165 : }
166 :
167 : // Verify that the device create correspond to api
168 0 : dev = (AVHWDeviceContext*) deviceCtx_->data;
169 0 : if (dev->type != hwType_) {
170 0 : JAMI_LOG("Device created as type {} has type {}.",
171 : av_hwdevice_get_type_name(hwType_),
172 : av_hwdevice_get_type_name(dev->type));
173 0 : av_buffer_unref(&deviceCtx_);
174 0 : return -1;
175 : }
176 0 : JAMI_LOG("Device type {} successfully created.", name);
177 :
178 0 : return 0;
179 : }
180 :
181 : int
182 348 : HardwareAccel::init_device_type(std::string& dev)
183 : {
184 : int err;
185 :
186 348 : const char* name = av_hwdevice_get_type_name(hwType_);
187 348 : if (!name) {
188 0 : JAMI_LOG("No name available for device type {}.", static_cast<int>(hwType_));
189 0 : return -1;
190 : }
191 :
192 348 : AVHWDeviceType check = av_hwdevice_find_type_by_name(name);
193 348 : if (check != hwType_) {
194 0 : JAMI_LOG("Type {} maps to name {} maps to type {}.", static_cast<int>(hwType_), name, static_cast<int>(check));
195 0 : return -1;
196 : }
197 :
198 348 : JAMI_WARNING("-- Starting {} init for {} with default device.",
199 : (type_ == CODEC_ENCODER) ? "encoding" : "decoding",
200 : name);
201 348 : if (possible_devices_->front().second != DeviceState::NOT_USABLE) {
202 19 : if (name_ == "qsv")
203 0 : err = init_device(name, "auto", 0);
204 : else
205 19 : err = init_device(name, nullptr, 0);
206 19 : if (err == 0) {
207 0 : JAMI_LOG("-- Init passed for {} with default device.", name);
208 0 : possible_devices_->front().second = DeviceState::USABLE;
209 0 : dev = "default";
210 0 : return 0;
211 : } else {
212 19 : possible_devices_->front().second = DeviceState::NOT_USABLE;
213 19 : JAMI_LOG("-- Init failed for {} with default device.", name);
214 : }
215 : }
216 :
217 1561 : for (auto& device : *possible_devices_) {
218 1213 : if (device.second == DeviceState::NOT_USABLE)
219 1152 : continue;
220 61 : JAMI_WARNING("-- Init {} for {} with device {}.",
221 : (type_ == CODEC_ENCODER) ? "encoding" : "decoding",
222 : name,
223 : device.first);
224 61 : err = init_device(name, device.first.c_str(), 0);
225 61 : if (err == 0) {
226 0 : JAMI_LOG("-- Init passed for {} with device {}.", name, device.first);
227 0 : device.second = DeviceState::USABLE;
228 0 : dev = device.first;
229 0 : return 0;
230 : } else {
231 61 : device.second = DeviceState::NOT_USABLE;
232 61 : JAMI_LOG("-- Init failed for {} with device {}.", name, device.first);
233 : }
234 : }
235 348 : return -1;
236 : }
237 :
238 : std::string
239 348 : HardwareAccel::getCodecName() const
240 : {
241 348 : if (type_ == CODEC_DECODER) {
242 696 : return avcodec_get_name(id_);
243 0 : } else if (type_ == CODEC_ENCODER) {
244 0 : return fmt::format("{}_{}", avcodec_get_name(id_), name_);
245 : }
246 0 : return {};
247 : }
248 :
249 : std::unique_ptr<VideoFrame>
250 0 : HardwareAccel::transfer(const VideoFrame& frame)
251 : {
252 0 : int ret = 0;
253 0 : if (type_ == CODEC_DECODER) {
254 0 : const auto* input = frame.pointer();
255 0 : if (input->format != format_) {
256 0 : JAMI_ERROR("Frame format mismatch: expected {}, got {}",
257 : av_get_pix_fmt_name(format_),
258 : av_get_pix_fmt_name(static_cast<AVPixelFormat>(input->format)));
259 0 : return nullptr;
260 : }
261 :
262 0 : return transferToMainMemory(frame, swFormat_);
263 0 : } else if (type_ == CODEC_ENCODER) {
264 0 : const auto* input = frame.pointer();
265 0 : if (input->format != swFormat_) {
266 0 : JAMI_ERROR("Frame format mismatch: expected {}, got {}",
267 : av_get_pix_fmt_name(swFormat_),
268 : av_get_pix_fmt_name(static_cast<AVPixelFormat>(input->format)));
269 0 : return nullptr;
270 : }
271 :
272 0 : auto framePtr = std::make_unique<VideoFrame>();
273 0 : auto* hwFrame = framePtr->pointer();
274 :
275 0 : if ((ret = av_hwframe_get_buffer(framesCtx_, hwFrame, 0)) < 0) {
276 0 : JAMI_ERROR("Failed to allocate hardware buffer: {}", libav_utils::getError(ret));
277 0 : return nullptr;
278 : }
279 :
280 0 : if (!hwFrame->hw_frames_ctx) {
281 0 : JAMI_ERROR("Failed to allocate hardware buffer: Unable to allocate memory");
282 0 : return nullptr;
283 : }
284 :
285 0 : if ((ret = av_hwframe_transfer_data(hwFrame, input, 0)) < 0) {
286 0 : JAMI_ERROR("Failed to push frame to GPU: {}", libav_utils::getError(ret));
287 0 : return nullptr;
288 : }
289 :
290 0 : hwFrame->pts = input->pts; // transfer does not copy timestamp
291 0 : return framePtr;
292 0 : } else {
293 0 : JAMI_ERROR("Invalid hardware accelerator");
294 0 : return nullptr;
295 : }
296 : }
297 :
298 : void
299 0 : HardwareAccel::setDetails(AVCodecContext* codecCtx)
300 : {
301 0 : if (type_ == CODEC_DECODER) {
302 0 : codecCtx->hw_device_ctx = av_buffer_ref(deviceCtx_);
303 0 : codecCtx->get_format = getFormatCb;
304 0 : } else if (type_ == CODEC_ENCODER) {
305 0 : if (framesCtx_)
306 : // encoder doesn't need a device context, only a frame context
307 0 : codecCtx->hw_frames_ctx = av_buffer_ref(framesCtx_);
308 : }
309 0 : }
310 :
311 : bool
312 0 : HardwareAccel::initFrame()
313 : {
314 0 : int ret = 0;
315 0 : if (!deviceCtx_) {
316 0 : JAMI_ERROR("Unable to initialize hardware frames without a valid hardware device");
317 0 : return false;
318 : }
319 :
320 0 : framesCtx_ = av_hwframe_ctx_alloc(deviceCtx_);
321 0 : if (!framesCtx_)
322 0 : return false;
323 :
324 0 : auto* ctx = reinterpret_cast<AVHWFramesContext*>(framesCtx_->data);
325 0 : ctx->format = format_;
326 0 : ctx->sw_format = swFormat_;
327 0 : ctx->width = width_;
328 0 : ctx->height = height_;
329 0 : ctx->initial_pool_size = 20; // TODO try other values
330 :
331 0 : if ((ret = av_hwframe_ctx_init(framesCtx_)) < 0) {
332 0 : JAMI_ERROR("Failed to initialize hardware frame context: {} ({})", libav_utils::getError(ret), ret);
333 0 : av_buffer_unref(&framesCtx_);
334 : }
335 :
336 0 : return ret >= 0;
337 : }
338 :
339 : bool
340 0 : HardwareAccel::linkHardware(AVBufferRef* framesCtx)
341 : {
342 0 : if (framesCtx) {
343 : // Force sw_format to match swFormat_. Frame is never transferred to main
344 : // memory when hardware is linked, so the sw_format doesn't matter.
345 0 : auto* hw = reinterpret_cast<AVHWFramesContext*>(framesCtx->data);
346 0 : hw->sw_format = swFormat_;
347 :
348 0 : if (framesCtx_)
349 0 : av_buffer_unref(&framesCtx_);
350 0 : framesCtx_ = av_buffer_ref(framesCtx);
351 0 : if ((linked_ = (framesCtx_ != nullptr))) {
352 0 : JAMI_LOG("Hardware transcoding pipeline successfully set up for encoder '{}'", getCodecName());
353 : }
354 0 : return linked_;
355 : } else {
356 0 : return false;
357 : }
358 : }
359 :
360 : std::unique_ptr<VideoFrame>
361 0 : HardwareAccel::transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
362 : {
363 0 : const auto* input = frame.pointer();
364 0 : if (not input)
365 0 : throw std::runtime_error("Unable to transfer null frame");
366 :
367 0 : const auto* desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format));
368 0 : if (!desc) {
369 0 : throw std::runtime_error("Unable to transfer frame with invalid format");
370 : }
371 :
372 0 : auto out = std::make_unique<VideoFrame>();
373 0 : if (not(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
374 0 : out->copyFrom(frame);
375 0 : return out;
376 : }
377 :
378 0 : auto* output = out->pointer();
379 0 : output->format = desiredFormat;
380 :
381 0 : int ret = av_hwframe_transfer_data(output, input, 0);
382 0 : if (ret < 0) {
383 0 : throw std::runtime_error("Unable to transfer the frame from GPU");
384 : }
385 :
386 0 : output->pts = input->pts;
387 0 : if (AVFrameSideData* side_data = av_frame_get_side_data(input, AV_FRAME_DATA_DISPLAYMATRIX))
388 0 : av_frame_new_side_data_from_buf(output, AV_FRAME_DATA_DISPLAYMATRIX, av_buffer_ref(side_data->buf));
389 0 : return out;
390 0 : }
391 :
392 : int
393 348 : HardwareAccel::initAPI(bool linkable, AVBufferRef* framesCtx)
394 : {
395 348 : const auto& codecName = getCodecName();
396 348 : std::string device;
397 348 : auto ret = init_device_type(device);
398 348 : if (ret == 0) {
399 0 : bool link = false;
400 0 : if (linkable && framesCtx)
401 0 : link = linkHardware(framesCtx);
402 : // we don't need frame context for videotoolbox
403 0 : if (hwType_ == AV_HWDEVICE_TYPE_VIDEOTOOLBOX || link || initFrame()) {
404 0 : return 0;
405 : }
406 : }
407 348 : return -1;
408 348 : }
409 :
410 : std::list<HardwareAccel>
411 411 : HardwareAccel::getCompatibleAccel(AVCodecID id, int width, int height, CodecType type)
412 : {
413 411 : std::list<HardwareAccel> l;
414 411 : const auto& list = (type == CODEC_ENCODER) ? &apiListEnc : &apiListDec;
415 2055 : for (auto& api : *list) {
416 1644 : const auto& it = std::find(api.supportedCodecs.begin(), api.supportedCodecs.end(), id);
417 1644 : if (it != api.supportedCodecs.end()) {
418 705 : auto hwtype = AV_HWDEVICE_TYPE_NONE;
419 2820 : while ((hwtype = av_hwdevice_iterate_types(hwtype)) != AV_HWDEVICE_TYPE_NONE) {
420 2115 : if (hwtype == api.hwType) {
421 348 : auto accel = HardwareAccel(id, api.name, api.hwType, api.format, api.swFormat, type, api.dynBitrate);
422 348 : accel.height_ = height;
423 348 : accel.width_ = width;
424 348 : accel.possible_devices_ = &api.possible_devices;
425 348 : l.emplace_back(std::move(accel));
426 348 : }
427 : }
428 : }
429 : }
430 822 : return l;
431 0 : }
432 :
433 : } // namespace video
434 : } // namespace jami
|
