Line data Source code
1 : /*
2 : * Copyright (C) 2004-2026 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #ifdef HAVE_CONFIG_H
19 : #include "config.h"
20 : #endif
21 :
22 : #include "sinkclient.h"
23 :
24 : #ifdef ENABLE_SHM
25 : #include "shm_header.h"
26 : #endif // ENABLE_SHM
27 :
28 : #include "media_buffer.h"
29 : #include "logger.h"
30 : #include "client/jami_signal.h"
31 : #include "jami/videomanager_interface.h"
32 : #include "libav_utils.h"
33 : #include "video_scaler.h"
34 : #include "media_filter.h"
35 : #include "filter_transpose.h"
36 :
37 : #ifdef ENABLE_HWACCEL
38 : #include "accel.h"
39 : #endif
40 :
41 : #ifndef _WIN32
42 : #include <sys/mman.h>
43 : #endif
44 :
45 : #include <fcntl.h>
46 : #include <cstdio>
47 : #include <unistd.h>
48 : #include <cerrno>
49 : #include <cstring>
50 : #include <stdexcept>
51 : #include <cmath>
52 :
53 : namespace jami {
54 : namespace video {
55 :
56 : const constexpr char FILTER_INPUT_NAME[] = "in";
57 :
58 : #ifdef ENABLE_SHM
59 : // RAII class helper on sem_wait/sem_post sempahore operations
60 : class SemGuardLock
61 : {
62 : public:
63 : explicit SemGuardLock(sem_t& mutex)
64 : : m_(mutex)
65 : {
66 : auto ret = ::sem_wait(&m_);
67 : if (ret < 0) {
68 : throw std::logic_error {fmt::format("SHM mutex@{} lock failed ({})", fmt::ptr(&m_), ret)};
69 : }
70 : }
71 :
72 : ~SemGuardLock() { ::sem_post(&m_); }
73 :
74 : private:
75 : sem_t& m_;
76 : };
77 :
78 : class ShmHolder
79 : {
80 : public:
81 : ShmHolder(const std::string& name = {});
82 : ~ShmHolder();
83 :
84 : std::string name() const noexcept { return openedName_; }
85 :
86 : void renderFrame(const VideoFrame& src) noexcept;
87 :
88 : private:
89 : bool resizeArea(std::size_t desired_length) noexcept;
90 : char* getShmAreaDataPtr() noexcept;
91 :
92 : void unMapShmArea() noexcept
93 : {
94 : if (area_ != MAP_FAILED and ::munmap(area_, areaSize_) < 0) {
95 : JAMI_ERROR("[ShmHolder:{}] munmap({}) failed with errno {}", openedName_, areaSize_, errno);
96 : }
97 : }
98 :
99 : SHMHeader* area_ {static_cast<SHMHeader*>(MAP_FAILED)};
100 : std::size_t areaSize_ {0};
101 : std::string openedName_;
102 : int fd_ {-1};
103 : };
104 :
105 : ShmHolder::ShmHolder(const std::string& name)
106 : {
107 : static constexpr int flags = O_RDWR | O_CREAT | O_TRUNC | O_EXCL;
108 : static constexpr int perms = S_IRUSR | S_IWUSR;
109 :
110 : static auto shmFailedWithErrno = [this](const std::string& what) {
111 : throw std::runtime_error {fmt::format("ShmHolder[{}]: {} failed, errno={}", openedName_, what, errno)};
112 : };
113 :
114 : if (not name.empty()) {
115 : openedName_ = name;
116 : fd_ = ::shm_open(openedName_.c_str(), flags, perms);
117 : if (fd_ < 0)
118 : shmFailedWithErrno("shm_open");
119 : } else {
120 : for (int i = 0; fd_ < 0; ++i) {
121 : openedName_ = fmt::format(PACKAGE_NAME "_shm_{}_{}", getpid(), i);
122 : fd_ = ::shm_open(openedName_.c_str(), flags, perms);
123 : if (fd_ < 0 and errno != EEXIST)
124 : shmFailedWithErrno("shm_open");
125 : }
126 : }
127 :
128 : // Set size enough for header only (no frame data)
129 : if (!resizeArea(0))
130 : shmFailedWithErrno("resizeArea");
131 :
132 : // Header fields initialization
133 : std::memset(area_, 0, areaSize_);
134 :
135 : if (::sem_init(&area_->mutex, 1, 1) < 0)
136 : shmFailedWithErrno("sem_init(mutex)");
137 :
138 : if (::sem_init(&area_->frameGenMutex, 1, 0) < 0)
139 : shmFailedWithErrno("sem_init(frameGenMutex)");
140 :
141 : JAMI_LOG("[ShmHolder:{}] New holder created", openedName_);
142 : }
143 :
144 : ShmHolder::~ShmHolder()
145 : {
146 : if (fd_ < 0)
147 : return;
148 :
149 : ::close(fd_);
150 : ::shm_unlink(openedName_.c_str());
151 :
152 : if (area_ == MAP_FAILED)
153 : return;
154 :
155 : ::sem_wait(&area_->mutex);
156 : area_->frameSize = 0;
157 : ::sem_post(&area_->mutex);
158 :
159 : ::sem_post(&area_->frameGenMutex); // unlock waiting client before leaving
160 : unMapShmArea();
161 : }
162 :
163 : bool
164 : ShmHolder::resizeArea(std::size_t frameSize) noexcept
165 : {
166 : // aligned on 16-byte boundary frameSize
167 : frameSize = (frameSize + 15) & ~15;
168 :
169 : if (area_ != MAP_FAILED and frameSize == area_->frameSize)
170 : return true;
171 :
172 : // full area size: +15 to take care of maximum padding size
173 : const auto areaSize = sizeof(SHMHeader) + 2 * frameSize + 15;
174 : JAMI_LOG("[ShmHolder:{}] New size: f={}, a={}", openedName_, frameSize, areaSize);
175 :
176 : unMapShmArea();
177 :
178 : if (::ftruncate(fd_, areaSize) < 0) {
179 : JAMI_ERROR("[ShmHolder:{}] ftruncate({}) failed with errno {}", openedName_, areaSize, errno);
180 : return false;
181 : }
182 :
183 : area_ = static_cast<SHMHeader*>(::mmap(nullptr, areaSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
184 :
185 : if (area_ == MAP_FAILED) {
186 : areaSize_ = 0;
187 : JAMI_ERROR("[ShmHolder:{}] mmap({}) failed with errno {}", openedName_, areaSize, errno);
188 : return false;
189 : }
190 :
191 : areaSize_ = areaSize;
192 :
193 : if (frameSize) {
194 : SemGuardLock lk {area_->mutex};
195 :
196 : area_->frameSize = frameSize;
197 : area_->mapSize = areaSize;
198 :
199 : // Compute aligned IO pointers
200 : // Note: we not using std::align as not implemented in 4.9
201 : // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57350
202 : auto p = reinterpret_cast<std::uintptr_t>(area_->data);
203 : area_->writeOffset = ((p + 15) & ~15) - p;
204 : area_->readOffset = area_->writeOffset + frameSize;
205 : }
206 :
207 : return true;
208 : }
209 :
210 : void
211 : ShmHolder::renderFrame(const VideoFrame& src) noexcept
212 : {
213 : const auto width = src.width();
214 : const auto height = src.height();
215 : const auto format = AV_PIX_FMT_BGRA;
216 : const auto frameSize = videoFrameSize(format, width, height);
217 :
218 : if (!resizeArea(frameSize)) {
219 : JAMI_ERROR("[ShmHolder:{}] Unable to resize area size: {}x{}, format: {}",
220 : openedName_,
221 : width,
222 : height,
223 : av_get_pix_fmt_name(format));
224 : return;
225 : }
226 :
227 : {
228 : VideoFrame dst;
229 : VideoScaler scaler;
230 :
231 : dst.setFromMemory(area_->data + area_->writeOffset, format, width, height);
232 : scaler.scale(src, dst);
233 : }
234 :
235 : {
236 : SemGuardLock lk {area_->mutex};
237 :
238 : ++area_->frameGen;
239 : std::swap(area_->readOffset, area_->writeOffset);
240 : ::sem_post(&area_->frameGenMutex);
241 : }
242 : }
243 :
244 : std::string
245 : SinkClient::openedName() const noexcept
246 : {
247 : if (shm_)
248 : return shm_->name();
249 : return {};
250 : }
251 :
252 : bool
253 : SinkClient::start() noexcept
254 : {
255 : if (not shm_) {
256 : try {
257 : char* envvar = getenv("JAMI_DISABLE_SHM");
258 : if (envvar) // Do not use SHM if set
259 : return true;
260 : shm_ = std::make_shared<ShmHolder>();
261 : JAMI_LOG("[Sink:{}] Shared memory [{}] created", fmt::ptr(this), openedName());
262 : } catch (const std::runtime_error& e) {
263 : JAMI_ERROR("[Sink:{}] Failed to create shared memory: {}", fmt::ptr(this), e.what());
264 : }
265 : }
266 :
267 : return static_cast<bool>(shm_);
268 : }
269 :
270 : bool
271 : SinkClient::stop() noexcept
272 : {
273 : setFrameSize(0, 0);
274 : setCrop(0, 0, 0, 0);
275 : shm_.reset();
276 : return true;
277 : }
278 :
279 : #else // ENABLE_SHM
280 :
281 : std::string
282 177 : SinkClient::openedName() const noexcept
283 : {
284 177 : return {};
285 : }
286 :
287 : bool
288 242 : SinkClient::start() noexcept
289 : {
290 242 : return true;
291 : }
292 :
293 : bool
294 274 : SinkClient::stop() noexcept
295 : {
296 274 : setFrameSize(0, 0);
297 274 : setCrop(0, 0, 0, 0);
298 274 : return true;
299 : }
300 :
301 : #endif // !ENABLE_SHM
302 :
303 219 : SinkClient::SinkClient(const std::string& id, bool mixer)
304 219 : : id_ {id}
305 219 : , mixer_(mixer)
306 219 : , scaler_(new VideoScaler())
307 : #ifdef DEBUG_FPS
308 : , frameCount_(0u)
309 : , lastFrameDebug_(std::chrono::steady_clock::now())
310 : #endif
311 : {
312 876 : JAMI_LOG("[Sink:{}] Sink [{}] created", fmt::ptr(this), getId());
313 219 : }
314 :
315 : libjami::FrameBuffer
316 0 : SinkClient::configureFrameDirect(const std::shared_ptr<jami::MediaFrame>& frame_p)
317 : {
318 0 : libjami::FrameBuffer outFrame(av_frame_alloc());
319 0 : av_frame_ref(outFrame.get(), std::static_pointer_cast<VideoFrame>(frame_p)->pointer());
320 :
321 0 : if (crop_.w || crop_.h) {
322 : #ifdef ENABLE_HWACCEL
323 0 : const auto* desc = av_pix_fmt_desc_get((AVPixelFormat) std::static_pointer_cast<VideoFrame>(frame_p)->format());
324 : /*
325 : Cropping does not work for hardware-decoded frames.
326 : They need to be transferred to main memory.
327 : */
328 0 : if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
329 0 : std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
330 : try {
331 0 : frame = HardwareAccel::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame_p),
332 0 : AV_PIX_FMT_NV12);
333 0 : } catch (const std::runtime_error& e) {
334 0 : JAMI_ERROR("[Sink:{}] Transfer to hardware acceleration memory failed: {}", fmt::ptr(this), e.what());
335 0 : return {};
336 0 : }
337 0 : if (not frame)
338 0 : return {};
339 0 : av_frame_unref(outFrame.get());
340 0 : av_frame_ref(outFrame.get(), frame->pointer());
341 0 : }
342 : #endif
343 0 : outFrame->crop_top = crop_.y;
344 0 : outFrame->crop_bottom = (size_t) outFrame->height - crop_.y - crop_.h;
345 0 : outFrame->crop_left = crop_.x;
346 0 : outFrame->crop_right = (size_t) outFrame->width - crop_.x - crop_.w;
347 0 : av_frame_apply_cropping(outFrame.get(), AV_FRAME_CROP_UNALIGNED);
348 : }
349 0 : return outFrame;
350 0 : }
351 :
352 : void
353 0 : SinkClient::sendFrameTransformed(AVFrame* frame)
354 : {
355 0 : if (frame->width > 0 and frame->height > 0) {
356 0 : if (auto buffer_ptr = target_.pull()) {
357 0 : scaler_->scale(frame, buffer_ptr.get());
358 0 : target_.push(std::move(buffer_ptr));
359 0 : }
360 : }
361 0 : }
362 :
363 : std::shared_ptr<VideoFrame>
364 0 : SinkClient::applyTransform(VideoFrame& frame_p)
365 : {
366 0 : std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
367 : #ifdef ENABLE_HWACCEL
368 0 : const auto* desc = av_pix_fmt_desc_get((AVPixelFormat) frame_p.format());
369 0 : if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
370 : try {
371 0 : frame = HardwareAccel::transferToMainMemory(frame_p, AV_PIX_FMT_NV12);
372 0 : } catch (const std::runtime_error& e) {
373 0 : JAMI_ERROR("[Sink:{}] Transfer to hardware acceleration memory failed: {}", fmt::ptr(this), e.what());
374 0 : return {};
375 0 : }
376 0 : } else
377 : #endif
378 0 : frame->copyFrom(frame_p);
379 :
380 0 : int angle = frame->getOrientation();
381 0 : if (angle != rotation_) {
382 0 : filter_ = getTransposeFilter(angle, FILTER_INPUT_NAME, frame->width(), frame->height(), frame->format(), false);
383 0 : rotation_ = angle;
384 : }
385 0 : if (filter_) {
386 0 : filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
387 0 : frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput()));
388 : }
389 0 : if (crop_.w || crop_.h) {
390 0 : frame->pointer()->crop_top = crop_.y;
391 0 : frame->pointer()->crop_bottom = (size_t) frame->height() - crop_.y - crop_.h;
392 0 : frame->pointer()->crop_left = crop_.x;
393 0 : frame->pointer()->crop_right = (size_t) frame->width() - crop_.x - crop_.w;
394 0 : av_frame_apply_cropping(frame->pointer(), AV_FRAME_CROP_UNALIGNED);
395 : }
396 0 : return frame;
397 0 : }
398 :
399 : void
400 8920 : SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/, const std::shared_ptr<MediaFrame>& frame_p)
401 : {
402 : #ifdef DEBUG_FPS
403 : auto currentTime = std::chrono::steady_clock::now();
404 : auto seconds = currentTime - lastFrameDebug_;
405 : ++frameCount_;
406 : if (seconds > std::chrono::seconds(1)) {
407 : auto fps = frameCount_ / std::chrono::duration<double>(seconds).count();
408 : JAMI_WARNING("Sink {}, {} FPS", id_, fps);
409 : frameCount_ = 0;
410 : lastFrameDebug_ = currentTime;
411 : }
412 : #endif
413 :
414 8920 : std::unique_lock lock(mtx_);
415 8920 : bool hasObservers = getObserversCount() != 0;
416 8920 : bool hasDirectListener = target_.push and not target_.pull;
417 8920 : bool hasTransformedListener = target_.push and target_.pull;
418 :
419 8920 : if (hasDirectListener) {
420 0 : notify(frame_p);
421 0 : auto outFrame = configureFrameDirect(frame_p);
422 0 : if (!outFrame) {
423 0 : return;
424 : }
425 0 : if (outFrame->height != height_ || outFrame->width != width_) {
426 0 : lock.unlock();
427 0 : setFrameSize(outFrame->width, outFrame->height);
428 0 : return;
429 : }
430 0 : target_.push(std::move(outFrame));
431 0 : return;
432 0 : }
433 :
434 8920 : bool doTransfer = hasTransformedListener or hasObservers;
435 : #ifdef ENABLE_SHM
436 : doTransfer |= (shm_ && doShmTransfer_);
437 : #endif
438 :
439 8920 : if (doTransfer) {
440 0 : auto frame = applyTransform(*std::static_pointer_cast<VideoFrame>(frame_p));
441 0 : if (not frame)
442 0 : return;
443 :
444 0 : notify(std::static_pointer_cast<MediaFrame>(frame));
445 :
446 0 : if (frame->height() != height_ || frame->width() != width_) {
447 0 : lock.unlock();
448 0 : setFrameSize(frame->width(), frame->height());
449 0 : return;
450 : }
451 : #ifdef ENABLE_SHM
452 : if (shm_ && doShmTransfer_)
453 : shm_->renderFrame(*frame);
454 : #endif
455 0 : if (hasTransformedListener)
456 0 : sendFrameTransformed(frame->pointer());
457 0 : }
458 8920 : }
459 :
460 : void
461 412 : SinkClient::setFrameSize(int width, int height)
462 : {
463 412 : width_ = width;
464 412 : height_ = height;
465 412 : if (width > 0 and height > 0) {
466 356 : JAMI_LOG("[Sink:{}] Started - size={}x{}, mixer={}", fmt::ptr(this), width, height, mixer_ ? "Yes" : "No");
467 89 : emitSignal<libjami::VideoSignal::DecodingStarted>(getId(), openedName(), width, height, mixer_);
468 89 : started_ = true;
469 412 : } else if (started_) {
470 352 : JAMI_LOG("[Sink:{}] Stopped - size={}x{}, mixer={}", fmt::ptr(this), width, height, mixer_ ? "Yes" : "No");
471 88 : emitSignal<libjami::VideoSignal::DecodingStopped>(getId(), openedName(), mixer_);
472 88 : started_ = false;
473 : }
474 412 : }
475 :
476 : void
477 274 : SinkClient::setCrop(int x, int y, int w, int h)
478 : {
479 274 : if (x != crop_.x || y != crop_.y || w != crop_.w || h != crop_.h) {
480 0 : JAMI_LOG("[Sink:{}] Change crop to [{}x{} at ({}, {})]", fmt::ptr(this), w, h, x, y);
481 : }
482 274 : crop_.x = x;
483 274 : crop_.y = y;
484 274 : crop_.w = w;
485 274 : crop_.h = h;
486 274 : }
487 :
488 : } // namespace video
489 : } // namespace jami
|
