Line data Source code
1 : /*
2 : * Copyright (C) 2004-2024 Savoir-faire Linux Inc.
3 : *
4 : * Author: Tristan Matthews <tristan.matthews@savoirfairelinux.com>
5 : * Author: Guillaume Roguez <guillaume.roguez@savoirfairelinux.com>
6 : * Author: Alexandre Lision <alexandre.lision@savoirfairelinux.com>
7 : *
8 : * This program is free software; you can redistribute it and/or modify
9 : * it under the terms of the GNU General Public License as published by
10 : * the Free Software Foundation; either version 3 of the License, or
11 : * (at your option) any later version.
12 : *
13 : * This program is distributed in the hope that it will be useful,
14 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 : * GNU General Public License for more details.
17 : *
18 : * You should have received a copy of the GNU General Public License
19 : * along with this program; if not, write to the Free Software
20 : * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 : */
22 :
23 : #ifdef HAVE_CONFIG_H
24 : #include "config.h"
25 : #endif
26 :
27 : #include "sinkclient.h"
28 :
29 : #ifdef ENABLE_SHM
30 : #include "shm_header.h"
31 : #endif // ENABLE_SHM
32 :
33 : #include "media_buffer.h"
34 : #include "logger.h"
35 : #include "noncopyable.h"
36 : #include "client/ring_signal.h"
37 : #include "jami/videomanager_interface.h"
38 : #include "libav_utils.h"
39 : #include "video_scaler.h"
40 : #include "media_filter.h"
41 : #include "filter_transpose.h"
42 :
43 : #ifdef RING_ACCEL
44 : #include "accel.h"
45 : #endif
46 :
47 : #ifndef _WIN32
48 : #include <sys/mman.h>
49 : #endif
50 : #include <ciso646> // fix windows compiler bug
51 : #include <fcntl.h>
52 : #include <cstdio>
53 : #include <sstream>
54 : #include <unistd.h>
55 : #include <cerrno>
56 : #include <cstring>
57 : #include <stdexcept>
58 : #include <cmath>
59 :
60 : namespace jami {
61 : namespace video {
62 :
63 : const constexpr char FILTER_INPUT_NAME[] = "in";
64 :
65 : #ifdef ENABLE_SHM
66 : // RAII class helper on sem_wait/sem_post sempahore operations
67 : class SemGuardLock
68 : {
69 : public:
70 0 : explicit SemGuardLock(sem_t& mutex)
71 0 : : m_(mutex)
72 : {
73 0 : auto ret = ::sem_wait(&m_);
74 0 : if (ret < 0) {
75 0 : std::ostringstream msg;
76 0 : msg << "SHM mutex@" << &m_ << " lock failed (" << ret << ")";
77 0 : throw std::logic_error {msg.str()};
78 0 : }
79 0 : }
80 :
81 0 : ~SemGuardLock() { ::sem_post(&m_); }
82 :
83 : private:
84 : sem_t& m_;
85 : };
86 :
87 : class ShmHolder
88 : {
89 : public:
90 : ShmHolder(const std::string& name = {});
91 : ~ShmHolder();
92 :
93 398 : std::string name() const noexcept { return openedName_; }
94 :
95 : void renderFrame(const VideoFrame& src) noexcept;
96 :
97 : private:
98 : bool resizeArea(std::size_t desired_length) noexcept;
99 : char* getShmAreaDataPtr() noexcept;
100 :
101 470 : void unMapShmArea() noexcept
102 : {
103 470 : if (area_ != MAP_FAILED and ::munmap(area_, areaSize_) < 0) {
104 0 : JAMI_ERR("[ShmHolder:%s] munmap(%zu) failed with errno %d",
105 : openedName_.c_str(),
106 : areaSize_,
107 : errno);
108 : }
109 470 : }
110 :
111 : SHMHeader* area_ {static_cast<SHMHeader*>(MAP_FAILED)};
112 : std::size_t areaSize_ {0};
113 : std::string openedName_;
114 : int fd_ {-1};
115 : };
116 :
117 235 : ShmHolder::ShmHolder(const std::string& name)
118 : {
119 : static constexpr int flags = O_RDWR | O_CREAT | O_TRUNC | O_EXCL;
120 : static constexpr int perms = S_IRUSR | S_IWUSR;
121 :
122 0 : static auto shmFailedWithErrno = [this](const std::string& what) {
123 0 : std::ostringstream msg;
124 0 : msg << "ShmHolder[" << openedName_ << "]: " << what << " failed, errno=" << errno;
125 0 : throw std::runtime_error {msg.str()};
126 235 : };
127 :
128 235 : if (not name.empty()) {
129 0 : openedName_ = name;
130 0 : fd_ = ::shm_open(openedName_.c_str(), flags, perms);
131 0 : if (fd_ < 0)
132 0 : shmFailedWithErrno("shm_open");
133 : } else {
134 645 : for (int i = 0; fd_ < 0; ++i) {
135 410 : std::ostringstream tmpName;
136 410 : tmpName << PACKAGE_NAME << "_shm_" << getpid() << "_" << i;
137 410 : openedName_ = tmpName.str();
138 410 : fd_ = ::shm_open(openedName_.c_str(), flags, perms);
139 410 : if (fd_ < 0 and errno != EEXIST)
140 0 : shmFailedWithErrno("shm_open");
141 410 : }
142 : }
143 :
144 : // Set size enough for header only (no frame data)
145 235 : if (!resizeArea(0))
146 0 : shmFailedWithErrno("resizeArea");
147 :
148 : // Header fields initialization
149 235 : std::memset(area_, 0, areaSize_);
150 :
151 235 : if (::sem_init(&area_->mutex, 1, 1) < 0)
152 0 : shmFailedWithErrno("sem_init(mutex)");
153 :
154 235 : if (::sem_init(&area_->frameGenMutex, 1, 0) < 0)
155 0 : shmFailedWithErrno("sem_init(frameGenMutex)");
156 :
157 235 : JAMI_DBG("[ShmHolder:%s] New holder created", openedName_.c_str());
158 235 : }
159 :
160 470 : ShmHolder::~ShmHolder()
161 : {
162 235 : if (fd_ < 0)
163 0 : return;
164 :
165 235 : ::close(fd_);
166 235 : ::shm_unlink(openedName_.c_str());
167 :
168 235 : if (area_ == MAP_FAILED)
169 0 : return;
170 :
171 235 : ::sem_wait(&area_->mutex);
172 235 : area_->frameSize = 0;
173 235 : ::sem_post(&area_->mutex);
174 :
175 235 : ::sem_post(&area_->frameGenMutex); // unlock waiting client before leaving
176 235 : unMapShmArea();
177 235 : }
178 :
179 : bool
180 235 : ShmHolder::resizeArea(std::size_t frameSize) noexcept
181 : {
182 : // aligned on 16-byte boundary frameSize
183 235 : frameSize = (frameSize + 15) & ~15;
184 :
185 235 : if (area_ != MAP_FAILED and frameSize == area_->frameSize)
186 0 : return true;
187 :
188 : // full area size: +15 to take care of maximum padding size
189 235 : const auto areaSize = sizeof(SHMHeader) + 2 * frameSize + 15;
190 235 : JAMI_DBG("[ShmHolder:%s] New size: f=%zu, a=%zu", openedName_.c_str(), frameSize, areaSize);
191 :
192 235 : unMapShmArea();
193 :
194 235 : if (::ftruncate(fd_, areaSize) < 0) {
195 0 : JAMI_ERR("[ShmHolder:%s] ftruncate(%zu) failed with errno %d",
196 : openedName_.c_str(),
197 : areaSize,
198 : errno);
199 0 : return false;
200 : }
201 :
202 235 : area_ = static_cast<SHMHeader*>(
203 235 : ::mmap(nullptr, areaSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
204 :
205 235 : if (area_ == MAP_FAILED) {
206 0 : areaSize_ = 0;
207 0 : JAMI_ERR("[ShmHolder:%s] mmap(%zu) failed with errno %d",
208 : openedName_.c_str(),
209 : areaSize,
210 : errno);
211 0 : return false;
212 : }
213 :
214 235 : areaSize_ = areaSize;
215 :
216 235 : if (frameSize) {
217 0 : SemGuardLock lk {area_->mutex};
218 :
219 0 : area_->frameSize = frameSize;
220 0 : area_->mapSize = areaSize;
221 :
222 : // Compute aligned IO pointers
223 : // Note: we not using std::align as not implemented in 4.9
224 : // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57350
225 0 : auto p = reinterpret_cast<std::uintptr_t>(area_->data);
226 0 : area_->writeOffset = ((p + 15) & ~15) - p;
227 0 : area_->readOffset = area_->writeOffset + frameSize;
228 0 : }
229 :
230 235 : return true;
231 : }
232 :
233 : void
234 0 : ShmHolder::renderFrame(const VideoFrame& src) noexcept
235 : {
236 0 : const auto width = src.width();
237 0 : const auto height = src.height();
238 0 : const auto format = AV_PIX_FMT_BGRA;
239 0 : const auto frameSize = videoFrameSize(format, width, height);
240 :
241 0 : if (!resizeArea(frameSize)) {
242 0 : JAMI_ERR("[ShmHolder:%s] Could not resize area size: %dx%d, format: %d",
243 : openedName_.c_str(),
244 : width,
245 : height,
246 : format);
247 0 : return;
248 : }
249 :
250 : {
251 0 : VideoFrame dst;
252 0 : VideoScaler scaler;
253 :
254 0 : dst.setFromMemory(area_->data + area_->writeOffset, format, width, height);
255 0 : scaler.scale(src, dst);
256 0 : }
257 :
258 : {
259 0 : SemGuardLock lk {area_->mutex};
260 :
261 0 : ++area_->frameGen;
262 0 : std::swap(area_->readOffset, area_->writeOffset);
263 0 : ::sem_post(&area_->frameGenMutex);
264 0 : }
265 : }
266 :
267 : std::string
268 398 : SinkClient::openedName() const noexcept
269 : {
270 398 : if (shm_)
271 398 : return shm_->name();
272 0 : return {};
273 : }
274 :
275 : bool
276 245 : SinkClient::start() noexcept
277 : {
278 245 : if (not shm_) {
279 : try {
280 235 : char* envvar = getenv("JAMI_DISABLE_SHM");
281 235 : if (envvar) // Do not use SHM if set
282 0 : return true;
283 235 : shm_ = std::make_shared<ShmHolder>();
284 235 : JAMI_DBG("[Sink:%p] Shared memory [%s] created", this, openedName().c_str());
285 0 : } catch (const std::runtime_error& e) {
286 0 : JAMI_ERR("[Sink:%p] Failed to create shared memory: %s", this, e.what());
287 0 : }
288 : }
289 :
290 245 : return static_cast<bool>(shm_);
291 : }
292 :
293 : bool
294 268 : SinkClient::stop() noexcept
295 : {
296 268 : setFrameSize(0, 0);
297 268 : setCrop(0, 0, 0, 0);
298 268 : shm_.reset();
299 268 : return true;
300 : }
301 :
302 : #else // ENABLE_SHM
303 :
304 : std::string
305 : SinkClient::openedName() const noexcept
306 : {
307 : return {};
308 : }
309 :
310 : bool
311 : SinkClient::start() noexcept
312 : {
313 : return true;
314 : }
315 :
316 : bool
317 : SinkClient::stop() noexcept
318 : {
319 : setFrameSize(0, 0);
320 : setCrop(0, 0, 0, 0);
321 : return true;
322 : }
323 :
324 : #endif // !ENABLE_SHM
325 :
326 232 : SinkClient::SinkClient(const std::string& id, bool mixer)
327 232 : : id_ {id}
328 232 : , mixer_(mixer)
329 232 : , scaler_(new VideoScaler())
330 : #ifdef DEBUG_FPS
331 : , frameCount_(0u)
332 : , lastFrameDebug_(std::chrono::steady_clock::now())
333 : #endif
334 : {
335 232 : JAMI_DBG("[Sink:%p] Sink [%s] created", this, getId().c_str());
336 232 : }
337 :
338 : void
339 0 : SinkClient::sendFrameDirect(const std::shared_ptr<jami::MediaFrame>& frame_p)
340 : {
341 0 : notify(frame_p);
342 :
343 0 : libjami::FrameBuffer outFrame(av_frame_alloc());
344 0 : av_frame_ref(outFrame.get(), std::static_pointer_cast<VideoFrame>(frame_p)->pointer());
345 :
346 0 : if (crop_.w || crop_.h) {
347 : #ifdef RING_ACCEL
348 0 : auto desc = av_pix_fmt_desc_get(
349 0 : (AVPixelFormat) std::static_pointer_cast<VideoFrame>(frame_p)->format());
350 : /*
351 : Cropping does not work for hardware-decoded frames.
352 : They need to be transferred to main memory.
353 : */
354 0 : if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
355 0 : std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
356 : try {
357 0 : frame = HardwareAccel::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(
358 0 : frame_p),
359 0 : AV_PIX_FMT_NV12);
360 0 : } catch (const std::runtime_error& e) {
361 0 : JAMI_ERR("[Sink:%p] Transfert to hardware acceleration memory failed: %s",
362 : this,
363 : e.what());
364 0 : return;
365 0 : }
366 0 : if (not frame)
367 0 : return;
368 0 : av_frame_unref(outFrame.get());
369 0 : av_frame_ref(outFrame.get(), frame->pointer());
370 0 : }
371 : #endif
372 0 : outFrame->crop_top = crop_.y;
373 0 : outFrame->crop_bottom = (size_t) outFrame->height - crop_.y - crop_.h;
374 0 : outFrame->crop_left = crop_.x;
375 0 : outFrame->crop_right = (size_t) outFrame->width - crop_.x - crop_.w;
376 0 : av_frame_apply_cropping(outFrame.get(), AV_FRAME_CROP_UNALIGNED);
377 : }
378 0 : if (outFrame->height != height_ || outFrame->width != width_) {
379 0 : setFrameSize(outFrame->width, outFrame->height);
380 0 : return;
381 : }
382 0 : target_.push(std::move(outFrame));
383 0 : }
384 :
385 : void
386 0 : SinkClient::sendFrameTransformed(AVFrame* frame)
387 : {
388 0 : if (frame->width > 0 and frame->height > 0) {
389 0 : if (auto buffer_ptr = target_.pull()) {
390 0 : scaler_->scale(frame, buffer_ptr.get());
391 0 : target_.push(std::move(buffer_ptr));
392 0 : }
393 : }
394 0 : }
395 :
396 : std::shared_ptr<VideoFrame>
397 0 : SinkClient::applyTransform(VideoFrame& frame_p)
398 : {
399 0 : std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
400 : #ifdef RING_ACCEL
401 0 : auto desc = av_pix_fmt_desc_get((AVPixelFormat) frame_p.format());
402 0 : if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
403 : try {
404 0 : frame = HardwareAccel::transferToMainMemory(frame_p, AV_PIX_FMT_NV12);
405 0 : } catch (const std::runtime_error& e) {
406 0 : JAMI_ERR("[Sink:%p] Transfert to hardware acceleration memory failed: %s",
407 : this,
408 : e.what());
409 0 : return {};
410 0 : }
411 0 : } else
412 : #endif
413 0 : frame->copyFrom(frame_p);
414 :
415 0 : int angle = frame->getOrientation();
416 0 : if (angle != rotation_) {
417 0 : filter_ = getTransposeFilter(angle,
418 : FILTER_INPUT_NAME,
419 : frame->width(),
420 : frame->height(),
421 : frame->format(),
422 0 : false);
423 0 : rotation_ = angle;
424 : }
425 0 : if (filter_) {
426 0 : filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
427 0 : frame = std::static_pointer_cast<VideoFrame>(
428 0 : std::shared_ptr<MediaFrame>(filter_->readOutput()));
429 : }
430 0 : if (crop_.w || crop_.h) {
431 0 : frame->pointer()->crop_top = crop_.y;
432 0 : frame->pointer()->crop_bottom = (size_t) frame->height() - crop_.y - crop_.h;
433 0 : frame->pointer()->crop_left = crop_.x;
434 0 : frame->pointer()->crop_right = (size_t) frame->width() - crop_.x - crop_.w;
435 0 : av_frame_apply_cropping(frame->pointer(), AV_FRAME_CROP_UNALIGNED);
436 : }
437 0 : return frame;
438 0 : }
439 :
440 : void
441 8645 : SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
442 : const std::shared_ptr<MediaFrame>& frame_p)
443 : {
444 : #ifdef DEBUG_FPS
445 : auto currentTime = std::chrono::steady_clock::now();
446 : auto seconds = currentTime - lastFrameDebug_;
447 : ++frameCount_;
448 : if (seconds > std::chrono::seconds(1)) {
449 : auto fps = frameCount_ / std::chrono::duration<double>(seconds).count();
450 : JAMI_WARNING("Sink {}, {} FPS", id_, fps);
451 : frameCount_ = 0;
452 : lastFrameDebug_ = currentTime;
453 : }
454 : #endif
455 :
456 8645 : std::unique_lock lock(mtx_);
457 8645 : bool hasObservers = getObserversCount() != 0;
458 8645 : bool hasDirectListener = target_.push and not target_.pull;
459 8645 : bool hasTransformedListener = target_.push and target_.pull;
460 :
461 8645 : if (hasDirectListener) {
462 0 : sendFrameDirect(frame_p);
463 0 : return;
464 : }
465 :
466 8645 : bool doTransfer = hasTransformedListener or hasObservers;
467 : #ifdef ENABLE_SHM
468 8645 : doTransfer |= (shm_ && doShmTransfer_);
469 : #endif
470 :
471 8645 : if (doTransfer) {
472 0 : auto frame = applyTransform(*std::static_pointer_cast<VideoFrame>(frame_p));
473 0 : if (not frame)
474 0 : return;
475 :
476 0 : notify(std::static_pointer_cast<MediaFrame>(frame));
477 :
478 0 : if (frame->height() != height_ || frame->width() != width_) {
479 0 : lock.unlock();
480 0 : setFrameSize(frame->width(), frame->height());
481 0 : return;
482 : }
483 : #ifdef ENABLE_SHM
484 0 : if (shm_ && doShmTransfer_)
485 0 : shm_->renderFrame(*frame);
486 : #endif
487 0 : if (hasTransformedListener)
488 0 : sendFrameTransformed(frame->pointer());
489 0 : }
490 8645 : }
491 :
492 : void
493 404 : SinkClient::setFrameSize(int width, int height)
494 : {
495 404 : width_ = width;
496 404 : height_ = height;
497 404 : if (width > 0 and height > 0) {
498 84 : JAMI_DBG("[Sink:%p] Started - size=%dx%d, mixer=%s",
499 : this,
500 : width,
501 : height,
502 : mixer_ ? "Yes" : "No");
503 84 : emitSignal<libjami::VideoSignal::DecodingStarted>(getId(),
504 84 : openedName(),
505 : width,
506 : height,
507 84 : mixer_);
508 84 : started_ = true;
509 320 : } else if (started_) {
510 79 : JAMI_DBG("[Sink:%p] Stopped - size=%dx%d, mixer=%s",
511 : this,
512 : width,
513 : height,
514 : mixer_ ? "Yes" : "No");
515 79 : emitSignal<libjami::VideoSignal::DecodingStopped>(getId(), openedName(), mixer_);
516 79 : started_ = false;
517 : }
518 404 : }
519 :
520 : void
521 275 : SinkClient::setCrop(int x, int y, int w, int h)
522 : {
523 275 : JAMI_DBG("[Sink:%p] Change crop to [%dx%d at (%d, %d)]", this, w, h, x, y);
524 275 : crop_.x = x;
525 275 : crop_.y = y;
526 275 : crop_.w = w;
527 275 : crop_.h = h;
528 275 : }
529 :
530 : } // namespace video
531 : } // namespace jami
|
