Line data Source code
1 : /*
2 : * Copyright (C) 2004-2024 Savoir-faire Linux Inc.
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 : */
17 :
18 : #ifdef HAVE_CONFIG_H
19 : #include "config.h"
20 : #endif
21 :
22 : #include "sinkclient.h"
23 :
24 : #ifdef ENABLE_SHM
25 : #include "shm_header.h"
26 : #endif // ENABLE_SHM
27 :
28 : #include "media_buffer.h"
29 : #include "logger.h"
30 : #include "noncopyable.h"
31 : #include "client/ring_signal.h"
32 : #include "jami/videomanager_interface.h"
33 : #include "libav_utils.h"
34 : #include "video_scaler.h"
35 : #include "media_filter.h"
36 : #include "filter_transpose.h"
37 :
38 : #ifdef RING_ACCEL
39 : #include "accel.h"
40 : #endif
41 :
42 : #ifndef _WIN32
43 : #include <sys/mman.h>
44 : #endif
45 : #include <ciso646> // fix windows compiler bug
46 : #include <fcntl.h>
47 : #include <cstdio>
48 : #include <sstream>
49 : #include <unistd.h>
50 : #include <cerrno>
51 : #include <cstring>
52 : #include <stdexcept>
53 : #include <cmath>
54 :
55 : namespace jami {
56 : namespace video {
57 :
58 : const constexpr char FILTER_INPUT_NAME[] = "in";
59 :
60 : #ifdef ENABLE_SHM
61 : // RAII class helper on sem_wait/sem_post sempahore operations
62 : class SemGuardLock
63 : {
64 : public:
65 0 : explicit SemGuardLock(sem_t& mutex)
66 0 : : m_(mutex)
67 : {
68 0 : auto ret = ::sem_wait(&m_);
69 0 : if (ret < 0) {
70 0 : std::ostringstream msg;
71 0 : msg << "SHM mutex@" << &m_ << " lock failed (" << ret << ")";
72 0 : throw std::logic_error {msg.str()};
73 0 : }
74 0 : }
75 :
76 0 : ~SemGuardLock() { ::sem_post(&m_); }
77 :
78 : private:
79 : sem_t& m_;
80 : };
81 :
82 : class ShmHolder
83 : {
84 : public:
85 : ShmHolder(const std::string& name = {});
86 : ~ShmHolder();
87 :
88 430 : std::string name() const noexcept { return openedName_; }
89 :
90 : void renderFrame(const VideoFrame& src) noexcept;
91 :
92 : private:
93 : bool resizeArea(std::size_t desired_length) noexcept;
94 : char* getShmAreaDataPtr() noexcept;
95 :
96 532 : void unMapShmArea() noexcept
97 : {
98 532 : if (area_ != MAP_FAILED and ::munmap(area_, areaSize_) < 0) {
99 0 : JAMI_ERR("[ShmHolder:%s] munmap(%zu) failed with errno %d",
100 : openedName_.c_str(),
101 : areaSize_,
102 : errno);
103 : }
104 532 : }
105 :
106 : SHMHeader* area_ {static_cast<SHMHeader*>(MAP_FAILED)};
107 : std::size_t areaSize_ {0};
108 : std::string openedName_;
109 : int fd_ {-1};
110 : };
111 :
112 266 : ShmHolder::ShmHolder(const std::string& name)
113 : {
114 : static constexpr int flags = O_RDWR | O_CREAT | O_TRUNC | O_EXCL;
115 : static constexpr int perms = S_IRUSR | S_IWUSR;
116 :
117 0 : static auto shmFailedWithErrno = [this](const std::string& what) {
118 0 : std::ostringstream msg;
119 0 : msg << "ShmHolder[" << openedName_ << "]: " << what << " failed, errno=" << errno;
120 0 : throw std::runtime_error {msg.str()};
121 266 : };
122 :
123 266 : if (not name.empty()) {
124 0 : openedName_ = name;
125 0 : fd_ = ::shm_open(openedName_.c_str(), flags, perms);
126 0 : if (fd_ < 0)
127 0 : shmFailedWithErrno("shm_open");
128 : } else {
129 732 : for (int i = 0; fd_ < 0; ++i) {
130 466 : std::ostringstream tmpName;
131 466 : tmpName << PACKAGE_NAME << "_shm_" << getpid() << "_" << i;
132 466 : openedName_ = tmpName.str();
133 466 : fd_ = ::shm_open(openedName_.c_str(), flags, perms);
134 466 : if (fd_ < 0 and errno != EEXIST)
135 0 : shmFailedWithErrno("shm_open");
136 466 : }
137 : }
138 :
139 : // Set size enough for header only (no frame data)
140 266 : if (!resizeArea(0))
141 0 : shmFailedWithErrno("resizeArea");
142 :
143 : // Header fields initialization
144 266 : std::memset(area_, 0, areaSize_);
145 :
146 266 : if (::sem_init(&area_->mutex, 1, 1) < 0)
147 0 : shmFailedWithErrno("sem_init(mutex)");
148 :
149 266 : if (::sem_init(&area_->frameGenMutex, 1, 0) < 0)
150 0 : shmFailedWithErrno("sem_init(frameGenMutex)");
151 :
152 266 : JAMI_DBG("[ShmHolder:%s] New holder created", openedName_.c_str());
153 266 : }
154 :
155 532 : ShmHolder::~ShmHolder()
156 : {
157 266 : if (fd_ < 0)
158 0 : return;
159 :
160 266 : ::close(fd_);
161 266 : ::shm_unlink(openedName_.c_str());
162 :
163 266 : if (area_ == MAP_FAILED)
164 0 : return;
165 :
166 266 : ::sem_wait(&area_->mutex);
167 266 : area_->frameSize = 0;
168 266 : ::sem_post(&area_->mutex);
169 :
170 266 : ::sem_post(&area_->frameGenMutex); // unlock waiting client before leaving
171 266 : unMapShmArea();
172 266 : }
173 :
174 : bool
175 266 : ShmHolder::resizeArea(std::size_t frameSize) noexcept
176 : {
177 : // aligned on 16-byte boundary frameSize
178 266 : frameSize = (frameSize + 15) & ~15;
179 :
180 266 : if (area_ != MAP_FAILED and frameSize == area_->frameSize)
181 0 : return true;
182 :
183 : // full area size: +15 to take care of maximum padding size
184 266 : const auto areaSize = sizeof(SHMHeader) + 2 * frameSize + 15;
185 266 : JAMI_DBG("[ShmHolder:%s] New size: f=%zu, a=%zu", openedName_.c_str(), frameSize, areaSize);
186 :
187 266 : unMapShmArea();
188 :
189 266 : if (::ftruncate(fd_, areaSize) < 0) {
190 0 : JAMI_ERR("[ShmHolder:%s] ftruncate(%zu) failed with errno %d",
191 : openedName_.c_str(),
192 : areaSize,
193 : errno);
194 0 : return false;
195 : }
196 :
197 266 : area_ = static_cast<SHMHeader*>(
198 266 : ::mmap(nullptr, areaSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
199 :
200 266 : if (area_ == MAP_FAILED) {
201 0 : areaSize_ = 0;
202 0 : JAMI_ERR("[ShmHolder:%s] mmap(%zu) failed with errno %d",
203 : openedName_.c_str(),
204 : areaSize,
205 : errno);
206 0 : return false;
207 : }
208 :
209 266 : areaSize_ = areaSize;
210 :
211 266 : if (frameSize) {
212 0 : SemGuardLock lk {area_->mutex};
213 :
214 0 : area_->frameSize = frameSize;
215 0 : area_->mapSize = areaSize;
216 :
217 : // Compute aligned IO pointers
218 : // Note: we not using std::align as not implemented in 4.9
219 : // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57350
220 0 : auto p = reinterpret_cast<std::uintptr_t>(area_->data);
221 0 : area_->writeOffset = ((p + 15) & ~15) - p;
222 0 : area_->readOffset = area_->writeOffset + frameSize;
223 0 : }
224 :
225 266 : return true;
226 : }
227 :
228 : void
229 0 : ShmHolder::renderFrame(const VideoFrame& src) noexcept
230 : {
231 0 : const auto width = src.width();
232 0 : const auto height = src.height();
233 0 : const auto format = AV_PIX_FMT_BGRA;
234 0 : const auto frameSize = videoFrameSize(format, width, height);
235 :
236 0 : if (!resizeArea(frameSize)) {
237 0 : JAMI_ERR("[ShmHolder:%s] Unable to resize area size: %dx%d, format: %d",
238 : openedName_.c_str(),
239 : width,
240 : height,
241 : format);
242 0 : return;
243 : }
244 :
245 : {
246 0 : VideoFrame dst;
247 0 : VideoScaler scaler;
248 :
249 0 : dst.setFromMemory(area_->data + area_->writeOffset, format, width, height);
250 0 : scaler.scale(src, dst);
251 0 : }
252 :
253 : {
254 0 : SemGuardLock lk {area_->mutex};
255 :
256 0 : ++area_->frameGen;
257 0 : std::swap(area_->readOffset, area_->writeOffset);
258 0 : ::sem_post(&area_->frameGenMutex);
259 0 : }
260 : }
261 :
262 : std::string
263 430 : SinkClient::openedName() const noexcept
264 : {
265 430 : if (shm_)
266 430 : return shm_->name();
267 0 : return {};
268 : }
269 :
270 : bool
271 276 : SinkClient::start() noexcept
272 : {
273 276 : if (not shm_) {
274 : try {
275 266 : char* envvar = getenv("JAMI_DISABLE_SHM");
276 266 : if (envvar) // Do not use SHM if set
277 0 : return true;
278 266 : shm_ = std::make_shared<ShmHolder>();
279 266 : JAMI_DBG("[Sink:%p] Shared memory [%s] created", this, openedName().c_str());
280 0 : } catch (const std::runtime_error& e) {
281 0 : JAMI_ERR("[Sink:%p] Failed to create shared memory: %s", this, e.what());
282 0 : }
283 : }
284 :
285 276 : return static_cast<bool>(shm_);
286 : }
287 :
288 : bool
289 298 : SinkClient::stop() noexcept
290 : {
291 298 : setFrameSize(0, 0);
292 298 : setCrop(0, 0, 0, 0);
293 298 : shm_.reset();
294 298 : return true;
295 : }
296 :
297 : #else // ENABLE_SHM
298 :
299 : std::string
300 : SinkClient::openedName() const noexcept
301 : {
302 : return {};
303 : }
304 :
305 : bool
306 : SinkClient::start() noexcept
307 : {
308 : return true;
309 : }
310 :
311 : bool
312 : SinkClient::stop() noexcept
313 : {
314 : setFrameSize(0, 0);
315 : setCrop(0, 0, 0, 0);
316 : return true;
317 : }
318 :
319 : #endif // !ENABLE_SHM
320 :
321 247 : SinkClient::SinkClient(const std::string& id, bool mixer)
322 247 : : id_ {id}
323 247 : , mixer_(mixer)
324 247 : , scaler_(new VideoScaler())
325 : #ifdef DEBUG_FPS
326 : , frameCount_(0u)
327 : , lastFrameDebug_(std::chrono::steady_clock::now())
328 : #endif
329 : {
330 247 : JAMI_DBG("[Sink:%p] Sink [%s] created", this, getId().c_str());
331 247 : }
332 :
333 : void
334 0 : SinkClient::sendFrameDirect(const std::shared_ptr<jami::MediaFrame>& frame_p)
335 : {
336 0 : notify(frame_p);
337 :
338 0 : libjami::FrameBuffer outFrame(av_frame_alloc());
339 0 : av_frame_ref(outFrame.get(), std::static_pointer_cast<VideoFrame>(frame_p)->pointer());
340 :
341 0 : if (crop_.w || crop_.h) {
342 : #ifdef RING_ACCEL
343 0 : auto desc = av_pix_fmt_desc_get(
344 0 : (AVPixelFormat) std::static_pointer_cast<VideoFrame>(frame_p)->format());
345 : /*
346 : Cropping does not work for hardware-decoded frames.
347 : They need to be transferred to main memory.
348 : */
349 0 : if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
350 0 : std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
351 : try {
352 0 : frame = HardwareAccel::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(
353 0 : frame_p),
354 0 : AV_PIX_FMT_NV12);
355 0 : } catch (const std::runtime_error& e) {
356 0 : JAMI_ERR("[Sink:%p] Transfert to hardware acceleration memory failed: %s",
357 : this,
358 : e.what());
359 0 : return;
360 0 : }
361 0 : if (not frame)
362 0 : return;
363 0 : av_frame_unref(outFrame.get());
364 0 : av_frame_ref(outFrame.get(), frame->pointer());
365 0 : }
366 : #endif
367 0 : outFrame->crop_top = crop_.y;
368 0 : outFrame->crop_bottom = (size_t) outFrame->height - crop_.y - crop_.h;
369 0 : outFrame->crop_left = crop_.x;
370 0 : outFrame->crop_right = (size_t) outFrame->width - crop_.x - crop_.w;
371 0 : av_frame_apply_cropping(outFrame.get(), AV_FRAME_CROP_UNALIGNED);
372 : }
373 0 : if (outFrame->height != height_ || outFrame->width != width_) {
374 0 : setFrameSize(outFrame->width, outFrame->height);
375 0 : return;
376 : }
377 0 : target_.push(std::move(outFrame));
378 0 : }
379 :
380 : void
381 0 : SinkClient::sendFrameTransformed(AVFrame* frame)
382 : {
383 0 : if (frame->width > 0 and frame->height > 0) {
384 0 : if (auto buffer_ptr = target_.pull()) {
385 0 : scaler_->scale(frame, buffer_ptr.get());
386 0 : target_.push(std::move(buffer_ptr));
387 0 : }
388 : }
389 0 : }
390 :
391 : std::shared_ptr<VideoFrame>
392 0 : SinkClient::applyTransform(VideoFrame& frame_p)
393 : {
394 0 : std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
395 : #ifdef RING_ACCEL
396 0 : auto desc = av_pix_fmt_desc_get((AVPixelFormat) frame_p.format());
397 0 : if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
398 : try {
399 0 : frame = HardwareAccel::transferToMainMemory(frame_p, AV_PIX_FMT_NV12);
400 0 : } catch (const std::runtime_error& e) {
401 0 : JAMI_ERR("[Sink:%p] Transfert to hardware acceleration memory failed: %s",
402 : this,
403 : e.what());
404 0 : return {};
405 0 : }
406 0 : } else
407 : #endif
408 0 : frame->copyFrom(frame_p);
409 :
410 0 : int angle = frame->getOrientation();
411 0 : if (angle != rotation_) {
412 0 : filter_ = getTransposeFilter(angle,
413 : FILTER_INPUT_NAME,
414 : frame->width(),
415 : frame->height(),
416 : frame->format(),
417 0 : false);
418 0 : rotation_ = angle;
419 : }
420 0 : if (filter_) {
421 0 : filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
422 0 : frame = std::static_pointer_cast<VideoFrame>(
423 0 : std::shared_ptr<MediaFrame>(filter_->readOutput()));
424 : }
425 0 : if (crop_.w || crop_.h) {
426 0 : frame->pointer()->crop_top = crop_.y;
427 0 : frame->pointer()->crop_bottom = (size_t) frame->height() - crop_.y - crop_.h;
428 0 : frame->pointer()->crop_left = crop_.x;
429 0 : frame->pointer()->crop_right = (size_t) frame->width() - crop_.x - crop_.w;
430 0 : av_frame_apply_cropping(frame->pointer(), AV_FRAME_CROP_UNALIGNED);
431 : }
432 0 : return frame;
433 0 : }
434 :
435 : void
436 8524 : SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
437 : const std::shared_ptr<MediaFrame>& frame_p)
438 : {
439 : #ifdef DEBUG_FPS
440 : auto currentTime = std::chrono::steady_clock::now();
441 : auto seconds = currentTime - lastFrameDebug_;
442 : ++frameCount_;
443 : if (seconds > std::chrono::seconds(1)) {
444 : auto fps = frameCount_ / std::chrono::duration<double>(seconds).count();
445 : JAMI_WARNING("Sink {}, {} FPS", id_, fps);
446 : frameCount_ = 0;
447 : lastFrameDebug_ = currentTime;
448 : }
449 : #endif
450 :
451 8524 : std::unique_lock lock(mtx_);
452 8524 : bool hasObservers = getObserversCount() != 0;
453 8524 : bool hasDirectListener = target_.push and not target_.pull;
454 8524 : bool hasTransformedListener = target_.push and target_.pull;
455 :
456 8524 : if (hasDirectListener) {
457 0 : sendFrameDirect(frame_p);
458 0 : return;
459 : }
460 :
461 8524 : bool doTransfer = hasTransformedListener or hasObservers;
462 : #ifdef ENABLE_SHM
463 8524 : doTransfer |= (shm_ && doShmTransfer_);
464 : #endif
465 :
466 8524 : if (doTransfer) {
467 0 : auto frame = applyTransform(*std::static_pointer_cast<VideoFrame>(frame_p));
468 0 : if (not frame)
469 0 : return;
470 :
471 0 : notify(std::static_pointer_cast<MediaFrame>(frame));
472 :
473 0 : if (frame->height() != height_ || frame->width() != width_) {
474 0 : lock.unlock();
475 0 : setFrameSize(frame->width(), frame->height());
476 0 : return;
477 : }
478 : #ifdef ENABLE_SHM
479 0 : if (shm_ && doShmTransfer_)
480 0 : shm_->renderFrame(*frame);
481 : #endif
482 0 : if (hasTransformedListener)
483 0 : sendFrameTransformed(frame->pointer());
484 0 : }
485 8524 : }
486 :
487 : void
488 437 : SinkClient::setFrameSize(int width, int height)
489 : {
490 437 : width_ = width;
491 437 : height_ = height;
492 437 : if (width > 0 and height > 0) {
493 85 : JAMI_DBG("[Sink:%p] Started - size=%dx%d, mixer=%s",
494 : this,
495 : width,
496 : height,
497 : mixer_ ? "Yes" : "No");
498 85 : emitSignal<libjami::VideoSignal::DecodingStarted>(getId(),
499 85 : openedName(),
500 : width,
501 : height,
502 85 : mixer_);
503 85 : started_ = true;
504 352 : } else if (started_) {
505 79 : JAMI_DBG("[Sink:%p] Stopped - size=%dx%d, mixer=%s",
506 : this,
507 : width,
508 : height,
509 : mixer_ ? "Yes" : "No");
510 79 : emitSignal<libjami::VideoSignal::DecodingStopped>(getId(), openedName(), mixer_);
511 79 : started_ = false;
512 : }
513 437 : }
514 :
515 : void
516 306 : SinkClient::setCrop(int x, int y, int w, int h)
517 : {
518 306 : JAMI_DBG("[Sink:%p] Change crop to [%dx%d at (%d, %d)]", this, w, h, x, y);
519 306 : crop_.x = x;
520 306 : crop_.y = y;
521 306 : crop_.w = w;
522 306 : crop_.h = h;
523 306 : }
524 :
525 : } // namespace video
526 : } // namespace jami
|
