Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2025 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software: you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation, either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #ifdef HAVE_CONFIG_H
      19             : #include "config.h"
      20             : #endif
      21             : 
      22             : #include "sinkclient.h"
      23             : 
      24             : #ifdef ENABLE_SHM
      25             : #include "shm_header.h"
      26             : #endif // ENABLE_SHM
      27             : 
      28             : #include "media_buffer.h"
      29             : #include "logger.h"
      30             : #include "noncopyable.h"
      31             : #include "client/ring_signal.h"
      32             : #include "jami/videomanager_interface.h"
      33             : #include "libav_utils.h"
      34             : #include "video_scaler.h"
      35             : #include "media_filter.h"
      36             : #include "filter_transpose.h"
      37             : 
      38             : #ifdef ENABLE_HWACCEL
      39             : #include "accel.h"
      40             : #endif
      41             : 
      42             : #ifndef _WIN32
      43             : #include <sys/mman.h>
      44             : #endif
      45             : #include <ciso646> // fix windows compiler bug
      46             : #include <fcntl.h>
      47             : #include <cstdio>
      48             : #include <sstream>
      49             : #include <unistd.h>
      50             : #include <cerrno>
      51             : #include <cstring>
      52             : #include <stdexcept>
      53             : #include <cmath>
      54             : 
      55             : namespace jami {
      56             : namespace video {
      57             : 
      58             : const constexpr char FILTER_INPUT_NAME[] = "in";
      59             : 
      60             : #ifdef ENABLE_SHM
      61             : // RAII class helper on sem_wait/sem_post sempahore operations
      62             : class SemGuardLock
      63             : {
      64             : public:
      65             :     explicit SemGuardLock(sem_t& mutex)
      66             :         : m_(mutex)
      67             :     {
      68             :         auto ret = ::sem_wait(&m_);
      69             :         if (ret < 0) {
      70             :             throw std::logic_error {fmt::format("SHM mutex@{} lock failed ({})", fmt::ptr(&m_), ret)};
      71             :         }
      72             :     }
      73             : 
      74             :     ~SemGuardLock() { ::sem_post(&m_); }
      75             : 
      76             : private:
      77             :     sem_t& m_;
      78             : };
      79             : 
      80             : class ShmHolder
      81             : {
      82             : public:
      83             :     ShmHolder(const std::string& name = {});
      84             :     ~ShmHolder();
      85             : 
      86             :     std::string name() const noexcept { return openedName_; }
      87             : 
      88             :     void renderFrame(const VideoFrame& src) noexcept;
      89             : 
      90             : private:
      91             :     bool resizeArea(std::size_t desired_length) noexcept;
      92             :     char* getShmAreaDataPtr() noexcept;
      93             : 
      94             :     void unMapShmArea() noexcept
      95             :     {
      96             :         if (area_ != MAP_FAILED and ::munmap(area_, areaSize_) < 0) {
      97             :             JAMI_ERR("[ShmHolder:%s] munmap(%zu) failed with errno %d",
      98             :                      openedName_.c_str(),
      99             :                      areaSize_,
     100             :                      errno);
     101             :         }
     102             :     }
     103             : 
     104             :     SHMHeader* area_ {static_cast<SHMHeader*>(MAP_FAILED)};
     105             :     std::size_t areaSize_ {0};
     106             :     std::string openedName_;
     107             :     int fd_ {-1};
     108             : };
     109             : 
     110             : ShmHolder::ShmHolder(const std::string& name)
     111             : {
     112             :     static constexpr int flags = O_RDWR | O_CREAT | O_TRUNC | O_EXCL;
     113             :     static constexpr int perms = S_IRUSR | S_IWUSR;
     114             : 
     115             :     static auto shmFailedWithErrno = [this](const std::string& what) {
     116             :         throw std::runtime_error {fmt::format("ShmHolder[{}]: {} failed, errno={}", openedName_, what, errno)};
     117             :     };
     118             : 
     119             :     if (not name.empty()) {
     120             :         openedName_ = name;
     121             :         fd_ = ::shm_open(openedName_.c_str(), flags, perms);
     122             :         if (fd_ < 0)
     123             :             shmFailedWithErrno("shm_open");
     124             :     } else {
     125             :         for (int i = 0; fd_ < 0; ++i) {
     126             :             openedName_ = fmt::format(PACKAGE_NAME "_shm_{}_{}", getpid(), i);
     127             :             fd_ = ::shm_open(openedName_.c_str(), flags, perms);
     128             :             if (fd_ < 0 and errno != EEXIST)
     129             :                 shmFailedWithErrno("shm_open");
     130             :         }
     131             :     }
     132             : 
     133             :     // Set size enough for header only (no frame data)
     134             :     if (!resizeArea(0))
     135             :         shmFailedWithErrno("resizeArea");
     136             : 
     137             :     // Header fields initialization
     138             :     std::memset(area_, 0, areaSize_);
     139             : 
     140             :     if (::sem_init(&area_->mutex, 1, 1) < 0)
     141             :         shmFailedWithErrno("sem_init(mutex)");
     142             : 
     143             :     if (::sem_init(&area_->frameGenMutex, 1, 0) < 0)
     144             :         shmFailedWithErrno("sem_init(frameGenMutex)");
     145             : 
     146             :     JAMI_DBG("[ShmHolder:%s] New holder created", openedName_.c_str());
     147             : }
     148             : 
     149             : ShmHolder::~ShmHolder()
     150             : {
     151             :     if (fd_ < 0)
     152             :         return;
     153             : 
     154             :     ::close(fd_);
     155             :     ::shm_unlink(openedName_.c_str());
     156             : 
     157             :     if (area_ == MAP_FAILED)
     158             :         return;
     159             : 
     160             :     ::sem_wait(&area_->mutex);
     161             :     area_->frameSize = 0;
     162             :     ::sem_post(&area_->mutex);
     163             : 
     164             :     ::sem_post(&area_->frameGenMutex); // unlock waiting client before leaving
     165             :     unMapShmArea();
     166             : }
     167             : 
     168             : bool
     169             : ShmHolder::resizeArea(std::size_t frameSize) noexcept
     170             : {
     171             :     // aligned on 16-byte boundary frameSize
     172             :     frameSize = (frameSize + 15) & ~15;
     173             : 
     174             :     if (area_ != MAP_FAILED and frameSize == area_->frameSize)
     175             :         return true;
     176             : 
     177             :     // full area size: +15 to take care of maximum padding size
     178             :     const auto areaSize = sizeof(SHMHeader) + 2 * frameSize + 15;
     179             :     JAMI_DBG("[ShmHolder:%s] New size: f=%zu, a=%zu", openedName_.c_str(), frameSize, areaSize);
     180             : 
     181             :     unMapShmArea();
     182             : 
     183             :     if (::ftruncate(fd_, areaSize) < 0) {
     184             :         JAMI_ERR("[ShmHolder:%s] ftruncate(%zu) failed with errno %d",
     185             :                  openedName_.c_str(),
     186             :                  areaSize,
     187             :                  errno);
     188             :         return false;
     189             :     }
     190             : 
     191             :     area_ = static_cast<SHMHeader*>(
     192             :         ::mmap(nullptr, areaSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
     193             : 
     194             :     if (area_ == MAP_FAILED) {
     195             :         areaSize_ = 0;
     196             :         JAMI_ERR("[ShmHolder:%s] mmap(%zu) failed with errno %d",
     197             :                  openedName_.c_str(),
     198             :                  areaSize,
     199             :                  errno);
     200             :         return false;
     201             :     }
     202             : 
     203             :     areaSize_ = areaSize;
     204             : 
     205             :     if (frameSize) {
     206             :         SemGuardLock lk {area_->mutex};
     207             : 
     208             :         area_->frameSize = frameSize;
     209             :         area_->mapSize = areaSize;
     210             : 
     211             :         // Compute aligned IO pointers
     212             :         // Note: we not using std::align as not implemented in 4.9
     213             :         // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57350
     214             :         auto p = reinterpret_cast<std::uintptr_t>(area_->data);
     215             :         area_->writeOffset = ((p + 15) & ~15) - p;
     216             :         area_->readOffset = area_->writeOffset + frameSize;
     217             :     }
     218             : 
     219             :     return true;
     220             : }
     221             : 
     222             : void
     223             : ShmHolder::renderFrame(const VideoFrame& src) noexcept
     224             : {
     225             :     const auto width = src.width();
     226             :     const auto height = src.height();
     227             :     const auto format = AV_PIX_FMT_BGRA;
     228             :     const auto frameSize = videoFrameSize(format, width, height);
     229             : 
     230             :     if (!resizeArea(frameSize)) {
     231             :         JAMI_ERR("[ShmHolder:%s] Unable to resize area size: %dx%d, format: %d",
     232             :                  openedName_.c_str(),
     233             :                  width,
     234             :                  height,
     235             :                  format);
     236             :         return;
     237             :     }
     238             : 
     239             :     {
     240             :         VideoFrame dst;
     241             :         VideoScaler scaler;
     242             : 
     243             :         dst.setFromMemory(area_->data + area_->writeOffset, format, width, height);
     244             :         scaler.scale(src, dst);
     245             :     }
     246             : 
     247             :     {
     248             :         SemGuardLock lk {area_->mutex};
     249             : 
     250             :         ++area_->frameGen;
     251             :         std::swap(area_->readOffset, area_->writeOffset);
     252             :         ::sem_post(&area_->frameGenMutex);
     253             :     }
     254             : }
     255             : 
     256             : std::string
     257             : SinkClient::openedName() const noexcept
     258             : {
     259             :     if (shm_)
     260             :         return shm_->name();
     261             :     return {};
     262             : }
     263             : 
     264             : bool
     265             : SinkClient::start() noexcept
     266             : {
     267             :     if (not shm_) {
     268             :         try {
     269             :             char* envvar = getenv("JAMI_DISABLE_SHM");
     270             :             if (envvar) // Do not use SHM if set
     271             :                 return true;
     272             :             shm_ = std::make_shared<ShmHolder>();
     273             :             JAMI_DBG("[Sink:%p] Shared memory [%s] created", this, openedName().c_str());
     274             :         } catch (const std::runtime_error& e) {
     275             :             JAMI_ERR("[Sink:%p] Failed to create shared memory: %s", this, e.what());
     276             :         }
     277             :     }
     278             : 
     279             :     return static_cast<bool>(shm_);
     280             : }
     281             : 
     282             : bool
     283             : SinkClient::stop() noexcept
     284             : {
     285             :     setFrameSize(0, 0);
     286             :     setCrop(0, 0, 0, 0);
     287             :     shm_.reset();
     288             :     return true;
     289             : }
     290             : 
     291             : #else // ENABLE_SHM
     292             : 
     293             : std::string
     294         198 : SinkClient::openedName() const noexcept
     295             : {
     296         198 :     return {};
     297             : }
     298             : 
     299             : bool
     300         276 : SinkClient::start() noexcept
     301             : {
     302         276 :     return true;
     303             : }
     304             : 
     305             : bool
     306         299 : SinkClient::stop() noexcept
     307             : {
     308         299 :     setFrameSize(0, 0);
     309         299 :     setCrop(0, 0, 0, 0);
     310         299 :     return true;
     311             : }
     312             : 
     313             : #endif // !ENABLE_SHM
     314             : 
     315         245 : SinkClient::SinkClient(const std::string& id, bool mixer)
     316         245 :     : id_ {id}
     317         245 :     , mixer_(mixer)
     318         245 :     , scaler_(new VideoScaler())
     319             : #ifdef DEBUG_FPS
     320             :     , frameCount_(0u)
     321             :     , lastFrameDebug_(std::chrono::steady_clock::now())
     322             : #endif
     323             : {
     324         245 :     JAMI_DBG("[Sink:%p] Sink [%s] created", this, getId().c_str());
     325         245 : }
     326             : 
     327             : void
     328           0 : SinkClient::sendFrameDirect(const std::shared_ptr<jami::MediaFrame>& frame_p)
     329             : {
     330           0 :     notify(frame_p);
     331             : 
     332           0 :     libjami::FrameBuffer outFrame(av_frame_alloc());
     333           0 :     av_frame_ref(outFrame.get(), std::static_pointer_cast<VideoFrame>(frame_p)->pointer());
     334             : 
     335           0 :     if (crop_.w || crop_.h) {
     336             : #ifdef ENABLE_HWACCEL
     337           0 :         auto desc = av_pix_fmt_desc_get(
     338           0 :             (AVPixelFormat) std::static_pointer_cast<VideoFrame>(frame_p)->format());
     339             :         /*
     340             :          Cropping does not work for hardware-decoded frames.
     341             :          They need to be transferred to main memory.
     342             :          */
     343           0 :         if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
     344           0 :             std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
     345             :             try {
     346           0 :                 frame = HardwareAccel::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(
     347           0 :                                                                 frame_p),
     348           0 :                                                             AV_PIX_FMT_NV12);
     349           0 :             } catch (const std::runtime_error& e) {
     350           0 :                 JAMI_ERR("[Sink:%p] Transfert to hardware acceleration memory failed: %s",
     351             :                          this,
     352             :                          e.what());
     353           0 :                 return;
     354           0 :             }
     355           0 :             if (not frame)
     356           0 :                 return;
     357           0 :             av_frame_unref(outFrame.get());
     358           0 :             av_frame_ref(outFrame.get(), frame->pointer());
     359           0 :         }
     360             : #endif
     361           0 :         outFrame->crop_top = crop_.y;
     362           0 :         outFrame->crop_bottom = (size_t) outFrame->height - crop_.y - crop_.h;
     363           0 :         outFrame->crop_left = crop_.x;
     364           0 :         outFrame->crop_right = (size_t) outFrame->width - crop_.x - crop_.w;
     365           0 :         av_frame_apply_cropping(outFrame.get(), AV_FRAME_CROP_UNALIGNED);
     366             :     }
     367           0 :     if (outFrame->height != height_ || outFrame->width != width_) {
     368           0 :         setFrameSize(outFrame->width, outFrame->height);
     369           0 :         return;
     370             :     }
     371           0 :     target_.push(std::move(outFrame));
     372           0 : }
     373             : 
     374             : void
     375           0 : SinkClient::sendFrameTransformed(AVFrame* frame)
     376             : {
     377           0 :     if (frame->width > 0 and frame->height > 0) {
     378           0 :         if (auto buffer_ptr = target_.pull()) {
     379           0 :             scaler_->scale(frame, buffer_ptr.get());
     380           0 :             target_.push(std::move(buffer_ptr));
     381           0 :         }
     382             :     }
     383           0 : }
     384             : 
     385             : std::shared_ptr<VideoFrame>
     386           4 : SinkClient::applyTransform(VideoFrame& frame_p)
     387             : {
     388           4 :     std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
     389             : #ifdef ENABLE_HWACCEL
     390           4 :     auto desc = av_pix_fmt_desc_get((AVPixelFormat) frame_p.format());
     391           4 :     if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
     392             :         try {
     393           0 :             frame = HardwareAccel::transferToMainMemory(frame_p, AV_PIX_FMT_NV12);
     394           0 :         } catch (const std::runtime_error& e) {
     395           0 :             JAMI_ERR("[Sink:%p] Transfert to hardware acceleration memory failed: %s",
     396             :                      this,
     397             :                      e.what());
     398           0 :             return {};
     399           0 :         }
     400           0 :     } else
     401             : #endif
     402           4 :         frame->copyFrom(frame_p);
     403             : 
     404           4 :     int angle = frame->getOrientation();
     405           4 :     if (angle != rotation_) {
     406           0 :         filter_ = getTransposeFilter(angle,
     407             :                                      FILTER_INPUT_NAME,
     408             :                                      frame->width(),
     409             :                                      frame->height(),
     410             :                                      frame->format(),
     411           0 :                                      false);
     412           0 :         rotation_ = angle;
     413             :     }
     414           4 :     if (filter_) {
     415           0 :         filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
     416           0 :         frame = std::static_pointer_cast<VideoFrame>(
     417           0 :             std::shared_ptr<MediaFrame>(filter_->readOutput()));
     418             :     }
     419           4 :     if (crop_.w || crop_.h) {
     420           0 :         frame->pointer()->crop_top = crop_.y;
     421           0 :         frame->pointer()->crop_bottom = (size_t) frame->height() - crop_.y - crop_.h;
     422           0 :         frame->pointer()->crop_left = crop_.x;
     423           0 :         frame->pointer()->crop_right = (size_t) frame->width() - crop_.x - crop_.w;
     424           0 :         av_frame_apply_cropping(frame->pointer(), AV_FRAME_CROP_UNALIGNED);
     425             :     }
     426           4 :     return frame;
     427           4 : }
     428             : 
     429             : void
     430        9291 : SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
     431             :                    const std::shared_ptr<MediaFrame>& frame_p)
     432             : {
     433             : #ifdef DEBUG_FPS
     434             :     auto currentTime = std::chrono::steady_clock::now();
     435             :     auto seconds = currentTime - lastFrameDebug_;
     436             :     ++frameCount_;
     437             :     if (seconds > std::chrono::seconds(1)) {
     438             :         auto fps = frameCount_ / std::chrono::duration<double>(seconds).count();
     439             :         JAMI_WARNING("Sink {}, {} FPS", id_, fps);
     440             :         frameCount_ = 0;
     441             :         lastFrameDebug_ = currentTime;
     442             :     }
     443             : #endif
     444             : 
     445        9291 :     std::unique_lock lock(mtx_);
     446        9291 :     bool hasObservers = getObserversCount() != 0;
     447        9291 :     bool hasDirectListener = target_.push and not target_.pull;
     448        9291 :     bool hasTransformedListener = target_.push and target_.pull;
     449             : 
     450        9291 :     if (hasDirectListener) {
     451           0 :         sendFrameDirect(frame_p);
     452           0 :         return;
     453             :     }
     454             : 
     455        9291 :     bool doTransfer = hasTransformedListener or hasObservers;
     456             : #ifdef ENABLE_SHM
     457             :     doTransfer |= (shm_ && doShmTransfer_);
     458             : #endif
     459             : 
     460        9291 :     if (doTransfer) {
     461           4 :         auto frame = applyTransform(*std::static_pointer_cast<VideoFrame>(frame_p));
     462           4 :         if (not frame)
     463           0 :             return;
     464             : 
     465           4 :         notify(std::static_pointer_cast<MediaFrame>(frame));
     466             : 
     467           4 :         if (frame->height() != height_ || frame->width() != width_) {
     468           1 :             lock.unlock();
     469           1 :             setFrameSize(frame->width(), frame->height());
     470           1 :             return;
     471             :         }
     472             : #ifdef ENABLE_SHM
     473             :         if (shm_ && doShmTransfer_)
     474             :             shm_->renderFrame(*frame);
     475             : #endif
     476           3 :         if (hasTransformedListener)
     477           0 :             sendFrameTransformed(frame->pointer());
     478           4 :     }
     479        9291 : }
     480             : 
     481             : void
     482         452 : SinkClient::setFrameSize(int width, int height)
     483             : {
     484         452 :     width_ = width;
     485         452 :     height_ = height;
     486         452 :     if (width > 0 and height > 0) {
     487         102 :         JAMI_DBG("[Sink:%p] Started - size=%dx%d, mixer=%s",
     488             :                  this,
     489             :                  width,
     490             :                  height,
     491             :                  mixer_ ? "Yes" : "No");
     492         102 :         emitSignal<libjami::VideoSignal::DecodingStarted>(getId(),
     493         102 :                                                           openedName(),
     494             :                                                           width,
     495             :                                                           height,
     496         102 :                                                           mixer_);
     497         102 :         started_ = true;
     498         350 :     } else if (started_) {
     499          96 :         JAMI_DBG("[Sink:%p] Stopped - size=%dx%d, mixer=%s",
     500             :                  this,
     501             :                  width,
     502             :                  height,
     503             :                  mixer_ ? "Yes" : "No");
     504          96 :         emitSignal<libjami::VideoSignal::DecodingStopped>(getId(), openedName(), mixer_);
     505          96 :         started_ = false;
     506             :     }
     507         452 : }
     508             : 
     509             : void
     510         306 : SinkClient::setCrop(int x, int y, int w, int h)
     511             : {
     512         306 :     JAMI_DBG("[Sink:%p] Change crop to [%dx%d at (%d, %d)]", this, w, h, x, y);
     513         306 :     crop_.x = x;
     514         306 :     crop_.y = y;
     515         306 :     crop_.w = w;
     516         306 :     crop_.h = h;
     517         306 : }
     518             : 
     519             : } // namespace video
     520             : } // namespace jami