Line data    Source code

       1             : /*
       2             :  * Copyright (C) 2004-2026 Savoir-faire Linux Inc.
       3             :  *
       4             :  * This program is free software: you can redistribute it and/or modify
       5             :  * it under the terms of the GNU General Public License as published by
       6             :  * the Free Software Foundation, either version 3 of the License, or
       7             :  * (at your option) any later version.
       8             :  *
       9             :  * This program is distributed in the hope that it will be useful,
      10             :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  * GNU General Public License for more details.
      13             :  *
      14             :  * You should have received a copy of the GNU General Public License
      15             :  * along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #ifdef HAVE_CONFIG_H
      19             : #include "config.h"
      20             : #endif
      21             : 
      22             : #include "sinkclient.h"
      23             : 
      24             : #ifdef ENABLE_SHM
      25             : #include "shm_header.h"
      26             : #endif // ENABLE_SHM
      27             : 
      28             : #include "media_buffer.h"
      29             : #include "logger.h"
      30             : #include "client/jami_signal.h"
      31             : #include "jami/videomanager_interface.h"
      32             : #include "libav_utils.h"
      33             : #include "video_scaler.h"
      34             : #include "media_filter.h"
      35             : #include "filter_transpose.h"
      36             : 
      37             : #ifdef ENABLE_HWACCEL
      38             : #include "accel.h"
      39             : #endif
      40             : 
      41             : #ifndef _WIN32
      42             : #include <sys/mman.h>
      43             : #endif
      44             : 
      45             : #include <fcntl.h>
      46             : #include <cstdio>
      47             : #include <unistd.h>
      48             : #include <cerrno>
      49             : #include <cstring>
      50             : #include <stdexcept>
      51             : #include <cmath>
      52             : 
      53             : namespace jami {
      54             : namespace video {
      55             : 
      56             : const constexpr char FILTER_INPUT_NAME[] = "in";
      57             : 
      58             : #ifdef ENABLE_SHM
      59             : // RAII class helper on sem_wait/sem_post sempahore operations
      60             : class SemGuardLock
      61             : {
      62             : public:
      63             :     explicit SemGuardLock(sem_t& mutex)
      64             :         : m_(mutex)
      65             :     {
      66             :         auto ret = ::sem_wait(&m_);
      67             :         if (ret < 0) {
      68             :             throw std::logic_error {fmt::format("SHM mutex@{} lock failed ({})", fmt::ptr(&m_), ret)};
      69             :         }
      70             :     }
      71             : 
      72             :     ~SemGuardLock() { ::sem_post(&m_); }
      73             : 
      74             : private:
      75             :     sem_t& m_;
      76             : };
      77             : 
      78             : class ShmHolder
      79             : {
      80             : public:
      81             :     ShmHolder(const std::string& name = {});
      82             :     ~ShmHolder();
      83             : 
      84             :     std::string name() const noexcept { return openedName_; }
      85             : 
      86             :     void renderFrame(const VideoFrame& src) noexcept;
      87             : 
      88             : private:
      89             :     bool resizeArea(std::size_t desired_length) noexcept;
      90             :     char* getShmAreaDataPtr() noexcept;
      91             : 
      92             :     void unMapShmArea() noexcept
      93             :     {
      94             :         if (area_ != MAP_FAILED and ::munmap(area_, areaSize_) < 0) {
      95             :             JAMI_ERR("[ShmHolder:%s] munmap(%zu) failed with errno %d", openedName_.c_str(), areaSize_, errno);
      96             :         }
      97             :     }
      98             : 
      99             :     SHMHeader* area_ {static_cast<SHMHeader*>(MAP_FAILED)};
     100             :     std::size_t areaSize_ {0};
     101             :     std::string openedName_;
     102             :     int fd_ {-1};
     103             : };
     104             : 
     105             : ShmHolder::ShmHolder(const std::string& name)
     106             : {
     107             :     static constexpr int flags = O_RDWR | O_CREAT | O_TRUNC | O_EXCL;
     108             :     static constexpr int perms = S_IRUSR | S_IWUSR;
     109             : 
     110             :     static auto shmFailedWithErrno = [this](const std::string& what) {
     111             :         throw std::runtime_error {fmt::format("ShmHolder[{}]: {} failed, errno={}", openedName_, what, errno)};
     112             :     };
     113             : 
     114             :     if (not name.empty()) {
     115             :         openedName_ = name;
     116             :         fd_ = ::shm_open(openedName_.c_str(), flags, perms);
     117             :         if (fd_ < 0)
     118             :             shmFailedWithErrno("shm_open");
     119             :     } else {
     120             :         for (int i = 0; fd_ < 0; ++i) {
     121             :             openedName_ = fmt::format(PACKAGE_NAME "_shm_{}_{}", getpid(), i);
     122             :             fd_ = ::shm_open(openedName_.c_str(), flags, perms);
     123             :             if (fd_ < 0 and errno != EEXIST)
     124             :                 shmFailedWithErrno("shm_open");
     125             :         }
     126             :     }
     127             : 
     128             :     // Set size enough for header only (no frame data)
     129             :     if (!resizeArea(0))
     130             :         shmFailedWithErrno("resizeArea");
     131             : 
     132             :     // Header fields initialization
     133             :     std::memset(area_, 0, areaSize_);
     134             : 
     135             :     if (::sem_init(&area_->mutex, 1, 1) < 0)
     136             :         shmFailedWithErrno("sem_init(mutex)");
     137             : 
     138             :     if (::sem_init(&area_->frameGenMutex, 1, 0) < 0)
     139             :         shmFailedWithErrno("sem_init(frameGenMutex)");
     140             : 
     141             :     JAMI_DBG("[ShmHolder:%s] New holder created", openedName_.c_str());
     142             : }
     143             : 
     144             : ShmHolder::~ShmHolder()
     145             : {
     146             :     if (fd_ < 0)
     147             :         return;
     148             : 
     149             :     ::close(fd_);
     150             :     ::shm_unlink(openedName_.c_str());
     151             : 
     152             :     if (area_ == MAP_FAILED)
     153             :         return;
     154             : 
     155             :     ::sem_wait(&area_->mutex);
     156             :     area_->frameSize = 0;
     157             :     ::sem_post(&area_->mutex);
     158             : 
     159             :     ::sem_post(&area_->frameGenMutex); // unlock waiting client before leaving
     160             :     unMapShmArea();
     161             : }
     162             : 
     163             : bool
     164             : ShmHolder::resizeArea(std::size_t frameSize) noexcept
     165             : {
     166             :     // aligned on 16-byte boundary frameSize
     167             :     frameSize = (frameSize + 15) & ~15;
     168             : 
     169             :     if (area_ != MAP_FAILED and frameSize == area_->frameSize)
     170             :         return true;
     171             : 
     172             :     // full area size: +15 to take care of maximum padding size
     173             :     const auto areaSize = sizeof(SHMHeader) + 2 * frameSize + 15;
     174             :     JAMI_DBG("[ShmHolder:%s] New size: f=%zu, a=%zu", openedName_.c_str(), frameSize, areaSize);
     175             : 
     176             :     unMapShmArea();
     177             : 
     178             :     if (::ftruncate(fd_, areaSize) < 0) {
     179             :         JAMI_ERR("[ShmHolder:%s] ftruncate(%zu) failed with errno %d", openedName_.c_str(), areaSize, errno);
     180             :         return false;
     181             :     }
     182             : 
     183             :     area_ = static_cast<SHMHeader*>(::mmap(nullptr, areaSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
     184             : 
     185             :     if (area_ == MAP_FAILED) {
     186             :         areaSize_ = 0;
     187             :         JAMI_ERR("[ShmHolder:%s] mmap(%zu) failed with errno %d", openedName_.c_str(), areaSize, errno);
     188             :         return false;
     189             :     }
     190             : 
     191             :     areaSize_ = areaSize;
     192             : 
     193             :     if (frameSize) {
     194             :         SemGuardLock lk {area_->mutex};
     195             : 
     196             :         area_->frameSize = frameSize;
     197             :         area_->mapSize = areaSize;
     198             : 
     199             :         // Compute aligned IO pointers
     200             :         // Note: we not using std::align as not implemented in 4.9
     201             :         // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57350
     202             :         auto p = reinterpret_cast<std::uintptr_t>(area_->data);
     203             :         area_->writeOffset = ((p + 15) & ~15) - p;
     204             :         area_->readOffset = area_->writeOffset + frameSize;
     205             :     }
     206             : 
     207             :     return true;
     208             : }
     209             : 
     210             : void
     211             : ShmHolder::renderFrame(const VideoFrame& src) noexcept
     212             : {
     213             :     const auto width = src.width();
     214             :     const auto height = src.height();
     215             :     const auto format = AV_PIX_FMT_BGRA;
     216             :     const auto frameSize = videoFrameSize(format, width, height);
     217             : 
     218             :     if (!resizeArea(frameSize)) {
     219             :         JAMI_ERR("[ShmHolder:%s] Unable to resize area size: %dx%d, format: %d",
     220             :                  openedName_.c_str(),
     221             :                  width,
     222             :                  height,
     223             :                  format);
     224             :         return;
     225             :     }
     226             : 
     227             :     {
     228             :         VideoFrame dst;
     229             :         VideoScaler scaler;
     230             : 
     231             :         dst.setFromMemory(area_->data + area_->writeOffset, format, width, height);
     232             :         scaler.scale(src, dst);
     233             :     }
     234             : 
     235             :     {
     236             :         SemGuardLock lk {area_->mutex};
     237             : 
     238             :         ++area_->frameGen;
     239             :         std::swap(area_->readOffset, area_->writeOffset);
     240             :         ::sem_post(&area_->frameGenMutex);
     241             :     }
     242             : }
     243             : 
     244             : std::string
     245             : SinkClient::openedName() const noexcept
     246             : {
     247             :     if (shm_)
     248             :         return shm_->name();
     249             :     return {};
     250             : }
     251             : 
     252             : bool
     253             : SinkClient::start() noexcept
     254             : {
     255             :     if (not shm_) {
     256             :         try {
     257             :             char* envvar = getenv("JAMI_DISABLE_SHM");
     258             :             if (envvar) // Do not use SHM if set
     259             :                 return true;
     260             :             shm_ = std::make_shared<ShmHolder>();
     261             :             JAMI_DBG("[Sink:%p] Shared memory [%s] created", this, openedName().c_str());
     262             :         } catch (const std::runtime_error& e) {
     263             :             JAMI_ERR("[Sink:%p] Failed to create shared memory: %s", this, e.what());
     264             :         }
     265             :     }
     266             : 
     267             :     return static_cast<bool>(shm_);
     268             : }
     269             : 
     270             : bool
     271             : SinkClient::stop() noexcept
     272             : {
     273             :     setFrameSize(0, 0);
     274             :     setCrop(0, 0, 0, 0);
     275             :     shm_.reset();
     276             :     return true;
     277             : }
     278             : 
     279             : #else // ENABLE_SHM
     280             : 
     281             : std::string
     282         174 : SinkClient::openedName() const noexcept
     283             : {
     284         174 :     return {};
     285             : }
     286             : 
     287             : bool
     288         238 : SinkClient::start() noexcept
     289             : {
     290         238 :     return true;
     291             : }
     292             : 
     293             : bool
     294         270 : SinkClient::stop() noexcept
     295             : {
     296         270 :     setFrameSize(0, 0);
     297         270 :     setCrop(0, 0, 0, 0);
     298         270 :     return true;
     299             : }
     300             : 
     301             : #endif // !ENABLE_SHM
     302             : 
     303         214 : SinkClient::SinkClient(const std::string& id, bool mixer)
     304         214 :     : id_ {id}
     305         214 :     , mixer_(mixer)
     306         214 :     , scaler_(new VideoScaler())
     307             : #ifdef DEBUG_FPS
     308             :     , frameCount_(0u)
     309             :     , lastFrameDebug_(std::chrono::steady_clock::now())
     310             : #endif
     311             : {
     312         214 :     JAMI_DBG("[Sink:%p] Sink [%s] created", this, getId().c_str());
     313         214 : }
     314             : 
     315             : libjami::FrameBuffer
     316           0 : SinkClient::configureFrameDirect(const std::shared_ptr<jami::MediaFrame>& frame_p)
     317             : {
     318           0 :     libjami::FrameBuffer outFrame(av_frame_alloc());
     319           0 :     av_frame_ref(outFrame.get(), std::static_pointer_cast<VideoFrame>(frame_p)->pointer());
     320             : 
     321           0 :     if (crop_.w || crop_.h) {
     322             : #ifdef ENABLE_HWACCEL
     323           0 :         const auto* desc = av_pix_fmt_desc_get((AVPixelFormat) std::static_pointer_cast<VideoFrame>(frame_p)->format());
     324             :         /*
     325             :          Cropping does not work for hardware-decoded frames.
     326             :          They need to be transferred to main memory.
     327             :          */
     328           0 :         if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
     329           0 :             std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
     330             :             try {
     331           0 :                 frame = HardwareAccel::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame_p),
     332           0 :                                                             AV_PIX_FMT_NV12);
     333           0 :             } catch (const std::runtime_error& e) {
     334           0 :                 JAMI_ERR("[Sink:%p] Transfer to hardware acceleration memory failed: %s", this, e.what());
     335           0 :                 return {};
     336           0 :             }
     337           0 :             if (not frame)
     338           0 :                 return {};
     339           0 :             av_frame_unref(outFrame.get());
     340           0 :             av_frame_ref(outFrame.get(), frame->pointer());
     341           0 :         }
     342             : #endif
     343           0 :         outFrame->crop_top = crop_.y;
     344           0 :         outFrame->crop_bottom = (size_t) outFrame->height - crop_.y - crop_.h;
     345           0 :         outFrame->crop_left = crop_.x;
     346           0 :         outFrame->crop_right = (size_t) outFrame->width - crop_.x - crop_.w;
     347           0 :         av_frame_apply_cropping(outFrame.get(), AV_FRAME_CROP_UNALIGNED);
     348             :     }
     349           0 :     return outFrame;
     350           0 : }
     351             : 
     352             : void
     353           0 : SinkClient::sendFrameTransformed(AVFrame* frame)
     354             : {
     355           0 :     if (frame->width > 0 and frame->height > 0) {
     356           0 :         if (auto buffer_ptr = target_.pull()) {
     357           0 :             scaler_->scale(frame, buffer_ptr.get());
     358           0 :             target_.push(std::move(buffer_ptr));
     359           0 :         }
     360             :     }
     361           0 : }
     362             : 
     363             : std::shared_ptr<VideoFrame>
     364           0 : SinkClient::applyTransform(VideoFrame& frame_p)
     365             : {
     366           0 :     std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
     367             : #ifdef ENABLE_HWACCEL
     368           0 :     const auto* desc = av_pix_fmt_desc_get((AVPixelFormat) frame_p.format());
     369           0 :     if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
     370             :         try {
     371           0 :             frame = HardwareAccel::transferToMainMemory(frame_p, AV_PIX_FMT_NV12);
     372           0 :         } catch (const std::runtime_error& e) {
     373           0 :             JAMI_ERR("[Sink:%p] Transfer to hardware acceleration memory failed: %s", this, e.what());
     374           0 :             return {};
     375           0 :         }
     376           0 :     } else
     377             : #endif
     378           0 :         frame->copyFrom(frame_p);
     379             : 
     380           0 :     int angle = frame->getOrientation();
     381           0 :     if (angle != rotation_) {
     382           0 :         filter_ = getTransposeFilter(angle, FILTER_INPUT_NAME, frame->width(), frame->height(), frame->format(), false);
     383           0 :         rotation_ = angle;
     384             :     }
     385           0 :     if (filter_) {
     386           0 :         filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
     387           0 :         frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput()));
     388             :     }
     389           0 :     if (crop_.w || crop_.h) {
     390           0 :         frame->pointer()->crop_top = crop_.y;
     391           0 :         frame->pointer()->crop_bottom = (size_t) frame->height() - crop_.y - crop_.h;
     392           0 :         frame->pointer()->crop_left = crop_.x;
     393           0 :         frame->pointer()->crop_right = (size_t) frame->width() - crop_.x - crop_.w;
     394           0 :         av_frame_apply_cropping(frame->pointer(), AV_FRAME_CROP_UNALIGNED);
     395             :     }
     396           0 :     return frame;
     397           0 : }
     398             : 
     399             : void
     400        8847 : SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/, const std::shared_ptr<MediaFrame>& frame_p)
     401             : {
     402             : #ifdef DEBUG_FPS
     403             :     auto currentTime = std::chrono::steady_clock::now();
     404             :     auto seconds = currentTime - lastFrameDebug_;
     405             :     ++frameCount_;
     406             :     if (seconds > std::chrono::seconds(1)) {
     407             :         auto fps = frameCount_ / std::chrono::duration<double>(seconds).count();
     408             :         JAMI_WARNING("Sink {}, {} FPS", id_, fps);
     409             :         frameCount_ = 0;
     410             :         lastFrameDebug_ = currentTime;
     411             :     }
     412             : #endif
     413             : 
     414        8847 :     std::unique_lock lock(mtx_);
     415        8847 :     bool hasObservers = getObserversCount() != 0;
     416        8847 :     bool hasDirectListener = target_.push and not target_.pull;
     417        8847 :     bool hasTransformedListener = target_.push and target_.pull;
     418             : 
     419        8847 :     if (hasDirectListener) {
     420           0 :         notify(frame_p);
     421           0 :         auto outFrame = configureFrameDirect(frame_p);
     422           0 :         if (!outFrame) {
     423           0 :             return;
     424             :         }
     425           0 :         if (outFrame->height != height_ || outFrame->width != width_) {
     426           0 :             lock.unlock();
     427           0 :             setFrameSize(outFrame->width, outFrame->height);
     428           0 :             return;
     429             :         }
     430           0 :         target_.push(std::move(outFrame));
     431           0 :         return;
     432           0 :     }
     433             : 
     434        8847 :     bool doTransfer = hasTransformedListener or hasObservers;
     435             : #ifdef ENABLE_SHM
     436             :     doTransfer |= (shm_ && doShmTransfer_);
     437             : #endif
     438             : 
     439        8847 :     if (doTransfer) {
     440           0 :         auto frame = applyTransform(*std::static_pointer_cast<VideoFrame>(frame_p));
     441           0 :         if (not frame)
     442           0 :             return;
     443             : 
     444           0 :         notify(std::static_pointer_cast<MediaFrame>(frame));
     445             : 
     446           0 :         if (frame->height() != height_ || frame->width() != width_) {
     447           0 :             lock.unlock();
     448           0 :             setFrameSize(frame->width(), frame->height());
     449           0 :             return;
     450             :         }
     451             : #ifdef ENABLE_SHM
     452             :         if (shm_ && doShmTransfer_)
     453             :             shm_->renderFrame(*frame);
     454             : #endif
     455           0 :         if (hasTransformedListener)
     456           0 :             sendFrameTransformed(frame->pointer());
     457           0 :     }
     458        8847 : }
     459             : 
     460             : void
     461         407 : SinkClient::setFrameSize(int width, int height)
     462             : {
     463         407 :     width_ = width;
     464         407 :     height_ = height;
     465         407 :     if (width > 0 and height > 0) {
     466          87 :         JAMI_DBG("[Sink:%p] Started - size=%dx%d, mixer=%s", this, width, height, mixer_ ? "Yes" : "No");
     467          87 :         emitSignal<libjami::VideoSignal::DecodingStarted>(getId(), openedName(), width, height, mixer_);
     468          87 :         started_ = true;
     469         320 :     } else if (started_) {
     470          87 :         JAMI_DBG("[Sink:%p] Stopped - size=%dx%d, mixer=%s", this, width, height, mixer_ ? "Yes" : "No");
     471          87 :         emitSignal<libjami::VideoSignal::DecodingStopped>(getId(), openedName(), mixer_);
     472          87 :         started_ = false;
     473             :     }
     474         407 : }
     475             : 
     476             : void
     477         270 : SinkClient::setCrop(int x, int y, int w, int h)
     478             : {
     479         270 :     if (x != crop_.x || y != crop_.y || w != crop_.w || h != crop_.h) {
     480           0 :         JAMI_DBG("[Sink:%p] Change crop to [%dx%d at (%d, %d)]", this, w, h, x, y);
     481             :     }
     482         270 :     crop_.x = x;
     483         270 :     crop_.y = y;
     484         270 :     crop_.w = w;
     485         270 :     crop_.h = h;
     486         270 : }
     487             : 
     488             : } // namespace video
     489             : } // namespace jami