Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2025 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software: you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation, either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #ifdef HAVE_CONFIG_H
      19             : #include "config.h"
      20             : #endif
      21             : 
      22             : #include "sinkclient.h"
      23             : 
      24             : #ifdef ENABLE_SHM
      25             : #include "shm_header.h"
      26             : #endif // ENABLE_SHM
      27             : 
      28             : #include "media_buffer.h"
      29             : #include "logger.h"
      30             : #include "noncopyable.h"
      31             : #include "client/ring_signal.h"
      32             : #include "jami/videomanager_interface.h"
      33             : #include "libav_utils.h"
      34             : #include "video_scaler.h"
      35             : #include "media_filter.h"
      36             : #include "filter_transpose.h"
      37             : 
      38             : #ifdef ENABLE_HWACCEL
      39             : #include "accel.h"
      40             : #endif
      41             : 
      42             : #ifndef _WIN32
      43             : #include <sys/mman.h>
      44             : #endif
      45             : #include <ciso646> // fix windows compiler bug
      46             : #include <fcntl.h>
      47             : #include <cstdio>
      48             : #include <sstream>
      49             : #include <unistd.h>
      50             : #include <cerrno>
      51             : #include <cstring>
      52             : #include <stdexcept>
      53             : #include <cmath>
      54             : 
      55             : namespace jami {
      56             : namespace video {
      57             : 
      58             : const constexpr char FILTER_INPUT_NAME[] = "in";
      59             : 
      60             : #ifdef ENABLE_SHM
      61             : // RAII class helper on sem_wait/sem_post sempahore operations
      62             : class SemGuardLock
      63             : {
      64             : public:
      65             :     explicit SemGuardLock(sem_t& mutex)
      66             :         : m_(mutex)
      67             :     {
      68             :         auto ret = ::sem_wait(&m_);
      69             :         if (ret < 0) {
      70             :             throw std::logic_error {fmt::format("SHM mutex@{} lock failed ({})", fmt::ptr(&m_), ret)};
      71             :         }
      72             :     }
      73             : 
      74             :     ~SemGuardLock() { ::sem_post(&m_); }
      75             : 
      76             : private:
      77             :     sem_t& m_;
      78             : };
      79             : 
      80             : class ShmHolder
      81             : {
      82             : public:
      83             :     ShmHolder(const std::string& name = {});
      84             :     ~ShmHolder();
      85             : 
      86             :     std::string name() const noexcept { return openedName_; }
      87             : 
      88             :     void renderFrame(const VideoFrame& src) noexcept;
      89             : 
      90             : private:
      91             :     bool resizeArea(std::size_t desired_length) noexcept;
      92             :     char* getShmAreaDataPtr() noexcept;
      93             : 
      94             :     void unMapShmArea() noexcept
      95             :     {
      96             :         if (area_ != MAP_FAILED and ::munmap(area_, areaSize_) < 0) {
      97             :             JAMI_ERR("[ShmHolder:%s] munmap(%zu) failed with errno %d", openedName_.c_str(), areaSize_, errno);
      98             :         }
      99             :     }
     100             : 
     101             :     SHMHeader* area_ {static_cast<SHMHeader*>(MAP_FAILED)};
     102             :     std::size_t areaSize_ {0};
     103             :     std::string openedName_;
     104             :     int fd_ {-1};
     105             : };
     106             : 
     107             : ShmHolder::ShmHolder(const std::string& name)
     108             : {
     109             :     static constexpr int flags = O_RDWR | O_CREAT | O_TRUNC | O_EXCL;
     110             :     static constexpr int perms = S_IRUSR | S_IWUSR;
     111             : 
     112             :     static auto shmFailedWithErrno = [this](const std::string& what) {
     113             :         throw std::runtime_error {fmt::format("ShmHolder[{}]: {} failed, errno={}", openedName_, what, errno)};
     114             :     };
     115             : 
     116             :     if (not name.empty()) {
     117             :         openedName_ = name;
     118             :         fd_ = ::shm_open(openedName_.c_str(), flags, perms);
     119             :         if (fd_ < 0)
     120             :             shmFailedWithErrno("shm_open");
     121             :     } else {
     122             :         for (int i = 0; fd_ < 0; ++i) {
     123             :             openedName_ = fmt::format(PACKAGE_NAME "_shm_{}_{}", getpid(), i);
     124             :             fd_ = ::shm_open(openedName_.c_str(), flags, perms);
     125             :             if (fd_ < 0 and errno != EEXIST)
     126             :                 shmFailedWithErrno("shm_open");
     127             :         }
     128             :     }
     129             : 
     130             :     // Set size enough for header only (no frame data)
     131             :     if (!resizeArea(0))
     132             :         shmFailedWithErrno("resizeArea");
     133             : 
     134             :     // Header fields initialization
     135             :     std::memset(area_, 0, areaSize_);
     136             : 
     137             :     if (::sem_init(&area_->mutex, 1, 1) < 0)
     138             :         shmFailedWithErrno("sem_init(mutex)");
     139             : 
     140             :     if (::sem_init(&area_->frameGenMutex, 1, 0) < 0)
     141             :         shmFailedWithErrno("sem_init(frameGenMutex)");
     142             : 
     143             :     JAMI_DBG("[ShmHolder:%s] New holder created", openedName_.c_str());
     144             : }
     145             : 
     146             : ShmHolder::~ShmHolder()
     147             : {
     148             :     if (fd_ < 0)
     149             :         return;
     150             : 
     151             :     ::close(fd_);
     152             :     ::shm_unlink(openedName_.c_str());
     153             : 
     154             :     if (area_ == MAP_FAILED)
     155             :         return;
     156             : 
     157             :     ::sem_wait(&area_->mutex);
     158             :     area_->frameSize = 0;
     159             :     ::sem_post(&area_->mutex);
     160             : 
     161             :     ::sem_post(&area_->frameGenMutex); // unlock waiting client before leaving
     162             :     unMapShmArea();
     163             : }
     164             : 
     165             : bool
     166             : ShmHolder::resizeArea(std::size_t frameSize) noexcept
     167             : {
     168             :     // aligned on 16-byte boundary frameSize
     169             :     frameSize = (frameSize + 15) & ~15;
     170             : 
     171             :     if (area_ != MAP_FAILED and frameSize == area_->frameSize)
     172             :         return true;
     173             : 
     174             :     // full area size: +15 to take care of maximum padding size
     175             :     const auto areaSize = sizeof(SHMHeader) + 2 * frameSize + 15;
     176             :     JAMI_DBG("[ShmHolder:%s] New size: f=%zu, a=%zu", openedName_.c_str(), frameSize, areaSize);
     177             : 
     178             :     unMapShmArea();
     179             : 
     180             :     if (::ftruncate(fd_, areaSize) < 0) {
     181             :         JAMI_ERR("[ShmHolder:%s] ftruncate(%zu) failed with errno %d", openedName_.c_str(), areaSize, errno);
     182             :         return false;
     183             :     }
     184             : 
     185             :     area_ = static_cast<SHMHeader*>(::mmap(nullptr, areaSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
     186             : 
     187             :     if (area_ == MAP_FAILED) {
     188             :         areaSize_ = 0;
     189             :         JAMI_ERR("[ShmHolder:%s] mmap(%zu) failed with errno %d", openedName_.c_str(), areaSize, errno);
     190             :         return false;
     191             :     }
     192             : 
     193             :     areaSize_ = areaSize;
     194             : 
     195             :     if (frameSize) {
     196             :         SemGuardLock lk {area_->mutex};
     197             : 
     198             :         area_->frameSize = frameSize;
     199             :         area_->mapSize = areaSize;
     200             : 
     201             :         // Compute aligned IO pointers
     202             :         // Note: we not using std::align as not implemented in 4.9
     203             :         // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57350
     204             :         auto p = reinterpret_cast<std::uintptr_t>(area_->data);
     205             :         area_->writeOffset = ((p + 15) & ~15) - p;
     206             :         area_->readOffset = area_->writeOffset + frameSize;
     207             :     }
     208             : 
     209             :     return true;
     210             : }
     211             : 
     212             : void
     213             : ShmHolder::renderFrame(const VideoFrame& src) noexcept
     214             : {
     215             :     const auto width = src.width();
     216             :     const auto height = src.height();
     217             :     const auto format = AV_PIX_FMT_BGRA;
     218             :     const auto frameSize = videoFrameSize(format, width, height);
     219             : 
     220             :     if (!resizeArea(frameSize)) {
     221             :         JAMI_ERR("[ShmHolder:%s] Unable to resize area size: %dx%d, format: %d",
     222             :                  openedName_.c_str(),
     223             :                  width,
     224             :                  height,
     225             :                  format);
     226             :         return;
     227             :     }
     228             : 
     229             :     {
     230             :         VideoFrame dst;
     231             :         VideoScaler scaler;
     232             : 
     233             :         dst.setFromMemory(area_->data + area_->writeOffset, format, width, height);
     234             :         scaler.scale(src, dst);
     235             :     }
     236             : 
     237             :     {
     238             :         SemGuardLock lk {area_->mutex};
     239             : 
     240             :         ++area_->frameGen;
     241             :         std::swap(area_->readOffset, area_->writeOffset);
     242             :         ::sem_post(&area_->frameGenMutex);
     243             :     }
     244             : }
     245             : 
     246             : std::string
     247             : SinkClient::openedName() const noexcept
     248             : {
     249             :     if (shm_)
     250             :         return shm_->name();
     251             :     return {};
     252             : }
     253             : 
     254             : bool
     255             : SinkClient::start() noexcept
     256             : {
     257             :     if (not shm_) {
     258             :         try {
     259             :             char* envvar = getenv("JAMI_DISABLE_SHM");
     260             :             if (envvar) // Do not use SHM if set
     261             :                 return true;
     262             :             shm_ = std::make_shared<ShmHolder>();
     263             :             JAMI_DBG("[Sink:%p] Shared memory [%s] created", this, openedName().c_str());
     264             :         } catch (const std::runtime_error& e) {
     265             :             JAMI_ERR("[Sink:%p] Failed to create shared memory: %s", this, e.what());
     266             :         }
     267             :     }
     268             : 
     269             :     return static_cast<bool>(shm_);
     270             : }
     271             : 
     272             : bool
     273             : SinkClient::stop() noexcept
     274             : {
     275             :     setFrameSize(0, 0);
     276             :     setCrop(0, 0, 0, 0);
     277             :     shm_.reset();
     278             :     return true;
     279             : }
     280             : 
     281             : #else // ENABLE_SHM
     282             : 
     283             : std::string
     284         175 : SinkClient::openedName() const noexcept
     285             : {
     286         175 :     return {};
     287             : }
     288             : 
     289             : bool
     290         250 : SinkClient::start() noexcept
     291             : {
     292         250 :     return true;
     293             : }
     294             : 
     295             : bool
     296         282 : SinkClient::stop() noexcept
     297             : {
     298         282 :     setFrameSize(0, 0);
     299         282 :     setCrop(0, 0, 0, 0);
     300         282 :     return true;
     301             : }
     302             : 
     303             : #endif // !ENABLE_SHM
     304             : 
     305         223 : SinkClient::SinkClient(const std::string& id, bool mixer)
     306         223 :     : id_ {id}
     307         223 :     , mixer_(mixer)
     308         223 :     , scaler_(new VideoScaler())
     309             : #ifdef DEBUG_FPS
     310             :     , frameCount_(0u)
     311             :     , lastFrameDebug_(std::chrono::steady_clock::now())
     312             : #endif
     313             : {
     314         223 :     JAMI_DBG("[Sink:%p] Sink [%s] created", this, getId().c_str());
     315         223 : }
     316             : 
     317             : libjami::FrameBuffer
     318           0 : SinkClient::configureFrameDirect(const std::shared_ptr<jami::MediaFrame>& frame_p)
     319             : {
     320           0 :     libjami::FrameBuffer outFrame(av_frame_alloc());
     321           0 :     av_frame_ref(outFrame.get(), std::static_pointer_cast<VideoFrame>(frame_p)->pointer());
     322             : 
     323           0 :     if (crop_.w || crop_.h) {
     324             : #ifdef ENABLE_HWACCEL
     325           0 :         auto desc = av_pix_fmt_desc_get((AVPixelFormat) std::static_pointer_cast<VideoFrame>(frame_p)->format());
     326             :         /*
     327             :          Cropping does not work for hardware-decoded frames.
     328             :          They need to be transferred to main memory.
     329             :          */
     330           0 :         if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
     331           0 :             std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
     332             :             try {
     333           0 :                 frame = HardwareAccel::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(frame_p),
     334           0 :                                                             AV_PIX_FMT_NV12);
     335           0 :             } catch (const std::runtime_error& e) {
     336           0 :                 JAMI_ERR("[Sink:%p] Transfert to hardware acceleration memory failed: %s", this, e.what());
     337           0 :                 return {};
     338           0 :             }
     339           0 :             if (not frame)
     340           0 :                 return {};
     341           0 :             av_frame_unref(outFrame.get());
     342           0 :             av_frame_ref(outFrame.get(), frame->pointer());
     343           0 :         }
     344             : #endif
     345           0 :         outFrame->crop_top = crop_.y;
     346           0 :         outFrame->crop_bottom = (size_t) outFrame->height - crop_.y - crop_.h;
     347           0 :         outFrame->crop_left = crop_.x;
     348           0 :         outFrame->crop_right = (size_t) outFrame->width - crop_.x - crop_.w;
     349           0 :         av_frame_apply_cropping(outFrame.get(), AV_FRAME_CROP_UNALIGNED);
     350             :     }
     351           0 :     return outFrame;
     352           0 : }
     353             : 
     354             : void
     355           0 : SinkClient::sendFrameTransformed(AVFrame* frame)
     356             : {
     357           0 :     if (frame->width > 0 and frame->height > 0) {
     358           0 :         if (auto buffer_ptr = target_.pull()) {
     359           0 :             scaler_->scale(frame, buffer_ptr.get());
     360           0 :             target_.push(std::move(buffer_ptr));
     361           0 :         }
     362             :     }
     363           0 : }
     364             : 
     365             : std::shared_ptr<VideoFrame>
     366           0 : SinkClient::applyTransform(VideoFrame& frame_p)
     367             : {
     368           0 :     std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
     369             : #ifdef ENABLE_HWACCEL
     370           0 :     auto desc = av_pix_fmt_desc_get((AVPixelFormat) frame_p.format());
     371           0 :     if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
     372             :         try {
     373           0 :             frame = HardwareAccel::transferToMainMemory(frame_p, AV_PIX_FMT_NV12);
     374           0 :         } catch (const std::runtime_error& e) {
     375           0 :             JAMI_ERR("[Sink:%p] Transfert to hardware acceleration memory failed: %s", this, e.what());
     376           0 :             return {};
     377           0 :         }
     378           0 :     } else
     379             : #endif
     380           0 :         frame->copyFrom(frame_p);
     381             : 
     382           0 :     int angle = frame->getOrientation();
     383           0 :     if (angle != rotation_) {
     384           0 :         filter_ = getTransposeFilter(angle, FILTER_INPUT_NAME, frame->width(), frame->height(), frame->format(), false);
     385           0 :         rotation_ = angle;
     386             :     }
     387           0 :     if (filter_) {
     388           0 :         filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
     389           0 :         frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput()));
     390             :     }
     391           0 :     if (crop_.w || crop_.h) {
     392           0 :         frame->pointer()->crop_top = crop_.y;
     393           0 :         frame->pointer()->crop_bottom = (size_t) frame->height() - crop_.y - crop_.h;
     394           0 :         frame->pointer()->crop_left = crop_.x;
     395           0 :         frame->pointer()->crop_right = (size_t) frame->width() - crop_.x - crop_.w;
     396           0 :         av_frame_apply_cropping(frame->pointer(), AV_FRAME_CROP_UNALIGNED);
     397             :     }
     398           0 :     return frame;
     399           0 : }
     400             : 
     401             : void
     402        9046 : SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/, const std::shared_ptr<MediaFrame>& frame_p)
     403             : {
     404             : #ifdef DEBUG_FPS
     405             :     auto currentTime = std::chrono::steady_clock::now();
     406             :     auto seconds = currentTime - lastFrameDebug_;
     407             :     ++frameCount_;
     408             :     if (seconds > std::chrono::seconds(1)) {
     409             :         auto fps = frameCount_ / std::chrono::duration<double>(seconds).count();
     410             :         JAMI_WARNING("Sink {}, {} FPS", id_, fps);
     411             :         frameCount_ = 0;
     412             :         lastFrameDebug_ = currentTime;
     413             :     }
     414             : #endif
     415             : 
     416        9046 :     std::unique_lock lock(mtx_);
     417        9046 :     bool hasObservers = getObserversCount() != 0;
     418        9046 :     bool hasDirectListener = target_.push and not target_.pull;
     419        9046 :     bool hasTransformedListener = target_.push and target_.pull;
     420             : 
     421        9046 :     if (hasDirectListener) {
     422           0 :         notify(frame_p);
     423           0 :         auto outFrame = configureFrameDirect(frame_p);
     424           0 :         if (!outFrame) {
     425           0 :             return;
     426             :         }
     427           0 :         if (outFrame->height != height_ || outFrame->width != width_) {
     428           0 :             lock.unlock();
     429           0 :             setFrameSize(outFrame->width, outFrame->height);
     430           0 :             return;
     431             :         }
     432           0 :         target_.push(std::move(outFrame));
     433           0 :         return;
     434           0 :     }
     435             : 
     436        9046 :     bool doTransfer = hasTransformedListener or hasObservers;
     437             : #ifdef ENABLE_SHM
     438             :     doTransfer |= (shm_ && doShmTransfer_);
     439             : #endif
     440             : 
     441        9046 :     if (doTransfer) {
     442           0 :         auto frame = applyTransform(*std::static_pointer_cast<VideoFrame>(frame_p));
     443           0 :         if (not frame)
     444           0 :             return;
     445             : 
     446           0 :         notify(std::static_pointer_cast<MediaFrame>(frame));
     447             : 
     448           0 :         if (frame->height() != height_ || frame->width() != width_) {
     449           0 :             lock.unlock();
     450           0 :             setFrameSize(frame->width(), frame->height());
     451           0 :             return;
     452             :         }
     453             : #ifdef ENABLE_SHM
     454             :         if (shm_ && doShmTransfer_)
     455             :             shm_->renderFrame(*frame);
     456             : #endif
     457           0 :         if (hasTransformedListener)
     458           0 :             sendFrameTransformed(frame->pointer());
     459           0 :     }
     460        9046 : }
     461             : 
     462             : void
     463         420 : SinkClient::setFrameSize(int width, int height)
     464             : {
     465         420 :     width_ = width;
     466         420 :     height_ = height;
     467         420 :     if (width > 0 and height > 0) {
     468          88 :         JAMI_DBG("[Sink:%p] Started - size=%dx%d, mixer=%s", this, width, height, mixer_ ? "Yes" : "No");
     469          88 :         emitSignal<libjami::VideoSignal::DecodingStarted>(getId(), openedName(), width, height, mixer_);
     470          88 :         started_ = true;
     471         332 :     } else if (started_) {
     472          87 :         JAMI_DBG("[Sink:%p] Stopped - size=%dx%d, mixer=%s", this, width, height, mixer_ ? "Yes" : "No");
     473          87 :         emitSignal<libjami::VideoSignal::DecodingStopped>(getId(), openedName(), mixer_);
     474          87 :         started_ = false;
     475             :     }
     476         420 : }
     477             : 
     478             : void
     479         282 : SinkClient::setCrop(int x, int y, int w, int h)
     480             : {
     481         282 :     JAMI_DBG("[Sink:%p] Change crop to [%dx%d at (%d, %d)]", this, w, h, x, y);
     482         282 :     crop_.x = x;
     483         282 :     crop_.y = y;
     484         282 :     crop_.w = w;
     485         282 :     crop_.h = h;
     486         282 : }
     487             : 
     488             : } // namespace video
     489             : } // namespace jami