Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2025 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software: you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation, either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : #pragma once
      18             : 
      19             : #ifdef HAVE_CONFIG_H
      20             : #include "config.h"
      21             : #endif
      22             : 
      23             : #include <chrono>
      24             : #include <set>
      25             : #include <string>
      26             : #include <memory>
      27             : #include <vector>
      28             : #include <string_view>
      29             : #include <map>
      30             : #include <functional>
      31             : 
      32             : #include "conference_protocol.h"
      33             : #include "media/audio/audio_input.h"
      34             : #include "media/media_attribute.h"
      35             : #include "media/recordable.h"
      36             : 
      37             : #ifdef ENABLE_PLUGIN
      38             : #include "plugin/streamdata.h"
      39             : #endif
      40             : 
      41             : #ifdef ENABLE_VIDEO
      42             : #include "media/video/sinkclient.h"
      43             : #endif
      44             : 
      45             : #include <json/json.h>
      46             : 
      47             : namespace jami {
      48             : 
      49             : class Call;
      50             : class Account;
      51             : 
      52             : #ifdef ENABLE_VIDEO
      53             : namespace video {
      54             : class VideoMixer;
      55             : }
      56             : #endif
      57             : 
      58             : // info for a stream
      59             : struct ParticipantInfo
      60             : {
      61             :     std::string uri;
      62             :     std::string device;
      63             :     std::string sinkId; // stream ID
      64             :     bool active {false};
      65             :     int x {0};
      66             :     int y {0};
      67             :     int w {0};
      68             :     int h {0};
      69             :     bool videoMuted {false};
      70             :     bool audioLocalMuted {false};
      71             :     bool audioModeratorMuted {false};
      72             :     bool isModerator {false};
      73             :     bool handRaised {false};
      74             :     bool voiceActivity {false};
      75             :     bool recording {false};
      76             : 
      77         617 :     void fromJson(const Json::Value& v)
      78             :     {
      79         617 :         uri = v["uri"].asString();
      80         617 :         device = v["device"].asString();
      81         617 :         sinkId = v["sinkId"].asString();
      82         617 :         active = v["active"].asBool();
      83         617 :         x = v["x"].asInt();
      84         617 :         y = v["y"].asInt();
      85         617 :         w = v["w"].asInt();
      86         617 :         h = v["h"].asInt();
      87         617 :         videoMuted = v["videoMuted"].asBool();
      88         617 :         audioLocalMuted = v["audioLocalMuted"].asBool();
      89         617 :         audioModeratorMuted = v["audioModeratorMuted"].asBool();
      90         617 :         isModerator = v["isModerator"].asBool();
      91         617 :         handRaised = v["handRaised"].asBool();
      92         617 :         voiceActivity = v["voiceActivity"].asBool();
      93         617 :         recording = v["recording"].asBool();
      94         617 :     }
      95             : 
      96         638 :     Json::Value toJson() const
      97             :     {
      98         638 :         Json::Value val;
      99         639 :         val["uri"] = uri;
     100         638 :         val["device"] = device;
     101         640 :         val["sinkId"] = sinkId;
     102         640 :         val["active"] = active;
     103         639 :         val["x"] = x;
     104         638 :         val["y"] = y;
     105         640 :         val["w"] = w;
     106         639 :         val["h"] = h;
     107         640 :         val["videoMuted"] = videoMuted;
     108         638 :         val["audioLocalMuted"] = audioLocalMuted;
     109         640 :         val["audioModeratorMuted"] = audioModeratorMuted;
     110         640 :         val["isModerator"] = isModerator;
     111         640 :         val["handRaised"] = handRaised;
     112         636 :         val["voiceActivity"] = voiceActivity;
     113         637 :         val["recording"] = recording;
     114         640 :         return val;
     115           0 :     }
     116             : 
     117        1062 :     std::map<std::string, std::string> toMap() const
     118             :     {
     119        1062 :         return {{"uri", uri},
     120        1062 :                 {"device", device},
     121        1062 :                 {"sinkId", sinkId},
     122        1062 :                 {"active", active ? "true" : "false"},
     123        2124 :                 {"x", std::to_string(x)},
     124        2124 :                 {"y", std::to_string(y)},
     125        2124 :                 {"w", std::to_string(w)},
     126        2124 :                 {"h", std::to_string(h)},
     127        1062 :                 {"videoMuted", videoMuted ? "true" : "false"},
     128        1062 :                 {"audioLocalMuted", audioLocalMuted ? "true" : "false"},
     129        1062 :                 {"audioModeratorMuted", audioModeratorMuted ? "true" : "false"},
     130        1062 :                 {"isModerator", isModerator ? "true" : "false"},
     131        1062 :                 {"handRaised", handRaised ? "true" : "false"},
     132        1062 :                 {"voiceActivity", voiceActivity ? "true" : "false"},
     133       29736 :                 {"recording", recording ? "true" : "false"}};
     134             :     }
     135             : 
     136           0 :     friend bool operator==(const ParticipantInfo& p1, const ParticipantInfo& p2)
     137             :     {
     138           0 :         return p1.uri == p2.uri and p1.device == p2.device and p1.sinkId == p2.sinkId and p1.active == p2.active
     139           0 :                and p1.x == p2.x and p1.y == p2.y and p1.w == p2.w and p1.h == p2.h and p1.videoMuted == p2.videoMuted
     140           0 :                and p1.audioLocalMuted == p2.audioLocalMuted and p1.audioModeratorMuted == p2.audioModeratorMuted
     141           0 :                and p1.isModerator == p2.isModerator and p1.handRaised == p2.handRaised
     142           0 :                and p1.voiceActivity == p2.voiceActivity and p1.recording == p2.recording;
     143             :     }
     144             : 
     145             :     friend bool operator!=(const ParticipantInfo& p1, const ParticipantInfo& p2) { return !(p1 == p2); }
     146             : };
     147             : 
     148             : struct ConfInfo : public std::vector<ParticipantInfo>
     149             : {
     150             :     int h {0};
     151             :     int w {0};
     152             :     int v {1}; // Supported conference protocol version
     153             :     int layout {0};
     154             : 
     155           0 :     friend bool operator==(const ConfInfo& c1, const ConfInfo& c2)
     156             :     {
     157           0 :         if (c1.h != c2.h or c1.w != c2.w)
     158           0 :             return false;
     159           0 :         if (c1.size() != c2.size())
     160           0 :             return false;
     161             : 
     162           0 :         for (auto& p1 : c1) {
     163           0 :             auto it = std::find_if(c2.begin(), c2.end(), [&p1](const ParticipantInfo& p2) { return p1 == p2; });
     164           0 :             if (it != c2.end())
     165           0 :                 continue;
     166             :             else
     167           0 :                 return false;
     168             :         }
     169           0 :         return true;
     170             :     }
     171             : 
     172           0 :     friend bool operator!=(const ConfInfo& c1, const ConfInfo& c2) { return !(c1 == c2); }
     173             : 
     174             :     std::vector<std::map<std::string, std::string>> toVectorMapStringString() const;
     175             :     std::string toString() const;
     176             : };
     177             : 
     178             : using CallIdSet = std::set<std::string>;
     179             : using clock = std::chrono::steady_clock;
     180             : 
     181             : class Conference : public Recordable, public std::enable_shared_from_this<Conference>
     182             : {
     183             : public:
     184             :     enum class State { ACTIVE_ATTACHED, ACTIVE_DETACHED, HOLD };
     185             : 
     186             :     /**
     187             :      * Constructor for this class, increment static counter
     188             :      */
     189             :     explicit Conference(const std::shared_ptr<Account>&, const std::string& confId = "");
     190             : 
     191             :     /**
     192             :      * Destructor for this class, decrement static counter
     193             :      */
     194             :     ~Conference();
     195             : 
     196             :     /**
     197             :      * Return the conference id
     198             :      */
     199        1883 :     const std::string& getConfId() const { return id_; }
     200             : 
     201         401 :     std::shared_ptr<Account> getAccount() const { return account_.lock(); }
     202             : 
     203             :     std::string getAccountId() const;
     204             : 
     205             :     /**
     206             :      * Return the current conference state
     207             :      */
     208             :     State getState() const;
     209             : 
     210             :     /**
     211             :      * Set conference state
     212             :      */
     213             :     void setState(State state);
     214             : 
     215             :     /**
     216             :      * Set a callback that will be called when the conference will be destroyed
     217             :      */
     218          14 :     void onShutdown(std::function<void(int)> cb) { shutdownCb_ = std::move(cb); }
     219             : 
     220             :     /**
     221             :      * Return a string description of the conference state
     222             :      */
     223         286 :     static constexpr const char* getStateStr(State state)
     224             :     {
     225         286 :         switch (state) {
     226         173 :         case State::ACTIVE_ATTACHED:
     227         173 :             return "ACTIVE_ATTACHED";
     228         113 :         case State::ACTIVE_DETACHED:
     229         113 :             return "ACTIVE_DETACHED";
     230           0 :         case State::HOLD:
     231           0 :             return "HOLD";
     232           0 :         default:
     233           0 :             return "";
     234             :         }
     235             :     }
     236             : 
     237         203 :     const char* getStateStr() const { return getStateStr(confState_); }
     238             : 
     239             :     /**
     240             :      * Set the mute state of the local host
     241             :      */
     242             :     void setLocalHostMuteState(MediaType type, bool muted);
     243             : 
     244             :     /**
     245             :      * Get the mute state of the local host
     246             :      */
     247             :     bool isMediaSourceMuted(MediaType type) const;
     248             : 
     249             :     /**
     250             :      * Process a media change request.
     251             :      * Used to change the media attributes of the host.
     252             :      *
     253             :      * @param remoteMediaList new media list from the remote
     254             :      * @return true on success
     255             :      */
     256             :     bool requestMediaChange(const std::vector<libjami::MediaMap>& mediaList);
     257             : 
     258             :     /**
     259             :      * Process incoming media change request.
     260             :      *
     261             :      * @param callId the call ID
     262             :      * @param remoteMediaList new media list from the remote
     263             :      */
     264             :     void handleMediaChangeRequest(const std::shared_ptr<Call>& call,
     265             :                                   const std::vector<libjami::MediaMap>& remoteMediaList);
     266             : 
     267             :     /**
     268             :      * Add a new subcall to the conference
     269             :      */
     270             :     void addSubCall(const std::string& callId);
     271             : 
     272             :     /**
     273             :      * Remove a subcall from the conference
     274             :      */
     275             :     void removeSubCall(const std::string& callId);
     276             : 
     277             :     /**
     278             :      * Attach host
     279             :      */
     280             :     void attachHost(const std::vector<libjami::MediaMap>& mediaList);
     281             : 
     282             :     /**
     283             :      * Detach local audio/video from the conference
     284             :      */
     285             :     void detachHost();
     286             : 
     287             :     /**
     288             :      * Get the participant list for this conference
     289             :      */
     290             :     CallIdSet getSubCalls() const;
     291             : 
     292             :     /**
     293             :      * Start/stop recording toggle
     294             :      */
     295             :     bool toggleRecording() override;
     296             : 
     297             :     void switchInput(const std::string& input);
     298             :     void setActiveParticipant(const std::string& participant_id);
     299             :     void setActiveStream(const std::string& streamId, bool state);
     300             :     void setLayout(int layout);
     301             : 
     302             :     void onConfOrder(const std::string& callId, const std::string& order);
     303             : 
     304             :     bool isVideoEnabled() const;
     305             : 
     306             : #ifdef ENABLE_VIDEO
     307             :     void createSinks(const ConfInfo& infos);
     308             :     std::shared_ptr<video::VideoMixer> getVideoMixer();
     309             :     std::string getVideoInput() const;
     310             : #endif
     311             : 
     312          64 :     std::vector<std::map<std::string, std::string>> getConferenceInfos() const
     313             :     {
     314          64 :         std::lock_guard lk(confInfoMutex_);
     315         128 :         return confInfo_.toVectorMapStringString();
     316          64 :     }
     317             : 
     318             :     void updateConferenceInfo(ConfInfo confInfo);
     319             :     void setModerator(const std::string& uri, const bool& state);
     320             :     void hangupParticipant(const std::string& accountUri, const std::string& deviceId = "");
     321             :     void setHandRaised(const std::string& uri, const bool& state);
     322             :     void setVoiceActivity(const std::string& streamId, const bool& newState);
     323             : 
     324             :     void muteParticipant(const std::string& uri, const bool& state);
     325             :     void muteLocalHost(bool is_muted, const std::string& mediaType);
     326             :     bool isRemoteParticipant(const std::string& uri);
     327             :     void mergeConfInfo(ConfInfo& newInfo, const std::string& peerURI);
     328             : 
     329             :     /**
     330             :      * The client shows one tile per stream (video/audio related to a media)
     331             :      * @note for now, in conferences we can only mute the audio of a call
     332             :      * @todo add a track (audio OR video) parameter to know what we want to mute
     333             :      * @param accountUri        Account of the stream
     334             :      * @param deviceId          Device of the stream
     335             :      * @param streamId          Stream to mute
     336             :      * @param state             True to mute, false to unmute
     337             :      */
     338             :     void muteStream(const std::string& accountUri,
     339             :                     const std::string& deviceId,
     340             :                     const std::string& streamId,
     341             :                     const bool& state);
     342             :     void updateMuted();
     343             :     void updateRecording();
     344             : 
     345             :     void updateVoiceActivity();
     346             : 
     347             :     std::shared_ptr<Call> getCallFromPeerID(std::string_view peerId);
     348             : 
     349             :     /**
     350             :      * Announce to the client that medias are successfully negotiated
     351             :      */
     352             :     void reportMediaNegotiationStatus();
     353             : 
     354             :     /**
     355             :      * Retrieve current medias list
     356             :      * @return current medias
     357             :      */
     358             :     std::vector<libjami::MediaMap> currentMediaList() const;
     359             : 
     360             :     /**
     361             :      * Return the last media list before the host was detached
     362             :      * @return Last media list
     363             :      */
     364          12 :     std::vector<libjami::MediaMap> getLastMediaList() const { return lastMediaList_; }
     365             : 
     366             :     // Update layout if recording changes
     367             :     void stopRecording() override;
     368             :     bool startRecording(const std::string& path) override;
     369             : 
     370             :     /**
     371             :      * @return Conference duration in milliseconds
     372             :      */
     373          14 :     std::chrono::milliseconds getDuration() const
     374             :     {
     375          14 :         return duration_start_ == clock::time_point::min()
     376           0 :                    ? std::chrono::milliseconds::zero()
     377          28 :                    : std::chrono::duration_cast<std::chrono::milliseconds>(clock::now() - duration_start_);
     378             :     }
     379             : 
     380             : private:
     381          69 :     std::weak_ptr<Conference> weak() { return std::static_pointer_cast<Conference>(shared_from_this()); }
     382             : 
     383             :     static std::shared_ptr<Call> getCall(const std::string& callId);
     384             :     bool isModerator(std::string_view uri) const;
     385             :     bool isHandRaised(std::string_view uri) const;
     386             :     bool isVoiceActive(std::string_view uri) const;
     387             :     void updateModerators();
     388             :     void updateHandsRaised();
     389             :     void muteHost(bool state);
     390             :     void muteCall(const std::string& callId, bool state);
     391             : 
     392             :     void foreachCall(const std::function<void(const std::shared_ptr<Call>& call)>& cb);
     393             : 
     394             :     std::string id_;
     395             :     std::weak_ptr<Account> account_;
     396             :     State confState_ {State::ACTIVE_DETACHED};
     397             :     mutable std::mutex subcallsMtx_ {};
     398             :     CallIdSet subCalls_;
     399             :     std::string mediaPlayerId_ {};
     400             : 
     401             :     mutable std::mutex confInfoMutex_ {};
     402             :     ConfInfo confInfo_ {};
     403             : 
     404             :     void sendConferenceInfos();
     405             :     std::shared_ptr<RingBuffer> ghostRingBuffer_;
     406             : 
     407             : #ifdef ENABLE_VIDEO
     408             :     bool videoEnabled_;
     409             :     std::shared_ptr<video::VideoMixer> videoMixer_;
     410             :     std::map<std::string, std::shared_ptr<video::SinkClient>> confSinksMap_ {};
     411             : #endif
     412             : 
     413             :     std::shared_ptr<jami::AudioInput> audioMixer_;
     414             :     std::set<std::string, std::less<>> moderators_ {};
     415             :     std::set<std::string, std::less<>> participantsMuted_ {};
     416             :     std::set<std::string, std::less<>> handsRaised_;
     417             : 
     418             :     // stream IDs
     419             :     std::set<std::string, std::less<>> streamsVoiceActive {};
     420             : 
     421             :     void initRecorder(std::shared_ptr<MediaRecorder>& rec);
     422             :     void deinitRecorder(std::shared_ptr<MediaRecorder>& rec);
     423             : 
     424             :     bool isMuted(std::string_view uri) const;
     425             : 
     426             :     ConfInfo getConfInfoHostUri(std::string_view localHostURI, std::string_view destURI);
     427             :     bool isHost(std::string_view uri) const;
     428             :     bool isHostDevice(std::string_view deviceId) const;
     429             : 
     430             :     /**
     431             :      * If the local host is participating in the conference (attached
     432             :      * mode ), this variable will hold the media source states
     433             :      * of the local host.
     434             :      */
     435             :     std::vector<MediaAttribute> hostSources_;
     436             :     // Because host doesn't have a call, we need to store the audio inputs
     437             :     std::map<std::string, std::shared_ptr<jami::AudioInput>> hostAudioInputs_;
     438             : 
     439             :     // Last media list before detaching from a conference
     440             :     std::vector<libjami::MediaMap> lastMediaList_ = {};
     441             : 
     442             :     bool localModAdded_ {false};
     443             : 
     444             :     std::map<std::string, ConfInfo> remoteHosts_;
     445             : #ifdef ENABLE_VIDEO
     446             :     void resizeRemoteParticipants(ConfInfo& confInfo, std::string_view peerURI);
     447             : #endif
     448             :     std::string_view findHostforRemoteParticipant(std::string_view uri, std::string_view deviceId = "");
     449             : 
     450             :     std::shared_ptr<Call> getCallWith(const std::string& accountUri, const std::string& deviceId);
     451             : 
     452             :     std::mutex sinksMtx_ {};
     453             : 
     454             : #ifdef ENABLE_PLUGIN
     455             :     /**
     456             :      * Call Streams and some typedefs
     457             :      */
     458             :     using AVMediaStream = Observable<std::shared_ptr<MediaFrame>>;
     459             :     using MediaStreamSubject = PublishMapSubject<std::shared_ptr<MediaFrame>, AVFrame*>;
     460             : 
     461             : #ifdef ENABLE_VIDEO
     462             :     /**
     463             :      *   Map: maps the VideoFrame to an AVFrame
     464             :      **/
     465             :     std::function<AVFrame*(const std::shared_ptr<jami::MediaFrame>&)> pluginVideoMap_ =
     466        3459 :         [](const std::shared_ptr<jami::MediaFrame>& m) -> AVFrame* {
     467        3459 :         return std::static_pointer_cast<VideoFrame>(m)->pointer();
     468             :     };
     469             : #endif // ENABLE_VIDEO
     470             : 
     471             :     /**
     472             :      * @brief createConfAVStream
     473             :      * Creates a conf AV stream like video input, video receive, audio input or audio receive
     474             :      * @param StreamData
     475             :      * @param streamSource
     476             :      * @param mediaStreamSubject
     477             :      */
     478             :     void createConfAVStream(const StreamData& StreamData,
     479             :                             AVMediaStream& streamSource,
     480             :                             const std::shared_ptr<MediaStreamSubject>& mediaStreamSubject,
     481             :                             bool force = false);
     482             :     /**
     483             :      * @brief createConfAVStreams
     484             :      * Creates all Conf AV Streams (2 if audio, 4 if audio video)
     485             :      */
     486             :     void createConfAVStreams();
     487             : 
     488             :     std::mutex avStreamsMtx_ {};
     489             :     std::map<std::string, std::shared_ptr<MediaStreamSubject>> confAVStreams;
     490             : #endif // ENABLE_PLUGIN
     491             : 
     492             :     ConfProtocolParser parser_;
     493             :     std::string getRemoteId(const std::shared_ptr<jami::Call>& call) const;
     494             : 
     495             :     std::function<void(int)> shutdownCb_;
     496             :     clock::time_point duration_start_;
     497             : 
     498             :     /**
     499             :      * Initialize sources for host (takes default camera/audio)
     500             :      */
     501             :     void initSourcesForHost();
     502             : 
     503             :     /**
     504             :      * Take over media control from the call.
     505             :      * When a call joins a conference, the media control (mainly mute/un-mute
     506             :      * state of the local media source) will be handled by the conference and
     507             :      * the mixer.
     508             :      */
     509             :     void takeOverMediaSourceControl(const std::string& callId);
     510             : 
     511             :     /**
     512             :      * Bind host's audio
     513             :      */
     514             :     void bindHostAudio();
     515             : 
     516             :     /**
     517             :      * Unbind host's audio
     518             :      */
     519             :     void unbindHostAudio();
     520             : 
     521             :     /**
     522             :      * Bind call's audio to the conference
     523             :      */
     524             :     void bindSubCallAudio(const std::string& callId);
     525             : 
     526             :     /**
     527             :      * Unbind call's audio from the conference
     528             :      */
     529             :     void unbindSubCallAudio(const std::string& callId);
     530             : 
     531             :     /**
     532             :      * Clear all participant data from the conference
     533             :      */
     534             :     void clearParticipantData(const std::string& callId);
     535             : };
     536             : 
     537             : } // namespace jami