Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2026 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software: you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation, either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : #pragma once
      18             : 
      19             : #ifdef HAVE_CONFIG_H
      20             : #include "config.h"
      21             : #endif
      22             : 
      23             : #include <chrono>
      24             : #include <set>
      25             : #include <string>
      26             : #include <memory>
      27             : #include <vector>
      28             : #include <string_view>
      29             : #include <map>
      30             : #include <functional>
      31             : 
      32             : #include "conference_protocol.h"
      33             : #include "media/audio/audio_input.h"
      34             : #include "media/media_attribute.h"
      35             : #include "media/recordable.h"
      36             : 
      37             : #ifdef ENABLE_PLUGIN
      38             : #include "plugin/streamdata.h"
      39             : #endif
      40             : 
      41             : #ifdef ENABLE_VIDEO
      42             : #include "media/video/sinkclient.h"
      43             : #endif
      44             : 
      45             : #include <json/json.h>
      46             : 
      47             : namespace jami {
      48             : 
      49             : class Call;
      50             : class Account;
      51             : class JamiAccount;
      52             : 
      53             : #ifdef ENABLE_VIDEO
      54             : namespace video {
      55             : class VideoMixer;
      56             : struct SourceInfo;
      57             : } // namespace video
      58             : #endif
      59             : 
      60             : // info for a stream
      61             : struct ParticipantInfo
      62             : {
      63             :     std::string uri;
      64             :     std::string device;
      65             :     std::string sinkId; // stream ID
      66             :     bool active {false};
      67             :     int x {0};
      68             :     int y {0};
      69             :     int w {0};
      70             :     int h {0};
      71             :     bool videoMuted {false};
      72             :     bool audioLocalMuted {false};
      73             :     bool audioModeratorMuted {false};
      74             :     bool isModerator {false};
      75             :     bool handRaised {false};
      76             :     bool voiceActivity {false};
      77             :     bool recording {false};
      78             : 
      79         616 :     void fromJson(const Json::Value& v)
      80             :     {
      81         616 :         uri = v["uri"].asString();
      82         616 :         device = v["device"].asString();
      83         616 :         sinkId = v["sinkId"].asString();
      84         616 :         active = v["active"].asBool();
      85         616 :         x = v["x"].asInt();
      86         616 :         y = v["y"].asInt();
      87         616 :         w = v["w"].asInt();
      88         616 :         h = v["h"].asInt();
      89         616 :         videoMuted = v["videoMuted"].asBool();
      90         616 :         audioLocalMuted = v["audioLocalMuted"].asBool();
      91         616 :         audioModeratorMuted = v["audioModeratorMuted"].asBool();
      92         616 :         isModerator = v["isModerator"].asBool();
      93         616 :         handRaised = v["handRaised"].asBool();
      94         616 :         voiceActivity = v["voiceActivity"].asBool();
      95         616 :         recording = v["recording"].asBool();
      96         616 :     }
      97             : 
      98         643 :     Json::Value toJson() const
      99             :     {
     100         643 :         Json::Value val;
     101         643 :         val["uri"] = uri;
     102         644 :         val["device"] = device;
     103         645 :         val["sinkId"] = sinkId;
     104         645 :         val["active"] = active;
     105         645 :         val["x"] = x;
     106         643 :         val["y"] = y;
     107         641 :         val["w"] = w;
     108         646 :         val["h"] = h;
     109         646 :         val["videoMuted"] = videoMuted;
     110         645 :         val["audioLocalMuted"] = audioLocalMuted;
     111         646 :         val["audioModeratorMuted"] = audioModeratorMuted;
     112         646 :         val["isModerator"] = isModerator;
     113         646 :         val["handRaised"] = handRaised;
     114         643 :         val["voiceActivity"] = voiceActivity;
     115         645 :         val["recording"] = recording;
     116         646 :         return val;
     117           0 :     }
     118             : 
     119        1061 :     std::map<std::string, std::string> toMap() const
     120             :     {
     121        1061 :         return {{"uri", uri},
     122        1061 :                 {"device", device},
     123        1061 :                 {"sinkId", sinkId},
     124        1060 :                 {"active", active ? "true" : "false"},
     125        2121 :                 {"x", std::to_string(x)},
     126        2122 :                 {"y", std::to_string(y)},
     127        2122 :                 {"w", std::to_string(w)},
     128        2122 :                 {"h", std::to_string(h)},
     129        1061 :                 {"videoMuted", videoMuted ? "true" : "false"},
     130        1061 :                 {"audioLocalMuted", audioLocalMuted ? "true" : "false"},
     131        1061 :                 {"audioModeratorMuted", audioModeratorMuted ? "true" : "false"},
     132        1060 :                 {"isModerator", isModerator ? "true" : "false"},
     133        1061 :                 {"handRaised", handRaised ? "true" : "false"},
     134        1061 :                 {"voiceActivity", voiceActivity ? "true" : "false"},
     135       29706 :                 {"recording", recording ? "true" : "false"}};
     136             :     }
     137             : 
     138           0 :     friend bool operator==(const ParticipantInfo& p1, const ParticipantInfo& p2)
     139             :     {
     140           0 :         return p1.uri == p2.uri and p1.device == p2.device and p1.sinkId == p2.sinkId and p1.active == p2.active
     141           0 :                and p1.x == p2.x and p1.y == p2.y and p1.w == p2.w and p1.h == p2.h and p1.videoMuted == p2.videoMuted
     142           0 :                and p1.audioLocalMuted == p2.audioLocalMuted and p1.audioModeratorMuted == p2.audioModeratorMuted
     143           0 :                and p1.isModerator == p2.isModerator and p1.handRaised == p2.handRaised
     144           0 :                and p1.voiceActivity == p2.voiceActivity and p1.recording == p2.recording;
     145             :     }
     146             : 
     147             :     friend bool operator!=(const ParticipantInfo& p1, const ParticipantInfo& p2) { return !(p1 == p2); }
     148             : };
     149             : 
     150             : struct ConfInfo : public std::vector<ParticipantInfo>
     151             : {
     152             :     int h {0};
     153             :     int w {0};
     154             :     int v {1}; // Supported conference protocol version
     155             :     int layout {0};
     156             : 
     157           0 :     friend bool operator==(const ConfInfo& c1, const ConfInfo& c2)
     158             :     {
     159           0 :         if (c1.h != c2.h or c1.w != c2.w)
     160           0 :             return false;
     161           0 :         if (c1.size() != c2.size())
     162           0 :             return false;
     163             : 
     164           0 :         for (auto& p1 : c1) {
     165           0 :             auto it = std::find_if(c2.begin(), c2.end(), [&p1](const ParticipantInfo& p2) { return p1 == p2; });
     166           0 :             if (it != c2.end())
     167           0 :                 continue;
     168             :             else
     169           0 :                 return false;
     170             :         }
     171           0 :         return true;
     172             :     }
     173             : 
     174           0 :     friend bool operator!=(const ConfInfo& c1, const ConfInfo& c2) { return !(c1 == c2); }
     175             : 
     176             :     std::vector<std::map<std::string, std::string>> toVectorMapStringString() const;
     177             :     std::string toString() const;
     178             : };
     179             : 
     180             : using CallIdSet = std::set<std::string>;
     181             : using clock = std::chrono::steady_clock;
     182             : 
     183             : class Conference : public Recordable, public std::enable_shared_from_this<Conference>
     184             : {
     185             : public:
     186             :     enum class State { ACTIVE_ATTACHED, ACTIVE_DETACHED, HOLD };
     187             : 
     188             :     /**
     189             :      * Constructor for this class, increment static counter
     190             :      */
     191             :     explicit Conference(const std::shared_ptr<Account>&, const std::string& confId = "");
     192             : 
     193             :     /**
     194             :      * Destructor for this class, decrement static counter
     195             :      */
     196             :     ~Conference();
     197             : 
     198             :     /**
     199             :      * Return the conference id
     200             :      */
     201        1928 :     const std::string& getConfId() const { return id_; }
     202             : 
     203         408 :     std::shared_ptr<Account> getAccount() const { return account_.lock(); }
     204             : 
     205             :     std::string getAccountId() const;
     206             : 
     207             :     /**
     208             :      * Return the current conference state
     209             :      */
     210             :     State getState() const;
     211             : 
     212             :     /**
     213             :      * Set conference state
     214             :      */
     215             :     void setState(State state);
     216             : 
     217             :     /**
     218             :      * Set a callback that will be called when the conference will be destroyed
     219             :      */
     220          14 :     void onShutdown(std::function<void(int)> cb) { shutdownCb_ = std::move(cb); }
     221             : 
     222             :     /**
     223             :      * Return a string description of the conference state
     224             :      */
     225         288 :     static constexpr const char* getStateStr(State state)
     226             :     {
     227         288 :         switch (state) {
     228         174 :         case State::ACTIVE_ATTACHED:
     229         174 :             return "ACTIVE_ATTACHED";
     230         114 :         case State::ACTIVE_DETACHED:
     231         114 :             return "ACTIVE_DETACHED";
     232           0 :         case State::HOLD:
     233           0 :             return "HOLD";
     234           0 :         default:
     235           0 :             return "";
     236             :         }
     237             :     }
     238             : 
     239         204 :     const char* getStateStr() const { return getStateStr(confState_); }
     240             : 
     241             :     /**
     242             :      * Set the mute state of the local host
     243             :      */
     244             :     void setLocalHostMuteState(MediaType type, bool muted);
     245             : 
     246             :     /**
     247             :      * Get the mute state of the local host
     248             :      */
     249             :     bool isMediaSourceMuted(MediaType type) const;
     250             : 
     251             :     /**
     252             :      * Process a media change request.
     253             :      * Used to change the media attributes of the host.
     254             :      *
     255             :      * @param remoteMediaList new media list from the remote
     256             :      * @return true on success
     257             :      */
     258             :     bool requestMediaChange(const std::vector<libjami::MediaMap>& mediaList);
     259             : 
     260             :     /**
     261             :      * Process incoming media change request.
     262             :      *
     263             :      * @param callId the call ID
     264             :      * @param remoteMediaList new media list from the remote
     265             :      */
     266             :     void handleMediaChangeRequest(const std::shared_ptr<Call>& call,
     267             :                                   const std::vector<libjami::MediaMap>& remoteMediaList);
     268             : 
     269             :     /**
     270             :      * Add a new subcall to the conference
     271             :      */
     272             :     void addSubCall(const std::string& callId);
     273             : 
     274             :     /**
     275             :      * Remove a subcall from the conference
     276             :      */
     277             :     void removeSubCall(const std::string& callId);
     278             : 
     279             :     /**
     280             :      * Attach host
     281             :      */
     282             :     void attachHost(const std::vector<libjami::MediaMap>& mediaList);
     283             : 
     284             :     /**
     285             :      * Detach local audio/video from the conference
     286             :      */
     287             :     void detachHost();
     288             : 
     289             :     /**
     290             :      * Get the participant list for this conference
     291             :      */
     292             :     CallIdSet getSubCalls() const;
     293             : 
     294             :     /**
     295             :      * Start/stop recording toggle
     296             :      */
     297             :     bool toggleRecording() override;
     298             : 
     299             :     void switchInput(const std::string& input);
     300             :     void setActiveParticipant(const std::string& participant_id);
     301             :     void setActiveStream(const std::string& streamId, bool state);
     302             :     void setLayout(int layout);
     303             : 
     304             :     void onConfOrder(const std::string& callId, const std::string& order);
     305             : 
     306             :     bool isVideoEnabled() const;
     307             : 
     308             : #ifdef ENABLE_VIDEO
     309             :     void createSinks(const ConfInfo& infos);
     310             :     std::shared_ptr<video::VideoMixer> getVideoMixer();
     311             :     std::string getVideoInput() const;
     312             : #endif
     313             : 
     314          64 :     std::vector<std::map<std::string, std::string>> getConferenceInfos() const
     315             :     {
     316          64 :         std::lock_guard lk(confInfoMutex_);
     317         128 :         return confInfo_.toVectorMapStringString();
     318          64 :     }
     319             : 
     320             :     void updateConferenceInfo(ConfInfo confInfo);
     321             :     void setModerator(const std::string& uri, const bool& state);
     322             :     void hangupParticipant(const std::string& accountUri, const std::string& deviceId = "");
     323             :     void setHandRaised(const std::string& uri, const bool& state);
     324             :     void setVoiceActivity(const std::string& streamId, const bool& newState);
     325             : 
     326             :     void muteParticipant(const std::string& uri, const bool& state);
     327             :     void muteLocalHost(bool is_muted, const std::string& mediaType);
     328             :     bool isRemoteParticipant(const std::string& uri);
     329             :     void mergeConfInfo(ConfInfo& newInfo, const std::string& peerURI);
     330             : 
     331             :     /**
     332             :      * The client shows one tile per stream (video/audio related to a media)
     333             :      * @note for now, in conferences we can only mute the audio of a call
     334             :      * @todo add a track (audio OR video) parameter to know what we want to mute
     335             :      * @param accountUri        Account of the stream
     336             :      * @param deviceId          Device of the stream
     337             :      * @param streamId          Stream to mute
     338             :      * @param state             True to mute, false to unmute
     339             :      */
     340             :     void muteStream(const std::string& accountUri,
     341             :                     const std::string& deviceId,
     342             :                     const std::string& streamId,
     343             :                     const bool& state);
     344             :     void updateMuted();
     345             :     void updateRecording();
     346             : 
     347             :     void updateVoiceActivity();
     348             : 
     349             :     std::shared_ptr<Call> getCallFromPeerID(std::string_view peerId);
     350             : 
     351             :     /**
     352             :      * Announce to the client that medias are successfully negotiated
     353             :      */
     354             :     void reportMediaNegotiationStatus();
     355             : 
     356             :     /**
     357             :      * Retrieve current medias list
     358             :      * @return current medias
     359             :      */
     360             :     std::vector<libjami::MediaMap> currentMediaList() const;
     361             : 
     362             :     /**
     363             :      * Return the last media list before the host was detached
     364             :      * @return Last media list
     365             :      */
     366          12 :     std::vector<libjami::MediaMap> getLastMediaList() const { return lastMediaList_; }
     367             : 
     368             :     // Update layout if recording changes
     369             :     void stopRecording() override;
     370             :     bool startRecording(const std::string& path) override;
     371             : 
     372             :     /**
     373             :      * @return Conference duration in milliseconds
     374             :      */
     375          14 :     std::chrono::milliseconds getDuration() const
     376             :     {
     377          14 :         return duration_start_ == clock::time_point::min()
     378           0 :                    ? std::chrono::milliseconds::zero()
     379          28 :                    : std::chrono::duration_cast<std::chrono::milliseconds>(clock::now() - duration_start_);
     380             :     }
     381             : 
     382             : private:
     383          70 :     std::weak_ptr<Conference> weak() { return std::static_pointer_cast<Conference>(shared_from_this()); }
     384             : 
     385             :     static std::shared_ptr<Call> getCall(const std::string& callId);
     386             :     bool isModerator(std::string_view uri) const;
     387             :     bool isHandRaised(std::string_view uri) const;
     388             :     bool isVoiceActive(std::string_view uri) const;
     389             :     void updateModerators();
     390             :     void updateHandsRaised();
     391             :     void muteHost(bool state);
     392             :     void muteCall(const std::string& callId, bool state);
     393             : 
     394             :     void foreachCall(const std::function<void(const std::shared_ptr<Call>& call)>& cb);
     395             : 
     396             :     std::string id_;
     397             :     std::weak_ptr<Account> account_;
     398             :     State confState_ {State::ACTIVE_DETACHED};
     399             :     mutable std::mutex subcallsMtx_ {};
     400             :     CallIdSet subCalls_;
     401             :     std::string mediaPlayerId_ {};
     402             : 
     403             :     mutable std::mutex confInfoMutex_ {};
     404             :     ConfInfo confInfo_ {};
     405             : 
     406             :     void sendConferenceInfos();
     407             :     std::shared_ptr<RingBuffer> ghostRingBuffer_;
     408             : 
     409             : #ifdef ENABLE_VIDEO
     410             :     bool videoEnabled_;
     411             :     std::shared_ptr<video::VideoMixer> videoMixer_;
     412             :     std::map<std::string, std::shared_ptr<video::SinkClient>> confSinksMap_ {};
     413             : #endif
     414             : 
     415             :     std::shared_ptr<jami::AudioInput> audioMixer_;
     416             :     std::set<std::string, std::less<>> moderators_ {};
     417             :     std::set<std::string, std::less<>> participantsMuted_ {};
     418             :     std::set<std::string, std::less<>> handsRaised_;
     419             : 
     420             :     // stream IDs
     421             :     std::set<std::string, std::less<>> streamsVoiceActive {};
     422             : 
     423             :     void initRecorder(std::shared_ptr<MediaRecorder>& rec);
     424             :     void deinitRecorder(std::shared_ptr<MediaRecorder>& rec);
     425             : 
     426             :     bool isMuted(std::string_view uri) const;
     427             : 
     428             :     ConfInfo getConfInfoHostUri(std::string_view localHostURI, std::string_view destURI);
     429             :     bool isHost(std::string_view uri) const;
     430             :     bool isHostDevice(std::string_view deviceId) const;
     431             : 
     432             :     /**
     433             :      * If the local host is participating in the conference (attached
     434             :      * mode ), this variable will hold the media source states
     435             :      * of the local host.
     436             :      */
     437             :     std::vector<MediaAttribute> hostSources_;
     438             :     // Because host doesn't have a call, we need to store the audio inputs
     439             :     std::map<std::string, std::shared_ptr<jami::AudioInput>> hostAudioInputs_;
     440             : 
     441             :     // Last media list before detaching from a conference
     442             :     std::vector<libjami::MediaMap> lastMediaList_ = {};
     443             : 
     444             :     bool localModAdded_ {false};
     445             : 
     446             :     std::map<std::string, ConfInfo> remoteHosts_;
     447             : #ifdef ENABLE_VIDEO
     448             :     void resizeRemoteParticipants(ConfInfo& confInfo, std::string_view peerURI);
     449             : #endif
     450             :     std::string_view findHostforRemoteParticipant(std::string_view uri, std::string_view deviceId = "");
     451             : 
     452             :     std::shared_ptr<Call> getCallWith(const std::string& accountUri, const std::string& deviceId);
     453             : 
     454             :     std::mutex sinksMtx_ {};
     455             : 
     456             : #ifdef ENABLE_PLUGIN
     457             :     /**
     458             :      * Call Streams and some typedefs
     459             :      */
     460             :     using AVMediaStream = Observable<std::shared_ptr<MediaFrame>>;
     461             :     using MediaStreamSubject = PublishMapSubject<std::shared_ptr<MediaFrame>, AVFrame*>;
     462             : 
     463             : #ifdef ENABLE_VIDEO
     464             :     /**
     465             :      *   Map: maps the VideoFrame to an AVFrame
     466             :      **/
     467             :     std::function<AVFrame*(const std::shared_ptr<jami::MediaFrame>&)> pluginVideoMap_ =
     468        3307 :         [](const std::shared_ptr<jami::MediaFrame>& m) -> AVFrame* {
     469        3307 :         return std::static_pointer_cast<VideoFrame>(m)->pointer();
     470             :     };
     471             : #endif // ENABLE_VIDEO
     472             : 
     473             :     /**
     474             :      * @brief createConfAVStream
     475             :      * Creates a conf AV stream like video input, video receive, audio input or audio receive
     476             :      * @param StreamData
     477             :      * @param streamSource
     478             :      * @param mediaStreamSubject
     479             :      */
     480             :     void createConfAVStream(const StreamData& StreamData,
     481             :                             AVMediaStream& streamSource,
     482             :                             const std::shared_ptr<MediaStreamSubject>& mediaStreamSubject,
     483             :                             bool force = false);
     484             :     /**
     485             :      * @brief createConfAVStreams
     486             :      * Creates all Conf AV Streams (2 if audio, 4 if audio video)
     487             :      */
     488             :     void createConfAVStreams();
     489             : 
     490             :     std::mutex avStreamsMtx_ {};
     491             :     std::map<std::string, std::shared_ptr<MediaStreamSubject>> confAVStreams;
     492             : #endif // ENABLE_PLUGIN
     493             : 
     494             :     ConfProtocolParser parser_;
     495             :     std::string getRemoteId(const std::shared_ptr<jami::Call>& call) const;
     496             : 
     497             :     std::function<void(int)> shutdownCb_;
     498             :     clock::time_point duration_start_;
     499             : 
     500             :     /**
     501             :      * Initialize sources for host (takes default camera/audio)
     502             :      */
     503             :     void initSourcesForHost();
     504             : 
     505             : #ifdef ENABLE_VIDEO
     506             :     /**
     507             :      * Setup video mixer and its callbacks.
     508             :      * Called during construction.
     509             :      */
     510             :     void setupVideoMixer();
     511             : 
     512             :     /**
     513             :      * Callback for when video sources are updated.
     514             :      * @param infos The updated source information
     515             :      */
     516             :     void onVideoSourcesUpdated(const std::vector<video::SourceInfo>& infos);
     517             : 
     518             :     /**
     519             :      * Create participant info for a remote call source.
     520             :      * @param info Source information from video mixer
     521             :      * @return Populated ParticipantInfo
     522             :      */
     523             :     ParticipantInfo createParticipantInfoFromRemoteSource(const video::SourceInfo& info);
     524             : 
     525             :     /**
     526             :      * Create participant info for a local/host source.
     527             :      * @param info Source information from video mixer
     528             :      * @param acc The JamiAccount for the host
     529             :      * @param hostAdded Output flag indicating if host was added
     530             :      * @return Populated ParticipantInfo
     531             :      */
     532             :     ParticipantInfo createParticipantInfoFromLocalSource(const video::SourceInfo& info,
     533             :                                                          const std::shared_ptr<JamiAccount>& acc,
     534             :                                                          bool& hostAdded);
     535             : #endif
     536             : 
     537             :     /**
     538             :      * Register all protocol message handlers.
     539             :      * Called during construction.
     540             :      */
     541             :     void registerProtocolHandlers();
     542             : 
     543             :     /**
     544             :      * Take over media control from the call.
     545             :      * When a call joins a conference, the media control (mainly mute/un-mute
     546             :      * state of the local media source) will be handled by the conference and
     547             :      * the mixer.
     548             :      */
     549             :     void takeOverMediaSourceControl(const std::string& callId);
     550             : 
     551             :     /**
     552             :      * Bind host's audio
     553             :      */
     554             :     void bindHostAudio();
     555             : 
     556             :     /**
     557             :      * Unbind host's audio
     558             :      */
     559             :     void unbindHostAudio();
     560             : 
     561             :     /**
     562             :      * Bind call's audio to the conference
     563             :      */
     564             :     void bindSubCallAudio(const std::string& callId);
     565             : 
     566             :     /**
     567             :      * Unbind call's audio from the conference
     568             :      */
     569             :     void unbindSubCallAudio(const std::string& callId);
     570             : 
     571             :     /**
     572             :      * Clear all participant data from the conference
     573             :      */
     574             :     void clearParticipantData(const std::string& callId);
     575             : 
     576             : #ifdef ENABLE_VIDEO
     577             :     /**
     578             :      * Make sure video is negotiated with all subcalls
     579             :      * When the conference needs video (host has video or any participant has video),
     580             :      * this method triggers renegotiation with subcalls that don't have video yet.
     581             :      * @param excludeCallId Optional call ID to exclude from renegotiation (e.g., the call that just added video)
     582             :      */
     583             :     void negotiateVideoWithSubcalls(const std::string& excludeCallId = "");
     584             : #endif
     585             : };
     586             : 
     587             : } // namespace jami