Line data    Source code

       1             : /*
       2             :  *  Copyright (C) 2004-2024 Savoir-faire Linux Inc.
       3             :  *
       4             :  *  This program is free software: you can redistribute it and/or modify
       5             :  *  it under the terms of the GNU General Public License as published by
       6             :  *  the Free Software Foundation, either version 3 of the License, or
       7             :  *  (at your option) any later version.
       8             :  *
       9             :  *  This program is distributed in the hope that it will be useful,
      10             :  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      12             :  *  GNU General Public License for more details.
      13             :  *
      14             :  *  You should have received a copy of the GNU General Public License
      15             :  *  along with this program. If not, see <https://www.gnu.org/licenses/>.
      16             :  */
      17             : #include "gitserver.h"
      18             : 
      19             : #include "fileutils.h"
      20             : #include "logger.h"
      21             : #include "gittransport.h"
      22             : #include "manager.h"
      23             : #include <opendht/thread_pool.h>
      24             : #include <dhtnet/multiplexed_socket.h>
      25             : #include <fmt/compile.h>
      26             : 
      27             : #include <charconv>
      28             : #include <ctime>
      29             : #include <fstream>
      30             : #include <git2.h>
      31             : #include <iomanip>
      32             : 
      33             : using namespace std::string_view_literals;
      34             : constexpr auto FLUSH_PKT = "0000"sv;
      35             : constexpr auto NAK_PKT = "0008NAK\n"sv;
      36             : constexpr auto DONE_CMD = "done\n"sv;
      37             : constexpr auto WANT_CMD = "want"sv;
      38             : constexpr auto HAVE_CMD = "have"sv;
      39             : constexpr auto SERVER_CAPABILITIES
      40             :     = " HEAD\0side-band side-band-64k shallow no-progress include-tag"sv;
      41             : 
      42             : namespace jami {
      43             : 
      44             : class GitServer::Impl
      45             : {
      46             : public:
      47         812 :     Impl(const std::string& repositoryId,
      48             :          const std::string& repository,
      49             :          const std::shared_ptr<dhtnet::ChannelSocket>& socket)
      50         812 :         : repositoryId_(repositoryId)
      51         812 :         , repository_(repository)
      52         812 :         , socket_(socket)
      53             :     {
      54             :         // Check at least if repository is correct
      55             :         git_repository* repo;
      56         812 :         if (git_repository_open(&repo, repository_.c_str()) != 0) {
      57           1 :             socket_->shutdown();
      58           1 :             return;
      59             :         }
      60         811 :         git_repository_free(repo);
      61             : 
      62         811 :         socket_->setOnRecv([this](const uint8_t* buf, std::size_t len) {
      63        6544 :             std::lock_guard lk(destroyMtx_);
      64        6544 :             if (isDestroying_)
      65           1 :                 return len;
      66        6542 :             if (parseOrder(std::string_view((const char*)buf, len)))
      67       15478 :                 while(parseOrder());
      68        6544 :             return len;
      69        6545 :         });
      70           0 :     }
      71         811 :     ~Impl() { stop(); }
      72        2061 :     void stop()
      73             :     {
      74        2061 :         std::lock_guard lk(destroyMtx_);
      75        2062 :         if (isDestroying_.exchange(true)) {
      76        1250 :             socket_->setOnRecv({});
      77        1249 :             socket_->shutdown();
      78             :         }
      79        2061 :     }
      80             :     bool parseOrder(std::string_view buf = {});
      81             : 
      82             :     void sendReferenceCapabilities(bool sendVersion = false);
      83             :     bool NAK();
      84             :     void ACKCommon();
      85             :     bool ACKFirst();
      86             :     void sendPackData();
      87             :     std::map<std::string, std::string> getParameters(std::string_view pkt_line);
      88             : 
      89             :     std::string repositoryId_ {};
      90             :     std::string repository_ {};
      91             :     std::shared_ptr<dhtnet::ChannelSocket> socket_ {};
      92             :     std::string wantedReference_ {};
      93             :     std::string common_ {};
      94             :     std::vector<std::string> haveRefs_ {};
      95             :     std::string cachedPkt_ {};
      96             :     std::mutex destroyMtx_ {};
      97             :     std::atomic_bool isDestroying_ {false};
      98             :     onFetchedCb onFetchedCb_ {};
      99             : };
     100             : 
     101             : bool
     102       22020 : GitServer::Impl::parseOrder(std::string_view buf)
     103             : {
     104       22020 :     std::string pkt = std::move(cachedPkt_);
     105       22020 :     if (!buf.empty())
     106        6541 :         pkt += buf;
     107             : 
     108             :     // Parse pkt len
     109             :     // Reference: https://github.com/git/git/blob/master/Documentation/technical/protocol-common.txt#L51
     110             :     // The first four bytes define the length of the packet and 0000 is a FLUSH pkt
     111             : 
     112       22018 :     unsigned int pkt_len = 0;
     113       22018 :     auto [p, ec] = std::from_chars(pkt.data(), pkt.data() + 4, pkt_len, 16);
     114       22015 :     if (ec != std::errc()) {
     115           0 :         JAMI_ERROR("Unable to parse packet size");
     116             :     }
     117       22015 :     if (pkt_len != pkt.size()) {
     118             :         // Store next packet part
     119       18469 :         if (pkt_len == 0) {
     120             :             // FLUSH_PKT
     121        4106 :             pkt_len = 4;
     122             :         }
     123       18469 :         cachedPkt_ = pkt.substr(pkt_len);
     124             :     }
     125             : 
     126       22020 :     auto pack = std::string_view(pkt).substr(4, pkt_len - 4);
     127       22016 :     if (pack == DONE_CMD) {
     128             :         // Reference:
     129             :         // https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L390 Do
     130             :         // not do multi-ack, just send ACK + pack file
     131             :         // In case of no common base, send NAK
     132        1109 :         JAMI_INFO("Peer negotiation is done. Answering to want order");
     133             :         bool sendData;
     134        1109 :         if (common_.empty())
     135         189 :             sendData = NAK();
     136             :         else
     137         920 :             sendData = ACKFirst();
     138        1109 :         if (sendData)
     139        1109 :             sendPackData();
     140        1109 :         return !cachedPkt_.empty();
     141       20902 :     } else if (pack.empty()) {
     142        4106 :         if (!haveRefs_.empty()) {
     143             :             // Reference:
     144             :             // https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L390
     145             :             // Do not do multi-ack, just send ACK + pack file In case of no common base ACK
     146         560 :             ACKCommon();
     147         560 :             NAK();
     148             :         }
     149        4106 :         return !cachedPkt_.empty();
     150             :     }
     151             : 
     152       16798 :     auto lim = pack.find(' ');
     153       16802 :     auto cmd = pack.substr(0, lim);
     154       16798 :     auto dat = (lim < pack.size()) ? pack.substr(lim+1) : std::string_view{};
     155       16802 :     if (cmd == UPLOAD_PACK_CMD) {
     156             :         // Cf: https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L166
     157             :         // References discovery
     158        2437 :         JAMI_INFO("Upload pack command detected.");
     159        2438 :         auto version = 1;
     160        2438 :         auto parameters = getParameters(dat);
     161        2438 :         auto versionIt = parameters.find("version");
     162        2438 :         bool sendVersion = false;
     163        2438 :         if (versionIt != parameters.end()) {
     164             :             try {
     165           0 :                 version = std::stoi(versionIt->second);
     166           0 :                 sendVersion = true;
     167           0 :             } catch (...) {
     168           0 :                 JAMI_WARNING("Invalid version detected: {}", versionIt->second);
     169           0 :             }
     170             :         }
     171        2438 :         if (version == 1) {
     172        2438 :             sendReferenceCapabilities(sendVersion);
     173             :         } else {
     174           0 :             JAMI_ERR("That protocol version is not yet supported (version: %u)", version);
     175             :         }
     176       16804 :     } else if (cmd == WANT_CMD) {
     177             :         // Reference:
     178             :         // https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L229
     179             :         // TODO can have more want
     180        1109 :         wantedReference_ = dat.substr(0, 40);
     181        1109 :         JAMI_INFO("Peer want ref: %s", wantedReference_.c_str());
     182       13252 :     } else if (cmd == HAVE_CMD) {
     183       13256 :         const auto& commit = haveRefs_.emplace_back(dat.substr(0, 40));
     184       13256 :         if (common_.empty()) {
     185             :             // Detect first common commit
     186             :             // Reference:
     187             :             // https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L390
     188             :             // TODO do not open repository every time
     189             :             git_repository* repo;
     190         920 :             if (git_repository_open(&repo, repository_.c_str()) != 0) {
     191           0 :                 JAMI_WARN("Unable to open %s", repository_.c_str());
     192           0 :                 return !cachedPkt_.empty();
     193             :             }
     194         920 :             GitRepository rep {repo, git_repository_free};
     195             :             git_oid commit_id;
     196         920 :             if (git_oid_fromstr(&commit_id, commit.c_str()) == 0) {
     197             :                 // Reference found
     198         920 :                 common_ = commit;
     199             :             }
     200         920 :         }
     201             :     } else {
     202           0 :         JAMI_WARNING("Unwanted packet received: {}", pkt);
     203             :     }
     204       16802 :     return !cachedPkt_.empty();
     205       22020 : }
     206             : 
     207             : std::string
     208       27551 : toGitHex(size_t value) {
     209       82652 :     return fmt::format(FMT_COMPILE("{:04x}"), value & 0x0FFFF);
     210             : }
     211             : 
     212             : void
     213        2438 : GitServer::Impl::sendReferenceCapabilities(bool sendVersion)
     214             : {
     215             :     // Get references
     216             :     // First, get the HEAD reference
     217             :     // https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L166
     218             :     git_repository* repo;
     219        2438 :     if (git_repository_open(&repo, repository_.c_str()) != 0) {
     220           0 :         JAMI_WARNING("Unable to open {}", repository_);
     221           0 :         socket_->shutdown();
     222           0 :         return;
     223             :     }
     224        2438 :     GitRepository rep {repo, git_repository_free};
     225             : 
     226             :     // Answer with the version number
     227             :     // **** When the client initially connects the server will immediately respond
     228             :     // **** with a version number (if "version=1" is sent as an Extra Parameter),
     229        2438 :     std::error_code ec;
     230        2438 :     if (sendVersion) {
     231           0 :         auto toSend = "000eversion 1\0"sv;
     232           0 :         socket_->write(reinterpret_cast<const unsigned char*>(toSend.data()),
     233             :                        toSend.size(),
     234             :                        ec);
     235           0 :         if (ec) {
     236           0 :             JAMI_WARNING("Unable to send data for {}: {}", repository_, ec.message());
     237           0 :             socket_->shutdown();
     238           0 :             return;
     239             :         }
     240             :     }
     241             : 
     242             :     git_oid commit_id;
     243        2438 :     if (git_reference_name_to_id(&commit_id, rep.get(), "HEAD") < 0) {
     244           0 :         JAMI_ERROR("Unable to get reference for HEAD");
     245           0 :         socket_->shutdown();
     246           0 :         return;
     247             :     }
     248        2438 :     std::string currentHead = git_oid_tostr_s(&commit_id);
     249             : 
     250             :     // Send references
     251        2438 :     std::ostringstream packet;
     252        2438 :     packet << toGitHex(5 + currentHead.size() + SERVER_CAPABILITIES.size());
     253        2438 :     packet << currentHead << SERVER_CAPABILITIES << "\n";
     254             : 
     255             :     // Now, add other references
     256             :     git_strarray refs;
     257        2438 :     if (git_reference_list(&refs, rep.get()) == 0) {
     258       25979 :         for (std::size_t i = 0; i < refs.count; ++i) {
     259       23541 :             std::string_view ref = refs.strings[i];
     260       23541 :             if (git_reference_name_to_id(&commit_id, rep.get(), ref.data()) < 0) {
     261           0 :                 JAMI_WARNING("Unable to get reference for {}", ref);
     262           0 :                 continue;
     263           0 :             }
     264       23543 :             currentHead = git_oid_tostr_s(&commit_id);
     265             : 
     266       23541 :             packet << toGitHex(6 /* size + space + \n */ + currentHead.size() + ref.size());
     267       23539 :             packet << currentHead << " " << ref << "\n";
     268             :         }
     269             :     }
     270        2438 :     git_strarray_dispose(&refs);
     271             : 
     272             :     // And add FLUSH
     273        2438 :     packet << FLUSH_PKT;
     274        2438 :     auto toSend = packet.str();
     275        2438 :     socket_->write(reinterpret_cast<const unsigned char*>(toSend.data()), toSend.size(), ec);
     276        2438 :     if (ec) {
     277           0 :         JAMI_WARNING("Unable to send data for {}: {}", repository_, ec.message());
     278           0 :         socket_->shutdown();
     279             :     }
     280        2437 : }
     281             : 
     282             : void
     283         560 : GitServer::Impl::ACKCommon()
     284             : {
     285         560 :     std::error_code ec;
     286             :     // Ack common base
     287         560 :     if (!common_.empty()) {
     288         560 :         auto toSend = fmt::format(FMT_COMPILE("{:04x}ACK {} continue\n"),
     289         560 :             18 + common_.size() /* size + ACK + space * 2 + continue + \n */, common_);
     290         560 :         socket_->write(reinterpret_cast<const unsigned char*>(toSend.c_str()), toSend.size(), ec);
     291         560 :         if (ec) {
     292           0 :             JAMI_WARNING("Unable to send data for {}: {}", repository_, ec.message());
     293           0 :             socket_->shutdown();
     294             :         }
     295         560 :     }
     296         560 : }
     297             : 
     298             : bool
     299         920 : GitServer::Impl::ACKFirst()
     300             : {
     301         920 :     std::error_code ec;
     302             :     // Ack common base
     303         920 :     if (!common_.empty()) {
     304         920 :         auto toSend = fmt::format(FMT_COMPILE("{:04x}ACK {}\n"),
     305         920 :             9 + common_.size() /* size + ACK + space + \n */, common_);
     306         920 :         socket_->write(reinterpret_cast<const unsigned char*>(toSend.c_str()), toSend.size(), ec);
     307         920 :         if (ec) {
     308           0 :             JAMI_WARNING("Unable to send data for {}: {}", repository_, ec.message());
     309           0 :             socket_->shutdown();
     310           0 :             return false;
     311             :         }
     312         920 :     }
     313         920 :     return true;
     314             : }
     315             : 
     316             : bool
     317         749 : GitServer::Impl::NAK()
     318             : {
     319         749 :     std::error_code ec;
     320             :     // NAK
     321         749 :     socket_->write(reinterpret_cast<const unsigned char*>(NAK_PKT.data()), NAK_PKT.size(), ec);
     322         749 :     if (ec) {
     323           0 :         JAMI_WARNING("Unable to send data for {}: {}", repository_, ec.message());
     324           0 :         socket_->shutdown();
     325           0 :         return false;
     326             :     }
     327         749 :     return true;
     328             : }
     329             : 
     330             : void
     331        1109 : GitServer::Impl::sendPackData()
     332             : {
     333             :     git_repository* repo_ptr;
     334        1109 :     if (git_repository_open(&repo_ptr, repository_.c_str()) != 0) {
     335           0 :         JAMI_WARN("Unable to open %s", repository_.c_str());
     336           0 :         return;
     337             :     }
     338        1109 :     GitRepository repo {repo_ptr, git_repository_free};
     339             : 
     340             :     git_packbuilder* pb_ptr;
     341        1109 :     if (git_packbuilder_new(&pb_ptr, repo.get()) != 0) {
     342           0 :         JAMI_WARNING("Unable to open packbuilder for {}", repository_);
     343           0 :         return;
     344             :     }
     345        1109 :     GitPackBuilder pb {pb_ptr, git_packbuilder_free};
     346             : 
     347        1109 :     std::string fetched = wantedReference_;
     348             :     git_oid oid;
     349        1109 :     if (git_oid_fromstr(&oid, fetched.c_str()) < 0) {
     350           0 :         JAMI_ERROR("Unable to get reference for commit {}", fetched);
     351           0 :         return;
     352             :     }
     353             : 
     354        1109 :     git_revwalk* walker_ptr = nullptr;
     355        1109 :     if (git_revwalk_new(&walker_ptr, repo.get()) < 0 || git_revwalk_push(walker_ptr, &oid) < 0) {
     356           0 :         if (walker_ptr)
     357           0 :             git_revwalk_free(walker_ptr);
     358           0 :         return;
     359             :     }
     360        1109 :     GitRevWalker walker {walker_ptr, git_revwalk_free};
     361        1109 :     git_revwalk_sorting(walker.get(), GIT_SORT_TOPOLOGICAL);
     362             :     // Add first commit
     363        1109 :     std::set<std::string> parents;
     364        1109 :     auto haveCommit = false;
     365             : 
     366        3064 :     while (!git_revwalk_next(&oid, walker.get())) {
     367             :         // log until have refs
     368        2875 :         std::string id = git_oid_tostr_s(&oid);
     369        2874 :         haveCommit |= std::find(haveRefs_.begin(), haveRefs_.end(), id) != haveRefs_.end();
     370        2875 :         auto itParents = std::find(parents.begin(), parents.end(), id);
     371        2875 :         if (itParents != parents.end())
     372        1766 :             parents.erase(itParents);
     373        2875 :         if (haveCommit && parents.size() == 0 /* We are sure that all commits are there */)
     374         920 :             break;
     375        1955 :         if (git_packbuilder_insert_commit(pb.get(), &oid) != 0) {
     376           0 :             JAMI_WARN("Unable to open insert commit %s for %s",
     377             :                       git_oid_tostr_s(&oid),
     378             :                       repository_.c_str());
     379           0 :             return;
     380             :         }
     381             : 
     382             :         // Get next commit to pack
     383             :         git_commit* commit_ptr;
     384        1955 :         if (git_commit_lookup(&commit_ptr, repo.get(), &oid) < 0) {
     385           0 :             JAMI_ERR("Unable to look up current commit");
     386           0 :             return;
     387             :         }
     388        1955 :         GitCommit commit {commit_ptr, git_commit_free};
     389        1955 :         auto parentsCount = git_commit_parentcount(commit.get());
     390        3727 :         for (unsigned int p = 0; p < parentsCount; ++p) {
     391             :             // make sure to explore all branches
     392        1773 :             const git_oid* pid = git_commit_parent_id(commit.get(), p);
     393        1773 :             if (pid)
     394        1773 :                 parents.emplace(git_oid_tostr_s(pid));
     395             :         }
     396        2874 :     }
     397             : 
     398        1109 :     git_buf data = {};
     399        1109 :     if (git_packbuilder_write_buf(&data, pb.get()) != 0) {
     400           0 :         JAMI_WARN("Unable to write pack data for %s", repository_.c_str());
     401           0 :         return;
     402             :     }
     403             : 
     404        1109 :     std::size_t sent = 0;
     405        1109 :     std::size_t len = data.size;
     406        1109 :     std::error_code ec;
     407        1109 :     std::vector<uint8_t> toSendData;
     408             :     do {
     409             :         // cf https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L166
     410             :         // In 'side-band-64k' mode it will send up to 65519 data bytes plus 1 control code, for a
     411             :         // total of up to 65520 bytes in a pkt-line.
     412        1576 :         std::size_t pkt_size = std::min(static_cast<std::size_t>(65515), len - sent);
     413        1575 :         std::string toSendHeader = toGitHex(pkt_size + 5);
     414        1576 :         toSendData.clear();
     415        1576 :         toSendData.reserve(pkt_size + 5);
     416        1575 :         toSendData.insert(toSendData.end(), toSendHeader.begin(), toSendHeader.end());
     417        1576 :         toSendData.push_back(0x1);
     418        1576 :         toSendData.insert(toSendData.end(), data.ptr + sent, data.ptr + sent + pkt_size);
     419             : 
     420        1575 :         socket_->write(reinterpret_cast<const unsigned char*>(toSendData.data()),
     421             :                        toSendData.size(),
     422             :                        ec);
     423        1576 :         if (ec) {
     424           0 :             JAMI_WARNING("Unable to send data for {}: {}", repository_, ec.message());
     425           0 :             git_buf_dispose(&data);
     426           0 :             return;
     427             :         }
     428        1576 :         sent += pkt_size;
     429        3152 :     } while (sent < len);
     430        1109 :     git_buf_dispose(&data);
     431        1109 :     toSendData = {};
     432             : 
     433             :     // And finish by a little FLUSH
     434        1109 :     socket_->write(reinterpret_cast<const uint8_t*>(FLUSH_PKT.data()), FLUSH_PKT.size(), ec);
     435        1109 :     if (ec) {
     436           0 :         JAMI_WARNING("Unable to send data for {}: {}", repository_, ec.message());
     437             :     }
     438             : 
     439             :     // Clear sent data
     440        1109 :     haveRefs_.clear();
     441        1109 :     wantedReference_.clear();
     442        1109 :     common_.clear();
     443        1109 :     if (onFetchedCb_)
     444        1109 :         onFetchedCb_(fetched);
     445        1109 : }
     446             : 
     447             : std::map<std::string, std::string>
     448        2438 : GitServer::Impl::getParameters(std::string_view pkt_line)
     449             : {
     450        2438 :     std::map<std::string, std::string> parameters;
     451        2438 :     std::string key, value;
     452        2438 :     auto isKey = true;
     453        2438 :     auto nullChar = 0;
     454      275494 :     for (auto letter: pkt_line) {
     455      273056 :         if (letter == '\0') {
     456             :             // parameters such as host or version are after the first \0
     457        4876 :             if (nullChar != 0 && !key.empty()) {
     458        2438 :                 parameters[std::move(key)] = std::move(value);
     459             :             }
     460        4876 :             nullChar += 1;
     461        4876 :             isKey = true;
     462        4876 :             key.clear();
     463        4876 :             value.clear();
     464      268180 :         } else if (letter == '=') {
     465        2438 :             isKey = false;
     466      265742 :         } else if (nullChar != 0) {
     467      165784 :             if (isKey) {
     468        9752 :                 key += letter;
     469             :             } else {
     470      156032 :                 value += letter;
     471             :             }
     472             :         }
     473             :     }
     474        4876 :     return parameters;
     475        2438 : }
     476             : 
     477         812 : GitServer::GitServer(const std::string& accountId,
     478             :                      const std::string& conversationId,
     479         812 :                      const std::shared_ptr<dhtnet::ChannelSocket>& client)
     480             : {
     481        1624 :     auto path = (fileutils::get_data_dir() / accountId / "conversations" / conversationId).string();
     482         812 :     pimpl_ = std::make_unique<GitServer::Impl>(conversationId, path, client);
     483         812 : }
     484             : 
     485         812 : GitServer::~GitServer()
     486             : {
     487         812 :     stop();
     488         812 :     pimpl_.reset();
     489         812 :     JAMI_INFO("GitServer destroyed");
     490         812 : }
     491             : 
     492             : void
     493         812 : GitServer::setOnFetched(const onFetchedCb& cb)
     494             : {
     495         812 :     if (!pimpl_)
     496           0 :         return;
     497         812 :     pimpl_->onFetchedCb_ = cb;
     498             : }
     499             : 
     500             : void
     501        1250 : GitServer::stop()
     502             : {
     503        1250 :     pimpl_->stop();
     504        1250 : }
     505             : 
     506             : } // namespace jami