Line data Source code
1 : /*
2 : * Copyright (C) 2019-2024 Savoir-faire Linux Inc.
3 : * Author: Sébastien Blin <sebastien.blin@savoirfairelinux.com>
4 : *
5 : * This program is free software; you can redistribute it and/or modify
6 : * it under the terms of the GNU General Public License as published by
7 : * the Free Software Foundation; either version 3 of the License, or
8 : * (at your option) any later version.
9 : *
10 : * This program is distributed in the hope that it will be useful,
11 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : * GNU General Public License for more details.
14 : *
15 : * You should have received a copy of the GNU General Public License
16 : * along with this program. If not, see <https://www.gnu.org/licenses/>.
17 : */
18 : #include "gitserver.h"
19 :
20 : #include "fileutils.h"
21 : #include "logger.h"
22 : #include "gittransport.h"
23 : #include "manager.h"
24 : #include <opendht/thread_pool.h>
25 : #include <dhtnet/multiplexed_socket.h>
26 :
27 : #include <charconv>
28 : #include <ctime>
29 : #include <fstream>
30 : #include <git2.h>
31 : #include <iomanip>
32 :
33 : using namespace std::string_view_literals;
34 : constexpr auto FLUSH_PKT = "0000"sv;
35 : constexpr auto NAK_PKT = "0008NAK\n"sv;
36 : constexpr auto DONE_CMD = "done\n"sv;
37 : constexpr auto WANT_CMD = "want"sv;
38 : constexpr auto HAVE_CMD = "have"sv;
39 : constexpr auto SERVER_CAPABILITIES
40 : = " HEAD\0side-band side-band-64k shallow no-progress include-tag"sv;
41 :
42 : namespace jami {
43 :
44 : class GitServer::Impl
45 : {
46 : public:
47 823 : Impl(const std::string& repositoryId,
48 : const std::string& repository,
49 : const std::shared_ptr<dhtnet::ChannelSocket>& socket)
50 823 : : repositoryId_(repositoryId)
51 823 : , repository_(repository)
52 823 : , socket_(socket)
53 : {
54 : // Check at least if repository is correct
55 : git_repository* repo;
56 823 : if (git_repository_open(&repo, repository_.c_str()) != 0) {
57 1 : socket_->shutdown();
58 1 : return;
59 : }
60 822 : git_repository_free(repo);
61 :
62 822 : socket_->setOnRecv([this](const uint8_t* buf, std::size_t len) {
63 5636 : std::lock_guard lk(destroyMtx_);
64 5635 : if (isDestroying_)
65 0 : return len;
66 5633 : if (parseOrder(std::string_view((const char*)buf, len)))
67 15463 : while(parseOrder());
68 5636 : return len;
69 5636 : });
70 0 : }
71 823 : ~Impl() { stop(); }
72 2054 : void stop()
73 : {
74 2054 : std::lock_guard lk(destroyMtx_);
75 2054 : if (isDestroying_.exchange(true)) {
76 1231 : socket_->setOnRecv({});
77 1231 : socket_->shutdown();
78 : }
79 2054 : }
80 : bool parseOrder(std::string_view buf = {});
81 :
82 : void sendReferenceCapabilities(bool sendVersion = false);
83 : bool NAK();
84 : void ACKCommon();
85 : bool ACKFirst();
86 : void sendPackData();
87 : std::map<std::string, std::string> getParameters(std::string_view pkt_line);
88 :
89 : std::string repositoryId_ {};
90 : std::string repository_ {};
91 : std::shared_ptr<dhtnet::ChannelSocket> socket_ {};
92 : std::string wantedReference_ {};
93 : std::string common_ {};
94 : std::vector<std::string> haveRefs_ {};
95 : std::string cachedPkt_ {};
96 : std::mutex destroyMtx_ {};
97 : std::atomic_bool isDestroying_ {false};
98 : onFetchedCb onFetchedCb_ {};
99 : };
100 :
101 : bool
102 21093 : GitServer::Impl::parseOrder(std::string_view buf)
103 : {
104 21093 : std::string pkt = std::move(cachedPkt_);
105 21091 : if (!buf.empty())
106 5632 : pkt += buf;
107 :
108 : // Parse pkt len
109 : // Reference: https://github.com/git/git/blob/master/Documentation/technical/protocol-common.txt#L51
110 : // The first four bytes define the length of the packet and 0000 is a FLUSH pkt
111 :
112 21091 : unsigned int pkt_len = 0;
113 21091 : auto [p, ec] = std::from_chars(pkt.data(), pkt.data() + 4, pkt_len, 16);
114 21086 : if (ec != std::errc()) {
115 0 : JAMI_ERROR("Can't parse packet size");
116 : }
117 21086 : if (pkt_len != pkt.size()) {
118 : // Store next packet part
119 17997 : if (pkt_len == 0) {
120 : // FLUSH_PKT
121 3651 : pkt_len = 4;
122 : }
123 17997 : cachedPkt_ = pkt.substr(pkt_len);
124 : }
125 :
126 21095 : auto pack = std::string_view(pkt).substr(4, pkt_len - 4);
127 21088 : if (pack == DONE_CMD) {
128 : // Reference:
129 : // https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L390 Do
130 : // not do multi-ack, just send ACK + pack file
131 : // In case of no common base, send NAK
132 1106 : JAMI_INFO("Peer negotiation is done. Answering to want order");
133 : bool sendData;
134 1106 : if (common_.empty())
135 190 : sendData = NAK();
136 : else
137 916 : sendData = ACKFirst();
138 1106 : if (sendData)
139 1106 : sendPackData();
140 1106 : return !cachedPkt_.empty();
141 19981 : } else if (pack.empty()) {
142 3651 : if (!haveRefs_.empty()) {
143 : // Reference:
144 : // https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L390
145 : // Do not do multi-ack, just send ACK + pack file In case of no common base ACK
146 560 : ACKCommon();
147 560 : NAK();
148 : }
149 3651 : return !cachedPkt_.empty();
150 : }
151 :
152 16332 : auto lim = pack.find(' ');
153 16336 : auto cmd = pack.substr(0, lim);
154 16335 : auto dat = (lim < pack.size()) ? pack.substr(lim+1) : std::string_view{};
155 16331 : if (cmd == UPLOAD_PACK_CMD) {
156 : // Cf: https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L166
157 : // References discovery
158 1985 : JAMI_INFO("Upload pack command detected.");
159 1985 : auto version = 1;
160 1985 : auto parameters = getParameters(dat);
161 1985 : auto versionIt = parameters.find("version");
162 1985 : bool sendVersion = false;
163 1985 : if (versionIt != parameters.end()) {
164 : try {
165 0 : version = std::stoi(versionIt->second);
166 0 : sendVersion = true;
167 0 : } catch (...) {
168 0 : JAMI_WARNING("Invalid version detected: {}", versionIt->second);
169 0 : }
170 : }
171 1985 : if (version == 1) {
172 1985 : sendReferenceCapabilities(sendVersion);
173 : } else {
174 0 : JAMI_ERR("That protocol version is not yet supported (version: %u)", version);
175 : }
176 16329 : } else if (cmd == WANT_CMD) {
177 : // Reference:
178 : // https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L229
179 : // TODO can have more want
180 1106 : wantedReference_ = dat.substr(0, 40);
181 1106 : JAMI_INFO("Peer want ref: %s", wantedReference_.c_str());
182 13243 : } else if (cmd == HAVE_CMD) {
183 13249 : const auto& commit = haveRefs_.emplace_back(dat.substr(0, 40));
184 13245 : if (common_.empty()) {
185 : // Detect first common commit
186 : // Reference:
187 : // https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L390
188 : // TODO do not open repository every time
189 : git_repository* repo;
190 916 : if (git_repository_open(&repo, repository_.c_str()) != 0) {
191 0 : JAMI_WARN("Couldn't open %s", repository_.c_str());
192 0 : return !cachedPkt_.empty();
193 : }
194 916 : GitRepository rep {repo, git_repository_free};
195 : git_oid commit_id;
196 916 : if (git_oid_fromstr(&commit_id, commit.c_str()) == 0) {
197 : // Reference found
198 916 : common_ = commit;
199 : }
200 916 : }
201 : } else {
202 0 : JAMI_WARNING("Unwanted packet received: {}", pkt);
203 : }
204 16336 : return !cachedPkt_.empty();
205 21094 : }
206 :
207 : void
208 1985 : GitServer::Impl::sendReferenceCapabilities(bool sendVersion)
209 : {
210 : // Get references
211 : // First, get the HEAD reference
212 : // https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L166
213 : git_repository* repo;
214 1985 : if (git_repository_open(&repo, repository_.c_str()) != 0) {
215 0 : JAMI_WARNING("Couldn't open {}", repository_);
216 0 : socket_->shutdown();
217 0 : return;
218 : }
219 1985 : GitRepository rep {repo, git_repository_free};
220 :
221 : // Answer with the version number
222 : // **** When the client initially connects the server will immediately respond
223 : // **** with a version number (if "version=1" is sent as an Extra Parameter),
224 1985 : std::string currentHead;
225 1985 : std::error_code ec;
226 1985 : std::stringstream packet;
227 1985 : if (sendVersion) {
228 0 : packet << "000eversion 1\0";
229 0 : socket_->write(reinterpret_cast<const unsigned char*>(packet.str().c_str()),
230 0 : packet.str().size(),
231 : ec);
232 0 : if (ec) {
233 0 : JAMI_WARNING("Couldn't send data for {}: {}", repository_, ec.message());
234 0 : socket_->shutdown();
235 0 : return;
236 : }
237 : }
238 :
239 : git_oid commit_id;
240 1985 : if (git_reference_name_to_id(&commit_id, rep.get(), "HEAD") < 0) {
241 0 : JAMI_ERROR("Cannot get reference for HEAD");
242 0 : socket_->shutdown();
243 0 : return;
244 : }
245 1985 : currentHead = git_oid_tostr_s(&commit_id);
246 :
247 : // Send references
248 1985 : std::string capStr = currentHead + SERVER_CAPABILITIES;
249 :
250 1985 : packet.str("");
251 1985 : packet << std::setw(4) << std::setfill('0') << std::hex << ((5 + capStr.size()) & 0x0FFFF);
252 1985 : packet << capStr << "\n";
253 :
254 : // Now, add other references
255 : git_strarray refs;
256 1985 : if (git_reference_list(&refs, rep.get()) == 0) {
257 21443 : for (std::size_t i = 0; i < refs.count; ++i) {
258 19452 : std::string ref = refs.strings[i];
259 19452 : if (git_reference_name_to_id(&commit_id, rep.get(), ref.c_str()) < 0) {
260 0 : JAMI_WARNING("Cannot get reference for {}", ref);
261 0 : continue;
262 0 : }
263 19459 : currentHead = git_oid_tostr_s(&commit_id);
264 :
265 19459 : packet << std::setw(4) << std::setfill('0') << std::hex
266 19452 : << ((6 /* size + space + \n */ + currentHead.size() + ref.size()) & 0x0FFFF);
267 19456 : packet << currentHead << " " << ref << "\n";
268 19457 : }
269 : }
270 1991 : git_strarray_dispose(&refs);
271 :
272 : // And add FLUSH
273 1985 : packet << FLUSH_PKT;
274 1985 : auto toSend = packet.str();
275 1985 : socket_->write(reinterpret_cast<const unsigned char*>(toSend.c_str()), toSend.size(), ec);
276 1985 : if (ec) {
277 0 : JAMI_WARNING("Couldn't send data for {}: {}", repository_, ec.message());
278 0 : socket_->shutdown();
279 : }
280 1985 : }
281 :
282 : void
283 560 : GitServer::Impl::ACKCommon()
284 : {
285 560 : std::error_code ec;
286 : // Ack common base
287 560 : if (!common_.empty()) {
288 560 : std::stringstream packet;
289 560 : packet << std::setw(4) << std::setfill('0') << std::hex
290 560 : << ((18 /* size + ACK + space * 2 + continue + \n */ + common_.size()) & 0x0FFFF);
291 560 : packet << "ACK " << common_ << " continue\n";
292 560 : auto toSend = packet.str();
293 560 : socket_->write(reinterpret_cast<const unsigned char*>(toSend.c_str()), toSend.size(), ec);
294 560 : if (ec) {
295 0 : JAMI_WARNING("Couldn't send data for {}: {}", repository_, ec.message());
296 0 : socket_->shutdown();
297 : }
298 560 : }
299 560 : }
300 :
301 : bool
302 916 : GitServer::Impl::ACKFirst()
303 : {
304 916 : std::error_code ec;
305 : // Ack common base
306 916 : if (!common_.empty()) {
307 916 : std::stringstream packet;
308 916 : packet << std::setw(4) << std::setfill('0') << std::hex
309 916 : << ((9 /* size + ACK + space + \n */ + common_.size()) & 0x0FFFF);
310 916 : packet << "ACK " << common_ << "\n";
311 916 : auto toSend = packet.str();
312 916 : socket_->write(reinterpret_cast<const unsigned char*>(toSend.c_str()), toSend.size(), ec);
313 916 : if (ec) {
314 0 : JAMI_WARNING("Couldn't send data for {}: {}", repository_, ec.message());
315 0 : socket_->shutdown();
316 0 : return false;
317 : }
318 916 : }
319 916 : return true;
320 : }
321 :
322 : bool
323 750 : GitServer::Impl::NAK()
324 : {
325 750 : std::error_code ec;
326 : // NAK
327 750 : socket_->write(reinterpret_cast<const unsigned char*>(NAK_PKT.data()), NAK_PKT.size(), ec);
328 750 : if (ec) {
329 0 : JAMI_WARNING("Couldn't send data for {}: {}", repository_, ec.message());
330 0 : socket_->shutdown();
331 0 : return false;
332 : }
333 750 : return true;
334 : }
335 :
336 : void
337 1106 : GitServer::Impl::sendPackData()
338 : {
339 : git_repository* repo_ptr;
340 1106 : if (git_repository_open(&repo_ptr, repository_.c_str()) != 0) {
341 0 : JAMI_WARN("Couldn't open %s", repository_.c_str());
342 0 : return;
343 : }
344 1106 : GitRepository repo {repo_ptr, git_repository_free};
345 :
346 : git_packbuilder* pb_ptr;
347 1106 : if (git_packbuilder_new(&pb_ptr, repo.get()) != 0) {
348 0 : JAMI_WARNING("Couldn't open packbuilder for {}", repository_);
349 0 : return;
350 : }
351 1106 : GitPackBuilder pb {pb_ptr, git_packbuilder_free};
352 :
353 1106 : std::string fetched = wantedReference_;
354 : git_oid oid;
355 1106 : if (git_oid_fromstr(&oid, wantedReference_.c_str()) < 0) {
356 0 : JAMI_ERROR("Cannot get reference for commit {}", wantedReference_);
357 0 : return;
358 : }
359 :
360 1106 : git_revwalk* walker_ptr = nullptr;
361 1106 : if (git_revwalk_new(&walker_ptr, repo.get()) < 0 || git_revwalk_push(walker_ptr, &oid) < 0) {
362 0 : if (walker_ptr)
363 0 : git_revwalk_free(walker_ptr);
364 0 : return;
365 : }
366 1106 : GitRevWalker walker {walker_ptr, git_revwalk_free};
367 1106 : git_revwalk_sorting(walker.get(), GIT_SORT_TOPOLOGICAL);
368 : // Add first commit
369 1106 : std::set<std::string> parents;
370 1106 : auto haveCommit = false;
371 :
372 3132 : while (!git_revwalk_next(&oid, walker.get())) {
373 : // log until have refs
374 2942 : std::string id = git_oid_tostr_s(&oid);
375 2941 : haveCommit |= std::find(haveRefs_.begin(), haveRefs_.end(), id) != haveRefs_.end();
376 2942 : auto itParents = std::find(parents.begin(), parents.end(), id);
377 2941 : if (itParents != parents.end())
378 1836 : parents.erase(itParents);
379 2941 : if (haveCommit && parents.size() == 0 /* We are sure that all commits are there */)
380 915 : break;
381 2026 : if (git_packbuilder_insert_commit(pb.get(), &oid) != 0) {
382 0 : JAMI_WARN("Couldn't open insert commit %s for %s",
383 : git_oid_tostr_s(&oid),
384 : repository_.c_str());
385 0 : return;
386 : }
387 :
388 : // Get next commit to pack
389 : git_commit* commit_ptr;
390 2026 : if (git_commit_lookup(&commit_ptr, repo.get(), &oid) < 0) {
391 0 : JAMI_ERR("Could not look up current commit");
392 0 : return;
393 : }
394 2026 : GitCommit commit {commit_ptr, git_commit_free};
395 2026 : auto parentsCount = git_commit_parentcount(commit.get());
396 3865 : for (unsigned int p = 0; p < parentsCount; ++p) {
397 : // make sure to explore all branches
398 1839 : const git_oid* pid = git_commit_parent_id(commit.get(), p);
399 1839 : if (pid)
400 1839 : parents.emplace(git_oid_tostr_s(pid));
401 : }
402 2941 : }
403 :
404 1106 : git_buf data = {};
405 1106 : if (git_packbuilder_write_buf(&data, pb.get()) != 0) {
406 0 : JAMI_WARN("Couldn't write pack data for %s", repository_.c_str());
407 0 : return;
408 : }
409 :
410 1106 : std::size_t sent = 0;
411 1106 : std::size_t len = data.size;
412 1106 : std::error_code ec;
413 : do {
414 : // cf https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt#L166
415 : // In 'side-band-64k' mode it will send up to 65519 data bytes plus 1 control code, for a
416 : // total of up to 65520 bytes in a pkt-line.
417 1571 : std::size_t pkt_size = std::min(static_cast<std::size_t>(65515), len - sent);
418 1572 : std::stringstream toSend;
419 1573 : toSend << std::setw(4) << std::setfill('0') << std::hex << ((pkt_size + 5) & 0x0FFFF);
420 1570 : toSend << "\x1" << std::string_view(data.ptr + sent, pkt_size);
421 1573 : std::string toSendStr = toSend.str();
422 :
423 1573 : socket_->write(reinterpret_cast<const unsigned char*>(toSendStr.c_str()),
424 : toSendStr.size(),
425 : ec);
426 1573 : if (ec) {
427 0 : JAMI_WARNING("Couldn't send data for {}: {}", repository_, ec.message());
428 0 : git_buf_dispose(&data);
429 0 : return;
430 : }
431 1573 : sent += pkt_size;
432 3146 : } while (sent < len);
433 1106 : git_buf_dispose(&data);
434 :
435 : // And finish by a little FLUSH
436 1106 : socket_->write(reinterpret_cast<const uint8_t*>(FLUSH_PKT.data()), FLUSH_PKT.size(), ec);
437 1106 : if (ec) {
438 0 : JAMI_WARNING("Couldn't send data for {}: {}", repository_, ec.message());
439 : }
440 :
441 : // Clear sent data
442 1106 : haveRefs_.clear();
443 1106 : wantedReference_.clear();
444 1106 : common_.clear();
445 1106 : if (onFetchedCb_)
446 1106 : onFetchedCb_(fetched);
447 1106 : }
448 :
449 : std::map<std::string, std::string>
450 1985 : GitServer::Impl::getParameters(std::string_view pkt_line)
451 : {
452 1985 : std::map<std::string, std::string> parameters;
453 1985 : std::string key, value;
454 1985 : auto isKey = true;
455 1985 : auto nullChar = 0;
456 224305 : for (auto letter: pkt_line) {
457 222320 : if (letter == '\0') {
458 : // parameters such as host or version are after the first \0
459 3970 : if (nullChar != 0 && !key.empty()) {
460 1985 : parameters[std::move(key)] = std::move(value);
461 : }
462 3970 : nullChar += 1;
463 3970 : isKey = true;
464 3970 : key.clear();
465 3970 : value.clear();
466 218350 : } else if (letter == '=') {
467 1985 : isKey = false;
468 216365 : } else if (nullChar != 0) {
469 134980 : if (isKey) {
470 7940 : key += letter;
471 : } else {
472 127040 : value += letter;
473 : }
474 : }
475 : }
476 3970 : return parameters;
477 1985 : }
478 :
479 823 : GitServer::GitServer(const std::string& accountId,
480 : const std::string& conversationId,
481 823 : const std::shared_ptr<dhtnet::ChannelSocket>& client)
482 : {
483 1646 : auto path = (fileutils::get_data_dir() / accountId / "conversations" / conversationId).string();
484 823 : pimpl_ = std::make_unique<GitServer::Impl>(conversationId, path, client);
485 823 : }
486 :
487 823 : GitServer::~GitServer()
488 : {
489 823 : stop();
490 823 : pimpl_.reset();
491 823 : JAMI_INFO("GitServer destroyed");
492 823 : }
493 :
494 : void
495 823 : GitServer::setOnFetched(const onFetchedCb& cb)
496 : {
497 823 : if (!pimpl_)
498 0 : return;
499 823 : pimpl_->onFetchedCb_ = cb;
500 : }
501 :
502 : void
503 1231 : GitServer::stop()
504 : {
505 1231 : pimpl_->stop();
506 1231 : }
507 :
508 : } // namespace jami
|