From: Eric Wong <e@80x24.org>
To: spew@80x24.org
Subject: [PATCH] lei_mirror: use fetch.hideRefs to speed up connectivity check
Date: Tue, 14 Feb 2023 09:13:11 +0000 [thread overview]
Message-ID: <20230214091311.2809825-1-e@80x24.org> (raw)
`git fetch' runs an expensive connectivity check against all
refs, which is unnecessarily expensive for incremental fetches
on RAM-constrained systems.
This depends on the proposal to support `fetch.hideRefs' for `git fetch':
https://public-inbox.org/git/20230212090426.M558990@dcvr/
---
lib/PublicInbox/LeiMirror.pm | 50 ++++++++++++++++++++++++++----------
1 file changed, 37 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 4dedac9b..fbc7c251 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -401,19 +401,20 @@ sub fgrp_fetch_all {
my $opt = {};
my @fetch = do {
local $self->{lei}->{opt}->{jobs} = 1;
- (fetch_args($self->{lei}, $opt), qw(--no-tags --multiple));
+ (fetch_args($self->{lei}, $opt), qw(--no-tags));
};
push(@fetch, "-j$j") if $j;
while (my ($osdir, $fgrp_old_new) = each %$todo) {
my $f = "$osdir/config";
return if !keep_going($self);
- my ($fgrpv, $new) = @$fgrp_old_new;
- @$fgrpv = sort { $b->{-sort} <=> $a->{-sort} } @$fgrpv;
- push @$fgrpv, @$new; # $new is ordered by references
-
+ my ($old, $new) = @$fgrp_old_new;
+ @$old = sort { $b->{-sort} <=> $a->{-sort} } @$old;
+ # $new is ordered by {references}
+ my $new0; # initial clone on empty objstore, only
my $cmd = ['git', "--git-dir=$osdir", qw(config -f), $f ];
- # clobber group from previous run atomically
- for ("remotes.$grp") { # TODO: hideRefs
+
+ # clobber settings from previous run atomically
+ for ("remotes.$grp", 'fetch.hideRefs') {
my $c = [ @$cmd, '--unset-all', $_ ];
$self->{lei}->qerr("# @$c");
next if $self->{dry_run};
@@ -424,7 +425,7 @@ sub fgrp_fetch_all {
# permanent configs:
my $cfg = PublicInbox::Config->git_config_dump($f);
- for my $fgrp (@$fgrpv) {
+ for my $fgrp (@$old, @$new) {
my $u = $fgrp->{-uri} // die 'BUG: no {-uri}';
my $rn = $fgrp->{-remote} // die 'BUG: no {-remote}';
for ("url=$u", "fetch=+refs/*:refs/remotes/$rn/*",
@@ -446,15 +447,38 @@ sub fgrp_fetch_all {
or die "open($f.lock): $!";
open my $fh, '>>', $f or die "open(>>$f): $!";
$fh->autoflush(1);
- my $buf = join('', "[remotes]\n",
- map { "\t$grp = $_->{-remote}\n" } @$fgrpv);
+ my $buf = "[fetch]\n\thideRefs = refs\n";
+ if (@$old) {
+ $buf .= join('', map {
+ "\thideRefs = !refs/remotes/" .
+ "$_->{-remote}/\n";
+ } @$old);
+ } else {
+ $new0 = shift @$new;
+ $buf .= "\thideRefs = !refs/remotes/" .
+ "$new0->{-remote}/\n";
+ }
+ $buf .= join('', "[remotes]\n",
+ (map { "\t$grp = $_->{-remote}\n" } @$old),
+ (map { "\t$grp = $_->{-remote}\n" } @$new));
print $fh $buf or die "print($f): $!";
close $fh or die "close($f): $!";
unlink("$f.lock") or die "unlink($f.lock): $!";
}
- $cmd = [ @git, "--git-dir=$osdir", @fetch, $grp ];
- my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $fgrpv);
- start_cmd($self, $cmd, $opt, $end);
+ my $m = [@git, "--git-dir=$osdir", @fetch, '--multiple', $grp];
+ push @$old, $new0 if defined($new0);
+ push @$old, @$new;
+ my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $old);
+ my $rest = $end;
+ if (defined($new0) && scalar(@$old) > 1) {
+ $rest = PublicInbox::OnDestroy->new($$,
+ \&start_cmd, $self, $m, $opt, $end);
+ }
+ if (defined $new0) {
+ $cmd = [ @git, "--git-dir=$osdir", @fetch,
+ $new0->{-remote} ];
+ start_cmd($self, $cmd, $opt, $rest);
+ } # else: $rest gets destroyed
}
}
next reply other threads:[~2023-02-14 9:13 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-02-14 9:13 Eric Wong [this message]
-- strict thread matches above, loose matches on Subject: below --
2023-02-14 11:09 [PATCH] lei_mirror: use fetch.hideRefs to speed up connectivity check Eric Wong
2023-02-15 5:33 Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230214091311.2809825-1-e@80x24.org \
--to=e@80x24.org \
--cc=spew@80x24.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).