From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <e@80x24.org>
X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on dcvr.yhbt.net
X-Spam-Level: 
X-Spam-ASN:  
X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,BAYES_00,
	DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no
	autolearn=ham autolearn_force=no version=3.4.6
Received: from localhost (dcvr.yhbt.net [127.0.0.1])
	by dcvr.yhbt.net (Postfix) with ESMTP id EF3291F626
	for <spew@80x24.org>; Tue, 14 Feb 2023 11:09:55 +0000 (UTC)
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org;
	s=selector1; t=1676372996;
	bh=ynSH3ZqATwSQBYMwBsyKqyxeUBwEL0a1ombaNJ9QNXM=;
	h=From:To:Subject:Date:From;
	b=Bk41EFeXiEJC0iaW6dAIg96uLNBFGGOqL6B6p1GKElmOpAwCA3cF9ue/qeKSfBskY
	 SX9owcesboudXuI1+68Jc8ydUkClhX4A4DBJzKmIFZJj8OpoCdgQ9nFjkQmu9OW7Mc
	 XvHoyAUAFoNmyejmAIGfKbnp08WmZaLyEOqXsmWQ=
From: Eric Wong <e@80x24.org>
To: spew@80x24.org
Subject: [PATCH] lei_mirror: use fetch.hideRefs to speed up connectivity check
Date: Tue, 14 Feb 2023 11:09:55 +0000
Message-Id: <20230214110955.2835814-1-e@80x24.org>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
List-Id: <spew.80x24.org>

`git fetch' runs an expensive connectivity check against all
refs, which is unnecessarily expensive for incremental fetches
on RAM-constrained systems.

This depends on the proposal to support `fetch.hideRefs' for `git fetch':
https://public-inbox.org/git/20230212090426.M558990@dcvr/
---
 lib/PublicInbox/LeiMirror.pm | 48 ++++++++++++++++++++++++++----------
 1 file changed, 35 insertions(+), 13 deletions(-)

diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 7e870df6..1e409f5b 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -401,19 +401,20 @@ sub fgrp_fetch_all {
 	my $opt = {};
 	my @fetch = do {
 		local $self->{lei}->{opt}->{jobs} = 1;
-		(fetch_args($self->{lei}, $opt), qw(--no-tags --multiple));
+		(fetch_args($self->{lei}, $opt), qw(--no-tags));
 	};
 	push(@fetch, "-j$j") if $j;
 	while (my ($osdir, $fgrp_old_new) = each %$todo) {
 		my $f = "$osdir/config";
 		return if !keep_going($self);
-		my ($fgrpv, $new) = @$fgrp_old_new;
-		@$fgrpv = sort { $b->{-sort} <=> $a->{-sort} } @$fgrpv;
-		push @$fgrpv, @$new; # $new is ordered by references
-
+		my ($old, $new) = @$fgrp_old_new;
+		@$old = sort { $b->{-sort} <=> $a->{-sort} } @$old;
+		# $new is ordered by {references}
+		my $new0; # initial clone on empty objstore, only
 		my $cmd = ['git', "--git-dir=$osdir", qw(config -f), $f ];
-		# clobber group from previous run atomically
-		for ("remotes.$grp") { # TODO: hideRefs
+
+		# clobber settings from previous run atomically
+		for ("remotes.$grp", 'fetch.hideRefs') {
 			my $c = [ @$cmd, '--unset-all', $_ ];
 			$self->{lei}->qerr("# @$c");
 			next if $self->{dry_run};
@@ -424,7 +425,7 @@ sub fgrp_fetch_all {
 
 		# permanent configs:
 		my $cfg = PublicInbox::Config->git_config_dump($f);
-		for my $fgrp (@$fgrpv) {
+		for my $fgrp (@$old, @$new) {
 			my $u = $fgrp->{-uri} // die 'BUG: no {-uri}';
 			my $rn = $fgrp->{-remote} // die 'BUG: no {-remote}';
 			for ("url=$u", "fetch=+refs/*:refs/remotes/$rn/*",
@@ -446,15 +447,36 @@ sub fgrp_fetch_all {
 				or die "open($f.lock): $!";
 			open my $fh, '>>', $f or die "open(>>$f): $!";
 			$fh->autoflush(1);
-			my $buf = join('', "[remotes]\n",
-				map { "\t$grp = $_->{-remote}\n" } @$fgrpv);
+			my $buf = "[fetch]\n\thideRefs = refs\n";
+			if (@$old) {
+				$buf .= join('', map {
+					"\thideRefs = !refs/remotes/" .
+						"$_->{-remote}/\n";
+				} @$old);
+			} else {
+				$new0 = shift @$new;
+				$buf .= "\thideRefs = !refs/remotes/" .
+					"$new0->{-remote}/\n";
+			}
+			$buf .= join('', "[remotes]\n",
+				(map { "\t$grp = $_->{-remote}\n" } @$old),
+				(map { "\t$grp = $_->{-remote}\n" } @$new));
 			print $fh $buf or die "print($f): $!";
 			close $fh or die "close($f): $!";
 			unlink("$f.lock") or die "unlink($f.lock): $!";
 		}
-		$cmd = [ @git, "--git-dir=$osdir", @fetch, $grp ];
-		my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $fgrpv);
-		start_cmd($self, $cmd, $opt, $end);
+		my $m = [@git, "--git-dir=$osdir", @fetch, '--multiple', $grp];
+		push @$old, $new0 if defined($new0);
+		push @$old, @$new;
+		my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $old);
+		my $rest = defined($new0) && scalar(@$old) == 1 ? $end :
+				PublicInbox::OnDestroy->new($$, \&start_cmd,
+							$self, $m, $opt, $end);
+		if (defined $new0) {
+			$cmd = [ @git, "--git-dir=$osdir", @fetch,
+				$new0->{-remote} ];
+			start_cmd($self, $cmd, $opt, $rest);
+		} # else: $rest gets destroyed
 	}
 }