From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <e@80x24.org>
X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on dcvr.yhbt.net
X-Spam-Level: 
X-Spam-ASN:  
X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00,
	DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,
	T_SCC_BODY_TEXT_LINE shortcircuit=no autolearn=ham autolearn_force=no
	version=3.4.6
Received: from localhost (dcvr.yhbt.net [127.0.0.1])
	by dcvr.yhbt.net (Postfix) with ESMTP id 08B731F406
	for <spew@80x24.org>; Mon, 20 Nov 2023 17:50:32 +0000 (UTC)
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org;
	s=selector1; t=1700502632;
	bh=RCSpuHDEB6QG2ZshMCwhXBr3IuWS2m2dmC16nYXGNXE=;
	h=From:To:Subject:Date:From;
	b=btxgIqb4EaenVMh0s198G5z7JbqAqG2RKaD18S70493DOz8amc9cOCi6XM1LHiYMz
	 vCoKLG4jrk/TK6uE/rmfRvleidoQxMe7T91x2vW7co2aQaJzk8kZFsXQc6No2HClBB
	 oQNQESS1J/YKq4PDxSz1u5qIXU9vnFbwCQV+OGY8=
From: Eric Wong <e@80x24.org>
To: spew@80x24.org
Subject: [PATCH] searchidx: run `git patch-id' in parallel
Date: Mon, 20 Nov 2023 17:50:31 +0000
Message-Id: <20231120175031.1669409-1-e@80x24.org>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
List-Id: <spew.80x24.org>

Informal benchmarks show a rough 5% indexing improvement on an
SMP system when there are idle cores due to Xapian shards being
I/O bound (since `git patch-id' is mainly CPU bound).

This is only parallelized on a per-patch basis.  Further
increasing parallelism would increase complexity and probably
not be worth it since `git patch-id'.
---
 lib/PublicInbox/CodeSearchIdx.pm |  2 +-
 lib/PublicInbox/SearchIdx.pm     | 24 +++++++++++-------------
 2 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index 5d619faf..fc7cdbde 100644
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -673,7 +673,7 @@ sub get_roots ($$) {
 		{ 0 => $refs, 1 => $roots_fh }, \&index_repo, $self, $git)
 }
 
-# for PublicInbox::SearchIdx::patch_id and with_umask
+# for PublicInbox::SearchIdx `git patch-id' call and with_umask
 sub git { $_[0]->{git} }
 
 sub load_existing ($) { # for -u/--update
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 9566b14d..32598b7c 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -22,7 +22,7 @@ use POSIX qw(strftime);
 use Fcntl qw(SEEK_SET);
 use Time::Local qw(timegm);
 use PublicInbox::OverIdx;
-use PublicInbox::Spawn qw(run_wait run_qx);
+use PublicInbox::Spawn qw(run_wait popen_rd);
 use PublicInbox::Git qw(git_unquote);
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
 use PublicInbox::Address;
@@ -350,20 +350,13 @@ sub index_diff ($$$) {
 	index_text($self, join("\n", @$xnq), 1, 'XNQ');
 }
 
-sub patch_id {
-	my ($self, $sref) = @_;
-	my $git = ($self->{ibx} // $self->{eidx} // $self)->git;
-	my $opt = { 0 => [ ':utf8', $sref ], 2 => \(my $err) };
-	my $id = run_qx($git->cmd(qw(patch-id --stable)), undef, $opt);
-	warn $err if $err;
-	$id =~ /\A([a-f0-9]{40,})/ ? $1 : undef;
-}
-
 sub index_body_text {
 	my ($self, $doc, $sref) = @_;
-	if ($$sref =~ /^(?:diff|---|\+\+\+) /ms) {
-		my $id = patch_id($self, $sref);
-		$doc->add_term('XDFID'.$id) if defined($id);
+	my $rd;
+	if ($$sref =~ /^(?:diff|---|\+\+\+) /ms) { # start patch-id in parallel
+		my $git = ($self->{ibx} // $self->{eidx} // $self)->git;
+		$rd = popen_rd($git->cmd(qw(patch-id --stable)), undef,
+				{ 0 => [ ':utf8', $sref ] });
 	}
 
 	# split off quoted and unquoted blocks:
@@ -387,6 +380,11 @@ sub index_body_text {
 		}
 		undef $txt; # free memory
 	}
+	if (defined $rd) { # reap `git patch-id'
+		(readline($rd) // '') =~ /\A([a-f0-9]{40,})/ and
+			$doc->add_term('XDFID'.$1);
+		$rd->close or warn "W: git patch-id failed: \$?=$? (non-fatal)"
+	}
 }
 
 sub index_xapian { # msg_iter callback