From 9c42ece6dfddf4156dc3016e2fa8835bf3d8aca1 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 24 Jun 2021 05:50:04 +0000 Subject: favor git(1) rather than libgit2 for ExtSearch While both git and libgit2 take around 16 minutes to load 100K alternates there's already a proposed patch to make git faster: It's also easier to patch and install git locally since the git.git build system defaults to prefix=$HOME and dealing with dynamic linking with libgit2 is more difficult for end users relying on Inline::C. libgit2 remains in use for the non-ALL.git case, but maybe it's not necessary (libgit2 is significantly slower than git in Debian 10 due to SHA-1 collision checking). --- lib/PublicInbox/GitAsyncCat.pm | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'lib/PublicInbox/GitAsyncCat.pm') diff --git a/lib/PublicInbox/GitAsyncCat.pm b/lib/PublicInbox/GitAsyncCat.pm index 7d1a13db..57c194d9 100644 --- a/lib/PublicInbox/GitAsyncCat.pm +++ b/lib/PublicInbox/GitAsyncCat.pm @@ -8,7 +8,7 @@ use strict; use parent qw(PublicInbox::DS Exporter); use POSIX qw(WNOHANG); use PublicInbox::Syscall qw(EPOLLIN EPOLLET); -our @EXPORT = qw(git_async_cat git_async_prefetch); +our @EXPORT = qw(ibx_async_cat ibx_async_prefetch); use PublicInbox::Git (); our $GCF2C; # singleton PublicInbox::Gcf2Client @@ -45,12 +45,16 @@ sub event_step { } } -sub git_async_cat ($$$$) { - my ($git, $oid, $cb, $arg) = @_; - if ($GCF2C //= eval { +sub ibx_async_cat ($$$$) { + my ($ibx, $oid, $cb, $arg) = @_; + my $git = $ibx->git; + # {topdir} means ExtSearch (likely [extindex "all"]) with potentially + # 100K alternates. git(1) has a proposed patch for 100K alternates: + # + if (!defined($ibx->{topdir}) && ($GCF2C //= eval { require PublicInbox::Gcf2Client; PublicInbox::Gcf2Client::new(); - } // 0) { # 0: do not retry if libgit2 or Inline::C are missing + } // 0)) { # 0: do not retry if libgit2 or Inline::C are missing $GCF2C->gcf2_async(\"$oid $git->{git_dir}\n", $cb, $arg); \undef; } else { # read-only end of git-cat-file pipe @@ -66,9 +70,10 @@ sub git_async_cat ($$$$) { # this is safe to call inside $cb, but not guaranteed to enqueue # returns true if successful, undef if not. -sub git_async_prefetch { - my ($git, $oid, $cb, $arg) = @_; - if ($GCF2C) { +sub ibx_async_prefetch { + my ($ibx, $oid, $cb, $arg) = @_; + my $git = $ibx->git; + if (!defined($ibx->{topdir}) && $GCF2C) { if (!$GCF2C->{wbuf}) { $oid .= " $git->{git_dir}\n"; return $GCF2C->gcf2_async(\$oid, $cb, $arg); # true -- cgit v1.2.3-24-ge0c7