cindex: dump cidx shards before inboxes

Since cidx shards used for associations are typically bigger
than individual inboxes, we'll dump them first to get better
work scheduling for xap_helper processes.

This gives roughly a 5% performance improvement with doing
a full associate on (git+lore).kernel.org

1 files changed, 7 insertions, 4 deletions

diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index 404d6826..a9a7d313 100644
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -511,8 +511,10 @@ sub assoc_max_init ($) {
         $max < 0 ? ((2 ** 31) - 1) : $max;
 }
 
-sub dump_roots_once {
+sub dump_roots_start {
         my ($self, $associate) = @_;
+        ($XHC, $XH_PID) = PublicInbox::XapClient::start_helper("-j$NPROC");
+        awaitpid($XH_PID, \&cmd_done, ['xap_helper', "-j$NPROC"]);
         $associate // die 'BUG: no $associate';
         $TODO{associating} = 1; # keep shards_active() happy
         progress($self, 'dumping IDs from coderepos');
@@ -571,7 +573,6 @@ sub dump_ibx_start {
         my $fold_pid = spawn(\@UNIQ_FOLD, $CMD_ENV, { 0 => $fold_r, 1 => $fh });
         awaitpid($sort_pid, \&cmd_done, \@sort, $associate);
         awaitpid($fold_pid, \&cmd_done, [@UNIQ_FOLD, '(ibx)'], $associate);
-        ($XHC, $XH_PID) = PublicInbox::XapClient::start_helper("-j$NPROC");
 }
 
 sub index_next ($) {
@@ -586,11 +587,11 @@ sub index_next ($) {
                 fp_start($self, $git, $prep_repo);
                 ct_start($self, $git, $prep_repo);
         } elsif ($TMPDIR) {
+                return if delete($TODO{dump_roots_start});
                 delete $TODO{dump_ibx_start}; # runs OnDestroy once
                 return dump_ibx($self, shift @IBXQ) if @IBXQ;
-                progress($self, 'done dumping inboxes') if $DUMP_IBX_WPIPE;
                 undef $DUMP_IBX_WPIPE; # done dumping inboxes, dump roots
-                dump_roots_once($self, delete($TODO{associate}) // return);
+                delete $TODO{associate};
         }
         # else: wait for shards_active (post_loop_do) callback
 }
@@ -934,6 +935,8 @@ EOM
         $TODO{associate} = PublicInbox::OnDestroy->new($$, \&associate, $self);
         $TODO{dump_ibx_start} = PublicInbox::OnDestroy->new($$,
                                 \&dump_ibx_start, $self, $TODO{associate});
+        $TODO{dump_roots_start} = PublicInbox::OnDestroy->new($$,
+                                \&dump_roots_start, $self, $TODO{associate});
         my $id = -1;
         @IBXQ = map { ++$id } @IBX;
 }