1 files changed, 53 insertions, 13 deletions

diff --git a/lib/PublicInbox/XapHelper.pm b/lib/PublicInbox/XapHelper.pm
index 2e20660e..ba41b5d2 100644
--- a/lib/PublicInbox/XapHelper.pm
+++ b/lib/PublicInbox/XapHelper.pm
@@ -18,7 +18,7 @@ use POSIX qw(:signal_h);
 use Fcntl qw(LOCK_UN LOCK_EX);
 use Carp qw(croak);
 my $X = \%PublicInbox::Search::X;
-our (%SRCH, %WORKERS, $nworker, $workerset, $in);
+our (%SRCH, %WORKERS, $nworker, $workerset, $in, $SHARD_NFD, $MY_FD_MAX);
 our $stderr = \*STDERR;
 
 sub cmd_test_inspect {
@@ -172,29 +172,65 @@ sub cmd_mset { # to be used by WWW + IMAP
         }
 }
 
+sub srch_init_extra ($) {
+        my ($req) = @_;
+        my $qp = $req->{srch}->{qp};
+        for (@{$req->{Q}}) {
+                my ($upfx, $m, $xpfx) = split /([:=])/;
+                $xpfx // die "E: bad -Q $_";
+                $m = $m eq '=' ? 'add_boolean_prefix' : 'add_prefix';
+                $qp->$m($upfx, $xpfx);
+        }
+        $req->{srch}->{qp_extra_done} = 1;
+}
+
 sub dispatch {
         my ($req, $cmd, @argv) = @_;
         my $fn = $req->can("cmd_$cmd") or return;
         $GLP->getoptionsfromarray(\@argv, $req, @PublicInbox::Search::XH_SPEC)
                 or return;
         my $dirs = delete $req->{d} or die 'no -d args';
-        my $key = join("\0", @$dirs);
-        $req->{srch} = $SRCH{$key} //= do {
-                my $new = { qp_flags => $PublicInbox::Search::QP_FLAGS };
+        my $key = "-d\0".join("\0-d\0", @$dirs);
+        $key .= "\0".join("\0", map { ('-Q', $_) } @{$req->{Q}}) if $req->{Q};
+        my $new;
+        $req->{srch} = $SRCH{$key} // do {
+                $new = { qp_flags => $PublicInbox::Search::QP_FLAGS };
+                my $nfd = scalar(@$dirs) * PublicInbox::Search::SHARD_COST;
+                $SHARD_NFD += $nfd;
+                if ($SHARD_NFD > $MY_FD_MAX) {
+                        $SHARD_NFD = $nfd;
+                        %SRCH = ();
+                }
                 my $first = shift @$dirs;
-                my $slow_phrase = -f "$first/iamchert";
-                $new->{xdb} = $X->{Database}->new($first);
-                for (@$dirs) {
-                        $slow_phrase ||= -f "$_/iamchert";
-                        $new->{xdb}->add_database($X->{Database}->new($_));
+                for my $retried (0, 1) {
+                        my $slow_phrase = -f "$first/iamchert";
+                        eval {
+                                $new->{xdb} = $X->{Database}->new($first);
+                                for (@$dirs) {
+                                        $slow_phrase ||= -f "$_/iamchert";
+                                        $new->{xdb}->add_database(
+                                                        $X->{Database}->new($_))
+                                }
+                        };
+                        last unless $@;
+                        if ($retried) {
+                                die "E: $@\n";
+                        } else { # may be EMFILE/ENFILE/ENOMEM....
+                                warn "W: $@, retrying...\n";
+                                %SRCH = ();
+                                $SHARD_NFD = $nfd;
+                        }
+                        $slow_phrase or $new->{qp_flags}
+                                |= PublicInbox::Search::FLAG_PHRASE();
                 }
-                $slow_phrase or
-                        $new->{qp_flags} |= PublicInbox::Search::FLAG_PHRASE();
                 bless $new, $req->{c} ? 'PublicInbox::CodeSearch' :
                                         'PublicInbox::Search';
                 $new->{qp} = $new->qparse_new;
-                $new;
+                $SRCH{$key} = $new;
         };
+        $req->{srch}->{xdb}->reopen unless $new;
+        $req->{Q} && !$req->{srch}->{qp_extra_done} and
+                srch_init_extra $req;
         my $timeo = $req->{K};
         alarm($timeo) if $timeo;
         $fn->($req, @argv);
@@ -288,7 +324,7 @@ sub start (@) {
         my $c = getsockopt(local $in = \*STDIN, SOL_SOCKET, SO_TYPE);
         unpack('i', $c) == SOCK_SEQPACKET or die 'stdin is not SOCK_SEQPACKET';
 
-        local (%SRCH, %WORKERS);
+        local (%SRCH, %WORKERS, $SHARD_NFD, $MY_FD_MAX);
         PublicInbox::Search::load_xapian();
         $GLP->getoptionsfromarray(\@argv, my $opt = { j => 1 }, 'j=i') or
                 die 'bad args';
@@ -297,6 +333,10 @@ sub start (@) {
         for (@PublicInbox::DS::UNBLOCKABLE, POSIX::SIGUSR1) {
                 $workerset->delset($_) or die "delset($_): $!";
         }
+        $MY_FD_MAX = PublicInbox::Search::ulimit_n //
+                die "E: unable to get RLIMIT_NOFILE: $!";
+        warn "W: RLIMIT_NOFILE=$MY_FD_MAX too low\n" if $MY_FD_MAX < 72;
+        $MY_FD_MAX -= 64;
 
         local $nworker = $opt->{j};
         return recv_loop() if $nworker == 0;