xap_helper: implement mset endpoint for WWW, IMAP, etc...

The C++ version will allow us to take full advantage of Xapian's
APIs for better queries, and the Perl bindings version can still
be advantageous in the future since we'll be able to support
timeouts effectively.

2 files changed, 95 insertions, 6 deletions

diff --git a/t/cindex.t b/t/cindex.t
index 261945bf..a9075092 100644
--- a/t/cindex.t
+++ b/t/cindex.t
@@ -121,22 +121,70 @@ my $no_metadata_set = sub {
 
 use_ok 'PublicInbox::CodeSearch';
 
+
+my @xh_args;
+my $exp = [ 'initial with NUL character', 'remove NUL character' ];
+my $zp_git = abs_path("$zp/.git");
 if ('multi-repo search') {
         my $csrch = PublicInbox::CodeSearch->new("$tmp/ext");
         my $mset = $csrch->mset('NUL');
         is(scalar($mset->items), 2, 'got results');
-        my $exp = [ 'initial with NUL character', 'remove NUL character' ];
         my @have = sort(map { $_->get_document->get_data } $mset->items);
         is_xdeeply(\@have, $exp, 'got expected subjects');
 
         $mset = $csrch->mset('NUL', { git_dir => "$tmp/wt0/.git" });
         is(scalar($mset->items), 0, 'no results with other GIT_DIR');
 
-        $mset = $csrch->mset('NUL', { git_dir => abs_path("$zp/.git") });
+        $mset = $csrch->mset('NUL', { git_dir => $zp_git });
         @have = sort(map { $_->get_document->get_data } $mset->items);
         is_xdeeply(\@have, $exp, 'got expected subjects w/ GIT_DIR filter');
         my @xdb = $csrch->xdb_shards_flat;
         $no_metadata_set->(0, ['indexlevel'], \@xdb);
+        @xh_args = $csrch->xh_args;
+}
+
+my $test_xhc = sub {
+        my ($xhc) = @_;
+        my $impl = $xhc->{impl};
+        my ($r, @l);
+        $r = $xhc->mkreq([], qw(mset -D -c -g), $zp_git, @xh_args, 'NUL');
+        chomp(@l = <$r>);
+        is(shift(@l), 'mset.size=2', "got expected header $impl");
+        my %docid2data;
+        my @got = sort map {
+                my @f = split /\0/;
+                is scalar(@f), 2, 'got 2 entries';
+                $docid2data{$f[0]} = $f[1];
+                $f[1];
+        } @l;
+        is_deeply(\@got, $exp, "expected doc_data $impl");
+
+        $r = $xhc->mkreq([], qw(mset -c -g), "$tmp/wt0/.git", @xh_args, 'NUL');
+        chomp(@l = <$r>);
+        is(shift(@l), 'mset.size=0', "got miss in wrong dir $impl");
+        is_deeply(\@l, [], "no extra lines $impl");
+
+        my $csrch = PublicInbox::CodeSearch->new("$tmp/ext");
+        while (my ($did, $expect) = each %docid2data) {
+                is_deeply($csrch->xdb->get_document($did)->get_data,
+                        $expect, "docid=$did data matches");
+        }
+        ok(!$xhc->{io}->close, "$impl close");
+        is($?, 66 << 8, "got EX_NOINPUT from $impl exit");
+};
+
+SKIP: {
+        require_mods('+SCM_RIGHTS', 1);
+        require PublicInbox::XapClient;
+        my $xhc = PublicInbox::XapClient::start_helper('-j0');
+        $test_xhc->($xhc);
+        skip 'PI_NO_CXX set', 1 if $ENV{PI_NO_CXX};
+        $xhc->{impl} =~ /Cxx/ or
+                skip 'C++ compiler or xapian development libs missing', 1;
+        skip 'TEST_XH_CXX_ONLY set', 1 if $ENV{TEST_XH_CXX_ONLY};
+        local $ENV{PI_NO_CXX} = 1; # force XS or SWIG binding test
+        $xhc = PublicInbox::XapClient::start_helper('-j0');
+        $test_xhc->($xhc);
 }
 
 if ('--update') {
diff --git a/t/xap_helper.t b/t/xap_helper.t
index e3abeded..ee25b2dc 100644
--- a/t/xap_helper.t
+++ b/t/xap_helper.t
@@ -40,6 +40,7 @@ my $v2 = create_inbox 'v2', indexlevel => 'medium', version => 2,
 };
 
 my @ibx_idx = glob("$v2->{inboxdir}/xap*/?");
+my @ibx_shard_args = map { ('-d', $_) } @ibx_idx;
 my (@int) = glob("$crepo/public-inbox-cindex/cidx*/?");
 my (@ext) = glob("$crepo/cidx-ext/cidx*/?");
 is(scalar(@ext), 2, 'have 2 external shards') or diag explain(\@ext);
@@ -76,8 +77,7 @@ my $test = sub {
         is($cinfo{has_threadid}, '0', 'has_threadid false for cindex');
         is($cinfo{pid}, $info{pid}, 'PID unchanged for cindex');
 
-        my @dump = (qw(dump_ibx -A XDFID), (map { ('-d', $_) } @ibx_idx),
-                        qw(13 rt:0..));
+        my @dump = (qw(dump_ibx -A XDFID), @ibx_shard_args, qw(13 rt:0..));
         $r = $doreq->($s, @dump);
         my @res;
         while (sysread($r, my $buf, 512) != 0) { push @res, $buf }
@@ -89,7 +89,8 @@ my $test = sub {
         my $res = do { local $/; <$r> };
         is(join('', @res), $res, 'got identical response w/ error pipe');
         my $stats = do { local $/; <$err_rd> };
-        is($stats, "mset.size=6 nr_out=6\n", 'mset.size reported');
+        is($stats, "mset.size=6 nr_out=6\n", 'mset.size reported') or
+                diag "res=$res";
 
         return wantarray ? ($ar, $s) : $ar if $cinfo{pid} == $pid;
 
@@ -198,7 +199,47 @@ for my $n (@NO_CXX) {
         is(scalar(@res), scalar(grep(/\A[0-9a-f]{40,} [0-9]+\n\z/, @res)),
                 'entries match format');
         $err = do { local $/; <$err_r> };
-        is($err, "mset.size=6 nr_out=5\n", "got expected status ($xhc->{impl})");
+        is $err, "mset.size=6 nr_out=5\n", "got expected status ($xhc->{impl})";
+
+        $r = $xhc->mkreq([], qw(mset -p -A XDFID -A Q), @ibx_shard_args,
+                                'dfn:lib/PublicInbox/Search.pm');
+        chomp((my $hdr, @res) = readline($r));
+        is $hdr, 'mset.size=1', "got expected header via mset ($xhc->{impl}";
+        is scalar(@res), 1, 'got one result';
+        @res = split /\0/, $res[0];
+        {
+                my $doc = $v2->search->xdb->get_document($res[0]);
+                my @q = PublicInbox::Search::xap_terms('Q', $doc);
+                is_deeply \@q, [ $mid ], 'docid usable';
+        }
+        ok $res[1] > 0 && $res[1] <= 100, 'pct > 0 && <= 100';
+        is $res[2], 'XDFID'.$dfid, 'XDFID result matches';
+        is $res[3], 'Q'.$mid, 'Q (msgid) mset result matches';
+        is scalar(@res), 4, 'only 4 columns in result';
+
+        $r = $xhc->mkreq([], qw(mset -p -A XDFID -A Q), @ibx_shard_args,
+                                'dt:19700101'.'000000..');
+        chomp(($hdr, @res) = readline($r));
+        is $hdr, 'mset.size=6',
+                "got expected header via multi-result mset ($xhc->{impl}";
+        is(scalar(@res), 6, 'got 6 rows');
+        for my $r (@res) {
+                my ($docid, $pct, @rest) = split /\0/, $r;
+                my $doc = $v2->search->xdb->get_document($docid);
+                ok $pct > 0 && $pct <= 100,
+                        "pct > 0 && <= 100 #$docid ($xhc->{impl})";
+                my %terms;
+                for (@rest) {
+                        s/\A([A-Z]+)// or xbail 'no prefix=', \@rest;
+                        push @{$terms{$1}}, $_;
+                }
+                while (my ($pfx, $vals) = each %terms) {
+                        @$vals = sort @$vals;
+                        my @q = PublicInbox::Search::xap_terms($pfx, $doc);
+                        is_deeply $vals, \@q,
+                                "#$docid $pfx as expected ($xhc->{impl})";
+                }
+        }
 }
 
 done_testing;