diff options
| -rw-r--r-- | lib/PublicInbox/CodeSearchIdx.pm | 15 | ||||
| -rw-r--r-- | lib/PublicInbox/SearchIdx.pm | 2 | ||||
| -rw-r--r-- | t/cindex.t | 45 |
3 files changed, 59 insertions, 3 deletions
diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm index 5f20325a..3a3fc03e 100644 --- a/lib/PublicInbox/CodeSearchIdx.pm +++ b/lib/PublicInbox/CodeSearchIdx.pm @@ -85,7 +85,6 @@ sub new { xpfx => "$dir/cidx". PublicInbox::CodeSearch::CIDX_SCHEMA_VER, cidx_dir => $dir, creat => 1, # TODO: get rid of this, should be implicit - indexlevel => $l, transact_bytes => 0, # for checkpoint total_bytes => 0, # for lock_release current_info => '', @@ -617,16 +616,28 @@ sub cidx_init ($) { } $self->lock_acquire; my @shards; + my $l = $self->{indexlevel} //= $self->{-opt}->{indexlevel}; + for my $n (0..($self->{nshard} - 1)) { my $shard = bless { %$self, shard => $n }, ref($self); delete @$shard{qw(lockfh lock_path)}; - $shard->idx_acquire; + my $xdb = $shard->idx_acquire; + if (!$n) { + if (($l // '') eq 'medium') { + $xdb->set_metadata('indexlevel', $l); + } elsif (($l // '') eq 'full') { + $xdb->set_metadata('indexlevel', ''); # unset + } + $l ||= $xdb->get_metadata('indexlevel') || 'full'; + } + $shard->{indexlevel} = $l; $shard->idx_release; $shard->wq_workers_start("cidx shard[$n]", 1, $SIGSET, { siblings => \@shards, # for ipc_atfork_child }, \&shard_done_wait, $self); push @shards, $shard; } + $self->{indexlevel} //= $l; # this warning needs to happen after idx_acquire state $once; warn <<EOM if $PublicInbox::Search::X{CLOEXEC_UNSET} && !$once++; diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index f36c8f97..699af432 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -90,7 +90,7 @@ sub new { $self; } -sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels } +sub need_xapian ($) { ($_[0]->{indexlevel} // 'full') =~ $xapianlevels } sub idx_release { my ($self, $wake) = @_; @@ -4,11 +4,13 @@ use v5.12; use PublicInbox::TestCommon; use Cwd qw(getcwd abs_path); +use List::Util qw(sum); require_mods(qw(json Search::Xapian)); use_ok 'PublicInbox::CodeSearchIdx'; require PublicInbox::Import; my ($tmp, $for_destroy) = tmpdir(); my $pwd = getcwd(); +my @unused_keys = qw(last_commit has_threadid skip_docdata); # I reworked CodeSearchIdx->shard_worker to handle empty trees # in the initial commit generated by cvs2svn for xapian.git @@ -71,7 +73,48 @@ ok(run_script([qw(-cindex --dangerous -q -d), "$tmp/ext", $zp, "$tmp/wt0"]), ok(-e "$tmp/ext/cidx.lock", 'external dir created'); ok(!-d "$zp/.git/public-inbox-cindex", 'no cindex in original coderepo'); +ok(run_script([qw(-cindex -L medium --dangerous -q -d), + "$tmp/med", $zp, "$tmp/wt0"]), 'cindex external medium'); + +my $no_metadata_set = sub { + my ($i, $extra, $xdb) = @_; + for my $xdb (@$xdb) { + for my $k (@unused_keys, @$extra) { + is($xdb->get_metadata($k) // '', '', + "metadata $k unset in shard #$i"); + } + ++$i; + } +}; + +{ + my $mid_size = sum(map { -s $_ } glob("$tmp/med/cidx*/*/*")); + my $full_size = sum(map { -s $_ } glob("$tmp/ext/cidx*/*/*")); + ok($full_size > $mid_size, 'full size > mid size') or + diag "full=$full_size mid=$mid_size"; + for my $l (qw(med ext)) { + ok(run_script([qw(-cindex -q --reindex -u -d), "$tmp/$l"]), + "reindex $l"); + } + $mid_size = sum(map { -s $_ } glob("$tmp/med/cidx*/*/*")); + $full_size = sum(map { -s $_ } glob("$tmp/ext/cidx*/*/*")); + ok($full_size > $mid_size, 'full size > mid size after reindex') or + diag "full=$full_size mid=$mid_size"; + my $csrch = PublicInbox::CodeSearch->new("$tmp/med"); + my ($xdb0, @xdb) = $csrch->xdb_shards_flat; + $no_metadata_set->(0, [], [ $xdb0 ]); + is($xdb0->get_metadata('indexlevel'), 'medium', + 'indexlevel set in shard #0'); + $no_metadata_set->(1, ['indexlevel'], \@xdb); + + ok(run_script([qw(-cindex -q -L full --reindex -u -d), "$tmp/med"]), + 'reindex medium as full'); + @xdb = $csrch->xdb_shards_flat; + $no_metadata_set->(0, ['indexlevel'], \@xdb); +} + use_ok 'PublicInbox::CodeSearch'; + if ('multi-repo search') { my $csrch = PublicInbox::CodeSearch->new("$tmp/ext"); my $mset = $csrch->mset('NUL'); @@ -86,6 +129,8 @@ if ('multi-repo search') { $mset = $csrch->mset('NUL', { git_dir => abs_path("$zp/.git") }); @have = sort(map { $_->get_document->get_data } $mset->items); is_xdeeply(\@have, $exp, 'got expected subjects w/ GIT_DIR filter'); + my @xdb = $csrch->xdb_shards_flat; + $no_metadata_set->(0, ['indexlevel'], \@xdb); } if ('--update') { |