From ab93e9dcb3bf1f439f1afea9e124b88ac830ca7f Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Wed, 14 Dec 2016 19:28:53 +0000 Subject: t/thread-cycle: no need for Xapian to run this test We don't actually use anything from SearchMsg, just the class name. --- t/thread-cycle.t | 1 - 1 file changed, 1 deletion(-) (limited to 't') diff --git a/t/thread-cycle.t b/t/thread-cycle.t index b0844490..9dd2aa3c 100644 --- a/t/thread-cycle.t +++ b/t/thread-cycle.t @@ -3,7 +3,6 @@ use strict; use warnings; use Test::More; -use_ok('PublicInbox::SearchMsg'); use_ok('PublicInbox::SearchThread'); use Email::Simple; my $mt = eval { -- cgit v1.2.3-24-ge0c7 From f084e94a4774b95eb45f55fc9f0dfda678522e54 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 17 Dec 2016 04:27:52 +0000 Subject: feed: support publicinbox..feedmax This allows users to customize by using smaller or larger Atom feeds than the default value of 25 entries. --- t/config.t | 2 ++ t/feed.t | 13 ++++--------- 2 files changed, 6 insertions(+), 9 deletions(-) (limited to 't') diff --git a/t/config.t b/t/config.t index 073d1d03..4bbbc838 100644 --- a/t/config.t +++ b/t/config.t @@ -30,6 +30,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1); 'url' => 'http://example.com/meta', -primary_address => 'meta@public-inbox.org', 'name' => 'meta', + feedmax => 100, -pi_config => $cfg, }, "lookup matches expected output"); @@ -45,6 +46,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1); 'mainrepo' => '/home/pi/test-main.git', 'domain' => 'public-inbox.org', 'name' => 'test', + feedmax => 100, 'url' => 'http://example.com/test', -pi_config => $cfg, }, "lookup matches expected output for test"); diff --git a/t/feed.t b/t/feed.t index 19a9ba09..b60273ed 100644 --- a/t/feed.t +++ b/t/feed.t @@ -46,6 +46,7 @@ my $ibx = PublicInbox::Inbox->new({ name => 'testbox', mainrepo => $git_dir, url => 'http://example.com/test', + feedmax => 3, }); my $git = $ibx->git; my $im = PublicInbox::Import->new($git, $ibx->{name}, 'test@example'); @@ -101,10 +102,7 @@ EOF { # check initial feed { - my $feed = string_feed({ - -inbox => $ibx, - max => 3 - }); + my $feed = string_feed({ -inbox => $ibx }); SKIP: { skip 'XML::Feed missing', 2 unless $have_xml_feed; my $p = XML::Feed->parse(\$feed); @@ -142,10 +140,7 @@ EOF # check spam shows up { - my $spammy_feed = string_feed({ - -inbox => $ibx, - max => 3 - }); + my $spammy_feed = string_feed({ -inbox => $ibx }); SKIP: { skip 'XML::Feed missing', 2 unless $have_xml_feed; my $p = XML::Feed->parse(\$spammy_feed); @@ -167,7 +162,7 @@ EOF # spam no longer shows up { - my $feed = string_feed({ -inbox => $ibx, max => 3 }); + my $feed = string_feed({ -inbox => $ibx }); SKIP: { skip 'XML::Feed missing', 2 unless $have_xml_feed; my $p = XML::Feed->parse(\$feed); -- cgit v1.2.3-24-ge0c7 From 7bf8dbcf615cc26f622f5a69192995dfa738a595 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 17 Dec 2016 05:50:30 +0000 Subject: t/config.t: fix feedmax default Oops :x --- t/config.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 't') diff --git a/t/config.t b/t/config.t index 4bbbc838..7271351b 100644 --- a/t/config.t +++ b/t/config.t @@ -30,7 +30,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1); 'url' => 'http://example.com/meta', -primary_address => 'meta@public-inbox.org', 'name' => 'meta', - feedmax => 100, + feedmax => 25, -pi_config => $cfg, }, "lookup matches expected output"); @@ -46,7 +46,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1); 'mainrepo' => '/home/pi/test-main.git', 'domain' => 'public-inbox.org', 'name' => 'test', - feedmax => 100, + feedmax => 25, 'url' => 'http://example.com/test', -pi_config => $cfg, }, "lookup matches expected output for test"); -- cgit v1.2.3-24-ge0c7 From 1a75ba282c16f8c15b7891090d0997628d7021dc Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 20 Dec 2016 03:03:56 +0000 Subject: tests: add thread-all testing for benchmarking I'll be using this to improve message threading performance. --- t/thread-all.t | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 t/thread-all.t (limited to 't') diff --git a/t/thread-all.t b/t/thread-all.t new file mode 100644 index 00000000..8ccf4f8c --- /dev/null +++ b/t/thread-all.t @@ -0,0 +1,38 @@ +# Copyright (C) 2016 all contributors +# License: AGPL-3.0+ +# +# real-world testing of search threading +use strict; +use warnings; +use Test::More; +use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC); +my $pi_dir = $ENV{GIANT_PI_DIR}; +plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir; +eval { require PublicInbox::Search; }; +plan skip_all => "Xapian missing for $0" if $@; +my $srch = eval { PublicInbox::Search->new($pi_dir) }; +plan skip_all => "$pi_dir not initialized for $0" if $@; + +require PublicInbox::View; +require PublicInbox::SearchThread; + +my $pfx = PublicInbox::Search::xpfx('thread'); +my $opts = { limit => 1000000, asc => 1 }; +my $t0 = clock_gettime(CLOCK_MONOTONIC); +my $elapsed; + +my $sres = $srch->_do_enquire(undef, $opts); +$elapsed = clock_gettime(CLOCK_MONOTONIC) - $t0; +diag "enquire: $elapsed"; + +$t0 = clock_gettime(CLOCK_MONOTONIC); +my $msgs = PublicInbox::View::load_results($srch, $sres); +$elapsed = clock_gettime(CLOCK_MONOTONIC) - $t0; +diag "load_results $elapsed"; + +$t0 = clock_gettime(CLOCK_MONOTONIC); +PublicInbox::View::thread_results($msgs); +$elapsed = clock_gettime(CLOCK_MONOTONIC) - $t0; +diag "thread_results $elapsed"; + +done_testing(); -- cgit v1.2.3-24-ge0c7 From 478d03688600a4c7b50e205d15d76113e019f3cd Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 20 Dec 2016 03:03:57 +0000 Subject: searchmsg: remove ensure_metadata Instead, only preload the ->mid field for threading, as we only need ->thread and ->path once in Search->get_thread (but we will need the ->mid field repeatedly). This more than doubles View->load_results performance on according to thread-all on an inbox with over 300K messages. --- t/search.t | 2 -- 1 file changed, 2 deletions(-) (limited to 't') diff --git a/t/search.t b/t/search.t index eed9c9b6..c16811d8 100644 --- a/t/search.t +++ b/t/search.t @@ -109,7 +109,6 @@ sub filter_mids { my $found = $ro->lookup_message(''); ok($found, "message found"); is($root_id, $found->{doc_id}, 'doc_id set correctly'); - $found->ensure_metadata; is($found->mid, 'root@s', 'mid set correctly'); ok(int($found->thread_id) > 0, 'thread_id is an integer'); @@ -290,7 +289,6 @@ sub filter_mids { body => "LOOP!\n")); ok($doc_id > 0, "doc_id defined with circular reference"); my $smsg = $rw->lookup_message('circle@a'); - $smsg->ensure_metadata; is($smsg->references, '', "no references created"); my $msg = PublicInbox::SearchMsg->load_doc($smsg->{doc}); is($s, $msg->subject, 'long subject not rewritten'); -- cgit v1.2.3-24-ge0c7 From 90b3d23352a0c37680ac266acaf4fef73a781bc9 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 20 Dec 2016 23:42:36 +0000 Subject: searchthread: simplify API and remove needless OO This simplifies callers to prevent errors and avoids needless object-orientation in favor of a single procedure call to handle threading and ordering. --- t/thread-cycle.t | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 't') diff --git a/t/thread-cycle.t b/t/thread-cycle.t index 9dd2aa3c..16ceee71 100644 --- a/t/thread-cycle.t +++ b/t/thread-cycle.t @@ -72,11 +72,11 @@ SKIP: { done_testing(); sub thread_to_s { - my $th = PublicInbox::SearchThread->new(shift); - $th->thread; - $th->order(sub { [ sort { $a->{id} cmp $b->{id} } @{$_[0]} ] }); + my ($msgs) = @_; + my $rootset = PublicInbox::SearchThread::thread($msgs, sub { + [ sort { $a->{id} cmp $b->{id} } @{$_[0]} ] }); my $st = ''; - my @q = map { (0, $_) } @{$th->{rootset}}; + my @q = map { (0, $_) } @$rootset; while (@q) { my $level = shift @q; my $node = shift @q or next; -- cgit v1.2.3-24-ge0c7