From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.6 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 1D7151F677 for ; Wed, 24 Apr 2024 01:34:22 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1713922462; bh=GJ9JmYW3TNz68cMlw4shw1Aw/sMMBoTL6f4X8IletSs=; h=From:To:Subject:Date:In-Reply-To:References:From; b=kxUfL0/EpJCGTq/DYYPJarfPS2PQRLHUA7LMnrratkf3hRsCoH/C/tjsJJ6/Zp4lR s8xQKfw/iXhIhYR97bzAQZYuVSt/CU46ZH3kcnpDRAQW7MYOVPQDJJPIAmE5mzHj/K lwFpW2/ruDZgeStFp6Wt+AZye7aVWTmSOKT0vdhI= From: Eric Wong To: spew@80x24.org Subject: [PATCH 7/8] WIP thread bad mbox Date: Wed, 24 Apr 2024 01:34:19 +0000 Message-ID: <20240424013420.476353-7-e@80x24.org> In-Reply-To: <20240424013420.476353-1-e@80x24.org> References: <20240424013420.476353-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: --- lib/PublicInbox/Isearch.pm | 2 +- lib/PublicInbox/Mbox.pm | 54 ++++++++++++++++---------------------- lib/PublicInbox/Search.pm | 5 +++- lib/PublicInbox/XhcMset.pm | 5 ++-- 4 files changed, 31 insertions(+), 35 deletions(-) diff --git a/lib/PublicInbox/Isearch.pm b/lib/PublicInbox/Isearch.pm index 20808d6d..ddc67f51 100644 --- a/lib/PublicInbox/Isearch.pm +++ b/lib/PublicInbox/Isearch.pm @@ -103,7 +103,7 @@ SELECT docid,xnum FROM xref3 WHERE ibx_id = ? AND docid IN ($qmarks) warn "W: $self->{es}->{topdir} may need to be reindexed\n"; @xnums = grep { defined } @xnums; } - \@xnums; + wantarray ? @xnums : \@xnums; } sub mset_to_smsg { diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index e58fb4b6..19dcba05 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -175,64 +175,56 @@ sub mbox_all_ids { PublicInbox::MboxGz::mbox_gz($ctx, \&all_ids_cb, 'all'); } -sub refill_result_ids ($) { - my ($ctx) = @_; +sub refill_ids_cb { # async_mset_cb + my ($ctx, $mset, $err) = @_; + if ($err) { + warn "E: $err"; + return; + } # refill result set, deprioritize since there's many results - my $srch = $ctx->{ibx}->isrch or return $ctx->gone('search'); - my $mset = $srch->mset($ctx->{query}, $ctx->{qopts}); my $size = $mset->size or return; $ctx->{qopts}->{offset} += $size; - $ctx->{ids} = $srch->mset_to_artnums($mset, $ctx->{qopts}); + $ctx->{ids} = $ctx->{srch}->mset_to_artnums($mset, $ctx->{qopts}); $ctx->{-low_prio} = 1; # true + results_cb($ctx); } -sub results_cb { +sub results_cb { # async_next or getline cb my ($ctx) = @_; my $over = $ctx->{ibx}->over or return $ctx->gone('over'); while (1) { - while (defined(my $num = shift(@{$ctx->{ids}}))) { + my $ids = $ctx->{xids} // $ctx->{ids}; + while (defined(my $num = shift(@$ids))) { my $smsg = $over->get_art($num) or next; return $smsg; } - refill_result_ids($ctx) or return; # refill ctx->{ids} - } -} - -sub results_thread_cb { - my ($ctx) = @_; - - my $over = $ctx->{ibx}->over or return $ctx->gone('over'); - while (1) { - while (defined(my $num = shift(@{$ctx->{xids}}))) { - my $smsg = $over->get_art($num) or next; - return $smsg; - } - next if $over->expand_thread($ctx); # refills ctx->{xids} - - refill_result_ids($ctx) or return; # refill ctx->{ids} + next if $ctx->{xids} && $over->expand_thread($ctx); + return if $ctx->{srch}->async_mset($ctx->{query}, $ctx->{qopts}, + \&refill_ids_cb, $ctx); } } sub mbox_qry_cb { # async_mset cb - my ($ctx, $opt, $q, $mset, $err) = @_; + my ($ctx, $q, $mset, $err) = @_; my $wcb = delete $ctx->{wcb}; if ($err) { warn "E: $err"; return $wcb->([500, [qw(Content-Type text/plain)], [ "Internal server error\n" ]]) } - $opt->{offset} = $mset->size or + $ctx->{qopts}->{offset} = $mset->size or return $wcb->([404, [qw(Content-Type text/plain)], ["No results found\n"]]); - $ctx->{ids} = $ctx->{srch}->mset_to_artnums($mset, $opt); - my @cb_fn; + $ctx->{ids} = $ctx->{srch}->mset_to_artnums($mset, $ctx->{qopts}); + my $fn; if ($q->{t} && $ctx->{srch}->has_threadid) { - @cb_fn = (\&results_thread_cb, "results-thread-$ctx->{query}"); + $ctx->{xids} = []; # triggers over->expand_thread + $fn = "results-thread-$ctx->{query}"; } else { - @cb_fn = (\&results_cb, "results-$ctx->{query}"); + $fn = "results-$ctx->{query}"; } require PublicInbox::MboxGz; - my $res = PublicInbox::MboxGz::mbox_gz($ctx, @cb_fn); + my $res = PublicInbox::MboxGz::mbox_gz($ctx, \&results_cb, $fn); ref($res) eq 'CODE' ? $res->($wcb) : $wcb->($res); } @@ -259,7 +251,7 @@ sub mbox_all { $ctx->{query} = $qstr; sub { $ctx->{wcb} = $_[0]; # PSGI server supplied write cb - $srch->async_mset($qstr, $opt, \&mbox_qry_cb, $ctx, $opt, $q); + $srch->async_mset($qstr, $opt, \&mbox_qry_cb, $ctx, $q); }; } diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index 5c417be3..60d12dbf 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -461,6 +461,8 @@ sub xh_opt ($) { @ret; } +# returns a true value if actually handled asynchronously, +# and a falsy value if handled synchronously sub async_mset { my ($self, $qry_str, $opt, $cb, @args) = @_; $XHC //= xhc_start_maybe; @@ -469,9 +471,10 @@ sub async_mset { my @margs = ($self->xh_args, xh_opt($opt)); my $rd = $XHC->mkreq(undef, 'mset', @margs, $qry_str); PublicInbox::XhcMset->maybe_new($rd, $self, $cb, @args); - } else { + } else { # synchronous my $mset = $self->mset($qry_str, $opt); $cb->(@args, $mset); + undef; } } diff --git a/lib/PublicInbox/XhcMset.pm b/lib/PublicInbox/XhcMset.pm index 2e6fa5f3..ac25eece 100644 --- a/lib/PublicInbox/XhcMset.pm +++ b/lib/PublicInbox/XhcMset.pm @@ -34,11 +34,12 @@ sub event_step { sub maybe_new { my (undef, $rd, $srch, @cb_args) = @_; my $self = bless { cb_args => \@cb_args, srch => $srch }, __PACKAGE__; - if ($PublicInbox::DS::in_loop) { + if ($PublicInbox::DS::in_loop) { # async $self->SUPER::new($rd, EPOLLIN|EPOLLONESHOT); - } else { + } else { # synchronous $self->{sock} = $rd; event_step($self); + undef; } }