From 6313d8bbc5f3816eb4936443fca57b411f4f6286 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 7 Jul 2022 09:40:30 +0000 Subject: lei: track seen messages to note duplicates This may help track down deduplication or other bugs in lei which lead to occasionally missing messages. Link: https://public-inbox.org/meta/CAL_JsqJH8xx_2NyZffNsRXbGXiv3kjmCETvKXt3Yfb0uToLm9Q@mail.gmail.com/ --- lib/PublicInbox/LeiXSearch.pm | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'lib/PublicInbox/LeiXSearch.pm') diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index 41e79856..6f877019 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2021 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # Combine any combination of PublicInbox::Search, @@ -163,8 +163,9 @@ sub mset_progress { } sub l2m_progress { - my ($lei, $nr) = @_; - $lei->{-nr_write} += $nr; + my ($lei, $nr_write, $nr_seen) = @_; + $lei->{-nr_write} += $nr_write; + $lei->{-nr_seen} += $nr_seen; } sub query_one_mset { # for --threads and l2m w/o sort @@ -447,13 +448,16 @@ Error closing $lei->{ovv}->{dst}: \$!=$! \$?=$? } if ($lei->{-progress}) { my $tot = $lei->{-mset_total} // 0; - my $nr = $lei->{-nr_write} // 0; + my $nr_w = $lei->{-nr_write} // 0; + my $d = ($lei->{-nr_seen} // 0) - $nr_w; + my $x = "$tot matches"; + $x .= ", $d duplicates" if $d; if ($l2m) { - my $m = "# $nr written to " . - "$lei->{ovv}->{dst} ($tot matches)"; - $nr ? $lei->qfin($m) : $lei->qerr($m); + my $m = "# $nr_w written to " . + "$lei->{ovv}->{dst} ($x)"; + $nr_w ? $lei->qfin($m) : $lei->qerr($m); } else { - $lei->qerr("# $tot matches"); + $lei->qerr("# $x"); } } $lei->start_mua if $l2m && !$l2m->lock_free; -- cgit v1.2.3-24-ge0c7