From 3d490f2b9a8b2934cb386a8649e034c8c6c70bc9 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Wed, 11 Oct 2023 07:20:57 +0000 Subject: lei import|tag|rm: support --commit-delay=SECONDS Delayed commits allows users to trade off immediate safety for throughput and reduced storage wear when running multiple discreet commands. This feature is currently useful for providing a way to make t/lei-store-fail.t reliable and for ensuring `lei blob' can retrieve messages which have not yet been committed. In the future, it'll also be useful for the FUSE layer to batch git activity. --- t/lei-import.t | 13 +++++++++++++ t/lei-store-fail.t | 20 +++++++++++++------- t/lei-tag.t | 15 ++++++++++++++- 3 files changed, 40 insertions(+), 8 deletions(-) (limited to 't') diff --git a/t/lei-import.t b/t/lei-import.t index 8b09d3aa..b2c1de9b 100644 --- a/t/lei-import.t +++ b/t/lei-import.t @@ -2,6 +2,7 @@ # Copyright (C) all contributors # License: AGPL-3.0+ use v5.12; use PublicInbox::TestCommon; +use PublicInbox::DS qw(now); use autodie qw(open close); test_lei(sub { ok(!lei(qw(import -F bogus), 't/plack-qp.eml'), 'fails with bogus format'); @@ -141,6 +142,18 @@ $res = json_utf8->decode($lei_out); is_deeply($res->[0]->{kw}, [qw(answered flagged seen)], 'keyword added'); is_deeply($res->[0]->{L}, [qw(boombox inbox)], 'labels preserved'); +lei_ok qw(import --commit-delay=1 +L:bin -F eml t/data/binary.patch); +lei_ok 'ls-label'; +unlike($lei_out, qr/\bbin\b/, 'commit-delay delays label'); +my $end = now + 10; +my $n = 1; +diag 'waiting for lei/store commit...'; +do { + tick $n; + $n = 0.1; +} until (!lei('ls-label') || $lei_out =~ /\bbin\b/ || now > $end); +like($lei_out, qr/\bbin\b/, 'commit-delay eventually commits'); + # see t/lei_to_mail.t for "import -F mbox*" }); done_testing; diff --git a/t/lei-store-fail.t b/t/lei-store-fail.t index fb0f2b75..c2f03148 100644 --- a/t/lei-store-fail.t +++ b/t/lei-store-fail.t @@ -9,8 +9,11 @@ use Fcntl qw(SEEK_SET); use File::Path qw(remove_tree); my $start_home = $ENV{HOME}; # bug guard +my $utf8_oid = '9bf1002c49eb075df47247b74d69bcd555e23422'; test_lei(sub { lei_ok qw(import -q t/plack-qp.eml); # start the store + ok(!lei(qw(blob --mail), $utf8_oid), 't/utf8.eml not imported, yet'); + my $opt; pipe($opt->{0}, my $in_w); open $opt->{1}, '+>', undef; @@ -20,27 +23,30 @@ test_lei(sub { my $tp = start_script($cmd, undef, $opt); close $opt->{0}; $in_w->autoflush(1); - for (1..500) { # need to fill up 64k read buffer - print $in_w < Date: Sat, 02 Oct 2010 00:00:00 +0000 Subject: hi -Message-ID: <$_\@t> +Message-ID: <0\@t> will this save? EOM - } - tick 0.2; # XXX ugh, this is so hacky + # import another message w/ delay while mboxrd import is still running + lei_ok qw(import -q --commit-delay=300 t/utf8.eml); + lei_ok qw(blob --mail), $utf8_oid, + \'blob immediately available despite --commit-delay'; + lei_ok qw(q m:testmessage@example.com); + is($lei_out, "[null]\n", 'delayed commit is unindexed'); - # make sto_done_request fail: + # make immediate ->sto_done_request fail from mboxrd import: remove_tree("$ENV{HOME}/.local/share/lei/store"); # subsequent lei commands are undefined behavior, # but we need to make sure the current lei command fails: close $in_w; # should trigger ->done $tp->join; - isnt($?, 0, 'lei import error code set on failure'); + isnt($?, 0, 'lei import -F mboxrd error code set on failure'); is(-s $opt->{1}, 0, 'nothing in stdout'); isnt(-s $opt->{2}, 0, 'stderr not empty'); seek($opt->{2}, 0, SEEK_SET); diff --git a/t/lei-tag.t b/t/lei-tag.t index cccf0af6..7278dfcd 100644 --- a/t/lei-tag.t +++ b/t/lei-tag.t @@ -1,9 +1,10 @@ #!perl -w # Copyright (C) 2021 all contributors # License: AGPL-3.0+ -use strict; use v5.10.1; use PublicInbox::TestCommon; +use v5.12; use PublicInbox::TestCommon; require_git 2.6; require_mods(qw(json DBD::SQLite Xapian)); +use PublicInbox::DS qw(now); my ($ro_home, $cfg_path) = setup_public_inboxes; my $check_kw = sub { my ($exp, %opt) = @_; @@ -104,5 +105,17 @@ test_lei(sub { lei_ok qw(tag +L:nope -F eml t/data/binary.patch); like $lei_err, qr/\b1 unimported messages/, 'noted unimported' or diag $lei_err; + + lei_ok qw(tag -F eml --commit-delay=1 t/utf8.eml +L:utf8); + lei_ok 'ls-label'; + unlike($lei_out, qr/\butf8\b/, 'commit-delay delays label'); + my $end = now + 10; + my $n = 1; + diag 'waiting for lei/store commit...'; + do { + tick $n; + $n = 0.1; + } until (!lei('ls-label') || $lei_out =~ /\butf8\b/ || now > $end); + like($lei_out, qr/\butf8\b/, 'commit-delay eventually commits'); }); done_testing; -- cgit v1.2.3-24-ge0c7