From fb8b16ff2b40ecd22ebbdea0d27069749e800077 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sun, 21 Feb 2021 07:41:29 +0000 Subject: lei q: support IMAP/IMAPS --output destinations Augment (and dedupe) aren't parallel, yet, so its more sensitive to high-latency networks. --- xt/net_writer-imap.t | 118 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 107 insertions(+), 11 deletions(-) (limited to 'xt') diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t index dfd765be..4832245a 100644 --- a/xt/net_writer-imap.t +++ b/xt/net_writer-imap.t @@ -7,6 +7,7 @@ use POSIX qw(strftime); use PublicInbox::OnDestroy; use PublicInbox::URIimap; use PublicInbox::Config; +use Fcntl qw(O_EXCL O_WRONLY O_CREAT); my $imap_url = $ENV{TEST_IMAP_WRITE_URL} or plan skip_all => 'TEST_IMAP_WRITE_URL unset'; my $uri = PublicInbox::URIimap->new($imap_url); @@ -19,30 +20,125 @@ my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!); my $folder = "INBOX.$base-$host-".strftime('%Y%m%d%H%M%S', gmtime(time)). "-$$-".sprintf('%x', int(rand(0xffffffff))); my $nwr = PublicInbox::NetWriter->new; -$imap_url .= '/' unless substr($imap_url, -1) eq '/'; +chop($imap_url) if substr($imap_url, -1) eq '/'; my $folder_uri = PublicInbox::URIimap->new("$imap_url/$folder"); is($folder_uri->mailbox, $folder, 'folder correct') or BAIL_OUT "BUG: bad $$uri"; $nwr->add_url($$folder_uri); is($nwr->errors, undef, 'no errors'); $nwr->{pi_cfg} = bless {}, 'PublicInbox::Config'; -my $mics = $nwr->imap_common_init; + +my $set_cred_helper = sub { + my ($f, $cred_set) = @_; + sysopen(my $fh, $f, O_CREAT|O_EXCL|O_WRONLY) or BAIL_OUT "open $f: $!"; + print $fh </dev/null`); +if ($cred_helper eq 'store') { + my $config = $ENV{XDG_CONFIG_HOME} // "$ENV{HOME}/.config"; + for my $f ("$ENV{HOME}/.git-credentials", "$config/git/credentials") { + next unless -f $f; + @cred_link = ($f, '/.git-credentials'); + last; + } + $cred_set = qq("$cred_helper"); +} elsif ($cred_helper =~ /\Acache(?:[ \t]|\z)/) { + my $cache = $ENV{XDG_CACHE_HOME} // "$ENV{HOME}/.cache"; + for my $d ("$ENV{HOME}/.git-credential-cache", + "$cache/git/credential") { + next unless -d $d; + @cred_link = ($d, '/.git-credential-cache'); + $cred_set = qq("$cred_helper"); + last; + } +} elsif (!$cred_helper) { # make the test less painful if no creds configured + ($tmpdir, $for_destroy) = tmpdir; + my $d = "$tmpdir/.git-credential-cache"; + mkdir($d, 0700) or BAIL_OUT $!; + $cred_set = "cache --timeout=60"; + @cred_link = ($d, '/.git-credential-cache'); +} else { + diag "credential.helper=$cred_helper will not be used for this test"; +} + +my $mics = do { + local $ENV{HOME} = $tmpdir // $ENV{HOME}; + if ($tmpdir && $cred_set) { + $set_cred_helper->("$ENV{HOME}/.gitconfig", $cred_set) + } + $nwr->imap_common_init; +}; my $mic = (values %$mics)[0]; -my $cleanup = PublicInbox::OnDestroy->new(sub { +my $cleanup = PublicInbox::OnDestroy->new($$, sub { + my $mic = $nwr->mic_get($imap_url); $mic->delete($folder) or fail "delete $folder <$folder_uri>: $@"; + if ($tmpdir && -f "$tmpdir/.gitconfig") { + local $ENV{HOME} = $tmpdir; + system(qw(git credential-cache exit)); + } }); my $imap_append = $nwr->can('imap_append'); my $smsg = bless { kw => [ 'seen' ] }, 'PublicInbox::Smsg'; $imap_append->($mic, $folder, undef, $smsg, eml_load('t/plack-qp.eml')); -my @res; $nwr->{quiet} = 1; -$nwr->imap_each($$folder_uri, sub { - my ($u, $uid, $kw, $eml, $arg) = @_; - push @res, [ $kw, $eml ]; -}); -is(scalar(@res), 1, 'got appended message'); -is_deeply(\@res, [ [ [ 'seen' ], eml_load('t/plack-qp.eml') ] ], +my $imap_slurp_all = sub { + my ($u, $uid, $kw, $eml, $res) = @_; + push @$res, [ $kw, $eml ]; +}; +$nwr->imap_each($$folder_uri, $imap_slurp_all, my $res = []); +is(scalar(@$res), 1, 'got appended message'); +my $plack_qp_eml = eml_load('t/plack-qp.eml'); +is_deeply($res, [ [ [ 'seen' ], $plack_qp_eml ] ], 'uploaded message read back'); +$res = $mic = $mics = undef; + +test_lei(sub { + my ($ro_home, $cfg_path) = setup_public_inboxes; + my $cfg = PublicInbox::Config->new($cfg_path); + $cfg->each_inbox(sub { + my ($ibx) = @_; + lei_ok qw(add-external -q), $ibx->{inboxdir} or BAIL_OUT; + }); + + # cred_link[0] may be on a different (hopefully encrypted) FS, + # we only symlink to it here, so we don't copy any sensitive data + # into the temporary directory + if (@cred_link && !symlink($cred_link[0], $ENV{HOME}.$cred_link[1])) { + diag "symlink @cred_link: $! (non-fatal)"; + $cred_set = undef; + } + $set_cred_helper->("$ENV{HOME}/.gitconfig", $cred_set) if $cred_set; + + lei_ok qw(q f:qp@example.com -o), $$folder_uri; + $nwr->imap_each($$folder_uri, $imap_slurp_all, my $res = []); + is(scalar(@$res), 1, 'got one deduped result') or diag explain($res); + is_deeply($res->[0]->[1], $plack_qp_eml, + 'lei q wrote expected result'); + + lei_ok qw(q f:matz -a -o), $$folder_uri; + $nwr->imap_each($$folder_uri, $imap_slurp_all, my $aug = []); + is(scalar(@$aug), 2, '2 results after augment') or diag explain($aug); + my $exp = $res->[0]->[1]->as_string; + is(scalar(grep { $_->[1]->as_string eq $exp } @$aug), 1, + 'original remains after augment'); + $exp = eml_load('t/iso-2202-jp.eml')->as_string; + is(scalar(grep { $_->[1]->as_string eq $exp } @$aug), 1, + 'new result shown after augment'); + + lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_uri->as_string; + $nwr->imap_each($$folder_uri, $imap_slurp_all, my $empty = []); + is(scalar(@$empty), 0, 'no results w/o augment'); + +}); -undef $cleanup; +undef $cleanup; # remove temporary folder done_testing; -- cgit v1.2.3-24-ge0c7