From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.6 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 67A1F1F454 for ; Sun, 8 Oct 2023 21:57:11 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1696802231; bh=54YmeVbLr5czmTXL8uGXfrn1cAJb8IiQBfI89mLseTI=; h=From:To:Subject:Date:From; b=GA06mcz8/uskVMw1+bMubVs9lE+p95uDdczCd98xH2CnpyXNil5WDQqzJbMT6nCVx 25/op/FH6xJbJW5cSDflcm7fA+ku+j13we7KYU0tsPT9aN1NajCRx0OKtmRC9x0Efa Tm8mFTSkTC/N0N7quFf8rOmyCdvBVKcvVl3NecXE= From: Eric Wong To: spew@80x24.org Subject: [PATCH] process_io: fix binmode and use it in lei_xsearch for curl Date: Sun, 8 Oct 2023 21:57:11 +0000 Message-ID: <20231008215711.2057108-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: The `binmode' perlop can only take two scalars, so passing `@_' blindly won't work since prototypes are checked. This means we can get IO::Uncompress::Gunzip working properly with ProcessIO. --- lib/PublicInbox/LeiXSearch.pm | 25 +++++++++++-------------- lib/PublicInbox/ProcessIO.pm | 4 ++-- t/lei-q-remote-import.t | 33 ++++++++++++++++++++------------- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index 4077191f..ef66aaea 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -21,6 +21,7 @@ use PublicInbox::LEI; use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR); use PublicInbox::ContentHash qw(git_sha); use POSIX qw(strftime); +use autodie qw(seek truncate); sub new { my ($class) = @_; @@ -353,29 +354,25 @@ sub query_remote_mboxrd { $uri->query_form(@qform, q => $q); my $cmd = $curl->for_uri($lei, $uri); $lei->qerr("# $cmd"); - my ($fh, $pid) = popen_rd($cmd, undef, $rdr); - my $reap_curl = PublicInbox::AutoReap->new($pid); - $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1); + my $cfh = popen_rd($cmd, undef, $rdr); + my $fh = IO::Uncompress::Gunzip->new($cfh, MultiStream => 1); PublicInbox::MboxReader->mboxrd($fh, \&each_remote_eml, $self, $lei, $each_smsg); $lei->sto_done_request if delete($self->{-sto_imported}); - $reap_curl->join; my $nr = delete $lei->{-nr_remote_eml} // 0; - if ($? == 0) { - # don't update if no results, maybe MTA is down + close $cfh; + if ($? == 0) { # don't update if no results, maybe MTA is down $lei->{lss}->cfg_set($key, $start) if $key && $nr; mset_progress($lei, $lei->{-current_url}, $nr, $nr); next; } my $err; - if (-s $cerr) { - seek($cerr, 0, SEEK_SET) // - warn "seek($cmd stderr): $!"; - $err = do { local $/; <$cerr> } // - warn "read($cmd stderr): $!"; - truncate($cerr, 0) // warn "truncate($cmd stderr): $!"; - } - $err //= ''; + eval { + seek($cerr, 0, SEEK_SET); + read($cerr, $err, -s $cerr); + truncate($cerr, 0); + }; + warn "E: $@ ($cmd stderr)" if $@; next if (($? >> 8) == 22 && $err =~ /\b404\b/); $uri->query_form(q => $qstr); $lei->child_error($?, "E: <$uri> $err"); diff --git a/lib/PublicInbox/ProcessIO.pm b/lib/PublicInbox/ProcessIO.pm index 5a81e3a6..f120edd0 100644 --- a/lib/PublicInbox/ProcessIO.pm +++ b/lib/PublicInbox/ProcessIO.pm @@ -33,8 +33,8 @@ sub TIEHANDLE { # for IO::Uncompress::Gunzip sub BINMODE { - my $self = shift; - binmode($self->{fh}, @_); + return binmode($_[0]->{fh}) if @_ == 1; + binmode $_[0]->{fh}, $_[1]; } sub READ { read($_[0]->{fh}, $_[1], $_[2], $_[3] || 0) } diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t index 92d8c9b6..885fa3e1 100644 --- a/t/lei-q-remote-import.t +++ b/t/lei-q-remote-import.t @@ -1,7 +1,8 @@ #!perl -w -# Copyright (C) 2021 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ -use strict; use v5.10.1; use PublicInbox::TestCommon; +use v5.12; use PublicInbox::TestCommon; +use autodie qw(open close unlink); require_mods(qw(lei -httpd)); require_cmd 'curl'; use PublicInbox::MboxReader; @@ -16,7 +17,7 @@ my $url = "http://$host_port/t2/"; my $exp1 = [ eml_load('t/plack-qp.eml') ]; my $exp2 = [ eml_load('t/iso-2202-jp.eml') ]; my $slurp_emls = sub { - open my $fh, '<', $_[0] or BAIL_OUT "open: $!"; + open my $fh, '<', $_[0]; my @eml; PublicInbox::MboxReader->mboxrd($fh, sub { my $eml = shift; @@ -31,33 +32,33 @@ test_lei({ tmpdir => $tmpdir }, sub { my @cmd = ('q', '-o', "mboxrd:$o", 'm:qp@example.com'); lei_ok(@cmd); ok(-f $o && !-s _, 'output exists but is empty'); - unlink $o or BAIL_OUT $!; + unlink $o; lei_ok(@cmd, '-I', $url); is_deeply($slurp_emls->($o), $exp1, 'got results after remote search'); - unlink $o or BAIL_OUT $!; + unlink $o; lei_ok(@cmd); ok(-f $o && -s _, 'output exists after import but is not empty') or diag $lei_err; is_deeply($slurp_emls->($o), $exp1, 'got results w/o remote search'); - unlink $o or BAIL_OUT $!; + unlink $o; $cmd[-1] = 'm:199707281508.AAA24167@hoyogw.example'; lei_ok(@cmd, '-I', $url, '--no-import-remote'); is_deeply($slurp_emls->($o), $exp2, 'got another after remote search'); - unlink $o or BAIL_OUT $!; + unlink $o; lei_ok(@cmd); ok(-f $o && !-s _, '--no-import-remote did not memoize'); open my $fh, '>', "$o.lock"; $cmd[-1] = 'm:qp@example.com'; - unlink $o or xbail("unlink $o $! cwd=".Cwd::getcwd()); + unlink $o; lei_ok(@cmd, '--lock=none'); ok(-f $o && -s _, '--lock=none respected') or diag $lei_err; - unlink $o or xbail("unlink $o $! cwd=".Cwd::getcwd()); + unlink $o; ok(!lei(@cmd, '--lock=dotlock,timeout=0.000001'), 'dotlock fails'); like($lei_err, qr/dotlock timeout/, 'timeout noted'); ok(-f $o && !-s _, 'nothing output on lock failure'); - unlink "$o.lock" or BAIL_OUT $!; + unlink "$o.lock"; lei_ok(@cmd, '--lock=dotlock,timeout=0.000001', \'succeeds after lock removal'); @@ -76,8 +77,8 @@ test_lei({ tmpdir => $tmpdir }, sub { 'm:testmessage@example.com'); is($lei_out, '', 'message not imported when in local external'); - open $fh, '>', $o or BAIL_OUT; - print $fh <<'EOF' or BAIL_OUT; + open $fh, '>', $o; + print $fh <<'EOF'; From a@z Mon Sep 17 00:00:00 2001 From: nobody@localhost Date: Sat, 13 Mar 2021 18:23:01 +0600 @@ -86,7 +87,7 @@ Status: OR whatever EOF - close $fh or BAIL_OUT; + close $fh; lei_ok(qw(q -o), "mboxrd:$o", 'm:testmessage@example.com'); is_deeply($slurp_emls->($o), [$exp], 'got expected result after clobber') or diag $lei_err; @@ -103,5 +104,11 @@ EOF lei_ok([qw(edit-search), "$ENV{HOME}/md"], $edit_env); like($lei_out, qr/^\Q[external "$url"]\E\n\s*lastresult = \d+/sm, 'lastresult set'); + + unlink $o; + lei_ok qw(q --no-save -q m:never2exist@example.com -o), "mboxrd:$o", + '--only', $url, + \'404 curl exit (22) does not influence lei(1)'; + is(-s $o, 0, 'empty result'); }); done_testing;