From b3b025134c71d00fa26435aec1d2cb8c196103ac Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Mon, 28 Nov 2022 05:32:25 +0000 Subject: lei_mirror: use curl -z/--timecond if manifest exists This lets us save cycles and avoid scanning + comparing manifest contents by relying on the Last-Modified HTTP response header. --- lib/PublicInbox/LeiMirror.pm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index b30cc519..cc5ea1d2 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -995,6 +995,7 @@ sub try_manifest { } my $ft = File::Temp->new(TEMPLATE => '.manifest-XXXX', %opt); my $cmd = $curl->for_uri($lei, $uri, qw(-f -R -o), $ft->filename); + push(@$cmd, '-z', $manifest) if -f $manifest; my $mf_url = "$uri"; %opt = map { $_ => $lei->{$_} } (0..2); my $cerr = run_reap($lei, $cmd, \%opt); @@ -1002,6 +1003,10 @@ sub try_manifest { return try_scrape($self) if ($cerr >> 8) == 22; # 404 missing return $lei->child_error($cerr, "@$cmd failed"); } + + # bail out if curl -z/--timecond hit 304 Not Modified, $ft will be empty + return $lei->qerr("# $manifest unchanged") if -f $manifest && !-s $ft; + my $m = eval { decode_manifest($ft, $ft, $uri) }; if ($@) { warn $@; -- cgit v1.2.3-24-ge0c7