From: Eric Wong <e@yhbt.net>
To: spew@80x24.org
Subject: [PATCH] wip
Date: Thu, 23 Apr 2020 04:27:56 +0000 [thread overview]
Message-ID: <20200423042756.30461-1-e@yhbt.net> (raw)
---
MANIFEST | 1 -
lib/PublicInbox/GzipFilter.pm | 17 ++--
lib/PublicInbox/Mbox.pm | 155 ++++++++++++++++++++--------------
lib/PublicInbox/MboxGz.pm | 49 -----------
4 files changed, 99 insertions(+), 123 deletions(-)
delete mode 100644 lib/PublicInbox/MboxGz.pm
diff --git a/MANIFEST b/MANIFEST
index b0f08567..857fab4c 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -131,7 +131,6 @@ lib/PublicInbox/MDA.pm
lib/PublicInbox/MID.pm
lib/PublicInbox/MIME.pm
lib/PublicInbox/Mbox.pm
-lib/PublicInbox/MboxGz.pm
lib/PublicInbox/MsgIter.pm
lib/PublicInbox/MsgTime.pm
lib/PublicInbox/Msgmap.pm
diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm
index d2eb4e66..3a3c4a6b 100644
--- a/lib/PublicInbox/GzipFilter.pm
+++ b/lib/PublicInbox/GzipFilter.pm
@@ -10,7 +10,15 @@ our @EXPORT_OK = qw(gzf_maybe);
my %OPT = (-WindowBits => 15 + 16, -AppendOutput => 1);
my @GZIP_HDRS = qw(Vary Accept-Encoding Content-Encoding gzip);
-sub new { bless {}, shift }
+sub gzip_or_die () {
+ my ($gz, $err) = Compress::Raw::Zlib::Deflate->new(%OPT);
+ $err == Z_OK or die "Deflate->new failed: $err";
+ $gz;
+}
+
+sub new { bless { gz => $_[1] ? undef : gzip_or_die() }, $_[0] }
+
+# for GetlineBody (via Qspawn) when NOT using $env->{'pi-httpd.async'}
# for Qspawn if using $env->{'pi-httpd.async'}
sub attach {
@@ -32,13 +40,6 @@ sub gzf_maybe ($$) {
bless { gz => $gz }, __PACKAGE__;
}
-sub gzip_or_die () {
- my ($gz, $err) = Compress::Raw::Zlib::Deflate->new(%OPT);
- $err == Z_OK or die "Deflate->new failed: $err";
- $gz;
-}
-
-# for GetlineBody (via Qspawn) when NOT using $env->{'pi-httpd.async'}
# Also used for ->getline callbacks
sub translate ($$) {
my $self = $_[0]; # $_[1] => input
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 97bec5e7..36f7aea8 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -9,17 +9,16 @@
# more common "push" model)
package PublicInbox::Mbox;
use strict;
-use warnings;
+use PublicInbox::GzipFilter;
+use PublicInbox::NoopFilter;
use PublicInbox::MID qw/mid_escape/;
use PublicInbox::Hval qw/to_filename/;
use PublicInbox::Smsg;
-use PublicInbox::WwwStream qw(html_oneshot);
use Email::Simple;
use Email::MIME::Encode;
sub subject_fn ($) {
- my ($hdr) = @_;
- my $fn = $hdr->header('Subject');
+ my ($fn) = @_;
return 'no-subject' if (!defined($fn) || $fn eq '');
# no need for full Email::MIME, here
@@ -31,63 +30,6 @@ sub subject_fn ($) {
$fn eq '' ? 'no-subject' : to_filename($fn);
}
-sub mb_stream {
- my ($more) = @_;
- bless $more, 'PublicInbox::Mbox';
-}
-
-# called by PSGI server as body response
-# this gets called twice for every message, once to return the header,
-# once to retrieve the body
-sub getline {
- my ($more) = @_; # self
- my ($ctx, $id, $prev, $next, $mref, $hdr) = @$more;
- if ($hdr) { # first message hits this, only
- pop @$more; # $hdr
- pop @$more; # $mref
- return msg_hdr($ctx, $hdr) . msg_body($$mref);
- }
- my $cur = $next or return;
- my $ibx = $ctx->{-inbox};
- $next = $ibx->over->next_by_mid($ctx->{mid}, \$id, \$prev);
- $mref = $ibx->msg_by_smsg($cur) or return;
- $hdr = Email::Simple->new($mref)->header_obj;
- @$more = ($ctx, $id, $prev, $next); # $next may be undef, here
- msg_hdr($ctx, $hdr) . msg_body($$mref);
-}
-
-sub close {} # noop
-
-# /$INBOX/$MESSAGE_ID/raw
-sub emit_raw {
- my ($ctx) = @_;
- my $mid = $ctx->{mid};
- my $ibx = $ctx->{-inbox};
- $ctx->{base_url} = $ibx->base_url($ctx->{env});
- my ($mref, $more, $id, $prev, $next);
- if (my $over = $ibx->over) {
- my $smsg = $over->next_by_mid($mid, \$id, \$prev) or return;
- $mref = $ibx->msg_by_smsg($smsg) or return;
- $next = $over->next_by_mid($mid, \$id, \$prev);
- } else {
- $mref = $ibx->msg_by_mid($mid) or return;
- }
- my $hdr = Email::Simple->new($mref)->header_obj;
- $more = [ $ctx, $id, $prev, $next, $mref, $hdr ]; # for ->getline
- my $fn = subject_fn($hdr);
- my @hdr = ('Content-Type');
- if ($ibx->{obfuscate}) {
- # obfuscation is stupid, but maybe scrapers are, too...
- push @hdr, 'application/mbox';
- $fn .= '.mbox';
- } else {
- push @hdr, 'text/plain';
- $fn .= '.txt';
- }
- push @hdr, 'Content-Disposition', "inline; filename=$fn";
- [ 200, \@hdr, mb_stream($more) ];
-}
-
sub msg_hdr ($$;$) {
my ($ctx, $header_obj, $mid) = @_;
@@ -135,6 +77,56 @@ sub msg_body ($) {
$_[0] .= "\n";
}
+sub emit_raw_cb {
+ my ($self, $ctx) = @_;
+ delete($self->{first}) // $ctx->{-inbox}->over->next_by_mid(
+ $ctx->{mid},
+ \($self->{cur_id}),
+ \($self->{prev}))
+}
+
+# /$INBOX/$MESSAGE_ID/raw
+sub emit_raw {
+ my ($ctx) = @_;
+ my $mid = $ctx->{mid};
+ my $ibx = $ctx->{-inbox};
+ $ctx->{base_url} = $ibx->base_url($ctx->{env});
+ my $h = [ 'Content-Type' => undef, 'Content-Disposition' => undef ];
+ my ($fn, $body);
+ my $gzf = gzf_maybe($h, $ctx->{env}) || PublicInbox::NoopFilter->new;
+ if (my $over = $ibx->over) {
+ $body = bless {
+ ctx => $ctx,
+ cb => \&emit_raw_cb,
+ gzf => $gzf,
+ cur_id => undef,
+ prev => undef
+ }, __PACKAGE__;
+ my $smsg = $over->next_by_mid($ctx->{mid},
+ \($body->{cur_id}),
+ \($body->{prev})) or return;
+ $body->{first} = $smsg;
+ $fn = $smsg->{subject};
+ } else {
+ my $mref = $ibx->msg_by_mid($ctx->{mid}) or return;
+ my $hdr = Email::Simple->new($mref)->header_obj;
+ $fn = $hdr->header('Subject');
+ $gzf->zmore(msg_hdr($ctx, $hdr, $ctx->{mid}));
+ $gzf->zflush(msg_body($$mref));
+ $body = [ $hdr ];
+ }
+ if ($ibx->{obfuscate}) {
+ # obfuscation is stupid, but maybe scrapers are, too...
+ $h->[1] = 'application/mbox';
+ $fn .= '.mbox';
+ } else {
+ $h->[1] = 'text/plain';
+ $fn .= '.txt';
+ }
+ $h->[3] = 'inline; filename='.subject_fn($fn);
+ [ 200, $h, $body ];
+}
+
sub thread_cb {
my ($ctx) = @_;
my $msgs = $ctx->{msgs};
@@ -150,6 +142,22 @@ sub thread_cb {
}
}
+sub mboxgz ($$$) {
+ my ($ctx, $cb, $fn) = @_;
+ $ctx->{base_url} = $ctx->{-inbox}->base_url($ctx->{env});
+ my $body = bless {
+ gzf => PublicInbox::GzipFilter->new(1),
+ cb => $cb,
+ ctx => $ctx,
+ }, __PACKAGE__;
+ # http://www.iana.org/assignments/media-types/application/gzip
+ $fn = defined($fn) && $fn ne '' ? to_filename($fn) : 'no-subject';
+ my $h = [ qw(Content-Type application/gzip),
+ 'Content-Disposition', "inline; filename=$fn.mbox.gz" ];
+ [ 200, $h, $body ];
+}
+
+
sub thread_mbox {
my ($ctx, $over, $sfx) = @_;
require PublicInbox::MboxGz;
@@ -157,7 +165,7 @@ sub thread_mbox {
return [404, [qw(Content-Type text/plain)], []] if !@$msgs;
$ctx->{prev} = $msgs->[-1];
$ctx->{over} = $over; # bump refcnt
- PublicInbox::MboxGz->response($ctx, \&thread_cb, $msgs->[0]->subject);
+ mboxgz($ctx, \&thread_cb, $msgs->[0]->subject);
}
sub emit_range {
@@ -195,7 +203,7 @@ sub mbox_all_ids {
return PublicInbox::WWW::need($ctx, 'Overview');
$ctx->{ids} = $ids;
$ctx->{prev} = $prev;
- return PublicInbox::MboxGz->response($ctx, \&all_ids_cb, 'all');
+ mboxgz($ctx, \&all_ids_cb, 'all');
}
sub results_cb {
@@ -220,7 +228,6 @@ sub results_cb {
sub mbox_all {
my ($ctx, $query) = @_;
- require PublicInbox::MboxGz;
return mbox_all_ids($ctx) if $query eq '';
my $qopts = $ctx->{qopts} = { mset => 2 };
my $srch = $ctx->{srch} = $ctx->{-inbox}->search or
@@ -231,7 +238,25 @@ sub mbox_all {
["No results found\n"]];
$ctx->{iter} = 0;
$ctx->{query} = $query;
- PublicInbox::MboxGz->response($ctx, \&results_cb, 'results-'.$query);
+ mboxgz($ctx, \&results_cb, 'results-'.$query);
}
+# called by Plack::Util::foreach or similar
+sub getline {
+ my ($self) = @_;
+ my $gzf = $self->{gzf} or return;
+ my $ctx = $self->{ctx};
+ while (my $smsg = $self->{cb}->($ctx)) {
+ my $mref = $ctx->{-inbox}->msg_by_smsg($smsg) or next;
+ my $h = Email::Simple->new($mref)->header_obj;
+ $gzf->zmore(msg_hdr($ctx, $h, $smsg->{mid}));
+ return $gzf->translate(msg_body($$mref));
+ }
+ # signal that we're done and can return undef next call:
+ delete $self->{gzf};
+ $gzf->zflush;
+}
+
+sub close {} # noop
+
1;
diff --git a/lib/PublicInbox/MboxGz.pm b/lib/PublicInbox/MboxGz.pm
deleted file mode 100644
index 7deabf54..00000000
--- a/lib/PublicInbox/MboxGz.pm
+++ /dev/null
@@ -1,49 +0,0 @@
-# Copyright (C) 2015-2020 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-package PublicInbox::MboxGz;
-use strict;
-use parent 'PublicInbox::GzipFilter';
-use Email::Simple;
-use PublicInbox::Hval qw/to_filename/;
-use PublicInbox::Mbox;
-
-sub new {
- my ($class, $ctx, $cb) = @_;
- $ctx->{base_url} = $ctx->{-inbox}->base_url($ctx->{env});
- bless {
- gz => PublicInbox::GzipFilter::gzip_or_die(),
- cb => $cb,
- ctx => $ctx
- }, $class;
-}
-
-sub response {
- my ($class, $ctx, $cb, $fn) = @_;
- my $body = $class->new($ctx, $cb);
- # http://www.iana.org/assignments/media-types/application/gzip
- $fn = defined($fn) && $fn ne '' ? to_filename($fn) : 'no-subject';
- my $h = [ qw(Content-Type application/gzip),
- 'Content-Disposition', "inline; filename=$fn.mbox.gz" ];
- [ 200, $h, $body ];
-}
-
-# called by Plack::Util::foreach or similar
-sub getline {
- my ($self) = @_;
- my $ctx = $self->{ctx} or return;
- while (my $smsg = $self->{cb}->($ctx)) {
- my $mref = $ctx->{-inbox}->msg_by_smsg($smsg) or next;
- my $h = Email::Simple->new($mref)->header_obj;
- $self->zmore(
- PublicInbox::Mbox::msg_hdr($ctx, $h, $smsg->{mid})
- );
- return $self->translate(PublicInbox::Mbox::msg_body($$mref));
- }
- # signal that we're done and can return undef next call:
- delete $self->{ctx};
- $self->zflush;
-}
-
-sub close {} # noop
-
-1;
next reply other threads:[~2020-04-23 4:28 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-04-23 4:27 Eric Wong [this message]
-- strict thread matches above, loose matches on Subject: below --
2021-10-27 20:16 [PATCH] wip Eric Wong
2021-06-05 19:58 Eric Wong
2021-04-05 7:42 Eric Wong
2021-03-08 7:11 Eric Wong
2021-01-21 4:24 [PATCH] WIP Eric Wong
2021-01-03 22:57 [PATCH] wip Eric Wong
2020-12-27 11:36 [PATCH] WIP Eric Wong
2020-11-15 7:35 [PATCH] wip Eric Wong
2020-04-20 7:14 Eric Wong
2020-01-13 9:24 [PATCH] WIP Eric Wong
2019-05-11 22:55 Eric Wong
2019-01-02 9:21 [PATCH] wip Eric Wong
2018-07-06 21:31 Eric Wong
2018-06-24 11:55 Eric Wong
2018-06-24 8:39 Eric Wong
2017-07-15 1:42 [PATCH] WIP Eric Wong
2017-04-12 20:17 [PATCH] wip Eric Wong
2017-04-05 18:40 Eric Wong
2016-08-23 20:07 Eric Wong
2016-08-18 2:16 Eric Wong
2016-06-26 3:46 Eric Wong
2015-12-22 0:15 Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200423042756.30461-1-e@yhbt.net \
--to=e@yhbt.net \
--cc=spew@80x24.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).