Merge remote-tracking branch 'origin/email-simple-mem' into master

* origin/email-simple-mem:
  nntp: reduce syscalls for ARTICLE and BODY
  mbox: split header and body processing
  mbox: use Email::Simple->new to do in-place modifications
  nntp: rework and simplify art_lookup response

2 files changed, 67 insertions, 65 deletions

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 15200d3a..0c3e52fe 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -16,8 +16,8 @@ use Email::Simple;
 use Email::MIME::Encode;
 
 sub subject_fn ($) {
-        my ($simple) = @_;
-        my $fn = $simple->header('Subject');
+        my ($hdr) = @_;
+        my $fn = $hdr->header('Subject');
         return 'no-subject' unless defined($fn);
 
         # no need for full Email::MIME, here
@@ -36,19 +36,26 @@ sub mb_stream {
 }
 
 # called by PSGI server as body response
+# this gets called twice for every message, once to return the header,
+# once to retrieve the body
 sub getline {
         my ($more) = @_; # self
-        my ($ctx, $id, $prev, $next, $cur) = @$more;
-        if ($cur) { # first
-                pop @$more;
-                return msg_str($ctx, $cur);
+        my ($ctx, $id, $prev, $next, $mref, $hdr) = @$more;
+        if ($hdr) { # first message hits this, only
+                pop @$more; # $hdr
+                return msg_hdr($ctx, $hdr);
         }
-        $cur = $next or return;
+        if ($mref) { # all messages hit this
+                pop @$more; # $mref
+                return msg_body($$mref);
+        }
+        my $cur = $next or return;
         my $ibx = $ctx->{-inbox};
         $next = $ibx->over->next_by_mid($ctx->{mid}, \$id, \$prev);
-        @$more = ($ctx, $id, $prev, $next); # $next may be undef, here
-        my $mref = $ibx->msg_by_smsg($cur) or return;
-        msg_str($ctx, Email::Simple->new($mref));
+        $mref = $ibx->msg_by_smsg($cur) or return;
+        $hdr = Email::Simple->new($mref)->header_obj;
+        @$more = ($ctx, $id, $prev, $next, $mref); # $next may be undef, here
+        msg_hdr($ctx, $hdr); # all but first message hits this
 }
 
 sub close {} # noop
@@ -57,22 +64,17 @@ sub emit_raw {
         my ($ctx) = @_;
         my $mid = $ctx->{mid};
         my $ibx = $ctx->{-inbox};
-        my $first;
-        my $more;
+        my ($mref, $more, $id, $prev, $next);
         if (my $over = $ibx->over) {
-                my ($id, $prev);
                 my $smsg = $over->next_by_mid($mid, \$id, \$prev) or return;
-                my $mref = $ibx->msg_by_smsg($smsg) or return;
-                $first = Email::Simple->new($mref);
-                my $next = $over->next_by_mid($mid, \$id, \$prev);
-                # $more is for ->getline
-                $more = [ $ctx, $id, $prev, $next, $first ] if $next;
+                $mref = $ibx->msg_by_smsg($smsg) or return;
+                $next = $over->next_by_mid($mid, \$id, \$prev);
         } else {
-                my $mref = $ibx->msg_by_mid($mid) or return;
-                $first = Email::Simple->new($mref);
+                $mref = $ibx->msg_by_mid($mid) or return;
         }
-        return unless defined $first;
-        my $fn = subject_fn($first);
+        my $hdr = Email::Simple->new($mref)->header_obj;
+        $more = [ $ctx, $id, $prev, $next, $mref, $hdr ]; # for ->getline
+        my $fn = subject_fn($hdr);
         my @hdr = ('Content-Type');
         if ($ibx->{obfuscate}) {
                 # obfuscation is stupid, but maybe scrapers are, too...
@@ -83,12 +85,11 @@ sub emit_raw {
                 $fn .= '.txt';
         }
         push @hdr, 'Content-Disposition', "inline; filename=$fn";
-        [ 200, \@hdr, $more ? mb_stream($more) : [ msg_str($ctx, $first) ] ];
+        [ 200, \@hdr, mb_stream($more) ];
 }
 
-sub msg_str {
-        my ($ctx, $simple, $mid) = @_; # Email::Simple object
-        my $header_obj = $simple->header_obj;
+sub msg_hdr ($$;$) {
+        my ($ctx, $header_obj, $mid) = @_;
 
         # drop potentially confusing headers, ssoma already should've dropped
         # Lines and Content-Length
@@ -104,7 +105,7 @@ sub msg_str {
                 'List-Archive', "<$base>",
                 'List-Post', "<mailto:$ibx->{-primary_address}>",
         );
-        my $crlf = $simple->crlf;
+        my $crlf = $header_obj->crlf;
         my $buf = "From mboxrd\@z Thu Jan  1 00:00:00 1970\n" .
                         $header_obj->as_string;
         for (my $i = 0; $i < @append; $i += 2) {
@@ -120,13 +121,13 @@ sub msg_str {
                 $buf .= "$k: $v$crlf" if defined $v;
         }
         $buf .= $crlf;
+}
 
+sub msg_body ($) {
         # mboxrd quoting style
         # ref: http://www.qmail.org/man/man5/mbox.html
-        my $body = $simple->body;
-        $body =~ s/^(>*From )/>$1/gm;
-        $buf .= $body;
-        $buf .= "\n";
+        $_[0] =~ s/^(>*From )/>$1/gm;
+        $_[0] .= "\n";
 }
 
 sub thread_mbox {
@@ -267,11 +268,13 @@ sub response {
 sub getline {
         my ($self) = @_;
         my $ctx = $self->{ctx} or return;
+        my $gz = $self->{gz};
         while (my $smsg = $self->{cb}->()) {
-                my $msg = $ctx->{-inbox}->msg_by_smsg($smsg) or next;
-                $msg = Email::Simple->new($msg);
-                $self->{gz}->write(PublicInbox::Mbox::msg_str($ctx, $msg,
-                                $smsg->{mid}));
+                my $mref = $ctx->{-inbox}->msg_by_smsg($smsg) or next;
+                my $h = Email::Simple->new($mref)->header_obj;
+                $gz->write(PublicInbox::Mbox::msg_hdr($ctx, $h, $smsg->{mid}));
+                $gz->write(PublicInbox::Mbox::msg_body($$mref));
+
                 my $bref = $self->{buf};
                 if (length($$bref) >= 8192) {
                         my $ret = $$bref; # copy :<
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 82762b1a..26bc679f 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -487,24 +487,23 @@ find_mid:
 found:
         my $smsg = $ng->over->get_art($n) or return $err;
         my $msg = $ng->msg_by_smsg($smsg) or return $err;
-        my $s = Email::Simple->new($msg);
-        if ($set_headers) {
-                set_nntp_headers($self, $s->header_obj, $ng, $n, $mid);
 
-                # must be last
-                $s->body_set('') if ($set_headers == 2);
-        }
-        [ $n, $mid, $s, $smsg->bytes, $smsg->lines, $ng ];
+        # Email::Simple->new will modify $msg in-place as documented
+        # in its manpage, so what's left is the body and we won't need
+        # to call Email::Simple::body(), later
+        my $hdr = Email::Simple->new($msg)->header_obj;
+        set_nntp_headers($self, $hdr, $ng, $n, $mid) if $set_headers;
+        [ $n, $mid, $msg, $hdr ];
 }
 
-sub simple_body_write ($$) {
-        my ($self, $s) = @_;
-        my $body = $s->body;
-        $s->body_set('');
-        $body =~ s/^\./../smg;
-        $body =~ s/(?<!\r)\n/\r\n/sg;
-        msg_more($self, $body);
-        msg_more($self, "\r\n") unless $body =~ /\r\n\z/s;
+sub msg_body_write ($$) {
+        my ($self, $msg) = @_;
+
+        # these can momentarily double the memory consumption :<
+        $$msg =~ s/^\./../smg;
+        $$msg =~ s/(?<!\r)\n/\r\n/sg; # Alpine barfs without this
+        $$msg .= "\r\n" unless $$msg =~ /\r\n\z/s;
+        msg_more($self, $$msg);
         '.'
 }
 
@@ -513,40 +512,40 @@ sub set_art {
         $self->{article} = $art if defined $art && $art =~ /\A[0-9]+\z/;
 }
 
-sub _header ($) {
-        my $hdr = $_[0]->header_obj->as_string;
+sub msg_hdr_write ($$$) {
+        my ($self, $hdr, $body_follows) = @_;
+        $hdr = $hdr->as_string;
         utf8::encode($hdr);
-        $hdr =~ s/(?<!\r)\n/\r\n/sg;
+        $hdr =~ s/(?<!\r)\n/\r\n/sg; # Alpine barfs without this
 
         # for leafnode compatibility, we need to ensure Message-ID headers
         # are only a single line.  We can't subclass Email::Simple::Header
         # and override _default_fold_at in here, either; since that won't
         # affect messages already in the archive.
         $hdr =~ s/^(Message-ID:)[ \t]*\r\n[ \t]+([^\r]+)\r\n/$1 $2\r\n/igsm;
-
-        $hdr
+        $hdr .= "\r\n" if $body_follows;
+        msg_more($self, $hdr);
 }
 
 sub cmd_article ($;$) {
         my ($self, $art) = @_;
         my $r = art_lookup($self, $art, 1);
         return $r unless ref $r;
-        my ($n, $mid, $s) = @$r;
+        my ($n, $mid, $msg, $hdr) = @$r;
         set_art($self, $art);
         more($self, "220 $n <$mid> article retrieved - head and body follow");
-        msg_more($self, _header($s));
-        msg_more($self, "\r\n");
-        simple_body_write($self, $s);
+        msg_hdr_write($self, $hdr, 1);
+        msg_body_write($self, $msg);
 }
 
 sub cmd_head ($;$) {
         my ($self, $art) = @_;
         my $r = art_lookup($self, $art, 2);
         return $r unless ref $r;
-        my ($n, $mid, $s) = @$r;
+        my ($n, $mid, undef, $hdr) = @$r;
         set_art($self, $art);
         more($self, "221 $n <$mid> article retrieved - head follows");
-        msg_more($self, _header($s));
+        msg_hdr_write($self, $hdr, 0);
         '.'
 }
 
@@ -554,17 +553,17 @@ sub cmd_body ($;$) {
         my ($self, $art) = @_;
         my $r = art_lookup($self, $art, 0);
         return $r unless ref $r;
-        my ($n, $mid, $s) = @$r;
+        my ($n, $mid, $msg) = @$r;
         set_art($self, $art);
         more($self, "222 $n <$mid> article retrieved - body follows");
-        simple_body_write($self, $s);
+        msg_body_write($self, $msg);
 }
 
 sub cmd_stat ($;$) {
         my ($self, $art) = @_;
         my $r = art_lookup($self, $art, 0);
         return $r unless ref $r;
-        my ($n, $mid, undef) = @$r;
+        my ($n, $mid) = @$r;
         set_art($self, $art);
         "223 $n <$mid> article retrieved - request text separately";
 }
@@ -792,7 +791,7 @@ sub hdr_mid_prefix ($$$$$) {
 }
 
 sub hdr_mid_response ($$$$$$) {
-        my ($self, $xhdr, $ng, $n, $mid, $v) = @_; # r: art_lookup result
+        my ($self, $xhdr, $ng, $n, $mid, $v) = @_;
         my $res = '';
         if ($xhdr) {
                 $res .= r221 . "\r\n";