diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/README | 2 | ||||
| -rwxr-xr-x | scripts/dc-dlvr | 4 | ||||
| -rwxr-xr-x | scripts/import_maildir | 20 | ||||
| -rwxr-xr-x | scripts/import_slrnspool | 26 | ||||
| -rw-r--r-- | scripts/import_vger_from_mbox | 6 | ||||
| -rwxr-xr-x | scripts/slrnspool2maildir | 90 |
6 files changed, 85 insertions, 63 deletions
diff --git a/scripts/README b/scripts/README index 3b9c37da..7ffbd93c 100644 --- a/scripts/README +++ b/scripts/README @@ -1,5 +1,5 @@ This directory contains informal scripts and random tools used -in the development of public-inbox. Some only exist only for +in the development of public-inbox. Some only exist for historical purposes, and some may not work anymore. See the "script/" directory (not "scripts/") for supported and diff --git a/scripts/dc-dlvr b/scripts/dc-dlvr index 935a8312..ef6033b9 100755 --- a/scripts/dc-dlvr +++ b/scripts/dc-dlvr @@ -47,9 +47,9 @@ then rm_list="$rm_list $PREMSG" set +e mv -f $TMPMSG $PREMSG - $spamc -E --headers <$PREMSG >$TMPMSG + $spamc -E <$PREMSG >$TMPMSG else - $spamc -E --headers <$CDMSG >$TMPMSG + $spamc -E <$CDMSG >$TMPMSG fi err=$? diff --git a/scripts/import_maildir b/scripts/import_maildir index 269f2550..7228a3ad 100755 --- a/scripts/import_maildir +++ b/scripts/import_maildir @@ -1,21 +1,29 @@ #!/usr/bin/perl -w -# Copyright (C) 2014, Eric Wong <e@80x24.org> and all contributors +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> -# -# Script to import a Maildir into a public-inbox =begin usage +Ancient script to import a Maildir into a v1 public-inbox + + # this is only if you want a v1 inbox export GIT_DIR=/path/to/your/repo.git export GIT_AUTHOR_EMAIL='list@example.com' export GIT_AUTHOR_NAME='list name' ./import_maildir /path/to/maildir/ + +For v2 (strongly recommended), use: + + lei convert /path/to/maildir -o /path/to/v2-inbox + # (and `lei daemon-kill' if you don't want the daemon to linger) =cut -use strict; -use warnings; +use v5.12; use Date::Parse qw/str2time/; use PublicInbox::Eml; use PublicInbox::Git; use PublicInbox::Import; -sub usage { "Usage:\n".join('', grep(/\t/, `head -n 24 $0`)) } +sub usage { + open my $fh, '<', __FILE__; + ("Usage:\n", grep { /^=begin usage/../^=cut/ and !/^=/m } <$fh>); +} my $dir = shift @ARGV or die usage(); my $git_dir = `git rev-parse --git-dir`; chomp $git_dir; diff --git a/scripts/import_slrnspool b/scripts/import_slrnspool index d9a35dfd..81df6c2e 100755 --- a/scripts/import_slrnspool +++ b/scripts/import_slrnspool @@ -1,20 +1,30 @@ #!/usr/bin/perl -w -# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org> +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> -# -# Incremental (or one-shot) importer of a slrnpull news spool =begin usage +Incremental (or one-shot) importer of a slrnpull news spool. + +Since the news spool can appear as an MH folder, you may also use +lei from public-inbox 2.0+ to convert it: + + lei convert mh:$SLRNPULL_ROOT/news/foo/bar -o v2:/path/to/inbox/ + # (and `lei daemon-kill' if you don't want the daemon to linger) + +But if you want to use this script: + export ORIGINAL_RECIPIENT=address@example.com - public-inbox-init $INBOX $GIT_DIR $HTTP_URL $ORIGINAL_RECIPIENT - ./import_slrnspool SLRNPULL_ROOT/news/foo/bar + public-inbox-init -V2 $INBOX $INBOX_DIR $HTTP_URL $ORIGINAL_RECIPIENT + ./import_slrnspool $SLRNPULL_ROOT/news/foo/bar =cut -use strict; -use warnings; +use v5.12; use PublicInbox::Config; use PublicInbox::Eml; use PublicInbox::Import; use PublicInbox::Git; -sub usage { "Usage:\n".join('',grep(/\t/, `head -n 10 $0`)) } +sub usage { + open my $fh, '<', __FILE__; + ("Usage:\n", grep { /^=begin usage/../^=cut/ and !/^=/m } <$fh>); +} my $exit = 0; my $sighandler = sub { $exit = 1 }; $SIG{INT} = $sighandler; diff --git a/scripts/import_vger_from_mbox b/scripts/import_vger_from_mbox index c33e42e4..40ccf50b 100644 --- a/scripts/import_vger_from_mbox +++ b/scripts/import_vger_from_mbox @@ -1,8 +1,8 @@ #!/usr/bin/perl -w -# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org> +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> -use strict; -use warnings; +# consider `lei convert' instead since it handles more formats +use v5.12; use Getopt::Long qw/:config gnu_getopt no_ignore_case auto_abbrev/; use PublicInbox::InboxWritable; my $usage = "usage: $0 NAME EMAIL DIR <MBOX\n"; diff --git a/scripts/slrnspool2maildir b/scripts/slrnspool2maildir index 8e2ba08a..ba0729ec 100755 --- a/scripts/slrnspool2maildir +++ b/scripts/slrnspool2maildir @@ -1,51 +1,55 @@ #!/usr/bin/perl -w -# Copyright (C) 2013-2021 all contributors <meta@public-inbox.org> +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> -# -# One-off script to convert an slrnpull news spool to Maildir =begin usage +One-off script to convert an slrnpull spool from gmane to Maildir +Note: this contains Gmane-specific header munging to workaround +the munging done by Gmane. + ./slrnspool2maildir SLRNPULL_ROOT/news/foo/bar /path/to/maildir/ -=cut -use strict; -use warnings; -use Email::Filter; -use Email::LocalDelivery; -use File::Glob qw(bsd_glob GLOB_NOSORT); -sub usage { "Usage:\n".join('',grep(/\t/, `head -n 12 $0`)) } -my $spool = shift @ARGV or die usage(); -my $dir = shift @ARGV or die usage(); --d $dir or die "$dir is not a directory\n"; -$dir .= '/' unless $dir =~ m!/\z!; -foreach my $sub (qw(cur new tmp)) { - my $nd = "$dir/$sub"; - -d $nd and next; - mkdir $nd or die "mkdir $nd failed: $!\n"; -} -foreach my $n (grep(/\d+\z/, bsd_glob("$spool/*", GLOB_NOSORT))) { - if (open my $fh, '<', $n) { - my $f = Email::Filter->new(data => do { local $/; <$fh> }); - my $s = $f->simple; +A generic replacement w/o Gmane-specific munging could treat +the slrnpull spool as an MH folder with lei: - # gmane rewrites Received headers, which increases spamminess - # Some older archives set Original-To - foreach my $x (qw(Received To)) { - my @h = $s->header("Original-$x"); - if (@h) { - $s->header_set($x, @h); - $s->header_set("Original-$x"); - } + lei convert mh:SLRNPULL_ROOT/news/foo/bar -o /path/to/maildir + # (and `lei daemon-kill' if you don't want the daemon to linger) +=cut +use v5.12; +use autodie; +# warning: unstable internal APIs: +use PublicInbox::Eml; +use PublicInbox::LeiToMail; +use PublicInbox::MHreader; +use PublicInbox::IO qw(read_all); +use File::Path qw(make_path); +use File::Spec (); +sub usage { + open my $fh, '<', __FILE__; + ("Usage:\n", grep { /^=begin usage/../^=cut/ and !/^=/m } <$fh>); +} +my $spool = shift @ARGV or die usage(); +my $dst = shift @ARGV or die usage(); +$dst .= '/' unless $dst =~ m!/\z!; +File::Path::make_path(map { $dst.$_ } qw(tmp new cur)); +$dst = File::Spec->rel2abs($dst).'/'; +opendir my $cwdfh, '.'; +my $mhr = PublicInbox::MHreader->new($spool, $cwdfh); +my $smsg; +$mhr->mh_each_eml(sub { + my ($d, $n, $kw, $eml) = @_; + # gmane rewrites Received headers, which increases spamminess + # Some older archives set Original-To + for my $x (qw(Received To)) { + my @h = $eml->header_raw("Original-$x"); + if (@h) { + $eml->header_set($x, @h); + $eml->header_set("Original-$x"); } - - # triggers for the SA HEADER_SPAM rule - foreach my $drop (qw(Approved)) { $s->header_set($drop) } - - # appears to be an old gmane bug: - $s->header_set('connect()'); - - $f->exit(0); - $f->accept($dir); - } else { - warn "Failed to open $n: $!\n"; } -} + # `Approved' triggers the SA HEADER_SPAM rule + # `connect()' appears to be an old gmane bug: + $eml->header_set($_) for ('Approved', 'connect()'); + my $buf = $eml->as_string; + $smsg->{blob} = $n; + PublicInbox::LeiToMail::_buf2maildir($dst, \$buf, $smsg, 'new/'); +}); |