imap: require ".$UID_MIN-$UID_END" suffix

Finish up the IMAP-only portion of iterative config reloading,
which allows us to create all sub-ranges of an inbox up front.
The InboxIdler still uses ->each_inbox which will struggle with
100K inboxes.

Having messages in the top-level newsgroup name of an inbox will
still waste bandwidth for clients which want to do full syncs
once there's a rollover to a new 50K range.  So instead, make
every inbox accessible exclusively via 50K slices in the form of
"$NEWSGROUP.$UID_MIN-$UID_END".

This introduces the DummyInbox, which makes $NEWSGROUP
and every parent component a selectable, empty inbox.
This aids navigation with mutt and possibly other MUAs.

Finally, the xt/perf-imap-list maintainer test is broken, now,
so remove it.  The grep perlfunc is already proven effective,
and we'll have separate tests for mocking out ~100k inboxes.

1 files changed, 41 insertions, 47 deletions

diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm
index a10fabff..b647c940 100644
--- a/lib/PublicInbox/IMAPD.pm
+++ b/lib/PublicInbox/IMAPD.pm
@@ -5,82 +5,75 @@
 # see script/public-inbox-imapd for how it is used
 package PublicInbox::IMAPD;
 use strict;
-use parent qw(PublicInbox::NNTPD);
+use PublicInbox::Config;
 use PublicInbox::InboxIdle;
 use PublicInbox::IMAP;
-# *UID_BLOCK = \&PublicInbox::IMAP::UID_BLOCK;
+use PublicInbox::DummyInbox;
+my $dummy = bless { uidvalidity => 0 }, 'PublicInbox::DummyInbox';
 
 sub new {
         my ($class) = @_;
         bless {
-                groups => {},
+                mailboxes => {},
                 err => \*STDERR,
                 out => \*STDOUT,
-                grouplist => [],
                 # accept_tls => { SSL_server => 1, ..., SSL_reuse_ctx => ... }
                 # pi_config => PublicInbox::Config
                 # idler => PublicInbox::InboxIdle
         }, $class;
 }
 
-sub refresh_inboxlist ($) {
-        my ($self) = @_;
-        my @names = map { $_->{newsgroup} } @{delete $self->{grouplist}};
-        my %ns; # "\Noselect \HasChildren"
-
-        if (my @uc = grep(/[A-Z]/, @names)) {
-                warn "Uppercase not allowed for IMAP newsgroup(s):\n",
-                        map { "\t$_\n" } @uc;
-                my %uc = map { $_ => 1 } @uc;
-                @names = grep { !$uc{$_} } @names;
-        }
-        for (@names) {
-                my $up = $_;
-                while ($up =~ s/\.[^\.]+\z//) {
-                        $ns{$up} = '\\Noselect \\HasChildren';
-                }
-        }
-        @names = map {;
-                my $at = delete($ns{$_}) ? '\\HasChildren' : '\\HasNoChildren';
-                qq[* LIST ($at) "." $_\r\n]
-        } @names;
-        push(@names, map { qq[* LIST ($ns{$_}) "." $_\r\n] } keys %ns);
-        @names = sort {
-                my ($xa) = ($a =~ / (\S+)\r\n/g);
-                my ($xb) = ($b =~ / (\S+)\r\n/g);
-                length($xa) <=> length($xb);
-        } @names;
-        $self->{inboxlist} = \@names;
-}
-
 sub imapd_refresh_ibx { # pi_config->each_inbox cb
         my ($ibx, $imapd) = @_;
         my $ngname = $ibx->{newsgroup} or return;
         if (ref $ngname) {
                 warn 'multiple newsgroups not supported: '.
                         join(', ', @$ngname). "\n";
+                return;
         } elsif ($ngname =~ m![^a-z0-9/_\.\-\~\@\+\=:]! ||
                  $ngname =~ /\.[0-9]+-[0-9]+\z/) {
-                warn "mailbox name invalid: `$ngname'\n";
+                warn "mailbox name invalid: newsgroup=`$ngname'\n";
+                return;
         }
-
+        $ibx->over or return;
+        $ibx->{over} = undef;
         my $mm = $ibx->mm or return;
         $ibx->{mm} = undef;
+
+        # RFC 3501 2.3.1.1 -  "A good UIDVALIDITY value to use in
+        # this case is a 32-bit representation of the creation
+        # date/time of the mailbox"
         defined($ibx->{uidvalidity} = $mm->created_at) or return;
-        $imapd->{tmp_groups}->{$ngname} = $ibx;
+        PublicInbox::IMAP::ensure_ranges_exist($imapd, $ibx, $mm->max // 1);
 
         # preload to avoid fragmentation:
         $ibx->description;
         $ibx->base_url;
-        # my $max = $mm->max // 0;
-        # my $uid_min = UID_BLOCK * int($max/UID_BLOCK) + 1;
+
+        # ensure dummies are selectable
+        my $dummies = $imapd->{dummies};
+        do {
+                $dummies->{$ngname} = $dummy;
+        } while ($ngname =~ s/\.[^\.]+\z//);
 }
 
 sub imapd_refresh_finalize {
         my ($imapd, $pi_config) = @_;
-        $imapd->{groups} = delete $imapd->{tmp_groups};
-        $imapd->{grouplist} = [ values %{$imapd->{groups}} ];
-        refresh_inboxlist($imapd);
+        my $mailboxes;
+        if (my $next = delete $imapd->{imapd_next}) {
+                $imapd->{mailboxes} = delete $next->{mailboxes};
+                $mailboxes = delete $next->{dummies};
+        } else {
+                $mailboxes = delete $imapd->{dummies};
+        }
+        %$mailboxes = (%$mailboxes, %{$imapd->{mailboxes}});
+        $imapd->{mailboxes} = $mailboxes;
+        $imapd->{inboxlist} = [
+                map {
+                        my $no = $mailboxes->{$_} == $dummy ? '' : 'No';
+                        qq[* LIST (\\Has${no}Children) "." $_\r\n]
+                } sort { length($a) <=> length($b) } keys %$mailboxes
+        ];
         $imapd->{pi_config} = $pi_config;
         if (my $idler = $imapd->{idler}) {
                 $idler->refresh($pi_config);
@@ -92,8 +85,8 @@ sub imapd_refresh_step { # pi_config->iterate_start cb
         if (defined($section)) {
                 return if $section !~ m!\Apublicinbox\.([^/]+)\z!;
                 my $ibx = $pi_config->lookup_name($1) or return;
-                imapd_refresh_ibx($ibx, $imapd);
-        } else { # "EOF"
+                imapd_refresh_ibx($ibx, $imapd->{imapd_next});
+        } else { # undef == "EOF"
                 imapd_refresh_finalize($imapd, $pi_config);
         }
 }
@@ -101,11 +94,12 @@ sub imapd_refresh_step { # pi_config->iterate_start cb
 sub refresh_groups {
         my ($self, $sig) = @_;
         my $pi_config = PublicInbox::Config->new;
-        $self->{tmp_groups} = {};
-        if (0 && $sig) { # SIGHUP
+        if ($sig) { # SIGHUP is handled through the event loop
+                $self->{imapd_next} = { dummies => {}, mailboxes => {} };
                 $pi_config->iterate_start(\&imapd_refresh_step, $self);
                 PublicInbox::DS::requeue($pi_config); # call event_step
-        } else { # initial start
+        } else { # initial start is synchronous
+                $self->{dummies} = {};
                 $pi_config->each_inbox(\&imapd_refresh_ibx, $self);
                 imapd_refresh_finalize($self, $pi_config);
         }