about summary refs log tree commit homepage
diff options
context:
space:
mode:
-rw-r--r--Documentation/RelNotes/v1.9.0.eml68
-rw-r--r--Documentation/RelNotes/v2.0.0.wip192
-rw-r--r--Documentation/dc-dlvr-spam-flow.txt2
-rw-r--r--Documentation/design_notes.txt10
-rw-r--r--Documentation/design_www.txt14
-rw-r--r--Documentation/flow.ge13
-rw-r--r--Documentation/flow.txt18
-rw-r--r--Documentation/hosted.txt41
-rw-r--r--Documentation/include.mk25
-rw-r--r--Documentation/lei-add-external.pod4
-rw-r--r--Documentation/lei-blob.pod2
-rw-r--r--Documentation/lei-config.pod33
-rw-r--r--Documentation/lei-convert.pod2
-rw-r--r--Documentation/lei-daemon-kill.pod4
-rw-r--r--Documentation/lei-edit-search.pod6
-rw-r--r--Documentation/lei-forget-search.pod4
-rw-r--r--Documentation/lei-import.pod12
-rw-r--r--Documentation/lei-inspect.pod4
-rw-r--r--Documentation/lei-lcat.pod4
-rw-r--r--Documentation/lei-ls-mail-source.pod2
-rw-r--r--Documentation/lei-ls-search.pod7
-rw-r--r--Documentation/lei-mail-formats.pod16
-rw-r--r--Documentation/lei-mail-sync-overview.pod2
-rw-r--r--Documentation/lei-overview.pod12
-rw-r--r--Documentation/lei-q.pod40
-rw-r--r--Documentation/lei-rediff.pod2
-rw-r--r--Documentation/lei-reindex.pod47
-rw-r--r--Documentation/lei-security.pod8
-rw-r--r--Documentation/lei-store-format.pod2
-rw-r--r--Documentation/lei-up.pod10
-rw-r--r--Documentation/lei.pod2
-rw-r--r--Documentation/marketing.txt13
-rwxr-xr-xDocumentation/mknews.perl3
-rw-r--r--Documentation/public-inbox-cindex.pod136
-rw-r--r--Documentation/public-inbox-clone.pod208
-rw-r--r--Documentation/public-inbox-config.pod108
-rw-r--r--Documentation/public-inbox-daemon.pod79
-rw-r--r--Documentation/public-inbox-extindex.pod51
-rw-r--r--Documentation/public-inbox-fetch.pod8
-rw-r--r--Documentation/public-inbox-glossary.pod10
-rw-r--r--Documentation/public-inbox-httpd.pod15
-rw-r--r--Documentation/public-inbox-imapd.pod6
-rw-r--r--Documentation/public-inbox-index.pod6
-rw-r--r--Documentation/public-inbox-learn.pod23
-rw-r--r--Documentation/public-inbox-mda.pod18
-rw-r--r--Documentation/public-inbox-netd.pod38
-rw-r--r--Documentation/public-inbox-nntpd.pod4
-rw-r--r--Documentation/public-inbox-overview.pod2
-rw-r--r--Documentation/public-inbox-pop3d.pod122
-rw-r--r--Documentation/public-inbox-purge.pod4
-rw-r--r--Documentation/public-inbox-tuning.pod40
-rw-r--r--Documentation/public-inbox-v2-format.pod6
-rw-r--r--Documentation/public-inbox-watch.pod29
-rw-r--r--Documentation/public-inbox.cgi.pod19
-rw-r--r--Documentation/reproducibility.txt4
-rwxr-xr-xDocumentation/standards.perl17
-rw-r--r--Documentation/technical/data_structures.txt30
-rw-r--r--Documentation/technical/ds.txt21
-rw-r--r--Documentation/technical/memory.txt10
-rw-r--r--Documentation/technical/weird-stuff.txt22
-rw-r--r--Documentation/technical/whyperl.txt20
-rwxr-xr-xDocumentation/txt2pre24
-rw-r--r--HACKING19
-rw-r--r--INSTALL126
-rw-r--r--MANIFEST101
-rw-r--r--Makefile.PL94
-rw-r--r--README36
-rw-r--r--TODO37
-rw-r--r--ci/README15
-rwxr-xr-xci/deps.perl267
-rwxr-xr-xci/profiles.perl34
-rwxr-xr-xci/profiles.sh79
-rwxr-xr-xci/run.sh18
-rw-r--r--contrib/completion/lei-completion.bash15
-rw-r--r--devel/README2
-rwxr-xr-xdevel/longest-tests7
-rwxr-xr-xdevel/syscall-list65
-rwxr-xr-xdevel/sysdefs-list188
-rw-r--r--examples/README4
-rw-r--r--examples/README.unsubscribe9
-rw-r--r--examples/apache2_cgi.conf34
-rw-r--r--examples/apache2_perl.conf25
-rw-r--r--examples/apache2_perl_old.conf38
-rw-r--r--examples/cgi-webrick.rb25
-rwxr-xr-xexamples/grok-pull.post_update_hook.sh2
-rw-r--r--examples/logrotate.conf4
-rw-r--r--examples/nginx_proxy9
-rw-r--r--examples/public-inbox-httpd.socket3
-rw-r--r--examples/public-inbox-httpd@.service8
-rw-r--r--examples/public-inbox-imap-onion.socket12
-rw-r--r--examples/public-inbox-imapd.socket17
-rw-r--r--examples/public-inbox-imapd@.service15
-rw-r--r--examples/public-inbox-imaps.socket12
-rw-r--r--examples/public-inbox-netd.socket45
-rw-r--r--examples/public-inbox-netd@.service62
-rw-r--r--examples/public-inbox-nntpd.socket21
-rw-r--r--examples/public-inbox-nntpd@.service12
-rw-r--r--examples/public-inbox-nntps.socket12
-rw-r--r--examples/public-inbox-watch.service2
-rw-r--r--examples/unsubscribe-milter@.service6
-rw-r--r--examples/unsubscribe.milter38
-rw-r--r--examples/varnish-4.vcl2
-rw-r--r--install/README16
-rwxr-xr-xinstall/deps.perl414
-rw-r--r--install/os.perl85
-rwxr-xr-xlei.sh2
-rw-r--r--lib/PublicInbox/Address.pm16
-rw-r--r--lib/PublicInbox/AddressPP.pm12
-rw-r--r--lib/PublicInbox/Admin.pm136
-rw-r--r--lib/PublicInbox/AdminEdit.pm4
-rw-r--r--lib/PublicInbox/Aspawn.pm34
-rw-r--r--lib/PublicInbox/AutoReap.pm7
-rw-r--r--lib/PublicInbox/Cgit.pm50
-rw-r--r--lib/PublicInbox/CidxComm.pm28
-rw-r--r--lib/PublicInbox/CidxLogP.pm28
-rw-r--r--lib/PublicInbox/CidxXapHelperAux.pm48
-rw-r--r--lib/PublicInbox/CmdIPC4.pm25
-rw-r--r--lib/PublicInbox/CodeSearch.pm372
-rw-r--r--lib/PublicInbox/CodeSearchIdx.pm1394
-rw-r--r--lib/PublicInbox/Compat.pm24
-rw-r--r--lib/PublicInbox/CompressNoop.pm4
-rw-r--r--lib/PublicInbox/Config.pm303
-rw-r--r--lib/PublicInbox/ContentDigestDbg.pm22
-rw-r--r--lib/PublicInbox/ContentHash.pm41
-rw-r--r--lib/PublicInbox/DS.pm493
-rw-r--r--lib/PublicInbox/DSKQXS.pm69
-rw-r--r--lib/PublicInbox/DSPoll.pm62
-rw-r--r--lib/PublicInbox/DSdeflate.pm (renamed from lib/PublicInbox/NNTPdeflate.pm)23
-rw-r--r--lib/PublicInbox/Daemon.pm586
-rw-r--r--lib/PublicInbox/DirIdle.pm36
-rw-r--r--lib/PublicInbox/EOFpipe.pm16
-rw-r--r--lib/PublicInbox/Emergency.pm34
-rw-r--r--lib/PublicInbox/Eml.pm36
-rw-r--r--lib/PublicInbox/Epoll.pm26
-rw-r--r--lib/PublicInbox/ExtMsg.pm4
-rw-r--r--lib/PublicInbox/ExtSearch.pm13
-rw-r--r--lib/PublicInbox/ExtSearchIdx.pm79
-rw-r--r--lib/PublicInbox/FakeInotify.pm200
-rw-r--r--lib/PublicInbox/Feed.pm46
-rw-r--r--lib/PublicInbox/Fetch.pm67
-rw-r--r--lib/PublicInbox/Filter/RubyLang.pm29
-rw-r--r--lib/PublicInbox/Gcf2.pm114
-rw-r--r--lib/PublicInbox/Gcf2Client.pm72
-rw-r--r--lib/PublicInbox/GetlineBody.pm46
-rw-r--r--lib/PublicInbox/GetlineResponse.pm40
-rw-r--r--lib/PublicInbox/Git.pm634
-rw-r--r--lib/PublicInbox/GitAsyncCat.pm88
-rw-r--r--lib/PublicInbox/GitCredential.pm24
-rw-r--r--lib/PublicInbox/GitHTTPBackend.pm44
-rw-r--r--lib/PublicInbox/GzipFilter.pm112
-rw-r--r--lib/PublicInbox/HTTP.pm51
-rw-r--r--lib/PublicInbox/HTTPD.pm62
-rw-r--r--lib/PublicInbox/HTTPD/Async.pm105
-rw-r--r--lib/PublicInbox/Hval.pm29
-rw-r--r--lib/PublicInbox/IMAP.pm199
-rw-r--r--lib/PublicInbox/IMAPD.pm148
-rw-r--r--lib/PublicInbox/IMAPdeflate.pm126
-rw-r--r--lib/PublicInbox/IMAPsearchqp.pm4
-rw-r--r--lib/PublicInbox/IO.pm150
-rw-r--r--lib/PublicInbox/IPC.pm254
-rw-r--r--lib/PublicInbox/IdxStack.pm20
-rw-r--r--lib/PublicInbox/Import.pm232
-rw-r--r--lib/PublicInbox/In2Tie.pm4
-rw-r--r--lib/PublicInbox/In3Event.pm24
-rw-r--r--lib/PublicInbox/In3Watch.pm20
-rw-r--r--lib/PublicInbox/Inbox.pm121
-rw-r--r--lib/PublicInbox/InboxIdle.pm24
-rw-r--r--lib/PublicInbox/InboxWritable.pm103
-rw-r--r--lib/PublicInbox/Inotify.pm47
-rw-r--r--lib/PublicInbox/Inotify3.pm115
-rw-r--r--lib/PublicInbox/InputPipe.pm54
-rw-r--r--lib/PublicInbox/Isearch.pm19
-rw-r--r--lib/PublicInbox/KQNotify.pm102
-rw-r--r--lib/PublicInbox/LEI.pm638
-rw-r--r--lib/PublicInbox/LI2Wrap.pm2
-rw-r--r--lib/PublicInbox/LeiALE.pm31
-rw-r--r--lib/PublicInbox/LeiAddWatch.pm11
-rw-r--r--lib/PublicInbox/LeiAuth.pm11
-rw-r--r--lib/PublicInbox/LeiBlob.pm52
-rw-r--r--lib/PublicInbox/LeiConfig.pm45
-rw-r--r--lib/PublicInbox/LeiConvert.pm24
-rw-r--r--lib/PublicInbox/LeiCurl.pm11
-rw-r--r--lib/PublicInbox/LeiDedupe.pm20
-rw-r--r--lib/PublicInbox/LeiExportKw.pm4
-rw-r--r--lib/PublicInbox/LeiExternal.pm71
-rw-r--r--lib/PublicInbox/LeiForgetExternal.pm11
-rw-r--r--lib/PublicInbox/LeiImport.pm53
-rw-r--r--lib/PublicInbox/LeiImportKw.pm6
-rw-r--r--lib/PublicInbox/LeiIndex.pm2
-rw-r--r--lib/PublicInbox/LeiInit.pm4
-rw-r--r--lib/PublicInbox/LeiInput.pm210
-rw-r--r--lib/PublicInbox/LeiInspect.pm32
-rw-r--r--lib/PublicInbox/LeiLcat.pm20
-rw-r--r--lib/PublicInbox/LeiLsExternal.pm4
-rw-r--r--lib/PublicInbox/LeiLsMailSource.pm12
-rw-r--r--lib/PublicInbox/LeiLsMailSync.pm10
-rw-r--r--lib/PublicInbox/LeiMailDiff.pm77
-rw-r--r--lib/PublicInbox/LeiMailSync.pm106
-rw-r--r--lib/PublicInbox/LeiMirror.pm1263
-rw-r--r--lib/PublicInbox/LeiNoteEvent.pm37
-rw-r--r--lib/PublicInbox/LeiOverview.pm9
-rw-r--r--lib/PublicInbox/LeiQuery.pm64
-rw-r--r--lib/PublicInbox/LeiRediff.pm63
-rw-r--r--lib/PublicInbox/LeiReindex.pm49
-rw-r--r--lib/PublicInbox/LeiRemote.pm14
-rw-r--r--lib/PublicInbox/LeiRmWatch.pm2
-rw-r--r--lib/PublicInbox/LeiSavedSearch.pm33
-rw-r--r--lib/PublicInbox/LeiSearch.pm33
-rw-r--r--lib/PublicInbox/LeiSelfSocket.pm14
-rw-r--r--lib/PublicInbox/LeiStore.pm128
-rw-r--r--lib/PublicInbox/LeiStoreErr.pm48
-rw-r--r--lib/PublicInbox/LeiSucks.pm13
-rw-r--r--lib/PublicInbox/LeiTag.pm13
-rw-r--r--lib/PublicInbox/LeiToMail.pm183
-rw-r--r--lib/PublicInbox/LeiUp.pm26
-rw-r--r--lib/PublicInbox/LeiViewText.pm9
-rw-r--r--lib/PublicInbox/LeiWatch.pm7
-rw-r--r--lib/PublicInbox/LeiXSearch.pm297
-rw-r--r--lib/PublicInbox/Limiter.pm50
-rw-r--r--lib/PublicInbox/Linkify.pm29
-rw-r--r--lib/PublicInbox/Listener.pm45
-rw-r--r--lib/PublicInbox/Lock.pm52
-rw-r--r--lib/PublicInbox/MHreader.pm104
-rw-r--r--lib/PublicInbox/MID.pm18
-rw-r--r--lib/PublicInbox/MailDiff.pm137
-rw-r--r--lib/PublicInbox/ManifestJsGz.pm8
-rw-r--r--lib/PublicInbox/Mbox.pm44
-rw-r--r--lib/PublicInbox/MboxGz.pm6
-rw-r--r--lib/PublicInbox/MboxLock.pm59
-rw-r--r--lib/PublicInbox/MboxReader.pm42
-rw-r--r--lib/PublicInbox/MdirReader.pm6
-rw-r--r--lib/PublicInbox/MdirSort.pm46
-rw-r--r--lib/PublicInbox/MiscIdx.pm12
-rw-r--r--lib/PublicInbox/MiscSearch.pm21
-rw-r--r--lib/PublicInbox/MsgTime.pm49
-rw-r--r--lib/PublicInbox/Msgmap.pm12
-rw-r--r--lib/PublicInbox/MultiGit.pm25
-rw-r--r--lib/PublicInbox/NNTP.pm449
-rw-r--r--lib/PublicInbox/NNTPD.pm33
-rw-r--r--lib/PublicInbox/NetNNTPSocks.pm25
-rw-r--r--lib/PublicInbox/NetReader.pm171
-rw-r--r--lib/PublicInbox/NetWriter.pm5
-rw-r--r--lib/PublicInbox/OnDestroy.pm5
-rw-r--r--lib/PublicInbox/Over.pm57
-rw-r--r--lib/PublicInbox/OverIdx.pm35
-rw-r--r--lib/PublicInbox/POP3.pm428
-rw-r--r--lib/PublicInbox/POP3D.pm277
-rw-r--r--lib/PublicInbox/PktOp.pm19
-rw-r--r--lib/PublicInbox/ProcessPipe.pm70
-rw-r--r--lib/PublicInbox/Qspawn.pm398
-rw-r--r--lib/PublicInbox/Reply.pm3
-rw-r--r--lib/PublicInbox/RepoAtom.pm128
-rw-r--r--lib/PublicInbox/RepoList.pm39
-rw-r--r--lib/PublicInbox/RepoSnapshot.pm87
-rw-r--r--lib/PublicInbox/RepoTree.pm99
-rw-r--r--lib/PublicInbox/SHA.pm67
-rw-r--r--lib/PublicInbox/SaPlugin/ListMirror.pm10
-rw-r--r--lib/PublicInbox/SaPlugin/ListMirror.pod27
-rw-r--r--lib/PublicInbox/Search.pm368
-rw-r--r--lib/PublicInbox/SearchIdx.pm193
-rw-r--r--lib/PublicInbox/SearchIdxShard.pm31
-rw-r--r--lib/PublicInbox/SearchQuery.pm15
-rw-r--r--lib/PublicInbox/SearchThread.pm2
-rw-r--r--lib/PublicInbox/SearchView.pm53
-rw-r--r--lib/PublicInbox/Select.pm43
-rw-r--r--lib/PublicInbox/SharedKV.pm4
-rw-r--r--lib/PublicInbox/Sigfd.pm34
-rw-r--r--lib/PublicInbox/Smsg.pm9
-rw-r--r--lib/PublicInbox/SolverGit.pm154
-rw-r--r--lib/PublicInbox/Spamcheck.pm12
-rw-r--r--lib/PublicInbox/Spamcheck/Spamc.pm23
-rw-r--r--lib/PublicInbox/Spawn.pm269
-rw-r--r--lib/PublicInbox/SpawnPP.pm45
-rw-r--r--lib/PublicInbox/Syscall.pm592
-rw-r--r--lib/PublicInbox/TLS.pm28
-rw-r--r--lib/PublicInbox/TailNotify.pm97
-rw-r--r--lib/PublicInbox/TestCommon.pm543
-rw-r--r--lib/PublicInbox/Tmpfile.pm10
-rw-r--r--lib/PublicInbox/URIimap.pm5
-rw-r--r--lib/PublicInbox/URInntps.pm4
-rw-r--r--lib/PublicInbox/Umask.pm70
-rw-r--r--lib/PublicInbox/V2Writable.pm76
-rw-r--r--lib/PublicInbox/View.pm559
-rw-r--r--lib/PublicInbox/ViewDiff.pm156
-rw-r--r--lib/PublicInbox/ViewVCS.pm654
-rw-r--r--lib/PublicInbox/WQBlocked.pm48
-rw-r--r--lib/PublicInbox/WWW.pm98
-rw-r--r--lib/PublicInbox/WWW.pod16
-rw-r--r--lib/PublicInbox/Watch.pm246
-rw-r--r--lib/PublicInbox/WwwAltId.pm21
-rw-r--r--lib/PublicInbox/WwwAtomStream.pm35
-rw-r--r--lib/PublicInbox/WwwCoderepo.pm377
-rw-r--r--lib/PublicInbox/WwwListing.pm63
-rw-r--r--lib/PublicInbox/WwwStatic.pm35
-rw-r--r--lib/PublicInbox/WwwStream.pm144
-rw-r--r--lib/PublicInbox/WwwText.pm119
-rw-r--r--lib/PublicInbox/WwwTopics.pm85
-rw-r--r--lib/PublicInbox/XapClient.pm48
-rw-r--r--lib/PublicInbox/XapHelper.pm313
-rw-r--r--lib/PublicInbox/XapHelperCxx.pm130
-rw-r--r--lib/PublicInbox/Xapcmd.pm214
-rw-r--r--lib/PublicInbox/xap_helper.h1049
-rw-r--r--lib/PublicInbox/xh_cidx.h277
-rw-r--r--lib/PublicInbox/xh_mset.h96
-rw-r--r--sa_config/README4
-rwxr-xr-xscript/lei19
-rwxr-xr-xscript/public-inbox-cindex102
-rwxr-xr-xscript/public-inbox-clone35
-rwxr-xr-xscript/public-inbox-compact20
-rwxr-xr-xscript/public-inbox-convert45
-rwxr-xr-xscript/public-inbox-edit8
-rwxr-xr-xscript/public-inbox-fetch4
-rwxr-xr-xscript/public-inbox-index9
-rwxr-xr-xscript/public-inbox-init39
-rwxr-xr-xscript/public-inbox-learn12
-rwxr-xr-xscript/public-inbox-mda21
-rwxr-xr-xscript/public-inbox-pop3d8
-rwxr-xr-xscript/public-inbox-purge6
-rwxr-xr-xscript/public-inbox-watch16
-rwxr-xr-xscript/public-inbox-xcpdb20
-rw-r--r--scripts/README2
-rwxr-xr-xscripts/dc-dlvr4
-rwxr-xr-xscripts/import_maildir20
-rwxr-xr-xscripts/import_slrnspool26
-rw-r--r--scripts/import_vger_from_mbox6
-rwxr-xr-xscripts/slrnspool2maildir90
-rw-r--r--t/address.t23
-rw-r--r--t/admin.t16
-rw-r--r--t/alt.psgi17
-rw-r--r--t/altid.t2
-rw-r--r--t/altid_v2.t12
-rw-r--r--t/check-www-inbox.perl4
-rw-r--r--t/cindex-join.t88
-rw-r--r--t/cindex.t299
-rwxr-xr-xt/clone-coderepo-puh1.sh6
-rwxr-xr-xt/clone-coderepo-puh2.sh6
-rw-r--r--t/clone-coderepo.psgi21
-rw-r--r--t/clone-coderepo.t220
-rw-r--r--t/cmd_ipc.t65
-rw-r--r--t/config.t193
-rw-r--r--t/config_limiter.t31
-rw-r--r--t/convert-compact.t29
-rw-r--r--t/data/attached-mbox-with-utf8.eml45
-rw-r--r--t/dir_idle.t25
-rw-r--r--t/ds-kqxs.t9
-rw-r--r--t/ds-leak.t17
-rw-r--r--t/ds-poll.t50
-rw-r--r--t/edit.t29
-rw-r--r--t/eml.t6
-rw-r--r--t/epoll.t23
-rw-r--r--t/extindex-psgi.t49
-rw-r--r--t/extsearch.t118
-rw-r--r--t/fake_inotify.t29
-rw-r--r--t/filter_rubylang.t16
-rw-r--r--t/gcf2.t5
-rw-r--r--t/gcf2_client.t32
-rw-r--r--t/git.t15
-rw-r--r--t/gzip_filter.t8
-rw-r--r--t/hl_mod.t4
-rw-r--r--t/httpd-corner.psgi45
-rw-r--r--t/httpd-corner.t121
-rw-r--r--t/httpd-https.t64
-rw-r--r--t/httpd-unix.t213
-rw-r--r--t/httpd.t6
-rw-r--r--t/imap.t13
-rw-r--r--t/imap_searchqp.t11
-rw-r--r--t/imapd-tls.t23
-rw-r--r--t/imapd.t95
-rw-r--r--t/import.t16
-rw-r--r--t/inbox_idle.t15
-rw-r--r--t/index-git-times.t12
-rw-r--r--t/indexlevels-mirror.t19
-rw-r--r--t/init.t29
-rw-r--r--t/inotify3.t17
-rw-r--r--t/io.t35
-rw-r--r--t/ipc.t33
-rw-r--r--t/kqnotify.t60
-rw-r--r--t/lei-auto-watch.t4
-rw-r--r--t/lei-convert.t96
-rw-r--r--t/lei-daemon.t7
-rw-r--r--t/lei-externals.t4
-rw-r--r--t/lei-import-nntp.t7
-rw-r--r--t/lei-import.t124
-rw-r--r--t/lei-index.t15
-rw-r--r--t/lei-mail-diff.t15
-rw-r--r--t/lei-mirror.t23
-rw-r--r--t/lei-p2q.t2
-rw-r--r--t/lei-q-kw.t23
-rw-r--r--t/lei-q-remote-import.t33
-rw-r--r--t/lei-q-save.t72
-rw-r--r--t/lei-q-thread.t2
-rw-r--r--t/lei-refresh-mail-sync.t28
-rw-r--r--t/lei-reindex.t12
-rw-r--r--t/lei-sigpipe.t34
-rw-r--r--t/lei-store-fail.t57
-rw-r--r--t/lei-tag.t20
-rw-r--r--t/lei-up.t43
-rw-r--r--t/lei-watch.t16
-rw-r--r--t/lei.t36
-rw-r--r--t/lei_dedupe.t6
-rw-r--r--t/lei_external.t20
-rw-r--r--t/lei_overview.t2
-rw-r--r--t/lei_store.t7
-rw-r--r--t/lei_to_mail.t39
-rw-r--r--t/lei_xsearch.t4
-rw-r--r--t/linkify.t5
-rw-r--r--t/mbox_reader.t6
-rw-r--r--t/mda.t100
-rw-r--r--t/mda_filter_rubylang.t2
-rw-r--r--t/mh_reader.t120
-rw-r--r--t/mime.t4
-rw-r--r--t/miscsearch.t2
-rw-r--r--t/net_reader-imap.t2
-rw-r--r--t/nntp.t74
-rw-r--r--t/nntpd-tls.t33
-rw-r--r--t/nntpd.t63
-rw-r--r--t/on_destroy.t8
-rw-r--r--t/plack.t200
-rw-r--r--t/pop3d-limit.t144
-rw-r--r--t/pop3d.t346
-rw-r--r--t/pop3d_lock.t16
-rw-r--r--t/psgi_attach.t13
-rw-r--r--t/psgi_bad_mids.t18
-rw-r--r--t/psgi_mount.t16
-rw-r--r--t/psgi_multipart_not.t18
-rw-r--r--t/psgi_scan_all.t32
-rw-r--r--t/psgi_search.t49
-rw-r--r--t/psgi_text.t21
-rw-r--r--t/psgi_v2.t98
-rw-r--r--t/qspawn.t17
-rw-r--r--t/replace.t6
-rw-r--r--t/reply.t10
-rw-r--r--t/search-thr-index.t2
-rw-r--r--t/search.t53
-rw-r--r--t/select.t4
-rw-r--r--t/sha.t25
-rw-r--r--t/sigfd.t42
-rw-r--r--t/solver_git.t221
-rw-r--r--t/spawn.t154
-rw-r--r--t/tail_notify.t38
-rw-r--r--t/v1-add-remove-add.t4
-rw-r--r--t/v1reindex.t2
-rw-r--r--t/v2-add-remove-add.t4
-rw-r--r--t/v2mda.t29
-rw-r--r--t/v2mirror.t11
-rw-r--r--t/v2reindex.t7
-rw-r--r--t/v2writable.t20
-rw-r--r--t/watch_filter_rubylang.t35
-rw-r--r--t/watch_imap.t20
-rw-r--r--t/watch_maildir.t85
-rw-r--r--t/watch_maildir_v2.t46
-rw-r--r--t/watch_mh.t120
-rw-r--r--t/watch_multiple_headers.t23
-rw-r--r--t/www_altid.t13
-rw-r--r--t/www_listing.t77
-rw-r--r--t/xap_helper.t281
-rw-r--r--t/xcpdb-reshard.t4
-rw-r--r--xt/check-debris.t30
-rwxr-xr-xxt/check-run.t (renamed from t/run.perl)42
-rw-r--r--xt/cmp-msgview.t94
-rw-r--r--xt/create-many-inboxes.t2
-rw-r--r--xt/eml_check_limits.t4
-rw-r--r--xt/git-http-backend.t42
-rw-r--r--xt/git_async_cmp.t14
-rw-r--r--xt/httpd-async-stream.t80
-rw-r--r--xt/imapd-mbsync-oimap.t15
-rw-r--r--xt/imapd-validate.t5
-rw-r--r--xt/lei-auth-fail.t7
-rw-r--r--xt/lei-onion-convert.t21
-rw-r--r--xt/mem-imapd-tls.t36
-rw-r--r--xt/mem-nntpd-tls.t254
-rw-r--r--xt/msgtime_cmp.t2
-rw-r--r--xt/net_writer-imap.t4
-rw-r--r--xt/nntpd-validate.t5
-rw-r--r--xt/perf-msgview.t30
-rw-r--r--xt/perf-obfuscate.t64
-rw-r--r--xt/pop3d-mpop.t76
-rw-r--r--xt/solver.t66
478 files changed, 23424 insertions, 9545 deletions
diff --git a/Documentation/RelNotes/v1.9.0.eml b/Documentation/RelNotes/v1.9.0.eml
new file mode 100644
index 00000000..08e16a66
--- /dev/null
+++ b/Documentation/RelNotes/v1.9.0.eml
@@ -0,0 +1,68 @@
+From: Eric Wong <e@80x24.org>
+To: meta@public-inbox.org
+Subject: [ANNOUNCE] public-inbox 1.9.0
+Date: Sun, 21 Aug 2022 02:36:59 +0000
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: inline
+Message-ID: <2022-08-21T023659Z-public-inbox-1.9.0-rele@sed>
+
+Upgrading:
+
+  lei users need to "lei daemon-kill" after installation to load
+  new code.  Normal daemons (read-only, and public-inbox-watch)
+  will also need restarts, of course, but there's no
+  backwards-incompatible data format changes so rolling back to
+  older versions is harmless.
+
+Major bugfixes:
+
+  * lei no longer freezes from inotify/EVFILT_VNODE handling,
+    user interrupts (Ctrl-C), nor excessive errors/warnings
+
+  * IMAP server fairness improved to avoid excessive blob prefetch
+
+New features:
+
+  * POP3 server support added, use either public-inbox-pop3d or
+    the new public-inbox-netd superserver
+
+  * public-inbox-netd superserver supporting any combination of HTTP,
+    IMAP, POP3, and NNTP services; simplifying management and allowing
+    more sharing of memory used for various data structures.
+
+  * public-inbox-httpd and -netd support per-listener .psgi files
+
+  * SIGHUP reloads TLS certs and keys in addition to config and .psgi files
+
+  * "lei reindex" command for lei users to update personal index
+    in ~/.local/share/lei/store for search improvements below:
+
+Search improvements:
+
+  These will require --reindex with public-inbox-index and/or
+  public-inbox-extindex for public inboxes.
+
+  * patchid: prefix search support added to WWW and lei for
+    "git patch-id --stable" support
+
+  * text inside base-85 binary patches is no longer indexed
+    to avoid false positives
+
+  * for lei users, "lei reindex" now exists and is required
+    to take advantage of aforementioned indexing changes
+
+Performance improvements:
+
+  * IMAP server startup is faster with many mailboxes when using
+    "public-inbox-extindex --all"
+
+  * NNTP group listings are also faster with many inboxes when
+    using "public-inbox-extindex --all"
+
+  * various small opcode and memory usage reductions
+
+Please report bugs via plain-text mail to: meta@public-inbox.org
+
+See archives at https://public-inbox.org/meta/ for all history.
+See https://public-inbox.org/TODO for what the future holds.
diff --git a/Documentation/RelNotes/v2.0.0.wip b/Documentation/RelNotes/v2.0.0.wip
new file mode 100644
index 00000000..4d872fd7
--- /dev/null
+++ b/Documentation/RelNotes/v2.0.0.wip
@@ -0,0 +1,192 @@
+To: meta@public-inbox.org
+Subject: [WIP] public-inbox 2.2.0
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: inline
+
+This release includes several new features and fixes; mostly
+around improved integration between inboxes and coderepos for
+solver.  Portability and reliability is also improved, especially
+in the internal process management of lei.
+
+public-inbox-cindex is a new command to index coderepos for
+WWW search and perform automatic associations between
+coderepos and inboxes.  This makes solver vastly more useful
+for the WWW UI as admins will no longer have to manually
+associate coderepos with inboxes.
+
+public-inbox-clone gains the ability to mirror entire (or partial)
+grokmirror-compatible manifests.
+
+Internal process and object management data structures are vastly
+simplified throughout and error handling made more robust.
+
+git SHA-256 support remains a work-in-progress for inboxes and
+extindex due to the need to interoperate with SHA-1 epochs.
+
+Upgrading:
+
+  lei users need to "lei daemon-kill" after installation to load
+  new code.  Normal daemons (read-only, and public-inbox-watch)
+  will also need restarts, of course, but there's no
+  backwards-incompatible data format changes so rolling back to
+  older versions is harmless.
+
+Compatibility:
+
+  Uppercase newsgroup names were always broken with IMAP, POP3, and
+  -extindex.  Uppercase names will now be lowercased by default and
+  warnings will be emitted.  Conflicting newsgroup names (and `inboxdir'
+  entries if `newsgroup' isn't specified) will also generate warnings
+  since they break -extindex and the new -cindex (coderepo index).
+
+New users + hackers:
+
+  The install/ directory includes tools to automate installation and
+  removal of dependencies for stripped-down or full setups.  See
+  install/README for more details.
+
+treewide
+
+  * support raw UTF-8 headers from SMTPUTF8 hosts
+
+  * standardize on `#' prefix for stderr diagnostics (previously `I:')
+
+  * SHA-256 coderepos are fully supported (but not inboxes, yet)
+
+  * jemalloc (tested as an LD_PRELOAD) is recommended to reduce fragmentation
+    in long-running daemon processes serving unpredictable traffic
+
+PublicInbox::WWW
+
+  * support `+' in inbox names
+
+  * support coderepo displays for systems without cgit
+
+  * improve display of git tags, commits and trees in $INBOX/$OID/s/ endpoint
+
+  * numerous memory usage reductions by avoiding Perl scratchpads
+
+  * add #related anchor and search form to find related patches
+    based on blob OIDs (IOW, exposing `lei p2q' to the web)
+
+  * fix footer in listing of >200 inboxes
+
+  * support dumb HTTP clones of SHA-256 git repos
+
+  * add /$INBOX/$MSGID/d/ endpoint to show diffs in reused Message-IDs
+    (`lei mail-diff' for the web)
+
+  * support POST /$INBOX/$MSGID/?x=m&q= to limit mbox results to a thread
+
+  * add topics_(new|active).(html|atom) endpoints
+
+  * linkify peer public-inbox addresses in To/Cc headers
+
+public-inbox-watch:
+
+  * watching MH folders is now supported
+
+lei
+
+  * use http.proxy / http.<remote>.proxy from system-wide git-config if
+    unconfigured for lei
+
+  * improve IMAP and NNTP error reporting
+
+  * reduce default IMAP connections to avoid overloading servers
+
+  * compatibility with SQLite <3.8.3 on CentOS 7.x
+
+  * fix `lei q -tt' on locally indexed messages (still broken for remotes:
+    https://public-inbox.org/meta/20230226170931.M947721@dcvr/ )
+
+  * `lei import' now sets labels+keywords consistently on all
+     already imported messages
+
+  * fix `lei up' on saved local queries which previously used -t/--threads
+
+  * `lei convert' output to v2 public-inboxes is now idempotent
+
+  * improved bash completion for labels (see contrib/completion)
+
+  * support for reading (but not writing) MH folders
+
+  * `lei index' accepts `+L:$LABEL' like `lei import' does
+
+solver (used by lei (rediff|blob), and PublicInbox::WWW)
+
+  * handle copies in patches properly
+
+  * no longer redundantly parallelized within each WWW process
+
+portability
+
+  * SIGWINCH is handled properly on less common architectures and OSes
+
+  * fix EINTR handling for kqueue users
+
+  * various fixes for CentOS 7.x
+
+  * fix excessive pipelining to `git cat-file' on systems with small
+    getdelim(3) buffers (mainly affects musl)
+
+  * support Alpine Linux, Dragonfly, NetBSD and OpenBSD.  This resulted
+    not only in bugfixes to our code, but also to Dragonfly and OpenBSD.
+
+  * Inline::C||Socket::MsgHdr no longer required for SCM_RIGHTS
+    with sendmsg/recvmsg on supported *BSDs.
+
+  * inotify support no longer requires Linux::Inotify2 XS package
+    for most architectures
+
+public-inbox-pop3d
+
+  * support `limit=NUM' and `initial_limit=NUM' query parameters
+    in mailbox names to limit results
+
+public-inbox-nntpd
+
+  * fix LISTGROUP with range (affects neomutt)
+
+public-inbox-clone / public-inbox-fetch / `lei add-external --mirror'
+
+  * mtime of downloaded manifest preserved
+
+public-inbox-clone:
+
+  * parallel mirroring of multiple inboxes/coderepos via manifest,
+    public-inbox-fetch is not used in this mode
+
+  * new flags to support manifest mirroring include:
+    --dry-run, --inbox-config=, --project-list=, --prune, --purge,
+    --keep-going, --jobs, --include=, --exclude=, --objstore=,
+    --manifest=, --remote-manifest=
+    See public-inbox-clone(1) man page for more details.
+
+PublicInbox::SaPlugin::ListMirror
+
+  * List-ID handling special-cased according to RFC 2919 rules
+
+Search improvements (lei and PublicInbox::WWW)
+
+  * quoted text inside base-85 binary patches is no longer indexed
+
+  * `public-inbox-cindex --join' prefers using Xapian's C++ API
+    directly to avoid Perl method dispatch overhead to get usable
+    performance associating ~300 inboxes with over 1K coderepos
+    (and vice versa).  Users requiring such performance will need
+    a C++ compiler, pkg-config, and the Xapian development files
+    (see INSTALL).
+
+    This C++ helper will be used more heavily in the future
+    to enable query parser customizations and other functionality
+    unavailable from the Xapian SWIG or XS bindings.
+
+Thanks to all the bug reporters and users who made this release
+possible, and thanks for bearing with my anxiety over making releases.
+
+Please report bugs via plain-text mail to: meta@public-inbox.org
+
+See archives at https://public-inbox.org/meta/ for all history.
+See https://public-inbox.org/TODO for what the future holds.
diff --git a/Documentation/dc-dlvr-spam-flow.txt b/Documentation/dc-dlvr-spam-flow.txt
index d151d272..6210fc7d 100644
--- a/Documentation/dc-dlvr-spam-flow.txt
+++ b/Documentation/dc-dlvr-spam-flow.txt
@@ -39,7 +39,7 @@ delivery path as well as removing the message from the git tree.
 
 * incron - run commands based on filesystem events: http://incron.aiken.cz/
 
-* sendmail / MTA - we use and recommend use postfix, which includes a
+* sendmail / MTA - we use and recommend postfix, which includes a
                    sendmail-compatible wrapper: http://www.postfix.org/
 
 * spamc / spamd - SpamAssassin: http://spamassassin.apache.org/
diff --git a/Documentation/design_notes.txt b/Documentation/design_notes.txt
index 3df5af3e..95f02556 100644
--- a/Documentation/design_notes.txt
+++ b/Documentation/design_notes.txt
@@ -52,15 +52,15 @@ Why email?
   There is no need to ask the NSA for backups of your mail archives :)
 
 * git, one of the most widely-used version control systems, includes many
-  tools for for email, including: git-format-patch(1), git-send-email(1),
+  tools for email, including: git-format-patch(1), git-send-email(1),
   git-am(1), git-imap-send(1).  Furthermore, the development of git itself
   is based on the git mailing list: https://public-inbox.org/git/
   (or
   http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/git/
-  for Tor users)
+  for Tor users).
 
 * Email is already the de-facto form of communication in many Free Software
-  communities..
+  communities.
 
 * Fallback/transition to private email and other lists, in case the
   public-inbox host becomes unavailable, users may still directly email
@@ -76,13 +76,13 @@ Why git?
 
 * As of 2016, git is widely used and known to nearly all Free Software
   developers.  For non-developers it is packaged for all major GNU/Linux
-  and *BSD distributions.  NNTP is not as widely-used nowadays, and
+  and *BSD distributions.  NNTP is not as widely used nowadays, and
   most IMAP clients do not have good support for read-only mailboxes.
 
 Why perl 5?
 -----------
 
-* Perl 5 is widely available on modern *nix systems with good a history
+* Perl 5 is widely available on modern *nix systems, with a good history
   of backwards and forward compatibility.
 
 * git and SpamAssassin both use it, so it should be one less thing for
diff --git a/Documentation/design_www.txt b/Documentation/design_www.txt
index b1f916dd..a0003f99 100644
--- a/Documentation/design_www.txt
+++ b/Documentation/design_www.txt
@@ -7,7 +7,7 @@ URL and anchor naming
 /$INBOX/?r=$GIT_COMMIT                 -> HTML only
 /$INBOX/new.atom                       -> Atom feed
 
-#### Optional, relies on Search::Xapian (or Xapian SWIG binding)
+#### Optional, relies on Xapian
 /$INBOX/$MESSAGE_ID/t/                 -> HTML content of thread (nested)
 /$INBOX/$MESSAGE_ID/T/                 -> HTML content of thread (flat)
         anchors:
@@ -102,7 +102,7 @@ We also set <title> to make window management easier.
 
 We favor <pre>-formatted text since public-inbox is intended as a place
 to share and discuss patches and code.  Unfortunately, long paragraphs
-tends to be less readable with fixed-width serif fonts which GUI
+tend to be less readable with fixed-width serif fonts which GUI
 browsers default to.
 
 * No graphics, images, or icons at all.  We tolerate, but do not
@@ -122,12 +122,12 @@ browsers default to.
   avoided as they do not render well with some displays or user-chosen
   fonts.
 
-* No JavaScript. JS is historically too buggy and insecure, and we will
+* No JavaScript.  JS is historically too buggy and insecure, and we will
   never expect our readers to do either of the following:
-  a) read and audit all our code for on every single page load
-  b) trust us and and run code without reading it
+  a) read and audit all our code on every single page load
+  b) trust us and run code without reading it
 
-* We only use CSS for one reason: wrapping pre-formatted text
+* We only use CSS for one reason: wrapping pre-formatted text.
   This is necessary because unfortunate GUI browsers tend to be
   prone to layout widening from unwrapped mailers.
   Do not expect CSS to be enabled, especially with scary things like:
@@ -141,4 +141,4 @@ CSS classes (for user-supplied CSS)
 -----------------------------------
 
 See examples in contrib/css/ and lib/PublicInbox/WwwText.pm
-(or https://public-inbox.org/meta/_/text/color/ soon)
+(or <https://public-inbox.org/meta/_/text/color/>)
diff --git a/Documentation/flow.ge b/Documentation/flow.ge
index 4308989a..5ad92fec 100644
--- a/Documentation/flow.ge
+++ b/Documentation/flow.ge
@@ -1,9 +1,11 @@
 # public-inbox data flow
 #
 # Note: choose either "delivery tools" OR "git mirroring tools"
-# for a given inboxdir.  Combining them for the SAME inboxdir
-# will cause conflicts.  Of course, different inboxdirs may
-# choose different means of getting mail into them.
+# for a given inboxdir.  Using them simultaneously for the
+# SAME inboxdir will cause conflicts.  Of course, different
+# inboxdirs may choose different means of getting mail into them.
+# You may fork any inbox by starting with "git mirroring tools",
+# and switching to "delivery tools".
 
 graph { flow: down }
 
@@ -13,6 +15,8 @@ graph { flow: down }
  public-inbox-learn] -> [inboxdir]
 
 [git mirroring tools:\n
+ public-inbox-clone,\n
+ public-inbox-fetch,\n
  grok-pull,\n
  various scripts
 ] -- git (clone|fetch) &&\n
@@ -20,9 +24,10 @@ graph { flow: down }
 
 [inboxdir] ->
 [read-only daemons:\n
+ public-inbox-netd\n
  public-inbox-httpd\n
  public-inbox-imapd\n
  public-inbox-nntpd]
 
-# Copyright 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
diff --git a/Documentation/flow.txt b/Documentation/flow.txt
index 1116a917..ed2dd80b 100644
--- a/Documentation/flow.txt
+++ b/Documentation/flow.txt
@@ -1,9 +1,11 @@
 # public-inbox data flow
 #
 # Note: choose either "delivery tools" OR "git mirroring tools"
-# for a given inboxdir.  Combining them for the SAME inboxdir
-# will cause conflicts.  Of course, different inboxdirs may
-# choose different means of getting mail into them.
+# for a given inboxdir.  Using them simultaneously for the
+# SAME inboxdir will cause conflicts.  Of course, different
+# inboxdirs may choose different means of getting mail into them.
+# You may fork any inbox by starting with "git mirroring tools",
+# and switching to "delivery tools".
 
                                                  +--------------------+
                                                  |  delivery tools:   |
@@ -15,8 +17,10 @@
                                                    |
                                                    v
 +----------------------+                         +--------------------+
-| git mirroring tools: |  git (clone|fetch) &&   |                    |
-|      grok-pull,      |  public-inbox-index     |      inboxdir      |
+| git mirroring tools: |                         |                    |
+| public-inbox-clone,  |                         |                    |
+| public-inbox-fetch,  |  git (clone|fetch) &&   |      inboxdir      |
+|      grok-pull,      |  public-inbox-index     |                    |
 |   various scripts    | ----------------------> |                    |
 +----------------------+                         +--------------------+
                                                    |
@@ -24,10 +28,12 @@
                                                    v
                                                  +--------------------+
                                                  | read-only daemons: |
+                                                 | public-inbox-netd  |
                                                  | public-inbox-httpd |
                                                  | public-inbox-imapd |
                                                  | public-inbox-nntpd |
                                                  +--------------------+
 
-# Copyright 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# This file was generated from flow.txt using Graph::Easy
diff --git a/Documentation/hosted.txt b/Documentation/hosted.txt
deleted file mode 100644
index 188ad254..00000000
--- a/Documentation/hosted.txt
+++ /dev/null
@@ -1,41 +0,0 @@
-unofficially hosted mirrors at public-inbox.org
-
-In addition to eating our own dogfood at <https://public-inbox.org/meta/>,
-public-inbox.org hosts unofficial archives for several other projects
-to further test our own software.
-
-These mirrors are NOT to be considered reliable or permanent.
-Interested parties are strongly encouraged to host their own mirrors.
-
-The presence of these archives does not imply these projects endorse
-public-inbox or public-inbox.org in any way.
-
-* https://public-inbox.org/bug-gnulib/
-  bug-gnulib@gnu.org
-  Discussion for Gnulib portability/common source project
-  https://lists.gnu.org/mailman/listinfo/bug-gnulib
-
-* https://public-inbox.org/git/
-  git@vger.kernel.org
-  Mailing list for the git version control system
-  http://vger.kernel.org/majordomo-info.html
-
-* https://public-inbox.org/libc-alpha/
-  libc-alpha@sourceware.org
-  Mailing list for GNU C library development
-  https://www.gnu.org/software/libc/involved.html
-
-* https://public-inbox.org/rack-devel/
-  rack-devel@googlegroups.com
-  Development list for the Ruby webserver interface
-  https://groups.google.com/group/rack-devel
-
-* https://public-inbox.org/sox-users/
-  sox-users@lists.sourceforge.net
-  Users' list for the SoX sound processing tool
-  https://lists.sourceforge.net/lists/listinfo/sox-users
-
-* https://public-inbox.org/sox-devel/
-  sox-devel@lists.sourceforge.net
-  Developers' list for the SoX sound processing tool
-  https://lists.sourceforge.net/lists/listinfo/sox-devel
diff --git a/Documentation/include.mk b/Documentation/include.mk
index bfbc495f..86851376 100644
--- a/Documentation/include.mk
+++ b/Documentation/include.mk
@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 all::
 
@@ -6,6 +6,8 @@ RSYNC = rsync
 RSYNC_DEST = public-inbox.org:/srv/public-inbox/
 AWK = awk
 MAN = man
+
+# part of `man-db' on Debian, not sure about other distros
 LEXGROG = lexgrog
 
 # this is "xml" on FreeBSD and maybe some other distros:
@@ -47,14 +49,14 @@ install-man: man
 
 doc_install :: install-man
 
-check :: check-man
+check : check-man
 check_man = $(AWK) \
         '{gsub(/\b./,"")}$$0 !~ /\.onion/&&length>80{print;e=1}END{exit(e)}' \
         >&2
 
-check-man :: $(check_80)
+check-man : $(check_80)
 
-check-lexgrog :: $(check_lexgrog)
+check-lexgrog : $(check_lexgrog)
 
 all :: $(docs)
 
@@ -67,13 +69,16 @@ Documentation/standards.txt : Documentation/standards.perl
 
 # flow.txt is checked into git since Graph::Easy isn't in many distros
 Documentation/flow.txt : Documentation/flow.ge
-        (sed -ne '1,/^$$/p' <Documentation/flow.ge; \
-                $(GRAPH_EASY) Documentation/flow.ge || \
-                        cat Documentation/flow.txt; \
+
+%.txt : %.ge
+        (sed -ne '1,/^$$/p' <$<; \
+                $(GRAPH_EASY) $< || grep -v '^#' $@; \
                 echo; \
-                sed -ne '/^# Copyright/,$$p' <Documentation/flow.ge \
+                sed -ne '/^# Copyright/,$$p' <$< \
                 ) >$@+
-        touch -r Documentation/flow.ge $@+
+        echo >>$@+ \
+          '# This file was generated from $(@F) using Graph::Easy'
+        touch -r $< $@+
         mv $@+ $@
 
 Documentation/lei-q.pod : lib/PublicInbox/Search.pm Documentation/common.perl
@@ -83,7 +88,7 @@ NEWS NEWS.atom NEWS.html : $(news_deps)
         $(PERL) -I lib -w Documentation/mknews.perl $@ $(RELEASES)
 
 # check for internal API changes:
-check :: NEWS .NEWS.atom.check NEWS.html
+check : NEWS .NEWS.atom.check NEWS.html
 
 .NEWS.atom.check: NEWS.atom
         $(XMLSTARLET) val NEWS.atom || \
diff --git a/Documentation/lei-add-external.pod b/Documentation/lei-add-external.pod
index 7afcad63..2a131b55 100644
--- a/Documentation/lei-add-external.pod
+++ b/Documentation/lei-add-external.pod
@@ -75,7 +75,9 @@ Default: C<auto>
 
 =item --inbox-version=NUM
 
-Force a public-inbox version (must be C<1> or C<2>).
+Force a remote public-inbox version (must be C<1> or C<2>).
+This is auto-detected by default, and this option exists mainly
+for testing.
 
 =back
 
diff --git a/Documentation/lei-blob.pod b/Documentation/lei-blob.pod
index e401bb47..558fc54c 100644
--- a/Documentation/lei-blob.pod
+++ b/Documentation/lei-blob.pod
@@ -86,7 +86,7 @@ reconstructed from patch emails.
 
 =item --no-torsocks
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 =back
 
diff --git a/Documentation/lei-config.pod b/Documentation/lei-config.pod
index 663404fe..699f45cb 100644
--- a/Documentation/lei-config.pod
+++ b/Documentation/lei-config.pod
@@ -4,7 +4,11 @@ lei-config - git-config wrapper for lei configuration file
 
 =head1 SYNOPSIS
 
-lei config [OPTIONS]
+lei config <name> [[<value>] [<value-pattern>]]
+
+lei config -l | --list
+
+lei config -e | --edit
 
 =head1 DESCRIPTION
 
@@ -60,8 +64,8 @@ L<https://rt.cpan.org/Ticket/Display.html?id=129967>
 
 Enable debugging output of underlying IMAP and NNTP libraries,
 currently L<Mail::IMAPClient> and L<Net::NNTP>, respectively.
-If using L<imap.proxy> or L<nntp.proxy> point to a SOCKS proxy,
-debugging output for L<IO::Socket::Socks> will be enabled, as
+If L<imap.proxy> or L<nntp.proxy> points to a SOCKS proxy,
+debugging output for L<IO::Socket::Socks> will be enabled as
 well.
 
 Disabling L<imap.compress> may be required for readability.
@@ -97,6 +101,27 @@ C<frag>, C<func>, and C<context>.
 
 =back
 
+=head1 OPTIONS
+
+Most L<git-config(1)> command-line switches are accepted by C<lei config>
+as is.  The most frequently used options are expected to be:
+
+=over 4
+
+=item -e
+
+=item --edit
+
+Open an editor to edit the lei config file.
+
+=item -l
+
+=item --list
+
+List all variables set in config file, along with their values.
+
+=back
+
 =head1 CONTACT
 
 Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
@@ -106,6 +131,6 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
diff --git a/Documentation/lei-convert.pod b/Documentation/lei-convert.pod
index c113db18..b3e29824 100644
--- a/Documentation/lei-convert.pod
+++ b/Documentation/lei-convert.pod
@@ -48,7 +48,7 @@ L<lei-q(1)>.
 
 =item --no-torsocks
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 =back
 
diff --git a/Documentation/lei-daemon-kill.pod b/Documentation/lei-daemon-kill.pod
index 48c237b8..50f75f4c 100644
--- a/Documentation/lei-daemon-kill.pod
+++ b/Documentation/lei-daemon-kill.pod
@@ -26,12 +26,12 @@ so another L<lei-daemon(8)> process can take its place.
 =item SIGKILL
 
 Kills L<lei-daemon(8)> immediately.  Some worker processes may
-remain running after a short while after this takes effect.
+remain running for a short while.
 
 =back
 
 =for comment
-SIGQUIT and SIGINT currently do what SIGTERM do, may change...
+SIGQUIT and SIGINT currently do what SIGTERM does, may change...
 
 =head1 CONTACT
 
diff --git a/Documentation/lei-edit-search.pod b/Documentation/lei-edit-search.pod
index 21cb11aa..7f447ca2 100644
--- a/Documentation/lei-edit-search.pod
+++ b/Documentation/lei-edit-search.pod
@@ -8,7 +8,9 @@ lei edit-search [OPTIONS] OUTPUT
 
 =head1 DESCRIPTION
 
-Invoke C<git config --edit> to edit the saved search at C<OUTPUT>.
+Invoke C<git config --edit> to edit the saved search at C<OUTPUT>,
+where C<OUTPUT> was supplied for argument of C<lei q -o OUTPUT ...>
+A listing of outputs is available via C<lei ls-search>.
 
 =head1 CONTACT
 
@@ -19,7 +21,7 @@ and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/Documentation/lei-forget-search.pod b/Documentation/lei-forget-search.pod
index adbe7638..5ff526f1 100644
--- a/Documentation/lei-forget-search.pod
+++ b/Documentation/lei-forget-search.pod
@@ -8,7 +8,9 @@ lei forget-search [OPTIONS] OUTPUT
 
 =head1 DESCRIPTION
 
-Forget a saved search at C<OUTPUT>.
+Forget a saved search at C<OUTPUT>,
+where C<OUTPUT> was supplied for argument of C<lei q -o OUTPUT ...>
+A listing of outputs is available via C<lei ls-search>.
 
 =head1 OPTIONS
 
diff --git a/Documentation/lei-import.pod b/Documentation/lei-import.pod
index ad769084..31d6db13 100644
--- a/Documentation/lei-import.pod
+++ b/Documentation/lei-import.pod
@@ -10,7 +10,8 @@ lei import [OPTIONS] (--stdin|-)
 
 =head1 DESCRIPTION
 
-Import messages into the local storage of L<lei(1)>.  C<LOCATION> is a
+Import messages into the local storage of L<lei(1)>
+(aka L<leiE<sol>store|lei-store-format(5)>).  C<LOCATION> is a
 source of messages: a directory (Maildir), a file, or a URL
 (C<imap://>, C<imaps://>, C<nntp://>, or C<nntps://>).  URLs requiring
 authentication use L<git-credential(1)> to
@@ -81,12 +82,17 @@ Whether to wrap L<git(1)> and L<curl(1)> commands with L<torsocks(1)>.
 
 Default: C<auto>
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 Use the specified proxy (e.g., C<socks5h://0:9050>).
 
+Consider L<imap.proxy> and L<nntp.proxy> which can be persistently
+configured on a per-host basis in L<lei-config(1)>.
+
 =back
 
+See L<lei-config(1)> for various C<imap.*> and C<nntp.*> options.
+
 =head1 CONTACT
 
 Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
@@ -102,4 +108,4 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
 =head1 SEE ALSO
 
-L<lei-index(1)>
+L<lei-config(1)>, L<lei-index(1)>, L<lei-store-format(5)>
diff --git a/Documentation/lei-inspect.pod b/Documentation/lei-inspect.pod
index 19dd8ab5..82b9651a 100644
--- a/Documentation/lei-inspect.pod
+++ b/Documentation/lei-inspect.pod
@@ -26,7 +26,7 @@ An inboxdir, extindex topdir, or Xapian shard
 
 =item --pretty
 
-Pretty print output.  If stdout is opened to a tty, C<--pretty> is
+Pretty-print output.  If stdout is opened to a tty, C<--pretty> is
 enabled by default.
 
 =item -
@@ -47,7 +47,7 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/Documentation/lei-lcat.pod b/Documentation/lei-lcat.pod
index e85e5e67..530b755e 100644
--- a/Documentation/lei-lcat.pod
+++ b/Documentation/lei-lcat.pod
@@ -31,7 +31,7 @@ C<-f text> when writing to stdout.
 
 Most commonly C<text> (the default) or C<reply> to
 display the message(s) in a format suitable for trimming
-and sending as a email reply.
+and sending as an email reply.
 
 =item --stdin
 
@@ -52,7 +52,7 @@ which lets you pipe arbitrary lines to arbitrary commands).
 
 =item --torsocks=auto|no|yes, --no-torsocks
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 =item -o MFOLDER, --output=MFOLDER
 
diff --git a/Documentation/lei-ls-mail-source.pod b/Documentation/lei-ls-mail-source.pod
index 59d14afe..0e485923 100644
--- a/Documentation/lei-ls-mail-source.pod
+++ b/Documentation/lei-ls-mail-source.pod
@@ -28,7 +28,7 @@ Format output as JSON and include more information.
 
 =item --pretty
 
-Pretty print JSON output.  If stdout is opened to a tty, C<--pretty>
+Pretty-print JSON output.  If stdout is opened to a tty, C<--pretty>
 is enabled by default.
 
 =item --ascii
diff --git a/Documentation/lei-ls-search.pod b/Documentation/lei-ls-search.pod
index a56611bf..367f4ad6 100644
--- a/Documentation/lei-ls-search.pod
+++ b/Documentation/lei-ls-search.pod
@@ -8,7 +8,8 @@ lei ls-search [OPTIONS] [PREFIX]
 
 =head1 DESCRIPTION
 
-List saved search queries.  If C<PREFIX> is given, restrict the output
+List saved search queries (generated from C<lei q -o OUTPUT>).
+If C<PREFIX> is given, restrict the output
 to entries that start with the specified value.
 
 =head1 OPTIONS
@@ -25,7 +26,7 @@ C<jsonl>, or C<concatjson>.
 
 =item --pretty
 
-Pretty print C<json> or C<concatjson> output.  If stdout is opened to
+Pretty-print C<json> or C<concatjson> output.  If stdout is opened to
 a tty and used as the C<--output> destination, C<--pretty> is enabled
 by default.
 
@@ -55,7 +56,7 @@ and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/Documentation/lei-mail-formats.pod b/Documentation/lei-mail-formats.pod
index 930c5d76..618bada2 100644
--- a/Documentation/lei-mail-formats.pod
+++ b/Documentation/lei-mail-formats.pod
@@ -83,9 +83,19 @@ mbox.
 
 =head1 MH
 
-Not yet supported, locking semantics (or lack thereof) appear to
-make it unsuitable for parallel access.  It is widely-supported
-by a variety of MUAs and mailing list managers, however.
+Preliminary support for reads as of 2.0.0.  Locking semantics differ
+incompatibly amongst existing writers: Python and nmh appear
+compatible with each other, while mutt appears racy and unsuitable
+for parallel access due to rename(2) potentially clobbering the
+C<.mh_sequences> file.  More info about other clients is greatly
+appreciated.
+
+Sequence numbers may be packed and reused by some writers, so lei
+users may need to run L<lei-refresh-mail-sync(1)> if inotify|kevent
+missed packing while L<lei-daemon(8)> wasn't running.
+
+lei is safe for reading mlmmj archives as MH since mlmmj neither
+packs nor uses a .mh_sequences file to store state.
 
 =head1 MMDF
 
diff --git a/Documentation/lei-mail-sync-overview.pod b/Documentation/lei-mail-sync-overview.pod
index e30674bb..7ae7e887 100644
--- a/Documentation/lei-mail-sync-overview.pod
+++ b/Documentation/lei-mail-sync-overview.pod
@@ -20,7 +20,7 @@ Future work will be done to improve it and add IMAP IDLE support.
   # dump "inbox" labeled files from the past week to a Maildir
   lei q L:inbox rt:last.week.. -o /tmp/results
 
-  # open /tmp/results in your favorite mail agent.  If inotify or kevent
+  # Open /tmp/results in your favorite mail agent.  If inotify or kevent
   # works, keyword changes (e.g. marking messages as `seen') are
   # synchronized automatically.
 
diff --git a/Documentation/lei-overview.pod b/Documentation/lei-overview.pod
index 7095b504..e9a97d64 100644
--- a/Documentation/lei-overview.pod
+++ b/Documentation/lei-overview.pod
@@ -119,11 +119,13 @@ code repository.
 
 =head1 PERFORMANCE NOTES
 
-L<Inline::C> is required, lei runs as a background daemon to reduce
-startup costs and can provide real-time L<kqueue(2)>/L<inotify(7)>
-Maildir monitoring.  L<IO::KQueue> (p5-IO-KQueue on FreeBSD) and
-L<Linux::Inotify2> (liblinux-inotify2-perl and perl-Linux-Inotify2 in
-.deb and .rpm-based distros, respectively) are recommended.
+L<Inline::C> is required on BSDs and can speed things up on Linux.
+
+lei runs as a background daemon to reduce startup costs and can
+provide real-time L<kqueue(2)>/L<inotify(7)> Maildir monitoring.
+L<IO::KQueue> (p5-IO-KQueue on FreeBSD) and L<Linux::Inotify2>
+(liblinux-inotify2-perl and perl-Linux-Inotify2 in .deb and .rpm-based
+distros, respectively) are recommended.
 
 L<Socket::MsgHdr> is optional (libsocket-msghdr-perl in Debian),
 and further improves startup performance.  Its effect is most felt
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 1cbffba4..4476a806 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -12,9 +12,6 @@ lei q [OPTIONS] (--stdin|-)
 
 Search for messages across the lei/store and externals.
 
-=for comment
-TODO: Give common prefixes, or at least a description/reference.
-
 =head1 OPTIONS
 
 =for comment
@@ -50,6 +47,10 @@ A prefix can specify the format of the output: C<maildir>,
 C<mboxrd>, C<mboxcl2>, C<mboxcl>, C<mboxo>.  For a description of
 mail formats, see L<lei-mail-formats(5)>.
 
+C<v2:/path/to/inbox> may be used to create a new inbox of
+L<public-inbox-v2-format(5)>.  The new inbox will not be configured
+in the L<public-inbox-config(5)> file.
+
 C<maildir> is the default for an existing directory or non-existing path.
 
 Default: C<-> (stdout)
@@ -76,7 +77,7 @@ Disable color (for C<-f reply> and C<-f text>).
 
 =item --pretty
 
-Pretty print C<json> or C<concatjson> output.  If stdout is opened to
+Pretty-print C<json> or C<concatjson> output.  If stdout is opened to
 a tty and used as the C<--output> destination, C<--pretty> is enabled
 by default.
 
@@ -107,8 +108,9 @@ Augment output destination instead of clobbering it.
 
 =item --no-import-before
 
-Do not import keywords before writing to an existing output
-destination.
+Do not import messages before writing to an existing output destination.
+Be certain you do not need existing data in your output before using
+this, it permanently erases data unless C<--augment> is used.
 
 =item --threads
 
@@ -124,6 +126,28 @@ of the same thread.
 TODO: Warning: this flag may become persistent and saved in
 lei/store unless an MUA unflags it!  (Behavior undecided)
 
+Caveat: C<-tt> only works on locally-indexed messages at the
+moment, and not on remote (HTTP(S)) endpoints.
+
+=item --jobs=QUERY_WORKERS[,WRITE_WORKERS]
+
+=item --jobs=,WRITE_WORKERS
+
+=item -j QUERY_WORKERS[,WRITE_WORKERS]
+
+=item -j ,WRITE_WORKERS
+
+Set the number of query and write worker processes for parallelism.
+
+C<QUERY_WORKERS> defaults to the number of CPUs available, but 4 per
+remote (HTTP/HTTPS) host.
+
+C<WRITE_WORKERS> defaults to 75% of the number of CPUs available for
+Maildir and mbox* destinations, but 4 per IMAP/IMAPS host.
+
+Omitting C<QUERY_WORKERS> but leaving the comma (C<,>) allows
+one to only set C<WRITE_WORKERS>
+
 =item --dedupe=STRATEGY
 
 =item -d STRATEGY
@@ -194,7 +218,7 @@ Default: fcntl,dotlock
 
 =item -n NUMBER
 
-Fuzzy limit the number of matches per-local external and lei/store.
+Fuzzy-limit the number of matches per local external and lei/store.
 Messages added by the L<--threads> switch do not count towards this
 limit, and there is no limit on remote externals.
 
@@ -241,7 +265,7 @@ Whether to wrap L<git(1)> and L<curl(1)> commands with L<torsocks(1)>.
 
 Default: C<auto>
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 =back
 
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index 4d5e8168..f18548d3 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -104,7 +104,7 @@ The options below, described in L<lei-q(1)>, are also supported.
 
 =item --torsocks=auto|no|yes, --no-torsocks
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 =back
 
diff --git a/Documentation/lei-reindex.pod b/Documentation/lei-reindex.pod
new file mode 100644
index 00000000..3a5861c4
--- /dev/null
+++ b/Documentation/lei-reindex.pod
@@ -0,0 +1,47 @@
+=head1 NAME
+
+lei-reindex - reindex messages already in lei/store
+
+=head1 SYNOPSIS
+
+lei reindex [OPTIONS]
+
+=head1 DESCRIPTION
+
+Forces a re-index of all messages previously-indexed by L<lei-import(1)>
+or L<lei-index(1)>.  This can be used for in-place upgrades and bugfixes
+while other processes are querying the store.  Keep in mind this roughly
+doubles the size of the already-large Xapian database.
+
+It does not re-index messages in externals, using the C<--reindex>
+switch of L<public-inbox-index(1)> or L<public-inbox-extindex(1)> is
+needed for that.
+
+=head1 OPTIONS
+
+=over
+
+=item -q
+
+=item --quiet
+
+Suppress feedback messages.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-index(1)>, L<lei-import(1)>
diff --git a/Documentation/lei-security.pod b/Documentation/lei-security.pod
index 104bfb48..e54cae90 100644
--- a/Documentation/lei-security.pod
+++ b/Documentation/lei-security.pod
@@ -4,7 +4,7 @@ lei - security information
 
 =head1 SYNOPSIS
 
-L<lei(1)> is intended for use with both publicly-archived
+L<lei(1)> is intended for use with both publicly archived
 and "private" mail in personal mailboxes.  This document is
 intended to give an overview of security implications and
 lower^Wmanage user expectations.
@@ -66,7 +66,7 @@ other users on the local system.
 
 =head1 CORE DUMPS
 
-In case any process crashes, a core dumps may contain passwords or
+In case any process crashes, a core dump may contain passwords or
 contents of sensitive messages.  Please report these so they can be
 fixed (see L</CONTACT>).
 
@@ -74,7 +74,7 @@ fixed (see L</CONTACT>).
 
 lei currently uses the L<curl(1)> and L<git(1)> executables in
 C<$PATH> for HTTP and HTTPS network access.  Interactive
-authentication for HTTP and HTTPS is not-yet-supported since all
+authentication for HTTP and HTTPS is not yet supported since all
 currently supported HTTP/HTTPS sources are L<PublicInbox::WWW>
 instances.
 
@@ -83,7 +83,7 @@ L<Net::NNTP> (standard library) is used for NNTP and NNTPS.
 
 L<Mail::IMAPClient> and L<Net::NNTP> will use L<IO::Socket::SSL>
 for TLS if available.  In turn, L<IO::Socket::SSL> uses the
-widely-installed OpenSSL library.
+widely installed OpenSSL library.
 
 STARTTLS will be attempted if advertised by the server
 unless IMAPS or NNTPS are used.  C<-c imap.starttls=0>
diff --git a/Documentation/lei-store-format.pod b/Documentation/lei-store-format.pod
index 625c60f4..d4bb42d5 100644
--- a/Documentation/lei-store-format.pod
+++ b/Documentation/lei-store-format.pod
@@ -67,7 +67,7 @@ the "same" message.
 
 =head2 mail_sync.sqlite3
 
-This SQLite database maintained for bidirectional mapping of
+This SQLite database is maintained for bidirectional mapping of
 git blobs to IMAP UIDs, Maildir file names, and NNTP article numbers.
 
 It is also used for retrieving messages from Maildirs indexed by
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index 8fba0953..8c426942 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -26,14 +26,14 @@ updates remote mailboxes (currently C<imap://> and C<imaps://>).
 
 Look for mail older than the time of the last successful query.
 Using a small interval will reduce bandwidth use.  A larger
-interval reduces the likelyhood of missing a result due to MTA
+interval reduces the likelihood of missing a result due to MTA
 delays or downtime.
 
 The time(s) of the last successful queries are the C<lastresult>
 values visible from L<lei-edit-search(1)>.
 
-Date formats understood by L<git-rev-parse(1)> may be used.
-e.g C<1.hour> or C<3.days>
+Date formats understood by L<git-rev-parse(1)> may be used,
+e.g., C<1.hour> or C<3.days>.
 
 Default: 2.days
 
@@ -64,7 +64,9 @@ specified via C<lei q --only>.
 
 =item --mua=CMD
 
-C<--lock>, C<--alert>, and C<--mua> are all supported and
+=item --jobs QUERY_WORKERS[,WRITE_WORKERS]
+
+C<--lock>, C<--alert>, C<--mua>, and C<--jobs> are all supported and
 documented in L<lei-q(1)>.
 
 C<--mua> is incompatible with C<--all>.
diff --git a/Documentation/lei.pod b/Documentation/lei.pod
index f01f506a..2b10f490 100644
--- a/Documentation/lei.pod
+++ b/Documentation/lei.pod
@@ -126,7 +126,7 @@ Other subcommands include
 
 =head1 FILES
 
-By default storage is located at C<$XDG_DATA_HOME/lei/store>.  The
+By default, storage is located at C<$XDG_DATA_HOME/lei/store>.  The
 configuration for lei resides at C<$XDG_CONFIG_HOME/lei/config>.
 
 =head1 ERRORS
diff --git a/Documentation/marketing.txt b/Documentation/marketing.txt
index 385e5172..8e4aa3b5 100644
--- a/Documentation/marketing.txt
+++ b/Documentation/marketing.txt
@@ -3,7 +3,9 @@ marketing guide for public-inbox
 TL; DR: Don't market this.
 
 If you must: don't be pushy and annoying about it.  Slow down.
-Please no superlatives, hype or BS.
+Please no superlatives, hype or BS.  Please keep all marketing
+materials text-only to be accessible to those on slow networks
+and ancient hardware.
 
 It's online and public, so it already markets itself.
 Being informative is not a bad thing, being insistent is.
@@ -25,3 +27,12 @@ than the adoption of any software.
 
 Every time somebody recognizes and rejects various forms of
 lock-in and centralization is already a victory for us.
+
+Please keep in mind:
+
+* Perl 5 is not a well-liked language
+* AGPL is not a well-liked license
+* maintainer is a shy introvert
+
+Be sure to mention these things in any marketing materials
+to avoid wasting time of people who hate Perl and/or AGPL.
diff --git a/Documentation/mknews.perl b/Documentation/mknews.perl
index 1936cea7..68866f44 100755
--- a/Documentation/mknews.perl
+++ b/Documentation/mknews.perl
@@ -1,5 +1,5 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Generates NEWS, NEWS.atom, and NEWS.html files using release emails
 # this uses unstable internal APIs of public-inbox, and this script
@@ -46,6 +46,7 @@ if ($dst eq 'NEWS') {
                 ibx => $ibx,
                 -upfx => "$base_url/",
                 -hr => 1,
+                zfh => $out,
         };
         if ($dst eq 'NEWS.html') {
                 html_start($out, $ctx);
diff --git a/Documentation/public-inbox-cindex.pod b/Documentation/public-inbox-cindex.pod
new file mode 100644
index 00000000..fdc2b82d
--- /dev/null
+++ b/Documentation/public-inbox-cindex.pod
@@ -0,0 +1,136 @@
+=head1 NAME
+
+public-inbox-cindex - create and update code repository search indices
+
+=head1 SYNOPSIS
+
+public-inbox-cindex [OPTIONS] -g GIT_DIR [-g GIT_DIR]...
+
+public-inbox-cindex [OPTIONS] --update
+
+=head1 DESCRIPTION
+
+public-inbox-cindex creates and updates the Xapian search index for
+git code repository (C<coderepo>) search.  It can also associate
+(fuzzy join) coderepos with Xapian-indexed inboxes.  It only indexes
+commit messages and diffs as they would show up in an email.  It
+does not index the contents of blobs directly.
+
+Like inbox indices, coderepo indices can either be internal or external
+to a coderepo.  Either way, they're both created and updated through
+public-inbox-cindex.
+
+Once the initial indices are created by public-inbox-cindex,
+the L</--update> switch will incrementally update them.
+
+=head1 OPTIONS
+
+=over
+
+=item -d EXTDIR
+
+Use the given directory as an external index.  External indices are
+generally recommended to internal indices since they do not need
+write access to any code repositories themselves.  They are highly
+recommended when many repositories share a common history or if
+there is an M:N relationship between inboxes and coderepos.
+
+=item -j JOBS
+
+=item --jobs=JOBS
+
+Influences the number of Xapian indexing shards.
+
+If the repo has not been indexed or initialized, C<JOBS - 1>
+shards will be created.
+
+Default: the number of existing Xapian shards
+
+=item --reindex
+
+Forces a re-index of all commits.  This can be used for in-place
+upgrades and bugfixes while read-only processes are utilizing the index.
+
+=item --update
+
+=item -u
+
+Incrementally index all previously-indexed coderepos.
+
+=item --prune
+
+Removes commits which are no longer accessible via git.
+Use this after L<git-gc(1)> (or L<git-prune(1)>).
+
+=item --no-fsync
+
+=item --dangerous
+
+=item --max-size SIZE
+
+=item --batch-size SIZE
+
+These affect the coderepo index the same way they affect
+inbox indices.  See L<public-inbox-index(1)>.
+
+=back
+
+=head1 FILES
+
+For internal indices, the Xapian DB is stored in
+C<$GIT_DIR/public-inbox-cindex>.
+
+External indices are stored wherever L</-d> EXTDIR points.
+
+=head1 CONFIGURATION
+
+=over 8
+
+=item publicinbox.indexMaxSize
+
+=item publicinbox.indexBatchSize
+
+These configuration knobs affect the coderepo index the same way
+they affect inbox indices.  See L<public-inbox-index(1)>.
+
+=back
+
+=head1 ENVIRONMENT
+
+=over 8
+
+=item PI_CONFIG
+
+Used to override the default "~/.public-inbox/config" value.
+
+=item XAPIAN_FLUSH_THRESHOLD
+
+The number of documents to update before committing changes to
+disk.  This variable is handled directly by Xapian, refer to
+Xapian API documentation for more details.
+
+Use C<publicinbox.indexBatchSize> instead.
+
+=back
+
+=head1 UPGRADING
+
+Occasionally, public-inbox will update its schema version and
+require a full reindex by running this command with L</--reindex>.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<public-inbox-index(1)>
diff --git a/Documentation/public-inbox-clone.pod b/Documentation/public-inbox-clone.pod
index c80c3c5f..64ee3138 100644
--- a/Documentation/public-inbox-clone.pod
+++ b/Documentation/public-inbox-clone.pod
@@ -4,7 +4,9 @@ public-inbox-clone - "git clone --mirror" wrapper
 
 =head1 SYNOPSIS
 
-public-inbox-clone INBOX_URL [INBOX_DIR]
+public-inbox-clone [OPTIONS] INBOX_URL [INBOX_DIR]
+
+public-inbox-clone [OPTIONS] ROOT_URL [DESTINATION] # public-inbox 2.0+
 
 =head1 DESCRIPTION
 
@@ -13,20 +15,29 @@ making the initial clone of a remote HTTP(S) public-inbox.  It
 allows cloning multi-epoch v2 inboxes with a single command and
 zero configuration.
 
+In public-inbox 2.0+, public-inbox-clone can create and maintain
+a mirror of multiple inboxes or code repositories using manifest.js.gz
+files like L<grok-pull(1)> from grokmirror.  L<public-inbox-fetch(1)> is
+NOT required when using this mode.
+
 It does not run L<public-inbox-init(1)> nor
 L<public-inbox-index(1)>.  Those commands must be run separately
 if serving/searching the mirror is required.  As-is,
-public-inbox-clone is suitable for creating a git-only backup.
+public-inbox-clone is suitable for creating a git-only backup
+without Xapian and SQLite indices.
 
-public-inbox-clone creates a Makefile with handy targets to update the
-inbox once indexed.  This Makefile may be edited by the user; it will
+When cloning a single inbox, public-inbox-clone creates a Makefile
+with handy targets to update the inbox once indexed.
+This Makefile may be edited by the user; it will
 not be rewritten by L<public-inbox-fetch(1)> unless it is removed
 completely.
 
 public-inbox-clone does not use nor require any extra
-configuration files (not even C<~/.public-inbox/config>).
+configuration files (not even C<~/.public-inbox/config>),
+but it can download snippets suitable for adding to any
+L<public-inbox-config(5)> file.
 
-L<public-inbox-fetch(1)> may be used to keep C<INBOX_DIR>
+L<public-inbox-fetch(1)> may be used to keep a single C<INBOX_DIR>
 up-to-date.
 
 For v2 inboxes, it will create a C<$INBOX_DIR/manifest.js.gz>
@@ -51,6 +62,157 @@ C<--epoch=~2..> clones the three latest epochs.
 Default: C<0..~0> or C<0..> or C<..~0>
 (all epochs, all three examples are equivalent)
 
+=item -I PATTERN
+
+=item --include=PATTERN
+
+When cloning a top-level with multiple inboxes via manifest,
+only clone inboxes and repositories matching a given wildcard pattern
+(using C<*?> and C<[]> is supported).
+
+This is a new option in public-inbox 2.0+
+
+=item --exclude=PATTERN
+
+When cloning a top-level with multiple inboxes via manifest,
+ignore inboxes and repositories matching the given wildcard pattern.
+Supports the same wildcards as L</--include>
+
+This is a new option in public-inbox 2.0+
+
+=item --inbox-config=always|v2|v1|never
+
+Whether or not to retrieve the C<$INBOX/_/text/config/raw> HTTP(S)
+endpoint when cloning.
+
+Since we can't deduce v1 inboxes from code repositories, setting this
+to C<v2> or C<never> can allow faster clones of code repositories if
+no v1 inboxes are present.
+
+Default: C<always>
+
+This is a new option in public-inbox 2.0+
+
+=item --inbox-version=NUM
+
+Force a remote public-inbox version (must be C<1> or C<2>).
+This is auto-detected by default, and this option exists mainly
+for testing.
+
+This is a new option in public-inbox 2.0+
+
+=item --objstore=DIR
+
+Enables space savings when the remote C<manifest.js.gz>
+includes C<forkgroup> entries as generated by grokmirror 2.x.
+
+If C<DIR> does not start with C</>, C<./>, or C<../>, it is treated
+as relative to the C<DESTINATION> directory.  If only C<--objstore=>
+is specified where C<DIR> is an empty string (C<"">), then C<objstore>
+(C<$DESTINATION/objstore>) is the implied value of C<DIR>.
+
+This is a new option in public-inbox 2.0+
+
+=item --manifest=FILE
+
+When incrementally updating an existing mirror, load the given
+manifest (typically C<manifest.js.gz>) to speed up updates.
+
+By default, public-inbox writes the retrieved manifest to
+C<$DESTINATION/manifest.js.gz>, this directive also
+changes the destination to the specified C<FILE>
+
+If C<FILE> does not start with C</>, C<./>, or C<../>, it is treated
+as relative to the C<DESTINATION> directory.  If only C<--manifest=>
+is specified where C<FILE> is an empty string (C<"">), then C<manifest.js.gz>
+(C<$DESTINATION/manifest.js.gz>) is the implied value of C<FILE>.
+
+When updating manifests with many forks using the same objstore,
+git 2.41+ is highly recommended for performance as we automatically
+use the C<fetch.hideRefs> feature to speed up negotiation.
+
+C<--manifest=> is a new option in public-inbox 2.0+
+
+=item --remote-manifest=URL|RELATIVE_PATH
+
+Use an alternate location for the remote manifest.js.gz file.
+This may be specified as a full absolute URL (e.g
+C<--remote-manifest=https://80x24.org/lore/pub/manifest.js.gz>),
+or a pathname relative to the ROOT_URL (e.g
+C<--remote-manifest=pub/manifest.js.gz> when ROOT_URL is
+C<https://80x24.org/lore/>
+
+By default, C<ROOT_URL/manifest.js.gz> is used.
+
+This is a new option in public-inbox 2.0+
+
+=item --project-list=FILE
+
+When cloning code repos from a manifest, generate a cgit-compatible
+project list.
+
+If C<FILE> does not start with C</>, C<./>, or C<../>, it is treated
+as relative to the C<DESTINATION> directory.  If only C<--project-list=>
+is specified where C<FILE> is an empty string (C<"">), then C<projects.list>
+(C<$DESTINATION/projects.list>) is the implied value of C<FILE>.
+
+This is a new option in public-inbox 2.0+
+
+=item --post-update-hook=COMMAND
+
+Hooks to run after a repository is cloned or updated, C<COMMAND> will
+have the bare git repository destination given as its first and only
+argument.
+
+For v2 inboxes, this operates on a per-epoch basis.
+
+May be specified multiple times to run multiple commands in the
+order specified on the command-line.
+
+This is a new option in public-inbox 2.0+
+
+=item -p
+
+=item --prune
+
+Pass the C<--prune> and C<--prune-tags> flags to L<git-fetch(1)>
+calls on incremental clones.
+
+This is a new option in public-inbox 2.0+
+
+=item --purge
+
+Deletes entire repos which no longer exist in the remote manifest,
+or are filtered out by C<--include=> or C<--exclude=>.
+
+This is only useful when using C<--manifest>
+
+This is a new option in public-inbox 2.0+
+
+=item --exit-code
+
+Exit with C<127> if no updates are done when relying on a manifest.
+Updates include fingerprint mismatches in the manifest, new symlinks,
+new repositories, and removed repositories from the L<--project-list>
+
+This is a new option in public-inbox 2.0+
+
+=item -k
+
+=item --keep-going
+
+Continue as much as possible after an error.
+
+This is a new option in public-inbox 2.0+
+
+=item -n
+
+=item --dry-run
+
+Show what would be done, without making any changes.
+
+This is a new option in public-inbox 2.0+
+
 =item -q
 
 =item --quiet
@@ -71,6 +233,40 @@ Whether to wrap L<git(1)> and L<curl(1)> commands with L<torsocks(1)>.
 
 Default: C<auto>
 
+=item -j JOBS
+
+=item --jobs=JOBS
+
+The number of parallel processes to spawn at once for various network
+operations using L<git(1)> and/or L<curl(1)>.
+
+=back
+
+=head1 EXAMPLES
+
+=for comment
+Sticking to smaller projects in examples to minimize load on servers
+
+=over
+
+=item To mirror the most recent epochs of dwarves and LTTng inboxes:
+
+  public-inbox-clone --epoch=~0 \
+        --include='*lttng*' --include='*dwarves' \
+        https://80x24.org/lore/ /path/to/inbox-mirror
+
+C<https://lore.kernel.org/> may be used instead of C<https://80x24.org/lore/>
+
+=item To mirror all code repos of the sparse project:
+
+  public-inbox-clone --objstore= --project-list= --prune \
+        --include='*sparse*' --inbox-config=never \
+        --remote-manifest=https://80x24.org/lore/pub/manifest.js.gz \
+        https://80x24.org/lore/ /path/to/code-mirror
+
+C<https://git.kernel.org/> may be used instead of C<https://80x24.org/lore/>
+and the C<--remote-manifest> option can be omitted.
+
 =back
 
 =head1 CONTACT
diff --git a/Documentation/public-inbox-config.pod b/Documentation/public-inbox-config.pod
index 43e54ed4..b4a1d94d 100644
--- a/Documentation/public-inbox-config.pod
+++ b/Documentation/public-inbox-config.pod
@@ -67,10 +67,17 @@ may be any newsgroup name with hierarchies delimited by C<.>.
 For example, the newsgroup for L<mailto:meta@public-inbox.org>
 is: C<inbox.comp.mail.public-inbox.meta>
 
-It also configures the folder hierarchy used by L<public-inbox-imapd(1)>.
+It also configures the folder hierarchy used by L<public-inbox-imapd(1)>
+as well as L<public-inbox-pop3d(1)>
 
 Omitting this for a given inbox will prevent the inbox from
-being served by L<public-inbox-nntpd(1)> and/or L<public-inbox-imapd(1)>.
+being served by L<public-inbox-nntpd(1)>,
+L<public-inbox-imapd(1)>, and/or L<public-inbox-pop3d(1)>
+
+Newsgroup names should be all lowercase.  Uppercase characters are
+converted to lowercase for compatibility with IMAP, POP3, and our
+L<public-inbox-extindex(1)> and L<public-inbox-cindex(1)> tools
+starting with public-inbox 2.0+ (they were unusable before).
 
 Default: none, optional
 
@@ -123,8 +130,8 @@ C<basic> only requires L<DBD::SQLite(3pm)> and provides all
 NNTP functionality along with thread-awareness in the WWW
 interface.
 
-C<medium> requires L<Search::Xapian(3pm)> to provide full-text
-term search functionality in the WWW UI.
+C<medium> requires L<Xapian(3pm)> or L<Search::Xapian(3pm)> to provide
+full-text term search functionality in the WWW UI.
 
 C<full> also includes positional information used by Xapian to
 allow for searching for phrases using quoted text.
@@ -189,11 +196,28 @@ Default: :all
 The local path name of a CSS file for the PSGI web interface.
 May contain the attributes "media", "title" and "href" which match
 the associated attributes of the HTML <style> tag.
-"href" may be specified to point to the URL of an remote CSS file
+"href" may be specified to point to the URL of a remote CSS file
 and the path may be "/dev/null" or any empty file.
 Multiple files may be specified and will be included in the
 order specified.
 
+=item publicinboxImport.dropUniqueUnsubscribe
+
+Drop C<List-Unsubscribe> headers if the message also includes
+the C<List-Unsubscribe-Post: List-Unsubscribe=One-Click> header
+to signal MUAs to support an instantaneous unsubscribe.  This
+is strongly recommended for users creating their own public
+archives of mailing lists they subscribe to, otherwise any
+archive reader can unsubscribe the archivist.
+
+This may break DKIM signatures if the C<List-Unsubscribe*>
+headers are signed, but breaking DKIM signatures is the
+lesser evil compared to allowing any reader to unsubscribe
+the archivist.
+
+This affects L<public-inbox-mda(1)>, L<public-inbox-watch(1)>,
+and L<public-inbox-learn(1)>
+
 =item publicinboxmda.spamcheck
 
 This may be set to C<none> to disable the use of SpamAssassin
@@ -226,6 +250,17 @@ L<public-inbox-nntpd(1)> instance.
 
 Default: none
 
+=item publicinbox.pop3server
+
+Same as C<publicinbox.imapserver>, but for the hostname(s) of the
+L<public-inbox-pop3d(1)> instance.
+
+Default: none
+
+=item publicinbox.pop3state
+
+See L<public-inbox-pop3d(1)/publicinbox.pop3state>
+
 =item publicinbox.<name>.feedmax
 
 The size of an Atom feed for the inbox.  If specified more than
@@ -238,7 +273,9 @@ Default: 25
 
 A comma-delimited list of listings to hide the inbox from.
 
-Valid values are currently C<www> and C<manifest>.
+Valid values are currently C<www> and C<manifest> for non-C<404>
+values of L</publicinbox.wwwListing> and L</publicinbox.grokManifest>,
+respectively
 
 Default: none
 
@@ -252,6 +289,10 @@ The URL of the cgit instance associated with the coderepo.
 
 Default: none
 
+=item coderepo.snapshots
+
+See C<snapshots> in L<cgitrc(5)>
+
 =item publicinbox.cgitrc
 
 A path to a L<cgitrc(5)> file.  "repo.url" directives in the cgitrc
@@ -274,18 +315,50 @@ Default: /var/www/htdocs/cgit/cgit.cgi or /usr/lib/cgit/cgit.cgi
 =item publicinbox.cgitdata
 
 A path to the data directory used by cgit for storing static files.
-Typically guessed based the location of C<cgit.cgi> (from
-C<publicinbox.cgitbin>, but may be overridden.
+Typically guessed based on the location of C<cgit.cgi> (from
+C<publicinbox.cgitbin>), but may be overridden.
 
-Default: basename of C<publicinbox.cgitbin>, /var/www/htdocs/cgit/
+Default: dirname of C<publicinbox.cgitbin>, /var/www/htdocs/cgit/
 or /usr/share/cgit/
 
+=item publicinbox.cgit
+
+Controls whether or not and how C<cgit> is used for serving coderepos.
+New in public-inbox 2.0.0 (PENDING).
+
+=over 8
+
+=item * first
+
+Try using C<cgit> as the first choice, this is the default.
+
+=item * fallback
+
+Fall back to using C<cgit> only if our native, inbox-aware
+git code repository viewer doesn't recognize the URL.
+
+=begin comment
+=for comment rewrite is not yet implemented
+=item * rewrite
+
+Rewrite C<cgit> URLs for our native, inbox-aware code repository viewer.
+This implies C<fallback> for URLs the native viewer does not recognize.
+
+=end comment
+
+=back
+
+Default: C<first>  (C<cgit> will be used iff C<publicinbox.cgitrc>
+is set and the C<cgit> binary exists).
+
 =item publicinbox.mailEditor
 
 See L<public-inbox-edit(1)>
 
 =item publicinbox.indexMaxSize
+
 =item publicinbox.indexBatchSize
+
 =item publicinbox.indexSequentialShard
 
 See L<public-inbox-index(1)>
@@ -314,6 +387,21 @@ TODO support showing cgit listing
 
 Default: C<404>
 
+=item publicinbox.nameIsUrl
+
+Treat the name of the public inbox as its unqualified URL when
+using C<publicInbox.wwwListing=all>.  That is, every
+C<[publicinbox "foo"]> section implicitly sets C<publicinbox.foo.url=foo>.
+
+This is a convenient alternative to specifying
+C<publicinbox.E<lt>nameE<gt>.url> for every single inbox if
+your inbox URLs are domain-agnostic when using
+C<publicInbox.wwwListing=all>
+
+Default: false
+
+New in public-inbox 2.0.0 (PENDING).
+
 =item publicinbox.grokmanifest
 
 Controls the generation of a grokmirror-compatible gzipped JSON file
@@ -463,7 +551,7 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2016-2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/Documentation/public-inbox-daemon.pod b/Documentation/public-inbox-daemon.pod
index f77fc3a9..6f1e3b53 100644
--- a/Documentation/public-inbox-daemon.pod
+++ b/Documentation/public-inbox-daemon.pod
@@ -4,16 +4,18 @@ public-inbox-daemon - common usage for public-inbox network daemons
 
 =head1 SYNOPSIS
 
+        public-inbox-netd
         public-inbox-httpd
         public-inbox-imapd
         public-inbox-nntpd
+        public-inbox-pop3d
 
 =head1 DESCRIPTION
 
 This manual describes common options and behavior for
 public-inbox network daemons.  Network daemons for public-inbox
-provide read-only NNTP, IMAP and HTTP access to public-inboxes.  Write
-access to a public-inbox will never be required to run these.
+provide read-only IMAP, HTTP, NNTP and POP3 access to public-inboxes.
+Write access to a public-inbox will never be required to run these.
 
 These daemons are implemented with a common core using
 non-blocking sockets and optimized for fairness; even with
@@ -29,9 +31,9 @@ processes to take advantage of multiple CPUs.
 
 =over
 
-=item -l ADDRESS
+=item -l [PROTOCOL://]ADDRESS[?opt1=val1,opt2=val2]
 
-=item --listen ADDRESS
+=item --listen [PROTOCOL://]ADDRESS[?opt1=val1,opt2=val2]
 
 This takes an absolute path to a Unix socket or HOST:PORT
 to listen on.  For example, to listen to TCP connections on
@@ -42,8 +44,14 @@ like L<nginx(8)> to use.
 May be specified multiple times to allow listening on multiple
 sockets.
 
-This does not need to be specified at all if relying on
-L<systemd.socket(5)> or similar
+Unless per-listener options are used (required for
+L<public-inbox-netd(1)>), this does not need to be specified at
+all if relying on L<systemd.socket(5)> or similar,
+
+Per-listener options may be specified after C<?> as C<KEY=VALUE>
+pairs delimited by C<,>.  See L<public-inbox-netd(1)> for
+documentation on the C<cert=>, C<key=>, C<env.NAME=VALUE>,
+C<out=>, C<err=>, and C<psgi=> options available.
 
 Default: server-dependent unless socket activation is used with
 L<systemd(1)> or similar (see L<systemd.socket(5)>).
@@ -57,7 +65,9 @@ Using this is preferable to setting up the redirect externally
 (e.g. E<gt>E<gt>/path/to/log in shell) since it allows
 SIGUSR1 to be handled (see L<SIGNALS/SIGNALS> below).
 
-Default: /dev/null
+C<out=> may also be specified on a per-listener basis.
+
+Default: /dev/null with C<--daemonize>, inherited otherwise
 
 =item -2 PATH
 
@@ -65,6 +75,10 @@ Default: /dev/null
 
 Like C<--stdout>, but for the stderr descriptor (2).
 
+C<err=> may also be specified on a per-listener basis.
+
+Default: /dev/null with C<--daemonize>, inherited otherwise
+
 =item -W
 
 =item --worker-processes
@@ -82,12 +96,47 @@ the master on crashes.
 
 Default: 1
 
+=item --cert /path/to/cert
+
+The default TLS certificate for HTTPS, IMAPS, NNTPS, POP3S and/or STARTTLS
+support if the C<cert> option is not given with C<--listen>.
+
+With this option, well-known TCP ports automatically get TLS or STARTTLS
+support if using systemd-compatible socket activation.  That is, ports
+443, 563, 993, and 995 support HTTPS, NNTPS, IMAPS, and POP3S,
+respectively; while ports 110, 119, and 143 support STARTTLS on POP3,
+NNTP, and IMAP, respectively.
+
+=item --key /path/to/key
+
+The default TLS certificate key for the default C<--cert> or
+per-listener C<cert=> option.  The private key may be
+concatenated into the cert file itself, in which case this
+option is not needed.
+
+=item --multi-accept INTEGER
+
+By default, each worker accepts one connection at a time to maximize
+fairness and minimize contention across multiple processes on a
+shared listen socket.  Accepting multiple connections at once may be
+useful in constrained deployments with few, heavily loaded workers.
+Negative values enables a worker to accept all available clients at
+once, possibly starving others in the process.  C<-1> behaves like
+C<multi_accept yes> in nginx; while C<0> (the default) is
+C<multi_accept no> in nginx.  Positive values allow
+fine-tuning without the runaway behavior of C<-1>.
+
+This may be specified on a per-listener basis via the C<multi-accept=>
+per-listener directive (e.g. C<-l http://127.0.0.1?multi-accept=1>).
+
+Default: 0
+
 =back
 
 =head1 SIGNALS
 
 Most of our signal handling behavior is copied from L<nginx(8)>
-and/or L<starman(1)>; so it is possible to reuse common scripts
+and/or L<starman(1)>, so it is possible to reuse common scripts
 for managing them.
 
 =over 8
@@ -108,7 +157,7 @@ Reload config files associated with the process.
 
 =item SIGTTIN
 
-Increase the number of running workers processes by one.
+Increase the number of running worker processes by one.
 
 =item SIGTTOU
 
@@ -116,7 +165,7 @@ Decrease the number of running worker processes by one.
 
 =item SIGWINCH
 
-Stop all running worker processes.   SIGHUP or SIGTTIN
+Stop all running worker processes.  SIGHUP or SIGTTIN
 may be used to restart workers.
 
 =item SIGQUIT
@@ -144,7 +193,7 @@ activation.  See L<systemd.socket(5)> and L<sd_listen_fds(3)>.
 
 =item PERL_INLINE_DIRECTORY
 
-Pointing this to point to a writable directory enables the use
+Pointing this to a writable directory enables the use
 of L<Inline> and L<Inline::C> extensions which may provide
 platform-specific performance improvements.  Currently, this
 enables the use of L<vfork(2)> which speeds up subprocess
@@ -161,8 +210,8 @@ created by a user. See L<Inline> and L<Inline::C> for more details.
 There are two ways to upgrade a running process.
 
 Users of process management systems with socket activation
-(L<systemd(1)> or similar) may rely on multiple instances For
-systemd, this means using two (or more) '@' instances for each
+(L<systemd(1)> or similar) may rely on multiple daemon instances.
+For systemd, this means using two (or more) '@' instances for each
 service (e.g. C<SERVICENAME@INSTANCE>) as documented in
 L<systemd.unit(5)>.
 
@@ -183,11 +232,11 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2013-2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
 =head1 SEE ALSO
 
 L<public-inbox-httpd(1)>, L<public-inbox-imapd(1)>,
-L<public-inbox-nntpd(1)>
+L<public-inbox-nntpd(1)>, L<public-inbox-pop3d(1)>, L<public-inbox-netd(1)>
diff --git a/Documentation/public-inbox-extindex.pod b/Documentation/public-inbox-extindex.pod
index f71a90e5..b53e45ed 100644
--- a/Documentation/public-inbox-extindex.pod
+++ b/Documentation/public-inbox-extindex.pod
@@ -13,7 +13,7 @@ public-inbox-extindex [OPTIONS] [EXTINDEX_DIR] --all
 public-inbox-extindex creates and updates an external search and
 overview database used by the read-only public-inbox PSGI (HTTP),
 NNTP, and IMAP interfaces.  This requires either the
-L<Search::Xapian> XS bindings OR the L<Xapian> SWIG bindings,
+L<Xapian> SWIG bindings OR or L<Search::Xapian> XS bindings
 along with L<DBD::SQLite> and L<DBI> Perl modules.
 
 =head1 OPTIONS
@@ -47,11 +47,26 @@ C<indexlevel> set to C<basic> and their respective Xapian
 public-inboxes where cross-posting is common, this allows
 significant space savings on Xapian indices.
 
+=item --dedupe=MSGID
+
+=item --dedupe
+
+Rerun deduplication on messages with the given Message-ID or
+all messages if no Message-ID is specified.  Deduplication rules may
+change and evolve over time, especially if filters are involved.
+
+C<--dedupe=MSGID> may be specified multiple times to deduplicate
+multiple Message-IDs.
+
+Use this if you see C<W: BUG? $MSGID not deduplicated properly>
+warnings from WWW logs.
+
 =item --gc
 
 Perform garbage collection instead of indexing.  Use this if
-inboxes are removed from the extindex, or if messages are
-purged or removed from some inboxes.
+inboxes are removed from the extindex, a newsgroup name is
+set or changed, or if messages are purged or removed from
+some inboxes.
 
 =item --reindex
 
@@ -60,10 +75,6 @@ used for in-place upgrades and bugfixes while read-only server
 processes are utilizing the index.  Keep in mind this roughly
 doubles the size of the already-large Xapian database.
 
-The extindex locks will be released roughly every 10s to
-allow L<public-inbox-mda(1)> and L<public-inbox-watch(1)>
-processes to write to the extindex.
-
 =item --fast
 
 Used with C<--reindex>, it will only look for new and stale
@@ -77,9 +88,9 @@ L<public-inbox-extindex-format(5)>
 
 =head1 CONFIGURATION
 
-public-inbox-extindex does not currently write to the
-L<public-inbox-config(5)> file, configuration may be entered
-manually.  The extindex name of C<all> is a special case which
+public-inbox-extindex does not write to the L<public-inbox-config(5)>
+file, it must be entered manually.
+The extindex name of C<all> is a special case which
 corresponds to indexing C<--all> inboxes.  An example for
 C<--all> is as follows:
 
@@ -89,6 +100,16 @@ C<--all> is as follows:
                 coderepo = foo
                 coderepo = bar
 
+Putting an C<extindex> entry in the config allows L<PublicInbox::WWW>.
+You can have any number of C<extentry.$NAME> sections where C<$NAME>
+is something other than C<all> to display a union of several inboxes.
+
+It is strongly recommended any public inboxes indexed by this command
+have a stable C<publicinbox.$NAME.newsgroup> entry (regardless of
+the presence of an NNTP or IMAP server).  Otherwise, public-inbox-extindex
+will use C<publicinbox.$NAME.inboxdir> as an internal key which can
+cause needless reindexing and require L<--gc> if inboxes are relocated.
+
 See L<public-inbox-config(5)> for more details.
 
 =head1 ENVIRONMENT
@@ -117,9 +138,17 @@ Default: none, uses C<publicinbox.indexBatchSize>
 
 =head1 UPGRADING
 
-Occasionally, public-inbox will update it's schema version and
+Occasionally, public-inbox will update its schema version and
 require a full index by running this command.
 
+=head1 LOCKING
+
+It is safe to use C<--dedupe>, C<--gc> and C<--reindex> while
+other processes are writing to covered inboxes or extindex.
+The extindex locks will be released roughly every 10s to
+allow L<public-inbox-mda(1)> and L<public-inbox-watch(1)>
+processes to write to the extindex.
+
 =head1 CONTACT
 
 Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
diff --git a/Documentation/public-inbox-fetch.pod b/Documentation/public-inbox-fetch.pod
index c78ffc0b..1ff0df44 100644
--- a/Documentation/public-inbox-fetch.pod
+++ b/Documentation/public-inbox-fetch.pod
@@ -61,6 +61,14 @@ there are no updates:
         public-inbox-fetch -q --exit-code && public-inbox-index
         test $? -eq 0 || exit $?
 
+=item -p
+
+=item --prune
+
+Pass the C<--prune> and C<--prune-tags> flags to L<git-fetch(1)> calls.
+
+This is a new option in public-inbox 2.0+
+
 =item -v
 
 =item --verbose
diff --git a/Documentation/public-inbox-glossary.pod b/Documentation/public-inbox-glossary.pod
index 710098c8..d88539c8 100644
--- a/Documentation/public-inbox-glossary.pod
+++ b/Documentation/public-inbox-glossary.pod
@@ -25,7 +25,7 @@ C<over.sqlite3>
 
 =item tid, THREADID
 
-A sequentially-assigned positive integer.  These integers are
+A sequentially assigned positive integer.  These integers are
 per-inbox or per-extindex.  In the future, this may be prefixed
 with C<T> for JMAP (RFC 8621) and RFC 8474.  This may not be
 strictly compliant with RFC 8621 since inboxes and extindices
@@ -40,8 +40,8 @@ RFC-(822|2822|5322) email message.
 
 =item IMAP EMAILID, JMAP Email Id
 
-To-be-decided.  This will likely be the git blob ID prefixed with C<g>
-rather than the numeric UID to accomodate the same blob showing
+To be decided.  This will likely be the git blob ID prefixed with C<g>
+rather than the numeric UID to accommodate the same blob showing
 up in both an extindex and inbox (or multiple extindices).
 
 =item newsgroup
@@ -50,7 +50,7 @@ The name of the NNTP newsgroup, see L<public-inbox-config(5)>.
 
 =item IMAP (folder|mailbox) slice
 
-A 50K slice of a newsgroup to accomodate the limitations of IMAP
+A 50K slice of a newsgroup to accommodate the limitations of IMAP
 clients with L<public-inbox-imapd(1)>.  This is the C<newsgroup>
 name with a C<.$INTEGER_SUFFIX>, e.g. a newsgroup named C<inbox.test>
 would have its first slice named C<inbox.test.0>, and second slice
@@ -87,7 +87,7 @@ but it imports drafts.
 
 For L<lei(1)> users only.  This will allow lei users to place
 the same email into one or more virtual folders for
-ease-of-filtering.  This is NOT tied to public-inbox names, as
+ease of filtering.  This is NOT tied to public-inbox names, as
 messages stored by lei may not be public.
 
 These are similar in spirit to arbitrary freeform "tags"
diff --git a/Documentation/public-inbox-httpd.pod b/Documentation/public-inbox-httpd.pod
index 6a8673d8..3ed48adc 100644
--- a/Documentation/public-inbox-httpd.pod
+++ b/Documentation/public-inbox-httpd.pod
@@ -20,6 +20,19 @@ L<Plack::Middleware::Head>
 
 This may point to a PSGI file for supporting generic PSGI apps.
 
+=head1 ENVIRONMENT
+
+=over 8
+
+=item GIT_HTTP_MAX_REQUEST_BUFFER
+
+Shared with L<git-http-backend(1)>, this governs the maximum upload
+size of an HTTP request.
+
+Default: 10m
+
+=back
+
 =head1 CONTACT
 
 Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
@@ -29,7 +42,7 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2013-2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/Documentation/public-inbox-imapd.pod b/Documentation/public-inbox-imapd.pod
index 23577a69..85bf3651 100644
--- a/Documentation/public-inbox-imapd.pod
+++ b/Documentation/public-inbox-imapd.pod
@@ -27,12 +27,12 @@ are supported and documented below.
 
 =over
 
-=item -l PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item -l PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
-=item --listen PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item --listen PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
 In addition to the normal C<-l>/C<--listen> switch described in
-L<public-inbox-daemon(8)>, the C<PROTO> prefix (e.g. C<imap://> or
+L<public-inbox-daemon(8)>, the C<PROTOCOL> prefix (e.g. C<imap://> or
 C<imaps://>) may be specified to force a given protocol.
 
 For STARTTLS and IMAPS support, the C<cert> and C<key> may be specified
diff --git a/Documentation/public-inbox-index.pod b/Documentation/public-inbox-index.pod
index 011ade3c..14f157a5 100644
--- a/Documentation/public-inbox-index.pod
+++ b/Documentation/public-inbox-index.pod
@@ -13,8 +13,8 @@ public-inbox-index [OPTIONS] --all
 public-inbox-index creates and updates the search, overview and
 NNTP article number database used by the read-only public-inbox
 HTTP and NNTP interfaces.  Currently, this requires
-L<DBD::SQLite> and L<DBI> Perl modules.  L<Search::Xapian>
-is optional, only to support the PSGI search interface.
+L<DBD::SQLite> and L<DBI> Perl modules.  L<Xapian> (or L<Search::Xapian>)
+are optional, only to support the PSGI search interface.
 
 Once the initial indices are created by public-inbox-index,
 L<public-inbox-mda(1)> and L<public-inbox-watch(1)> will
@@ -319,7 +319,7 @@ Default: none, uses C<publicinbox.indexBatchSize>
 
 =head1 UPGRADING
 
-Occasionally, public-inbox will update it's schema version and
+Occasionally, public-inbox will update its schema version and
 require a full index by running this command.
 
 =head1 CONTACT
diff --git a/Documentation/public-inbox-learn.pod b/Documentation/public-inbox-learn.pod
index 3c92b1cc..b08e4bc8 100644
--- a/Documentation/public-inbox-learn.pod
+++ b/Documentation/public-inbox-learn.pod
@@ -54,7 +54,7 @@ This is similar to the C<spam> command above, but does
 not feed the message to L<spamc(1)> and only removes messages
 which match on any of the C<To:>, C<Cc:>, and C<List-ID:> headers.
 
-The C<--all> option may be used match C<spam> semantics in removing
+The C<--all> option may be used to match C<spam> semantics in removing
 the message from all configured inboxes.  C<--all> is only
 available in public-inbox 1.6.0+.
 
@@ -73,6 +73,25 @@ Default: ~/.public-inbox/config
 
 =back
 
+=head1 CONFIGURATION
+
+These configuration knobs should be used in the
+L<public-inbox-config(5)> file.
+
+=over 8
+
+=item publicinboxImport.dropUniqueUnsubscribe
+
+=item publicinbox.<name>.address
+
+=item publicinbox.<name>.listid
+
+=item publicinboxmda.spamcheck
+
+See L<public-inbox-config(5)> for descriptions of these options
+
+=back
+
 =head1 CONTACT
 
 Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
@@ -82,7 +101,7 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2019-2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/Documentation/public-inbox-mda.pod b/Documentation/public-inbox-mda.pod
index 93cb0e9c..edc90287 100644
--- a/Documentation/public-inbox-mda.pod
+++ b/Documentation/public-inbox-mda.pod
@@ -68,6 +68,22 @@ Default: ~/.public-inbox/emergency/
 
 =back
 
+=head1 CONFIGURATION
+
+Various configuration knobs should be used in the
+L<public-inbox-config(5)> file.
+
+=over 8
+
+=item publicinboxImport.dropUniqueUnsubscribe
+
+=item publicinbox.<name>.address
+
+=item publicinbox.<name>.listid
+
+See L<public-inbox-config(5)> for descriptions of these options
+
+=back
 
 =head1 CONTACT
 
@@ -78,7 +94,7 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2013-2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/Documentation/public-inbox-netd.pod b/Documentation/public-inbox-netd.pod
index dcf4d5b0..71425e3c 100644
--- a/Documentation/public-inbox-netd.pod
+++ b/Documentation/public-inbox-netd.pod
@@ -8,9 +8,10 @@ public-inbox-netd - read-only network daemon for sharing public-inboxes
 
 =head1 DESCRIPTION
 
-public-inbox-netd provides a read-only HTTP/IMAP/NNTP/POP3 daemon for
-public-inbox.  It uses options and environment variables common
-to all L<public-inbox-daemon(8)> implementations.
+public-inbox-netd provides a read-only multi-protocol
+(HTTP/IMAP/NNTP/POP3) daemon for public-inbox.  It uses options
+and environment variables common to all
+L<public-inbox-daemon(8)> implementations.
 
 The default configuration will never require write access
 to the directory where the public-inbox is stored, so it
@@ -24,25 +25,38 @@ See common options in L<public-inbox-daemon(8)/OPTIONS>.
 
 =over
 
-=item -l PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item -l PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
-=item --listen PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item --listen PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+
+=item -l http://ADDRESS/?env.PI_CONFIG=/path/to/cfg,psgi=/path/to/app.psgi
 
 In addition to the normal C<-l>/C<--listen> switch described in
 L<public-inbox-daemon(8)>, the protocol prefix (e.g. C<nntp://> or
 C<nntps://>) may be specified to force a given protocol.
 
+Environment variable overrides in effect during loading and
+reloading (SIGHUP) can be specified as C<env.NAME=VALUE> for
+all protocols.
+
+HTTP(S) listeners may also specify C<psgi=> to use a different
+C<.psgi> file for each listener.
+
+C<err=/path/to/errors.log> may be used to isolate error/debug output
+for a particular listener away from C<--stderr>.
+
+Non-HTTP(S) listeners may also specify C<out=> for logging to
+C<stdout>.  HTTP(S) users are encouraged to configure
+L<Plack::Middleware::AccessLog> or
+L<Plack::Middleware::AccessLog::Timed>, instead.
+
 =item --cert /path/to/cert
 
-The default TLS certificate for optional TLS support
-if the C<cert> option is not given with C<--listen>.
+See L<public-inbox-daemon(1)>.
 
 =item --key /path/to/key
 
-The default private TLS certificate key for optional TLS support
-if the C<key> option is not given with C<--listen>.  The private
-key may be concatenated into the path used by C<--cert>, in which case this
-option is not needed.
+See L<public-inbox-daemon(1)>.
 
 =back
 
@@ -57,6 +71,8 @@ L<public-inbox-config(5)>.
 
 =item publicinbox.nntpserver
 
+=item publicinbox.pop3state
+
 =back
 
 See L<public-inbox-config(5)> for documentation on them.
diff --git a/Documentation/public-inbox-nntpd.pod b/Documentation/public-inbox-nntpd.pod
index cf53da59..59111f92 100644
--- a/Documentation/public-inbox-nntpd.pod
+++ b/Documentation/public-inbox-nntpd.pod
@@ -26,9 +26,9 @@ are supported and documented below.
 
 =over
 
-=item -l PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item -l PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
-=item --listen PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item --listen PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
 In addition to the normal C<-l>/C<--listen> switch described in
 L<public-inbox-daemon(8)>, the protocol prefix (e.g. C<nntp://> or
diff --git a/Documentation/public-inbox-overview.pod b/Documentation/public-inbox-overview.pod
index d4318576..35917ccc 100644
--- a/Documentation/public-inbox-overview.pod
+++ b/Documentation/public-inbox-overview.pod
@@ -48,7 +48,7 @@ that inbox.  The instructions are roughly:
 
   # Optional but strongly recommended for hosting HTTP
   # (and required for NNTP)
-  # enable overview (requires DBD::SQLite) and, if Search::Xapian is
+  # enable overview (requires DBD::SQLite) and, if Xapian is
   # available, search:
   public-inbox-index INBOX_DIR
 
diff --git a/Documentation/public-inbox-pop3d.pod b/Documentation/public-inbox-pop3d.pod
new file mode 100644
index 00000000..fb16fb96
--- /dev/null
+++ b/Documentation/public-inbox-pop3d.pod
@@ -0,0 +1,122 @@
+=head1 NAME
+
+public-inbox-pop3d - POP3 server for sharing public-inboxes
+
+=head1 SYNOPSIS
+
+  public-inbox-pop3d [OPTIONS]
+
+=head1 DESCRIPTION
+
+public-inbox-pop3d provides a POP3 daemon for public-inbox.
+It uses options and environment variables common to all
+read-only L<public-inbox-daemon(8)> implementations,
+but requires additional read-write storage to keep track
+of deleted messages on a per-user basis.
+
+Like L<public-inbox-imapd(1)>, C<public-inbox-pop3d> will never
+require write access to the directory where the public-inboxes
+are stored.
+
+It is designed for anonymous access, thus the password is
+always C<anonymous> (all lower-case).
+
+Usernames are of the format:
+
+        C<$UUID@$NEWSGROUP_NAME>
+
+Where C<$UUID> is the output of the L<uuidgen(1)> command.  Dash
+(C<->) characters in UUIDs are ignored, and C<[A-F]> hex
+characters are case-insensitive.  Users should keep their UUIDs
+private to prevent others from deleting unretrieved messages.
+Users may switch to a new UUID at any time to retrieve
+previously-retrieved messages.
+
+Historical slices of 50K messages are available
+by suffixing the integer L<$SLICE>, where C<0> is the oldest.
+
+        C<$UUID@$NEWSGROUP_NAME.$SLICE>
+
+It may be run as a different user than the user running
+L<public-inbox-watch(1)>, L<public-inbox-mda(1)>, or
+L<public-inbox-fetch(1)>.
+
+To save storage, L</publicinbox.pop3state> only stores
+the highest-numbered deleted message
+
+=head1 OPTIONS
+
+See common options in L<public-inbox-daemon(8)/OPTIONS>.
+
+=over
+
+=item -l PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+
+=item --listen PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+
+In addition to the normal C<-l>/C<--listen> switch described in
+L<public-inbox-daemon(8)>, the C<PROTOCOL> prefix (e.g. C<pop3://> or
+C<pop3s://>) may be specified to force a given protocol.
+
+For STARTTLS and POP3S support, the C<cert> and C<key> may be specified
+on a per-listener basis after a C<?> character and separated by C<,>.
+These directives are per-directive, and it's possible to use a different
+cert for every listener.
+
+=item --cert /path/to/cert
+
+The default TLS certificate for optional STARTTLS and POP3S support
+if the C<cert> option is not given with C<--listen>.
+
+If using systemd-compatible socket activation and a TCP listener on port
+995 is inherited, it is automatically POP3S when this option is given.
+When a listener on port 110 is inherited and this option is given, it
+automatically gets STARTTLS support.
+
+=item --key /path/to/key
+
+The default private TLS certificate key for optional STARTTLS and POP3S
+support if the C<key> option is not given with C<--listen>.  The private
+key may be concatenated into the path used by C<--cert>, in which case this
+option is not needed.
+
+=back
+
+=head1 CONFIGURATION
+
+Aside from C<publicinbox.pop3state>, C<public-inbox-pop3d> uses the
+same configuration knobs as L<public-inbox-nntpd(1)>,
+see L<public-inbox-nntpd(1)> and L<public-inbox-config(5)>.
+
+=over 8
+
+=item publicInbox.pop3state
+
+A directory containing per-user/mailbox account information;
+must be writable to the C<public-inbox-pop3d> process.
+
+=item publicInbox.<name>.newsgroup
+
+The newsgroup name maps to a POP3 folder name.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>, and
+L<nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta>,
+L<nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<git(1)>, L<git-config(1)>, L<public-inbox-daemon(8)>,
+L<public-inbox-config(5)>, L<public-inbox-nntpd(1)>,
+L<uuidgen(1)>
diff --git a/Documentation/public-inbox-purge.pod b/Documentation/public-inbox-purge.pod
index 945286c6..1223b577 100644
--- a/Documentation/public-inbox-purge.pod
+++ b/Documentation/public-inbox-purge.pod
@@ -31,7 +31,7 @@ leads to discontiguous git history.
 =item --all
 
 Purge the message in all inboxes configured in ~/.public-inbox/config.
-This is an alternative to specifying individual inboxes directories
+This is an alternative to specifying individual inbox directories
 on the command-line.
 
 =back
@@ -74,7 +74,7 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2019-2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/Documentation/public-inbox-tuning.pod b/Documentation/public-inbox-tuning.pod
index 53668ecc..73246144 100644
--- a/Documentation/public-inbox-tuning.pod
+++ b/Documentation/public-inbox-tuning.pod
@@ -42,6 +42,14 @@ Other OS tuning knobs
 
 Scalability to many inboxes
 
+=item 9
+
+public-inbox-cindex --join performance
+
+=item 10
+
+public-inbox-clone with shared object stores
+
 =back
 
 =head2 New inboxes: public-inbox-init -V2
@@ -79,8 +87,8 @@ RAM.  Attempts to parallelize random I/O on HDDs leads to pathological
 slowdowns as inboxes grow.
 
 While C<-V2> introduced Xapian shards as a parallelization
-mechanism for SSDs; enabling C<publicInbox.indexSequentialShard>
-repurposes sharding as mechanism to reduce the kernel page cache
+mechanism for SSDs, enabling C<publicInbox.indexSequentialShard>
+repurposes sharding as a mechanism to reduce the kernel page cache
 footprint when indexing on HDDs.
 
 Initializing a mirror with a high C<--jobs> count to create more
@@ -108,7 +116,7 @@ indices on btrfs to achieve acceptable performance (even on SSD).
 Disabling copy-on-write also disables checksumming, thus C<raid1>
 (or higher) configurations may be corrupt after unsafe shutdowns.
 
-Fortunately, these SQLite and Xapian indices are designed to
+Fortunately, these SQLite and Xapian indices are designed to be
 recoverable from git if missing.
 
 Disabling CoW does not prevent all fragmentation.  Large values
@@ -125,7 +133,7 @@ C<btrfs filesystem defragment -fr $INBOX_DIR> may be necessary.
 Large filesystems benefit significantly from the C<space_cache=v2>
 mount option documented in L<btrfs(5)>.
 
-Older, non-CoW filesystems are generally work well out-of-the-box
+Older, non-CoW filesystems generally work well out of the box
 for our Xapian and SQLite indices.
 
 =head2 Performance on solid state drives
@@ -152,9 +160,14 @@ C<LimitNOFILE=> in L<systemd.exec(5)>) may need to be raised to
 accommodate many concurrent clients.
 
 Transport Layer Security (IMAPS, NNTPS, or via STARTTLS) significantly
-increases memory use of client sockets, sure to account for that in
+increases memory use of client sockets, be sure to account for that in
 capacity planning.
 
+Bursts of small object allocations late in process life contribute to
+fragmentation of the heap due to arenas (slabs) used internally by Perl.
+jemalloc (tested as an LD_PRELOAD on GNU/Linux) appears to reduce
+overall fragmentation compared to glibc malloc in long-lived processes.
+
 =head2 Other OS tuning knobs
 
 Linux users: the C<sys.vm.max_map_count> sysctl may need to be increased if
@@ -168,13 +181,28 @@ Other OSes may have similar tuning knobs (patches appreciated).
 L<public-inbox-extindex(1)> allows any number of public-inboxes
 to share the same Xapian indices.
 
-git 2.33+ startup time is orders-of-magnitude faster and uses
+git 2.33+ startup time is orders of magnitude faster and uses
 less memory when dealing with thousands of alternates required
 for thousands of inboxes with L<public-inbox-extindex(1)>.
 
 Frequent packing (via L<git-gc(1)>) both improves performance
 and reduces the need to increase C<sys.vm.max_map_count>.
 
+=head2 public-inbox-cindex --join performance
+
+A C++ compiler and the Xapian development files makes C<--join> or
+C<--join=aggressive> orders of magnitude faster in L<public-inbox-cindex(1)>.
+On Debian-based systems this is C<libxapian-dev>.  RPM-based distros have
+these in C<xapian-core-devel> or C<xapian14-core-libs>.  *BSDs typically
+package development files together with runtime libraries, so the C<xapian>
+or C<xapian-core> package will already have the development files.
+
+=head2 public-inbox-clone with shared object stores
+
+When mirroring manifests with many forks using the same objstore,
+git 2.41+ is highly recommended for performance as we automatically
+use the C<fetch.hideRefs> feature to speed up negotiation.
+
 =head1 CONTACT
 
 Feedback encouraged via plain-text mail to L<mailto:meta@public-inbox.org>
diff --git a/Documentation/public-inbox-v2-format.pod b/Documentation/public-inbox-v2-format.pod
index e93d7fc7..de3b0bfd 100644
--- a/Documentation/public-inbox-v2-format.pod
+++ b/Documentation/public-inbox-v2-format.pod
@@ -30,7 +30,7 @@ databases for parallelism by "shards".
   - all.git                         # empty, alternates to $EPOCH.git
   - xap$SCHEMA_VERSION/$SHARD       # per-shard Xapian DB
   - xap$SCHEMA_VERSION/over.sqlite3 # OVER-view DB for NNTP, threading
-  - msgmap.sqlite3                  # same the v1 msgmap
+  - msgmap.sqlite3                  # same as the v1 msgmap
 
 For blob lookups, the reader only needs to open the "all.git"
 repository with $GIT_DIR/objects/info/alternates which references
@@ -89,7 +89,7 @@ After-the-fact invocations of L<public-inbox-index> will ignore
 messages written to 'd' after they are written to 'm'.
 
 Deltafication is not significantly improved over v1, but overall
-storage for trees is made as as small as possible.  Initial
+storage for trees is made as small as possible.  Initial
 statistics and benchmarks showing the benefits of this approach
 are documented at:
 
@@ -97,7 +97,7 @@ L<https://public-inbox.org/meta/20180209205140.GA11047@dcvr/>
 
 =head2 XAPIAN SHARDS
 
-Another second scalability problem in v1 was the inability to
+Another scalability problem in v1 was the inability to
 utilize multiple CPU cores for Xapian indexing.  This is
 addressed by using shards in Xapian to perform import
 indexing in parallel.
diff --git a/Documentation/public-inbox-watch.pod b/Documentation/public-inbox-watch.pod
index e8f97c80..6e2142fe 100644
--- a/Documentation/public-inbox-watch.pod
+++ b/Documentation/public-inbox-watch.pod
@@ -41,16 +41,18 @@ importing them into public-inbox git repositories and indices.
 public-inbox-watch is useful in situations when a user wishes to
 mirror an existing mailing list, but has no access to run
 L<public-inbox-mda(1)> on a server.  Unlike public-inbox-mda
-which is invoked once per-message, public-inbox-watch is a
+which is invoked once per message, public-inbox-watch is a
 persistent process, making it faster for after-the-fact imports
 of large Maildirs.
 
 Upon startup, it scans the mailbox for new messages to be
 imported while it was not running.
 
-As of public-inbox 1.6.0, Maildirs, IMAP folders, and NNTP
-newsgroups are supported.  Previous versions of public-inbox
-only supported Maildirs.
+All versions of public-inbox-watch support Maildirs.  public-inbox
+1.6.0 added support for IMAP folders and NNTP newsgroups.
+public-inbox 2.0 adds support for MH directories.  There are no
+plans to support the mbox family since new messages are expensive
+to detect in large mboxes.
 
 public-inbox-watch should be run inside a L<screen(1)> session
 or as a L<systemd(1)> service.  Errors are emitted to stderr.
@@ -62,10 +64,14 @@ public-inbox-watch takes no command-line options.
 =head1 CONFIGURATION
 
 These configuration knobs should be used in the
-L<public-inbox-config(5)> file
+L<public-inbox-config(5)> file.
 
 =over 8
 
+=item publicinboxImport.dropUniqueUnsubscribe
+
+See L<public-inbox-config(5)/publicinboxImport.dropUniqueUnsubscribe>
+
 =item publicinbox.<name>.watch
 
 A location to watch.  public-inbox 1.5.0 and earlier only supported
@@ -74,17 +80,24 @@ C<maildir:> paths:
         [publicinbox "test"]
                 watch = maildir:/path/to/maildirs/.INBOX.test/
 
-public-inbox 1.6.0 supports C<nntp://>, C<nntps://>,
+public-inbox 1.6.0+ supports C<nntp://>, C<nntps://>,
 C<imap://> and C<imaps://> URLs:
 
                 watch = nntp://news.example.com/inbox.test.group
                 watch = imaps://user@mail.example.com/INBOX.test
 
+2.0+ supports MH:
+
+                watch = mh:/path/to/MH/inbox.test
+
 This may be specified multiple times to combine several mailboxes
 into a single public-inbox.  URLs requiring authentication
 will require L<netrc(5)> and/or L<git-credential(1)> (preferred) to fill
 in the username and password.
 
+public-inbox 2.0+ also supports boolean C<false> to prevent the global
+L</publicinboxwatch.watchspam> directive from writing to the inbox.
+
 Default: none
 
 =item publicinbox.<name>.watchheader
@@ -120,7 +133,7 @@ Messages without the (S)een flag are not considered for hiding.
 This hiding affects all configured public-inboxes in PI_CONFIG.
 
 As with C<publicinbox.$NAME.watch>, C<imap://> and C<imaps://> URLs
-are supported in public-inbox 1.6.0+.
+are supported in public-inbox 1.6.0+, and C<MH> in 2.0+.
 
 Default: none; only for L<public-inbox-watch(1)> users
 
@@ -201,7 +214,7 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2016-2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/Documentation/public-inbox.cgi.pod b/Documentation/public-inbox.cgi.pod
index 71f8a6f5..58d59ba2 100644
--- a/Documentation/public-inbox.cgi.pod
+++ b/Documentation/public-inbox.cgi.pod
@@ -4,7 +4,7 @@ public-inbox.cgi - CGI wrapper for PublicInbox::WWW
 
 =head1 SYNOPSIS
 
-You generally want to run public-inbox-httpd, instead
+You generally want to run public-inbox-netd or public-inbox-httpd, instead
 
 =head1 DESCRIPTION
 
@@ -12,9 +12,15 @@ public-inbox.cgi provides a CGI interface wrapper on top of the
 PSGI/Plack L<PublicInbox::WWW> module.  It is only provided for
 compatibility reasons and NOT recommended.
 
-CGI with Perl is slow due to code loading overhead and web servers lack
-the scheduling fairness of L<public-inbox-httpd(1)> for handling git
-clones and streaming large mbox downloads.
+CGI with Perl is slow due to code loading overhead and
+web servers lack the scheduling fairness of L<public-inbox-netd(1)>
+and L<public-inbox-httpd(1)> for handling git clones and
+streaming large mbox downloads.
+
+=head1 COMPATIBILITY NOTE
+
+When using the CGI with Apache, make sure to set AllowEncodedSlashes to On, as
+public-inbox makes heavy use of encoded slashes.
 
 =head1 CONTACT
 
@@ -25,10 +31,11 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2019-2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
 =head1 SEE ALSO
 
-L<public-inbox-httpd(1)>, L<PublicInbox::WWW>, L<public-inbox-daemon(8)>,
+L<public-inbox-netd(1)>, L<public-inbox-httpd(1)>,
+L<PublicInbox::WWW>, L<public-inbox-daemon(8)>,
diff --git a/Documentation/reproducibility.txt b/Documentation/reproducibility.txt
index 4e56ada4..3336de73 100644
--- a/Documentation/reproducibility.txt
+++ b/Documentation/reproducibility.txt
@@ -12,7 +12,7 @@ reproducible.
 Keeping all communications as email ensures the full history
 of the entire project can be mirrored by anyone with the
 resources to do so.  Compact, low-complexity data requires
-less resources to mirror, so sticking with plain-text
+less resources to mirror, so sticking with plain text
 ensures more parties can mirror and potentially fork the
 project with all its data.
 
@@ -26,4 +26,4 @@ If these things make power hungry project leaders and admins
 uncomfortable, good.  That was the point.  It's how checks
 and balances ought to work.
 
-Comments, corrections, etc welcome: meta@public-inbox.org
+Comments, corrections, etc. welcome: meta@public-inbox.org
diff --git a/Documentation/standards.perl b/Documentation/standards.perl
index 69568ceb..743cdee1 100755
--- a/Documentation/standards.perl
+++ b/Documentation/standards.perl
@@ -1,6 +1,6 @@
 #!/usr/bin/perl -w
-use strict;
-# Copyright 2019-2021 all contributors <meta@public-inbox.org>
+use v5.12;
+# Copyright all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 print <<EOF;
@@ -11,11 +11,11 @@ Non-exhaustive list of standards public-inbox software attempts or
 intends to implement.  This list is intended to be a quick reference
 for hackers and users.
 
-Given the goals of interoperability and accessibility; strict
+Given the goals of interoperability and accessibility, strict
 conformance to standards is not always possible, but rather
 best-effort taking into account real-world cases.  In particular,
 "obsolete" standards remain relevant as long as clients and
-data exists.
+data using them exist.
 
 IETF RFCs
 ---------
@@ -66,8 +66,13 @@ my $rfcs = [
         2369 => 'URLs as Meta-Syntax for Core Mail List Commands',
         8058 => 'Signaling One-Click Functionality for List Email Headers',
 
-        # TODO: flesh this out
+        1081 => 'Post Office Protocol – Version 3',
+        1939 => 'Post Office Protocol – Version 3 (STD 53)',
+        2449 => 'POP3 extension mechanism',
+        2595 => 'STARTTLS for IMAP and POP3',
+        2384 => 'POP URL Scheme',
 
+        # TODO: flesh this out
 ];
 
 my @rfc_urls = qw(tools.ietf.org/html/rfc%d
@@ -102,6 +107,6 @@ Other relevant documentation
 Copyright
 ---------
 
-Copyright (C) 2019-2020 all contributors <meta@public-inbox.org>
+Copyright (C) all contributors <meta@public-inbox.org>
 License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 EOF
diff --git a/Documentation/technical/data_structures.txt b/Documentation/technical/data_structures.txt
index 4dcf9ce6..11f78041 100644
--- a/Documentation/technical/data_structures.txt
+++ b/Documentation/technical/data_structures.txt
@@ -32,19 +32,19 @@ Per-message classes
   Common abbreviation: $mime, $eml
   Used by: PublicInbox::WWW, PublicInbox::SearchIdx
 
-  An representation of an entire email, multipart or not.
+  A representation of an entire email, multipart or not.
   An option to use libgmime or libmailutils may be supported
   in the future for performance and memory use.
 
   This can be a memory hog with big messages and giant
   attachments, so our PublicInbox::WWW interface only keeps
-  one object of this class in memory at-a-time.
+  one object of this class in memory at a time.
 
   In other words, this is the "meat" of the message, whereas
   $smsg (below) is just the "skeleton".
 
   Our PublicInbox::V2Writable class may have two objects of this
-  type in memory at-a-time for deduplication.
+  type in memory at a time for deduplication.
 
   In public-inbox 1.4 and earlier, Email::MIME and its subclass,
   PublicInbox::MIME were used.  Despite still slurping,
@@ -61,10 +61,10 @@ Per-message classes
 
   This is loaded from either the overview DB (over.sqlite3) or
   the Xapian DB (docdata.glass), though the Xapian docdata
-  is won't hold NNTP-only fields (Cc:/To:)
+  won't hold NNTP-only fields (Cc:/To:).
 
   There may be hundreds or thousands of these objects in memory
-  at-a-time, so fields are pruned if unneeded.
+  at a time, so fields are pruned if unneeded.
 
 * PublicInbox::SearchThread::Msg - subclass of Smsg
   Common abbreviation: $cont or $node
@@ -75,9 +75,9 @@ Per-message classes
   Nowadays, this is a re-blessed $smsg with additional fields.
 
   As with $smsg objects, there may be hundreds or thousands
-  of these objects in memory at-a-time.
+  of these objects in memory at a time.
 
-  We also do not use a linked-list for storing children as JWZ
+  We also do not use a linked list for storing children as JWZ
   describes, but instead a Perl hashref for {children} which
   becomes an arrayref upon sorting.
 
@@ -88,7 +88,7 @@ Per-inbox classes
 
 * PublicInbox::Inbox - represents a single public-inbox
   Common abbreviation: $ibx
-  Used everywhere
+  Used everywhere.
 
   This represents a "publicinbox" section in the config
   file, see public-inbox-config(5) for details.
@@ -117,7 +117,7 @@ Per-inbox classes
 
 * PublicInbox::Search - Xapian read-only interface
   Common abbreviation: $srch, $ibx->search
-  Used everywhere if Search::Xapian (or Xapian.pm) is available.
+  Used everywhere if Xapian is available.
 
   Each indexed inbox has one of these, see
   public-inbox-v1-format(5) and public-inbox-v2-format(5)
@@ -152,7 +152,7 @@ ad-hoc structures shared across packages
   This holds the PSGI $env as well as any internal variables
   used by various modules of PublicInbox::WWW.
 
-  As with the PSGI $env, there is one per-active WWW
+  As with the PSGI $env, there is one per active WWW
   request+response cycle.  It does not exist for idle HTTP
   clients.
 
@@ -174,8 +174,8 @@ daemon classes
   Common abbreviation: $http
   Used by: PublicInbox::DS, public-inbox-httpd
 
-  Unlike PublicInbox::NNTP, this class no knowledge of any of
-  the email or git-specific parts of public-inbox, only PSGI.
+  Unlike PublicInbox::NNTP, this class has no knowledge of any of
+  the email- or git-specific parts of public-inbox, only PSGI.
   However, it supports APIs and behaviors (e.g. streaming large
   responses) which PublicInbox::WWW may take advantage of.
 
@@ -188,7 +188,7 @@ daemon classes
 
   This class calls non-blocking accept(2) or accept4(2) on a
   listen socket to create new PublicInbox::HTTP and
-  PublicInbox::HTTP instances.
+  PublicInbox::NNTP instances.
 
 * PublicInbox::HTTPD
   Common abbreviation: $httpd
@@ -197,9 +197,9 @@ daemon classes
   wrappers around client sockets accepted from
   PublicInbox::Listener.
 
-  Since the SERVER_NAME and SERVER_PORT PSGI variables needs to be
+  Since the SERVER_NAME and SERVER_PORT PSGI variables need to be
   exposed for HTTP/1.0 requests when Host: headers are missing,
-  this is per-Listener socket.
+  this is per Listener socket.
 
 * PublicInbox::HTTPD::Async
   Common abbreviation: $async
diff --git a/Documentation/technical/ds.txt b/Documentation/technical/ds.txt
index 5a1655a1..afead2f1 100644
--- a/Documentation/technical/ds.txt
+++ b/Documentation/technical/ds.txt
@@ -1,9 +1,14 @@
 PublicInbox::DS - event loop and async I/O base class
 
-Our PublicInbox::DS event loop which powers public-inbox-nntpd
-and public-inbox-httpd diverges significantly from the
-unmaintained Danga::Socket package we forked from.  In fact,
-it's probably different from most other event loops out there.
+Our PublicInbox::DS event loop which powers most of our long-lived
+processes(*) diverges significantly from the unmaintained Danga::Socket
+package we forked from.  In fact, it's probably different from most
+other event loops out there.
+
+Most notably, it uses one-shot, level-trigger, and edge-trigger mode
+modes of kqueue|epoll depending on the situation.
+
+(*) public-inbox-netd,(-httpd,-imapd,-nntpd,-pop3d,-watch) + lei-daemon
 
 Most notably:
 
@@ -14,7 +19,7 @@ Most notably:
   triggers a call.
 
   The lack of read/write callback distinction is driven by the
-  fact TLS libraries (e.g. OpenSSL via IO::Socket::SSL) may
+  fact that TLS libraries (e.g. OpenSSL via IO::Socket::SSL) may
   declare SSL_WANT_READ on SSL_write(), and SSL_WANT_READ on
   SSL_read().  So we end up having to let each user object decide
   whether it wants to make read or write calls depending on its
@@ -30,7 +35,7 @@ Most notably:
   Reducing the user-supplied code down to a single callback allows
   subclasses to keep their logic self-contained.  The combination
   of this change and one-shot wakeups (see below) for bidirectional
-  data flows make asynchronous code easier to reason about.
+  data flows makes asynchronous code easier to reason about.
 
 Other divergences:
 
@@ -48,7 +53,7 @@ Other divergences:
 
 Augmented features:
 
-* obj->write(CODEREF) passes the object itself to the CODEREF
+* obj->write(CODEREF) passes the object itself to the CODEREF.
   Being able to enqueue subroutine calls is a powerful feature in
   Danga::Socket for keeping linear logic in an asynchronous environment.
   Unfortunately, each subroutine takes several kilobytes of memory.
@@ -81,7 +86,7 @@ New features
 
 * IO::Socket::SSL support (for NNTPS, STARTTLS+NNTP, HTTPS)
 
-* dwaitpid (waitpid wrapper) support for reaping dead children
+* awaitpid (waitpid wrapper) support for reaping dead children
 
 * reliable signal wakeups are supported via signalfd on Linux,
   EVFILT_SIGNAL on *BSDs via IO::KQueue.
diff --git a/Documentation/technical/memory.txt b/Documentation/technical/memory.txt
index bb1c92fd..039694c3 100644
--- a/Documentation/technical/memory.txt
+++ b/Documentation/technical/memory.txt
@@ -8,12 +8,12 @@ memory-efficient.
 We strive to keep processes small to improve locality, allow
 the kernel to cache more files, and to be a good neighbor to
 other processes running on the machine.  Taking advantage of
-automatic reference counting (ARC) in Perl allows us
+automatic reference counting (ARC) in Perl allows us to
 deterministically release memory back to the heap.
 
 We start with a simple data model with few circular
 references.  This both eases human understanding and reduces
-the likelyhood of bugs.
+the likelihood of bugs.
 
 Knowing the relative sizes and quantities of our data
 structures, we limit the scope of allocations as much as
@@ -48,3 +48,9 @@ In the future, our internal data model will be further
 flattened and simplified to reduce the overhead imposed by
 small objects.  Large allocations may also be avoided by
 optionally using Inline::C.
+
+Finally, the mwrap-perl LD_PRELOAD wrapper was ported to Perl 5
+and enhanced to provide live memory usage tracking on 64-bit systems
+with minimal performance impact on production traffic:
+
+        git clone https://80x24.org/mwrap-perl.git
diff --git a/Documentation/technical/weird-stuff.txt b/Documentation/technical/weird-stuff.txt
new file mode 100644
index 00000000..0c8d6891
--- /dev/null
+++ b/Documentation/technical/weird-stuff.txt
@@ -0,0 +1,22 @@
+There's a lot of weird code in public-inbox which may be daunting
+to new hackers.
+
+* The event loop (PublicInbox::DS) is an evolution of a fairly standard
+  C10K event loop.  See ds.txt in this directory for more.
+
+Things got weirder in 2021:
+
+* The lei command-line tool is backed by a daemon.  This was done to
+  improve startup time for shell completion and manage git/SQLite/Xapian
+  single-writer during long, parallel imports.  It may eventually become
+  a read-write IMAP/JMAP server.
+
+* SOCK_SEQPACKET is used extensively in lei, and will likely make its
+  way into more places, still.
+
+And even more so in 2022:
+
+* public-inbox-clone / PublicInbox::LeiMirror relies on ->DESTROY
+  for make-like dependency management while providing parallelism.
+
+More to come, lei will expose Maildirs via FUSE 3...
diff --git a/Documentation/technical/whyperl.txt b/Documentation/technical/whyperl.txt
index fbe2e1b1..db1d9793 100644
--- a/Documentation/technical/whyperl.txt
+++ b/Documentation/technical/whyperl.txt
@@ -21,7 +21,7 @@ Good Things
 
   Perl 5 is installed on many, if not most GNU/Linux and
   BSD-based servers and workstations.  It is likely the most
-  widely-installed programming environment that offers a
+  widely installed programming environment that offers a
   significant amount of POSIX functionality.  Users won't
   have to waste bandwidth or space with giant toolchains or
   architecture-specific binaries.
@@ -47,8 +47,8 @@ Good Things
 
 * Predictable performance
 
-  While Perl is neither fast or memory-efficient, its
-  performance and memory use are predictable and does not
+  While Perl is neither fast nor memory-efficient, its
+  performance and memory use are predictable and do not
   require GC tuning by the user.
 
   public-inbox is developed for (and mostly on) old
@@ -56,7 +56,7 @@ Good Things
   late 1990s, and any cheap VPS today has more than enough
   RAM and CPU for handling plain-text email.
 
-  Low hardware requirements increases the reach of our software
+  Low hardware requirements increase the reach of our software
   to more users, improving centralization resistance.
 
 * Compatibility
@@ -86,7 +86,7 @@ Good Things
 
   There should be no need to rely on language-specific
   package managers such as cpan(1), those systems increase
-  the learning curve for users and systems administrators.
+  the learning curve for users and system administrators.
 
 * Compactness and terseness
 
@@ -98,7 +98,7 @@ Good Things
 * Performance ceiling and escape hatch
 
   With optional Inline::C, we can be "as fast as C" in some
-  cases.  Inline::C is widely-packaged by distros and it
+  cases.  Inline::C is widely packaged by distros and it
   gives us an escape hatch for dealing with missing bindings
   or performance problems should they arise.  Inline::C use
   (as opposed to XS) also preserves the software freedom and
@@ -135,7 +135,7 @@ Bad Things
   (m//, substr(), index(), etc.) still require memory copies
   into userspace, negating a benefit of zero-copy.
 
-* The XS/C API make it difficult to improve internals while
+* The XS/C API makes it difficult to improve internals while
   preserving compatibility.
 
 * Lack of optional type checking.  This may be a blessing in
@@ -161,14 +161,14 @@ Red herrings to ignore when evaluating other runtimes
 -----------------------------------------------------
 
 These don't discount a language or runtime from being
-being used, they're just not interesting.
+used, they're just not interesting.
 
 * Lightweight threading
 
   While lightweight threading implementations are
-  convenient, they tend to be significantly heavier than a
+  convenient, they tend to be significantly heavier than
   pure event-loop systems (or multi-threaded event-loop
-  systems)
+  systems).
 
   Lightweight threading implementations have stack overhead
   and growth typically measured in kilobytes.  The userspace
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 3ecd9100..b45c52e8 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -1,15 +1,15 @@
-#!/usr/bin/env perl
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# n.b. this is invoked via $(PERL) in makefiles
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Stupid script to make HTML from preformatted, utf-8 text versions,
 # only generating links for http(s).  Markdown does too much
 # and requires indentation to output preformatted text.
-use strict;
-use warnings;
+use v5.12;
 use PublicInbox::Linkify;
 use PublicInbox::Hval qw(ascii_html);
-my %xurls;
+my (%xurls, %lei);
 for (qw[lei(1)
         lei-add-external(1)
         lei-add-watch(1)
@@ -42,6 +42,7 @@ for (qw[lei(1)
         lei-q(1)
         lei-rediff(1)
         lei-refresh-mail-sync(1)
+        lei-reindex(1)
         lei-rm(1)
         lei-rm-watch(1)
         lei-security(7)
@@ -49,12 +50,13 @@ for (qw[lei(1)
         lei-tag(1)
         lei-up(1)
         public-inbox.cgi(1)
+        public-inbox-cindex(1)
         public-inbox-clone(1)
         public-inbox-config(5)
-        public-inbox-config(5)
         public-inbox-convert(1)
         public-inbox-daemon(8)
         public-inbox-edit(1)
+        public-inbox-extindex(1)
         public-inbox-fetch(1)
         public-inbox-glossary(7)
         public-inbox-httpd(1)
@@ -63,8 +65,10 @@ for (qw[lei(1)
         public-inbox-init(1)
         public-inbox-learn(1)
         public-inbox-mda(1)
+        public-inbox-netd(1)
         public-inbox-nntpd(1)
         public-inbox-overview(7)
+        public-inbox-pop3d(1)
         public-inbox-purge(1)
         public-inbox-v1-format(5)
         public-inbox-v2-format(5)
@@ -74,8 +78,11 @@ for (qw[lei(1)
         my ($n) = (/([\w\-\.]+)/);
         $xurls{$_} = "$n.html";
         $xurls{$n} = "$n.html";
+        /\Alei-(.+?)\(1\)\z/ and $xurls{"lei $1"} = "$n.html";
 }
 
+$xurls{'lei/store'} = 'lei-store-format.html';
+
 for (qw[make(1) flock(2) setrlimit(2) vfork(2) tmpfs(5) inotify(7) unix(7)
                 syslog(3)]) {
         my ($n, $s) = (/([\w\-]+)\((\d)\)/);
@@ -141,6 +148,8 @@ $xurls{'copydatabase(1)'} =
  'https://manpages.debian.org/stable/xapian-tools/copydatabase.1.en.html';
 $xurls{'xapian-compact(1)'} =
  'https://manpages.debian.org/stable/xapian-tools/xapian-compact.1.en.html';
+$xurls{'xapian-delve(1)'} =
+ 'https://manpages.debian.org/stable/xapian-tools/xapian-delve.1.en.html';
 $xurls{'gzip(1)'} = 'https://manpages.debian.org/stable/gzip/gzip.1.en.html';
 $xurls{'chmod(1)'} =
         'https://manpages.debian.org/stable/coreutils/chmod.1.en.html';
@@ -158,6 +167,9 @@ if ($str =~ /^NAME\n\s+([^\n]+)/sm) {
         if ($title =~ /([\w\.\-]+)/) {
                 delete $xurls{$1};
         }
+        if ($title =~ /\blei-([\w\-]+)\b/) {
+                delete $xurls{"lei $1"};
+        }
 }
 $title = ascii_html($title);
 my $l = PublicInbox::Linkify->new;
diff --git a/HACKING b/HACKING
index 1070d3ff..18ec7420 100644
--- a/HACKING
+++ b/HACKING
@@ -7,7 +7,7 @@ It is archived at: https://public-inbox.org/meta/
 and http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/ (using Tor)
 
 Contributions are email-driven, just like contributing to git
-itself or the Linux kernel; however anonymous and pseudonymous
+itself or the Linux kernel; nevertheless, anonymous and pseudonymous
 contributions will always be welcome.
 
 Please consider our goals in mind:
@@ -15,24 +15,23 @@ Please consider our goals in mind:
         Decentralization, Accessibility, Compatibility, Performance
 
 These goals apply to everyone: users viewing over the web or NNTP,
-sysadmins running public-inbox, and other hackers working public-inbox.
+sysadmins running public-inbox, and other hackers working on public-inbox.
 
 We will reject any feature which advocates or contributes to any
-particular instance of a public-inbox becoming a single point of failure.
+particular instance of public-inbox becoming a single point of failure.
 Things we've considered but rejected include:
 
 * exposing article serial numbers outside of NNTP
 * allowing readers to inject metadata (e.g. votes)
 
 We care about being accessible to folks with vision problems and/or
-lack the computing resources to view so-called "modern" websites.
+lacking the computing resources to view so-called "modern" websites.
 This includes folks on slow connections and ancient browsers which
 may be too difficult to upgrade due to resource demands.
 
 Only depend on Free Software packages which exist in the "main"
-section of Debian "stable" distribution.  That is Debian 9.x
-("stretch") as of this writing, but "oldstable" (8.x, "jessie")
-remains supported for v1 inboxes.
+section of Debian "stable" distribution; but continue to support
+any LTS distros within their support window.
 
 In general, we favor mature and well-tested old things rather than
 the shiny new.
@@ -46,7 +45,7 @@ Just-Ahead-of-Time-compiled C (via Inline::C)
 Do not recurse on user-supplied data.  Neither Perl or C handle
 deep recursion gracefully.  See lib/PublicInbox/SearchThread.pm
 and lib/PublicInbox/MsgIter.pm for examples of non-recursive
-alternatives to previously-recursive algorithms.
+alternatives to previously recursive algorithms.
 
 Performance should be reasonably good for server administrators, too,
 and we will sacrifice features to achieve predictable performance.
@@ -62,8 +61,6 @@ on specific topics, in particular data_structures.txt
 Optional packages for testing and development
 ---------------------------------------------
 
-Optional packages testing and development:
-
 - Plack::Test                      deb: libplack-test-perl
                                    pkg: p5-Plack
                                    rpm: perl-Plack-Test
@@ -108,6 +105,6 @@ Perl notes
 ----------
 
 * \w, \s, \d character classes all match Unicode characters;
-  so write out class ranges (e.g "[0-9]") if you only intend to
+  so write out class ranges (e.g., "[0-9]") if you only intend to
   match ASCII.  Do not use the "/a" (ASCII) modifier, that requires
   Perl 5.14 and we're only depending on 5.10.1 at the moment.
diff --git a/INSTALL b/INSTALL
index 0974028d..aa2b2b84 100644
--- a/INSTALL
+++ b/INSTALL
@@ -1,25 +1,35 @@
-public-inbox (server-side) installation
----------------------------------------
+public-inbox / lei installation
+-------------------------------
 
-This is for folks who want to setup their own public-inbox instance.
-Clients should use normal git-clone/git-fetch, IMAP or NNTP clients
-if they want to import mail into their personal inboxes.
+This is for people who want to run public-inbox on their server or
+lei as a command-line tool.  Any HTTP, IMAP, NNTP, or POP3 client can
+access public-inbox servers, as can git-{clone,fetch} on the HTTP(S)
+endpoint.
 
-As of 2021, public-inbox is packaged by several OS distributions,
+Since our marketing sucks, ease of installation is an important goal
+for this project and we only depend on distro-provided packages.
+
+As of 2024, public-inbox is packaged by several OS distributions,
 listed in alphabetical order: Debian, GNU Guix, NixOS, and Void Linux.
 
 public-inbox is developed on Debian GNU/Linux systems and will
 never depend on packages outside of the "main" component of
-the "stable" distribution, currently Debian 10.x ("buster"),
-but older versions of Debian remain supported.
+the "oldstable" distribution, currently Debian 11.x ("bullseye"),
+but older versions of Debian remain supported (as are newer ones).
 
-Most packages are available in other GNU/Linux distributions
-and FreeBSD.  CentOS 7.x users will likely want newer git and
-Xapian packages for better performance and v2 inbox support:
+Most packages are available in other GNU/Linux distributions,
+Alpine Linux, FreeBSD, NetBSD, OpenBSD, and DragonflyBSD.
+CentOS 7.x users will likely want newer git and Xapian for
+better performance and v2 inbox support:
 https://public-inbox.org/meta/20210421151308.yz5hzkgm75klunpe@nitro.local/
 
-TODO: this still needs to be documented better,
-also see the scripts/ and sa_config/ directories in the source tree
+As of 2.0, install/deps.perl makes it easier to install target
+dependencies needed for certain features.  See install/README in
+the source tree for more info.
+
+Also see sa_config/ directories in the source tree for recommended
+SpamAssassin configuration examples if using public-inbox-mda or
+public-inbox-watch.
 
 Requirements
 ------------
@@ -28,7 +38,7 @@ public-inbox requires a number of other packages to access its full
 functionality.  The core tools are, of course:
 
 * Git (1.8.0+, 2.6+ for writing v2 inboxes)
-* Perl 5.10.1+
+* Perl 5.12.0+
 * DBD::SQLite (needed for IMAP, NNTP, message threading, and v2 inboxes)
 
 To accept incoming mail into a public inbox, you'll likely want:
@@ -43,61 +53,59 @@ Beyond that, there is one non-standard Perl package required:
                                    rpm: perl-URI
                                    (for HTML/Atom generation)
 
-Plack and Date::Parse are optional as of public-inbox v1.3.0,
-but required for older releases:
-
-* Plack                            deb: libplack-perl
-                                   pkg: p5-Plack
-                                   rpm: perl-Plack, perl-Plack-Test,
-                                   (for HTML/Atom generation)
-
-- Date::Parse                      deb: libtimedate-perl
-                                   pkg: p5-TimeDate
-                                   rpm: perl-TimeDate
-                                   (for broken, mostly historical emails)
-
 Where "deb" indicates package names for Debian-derived distributions,
-"pkg" is for the FreeBSD package (maybe other common BSDs, too), and
-"rpm" is for RPM-based distributions (only known to work on Fedora).
+"pkg" is for the FreeBSD package (and some other common BSDs, too),
+"pkgin" for NetBSD, "apk" for Alpine Linux and "rpm" is for RPM-based
+distributions (only known to work on Fedora).
 
-Numerous optional modules are likely to be useful as well:
+Most users will likely also want the following:
 
 - DBD::SQLite                      deb: libdbd-sqlite3-perl
                                    pkg: p5-DBD-SQLite
                                    rpm: perl-DBD-SQLite
                                    (for v2, IMAP, NNTP, or gzipped mboxes)
 
-- Search::Xapian or Xapian(.pm)    deb: libsearch-xapian-perl
-                                   pkg: p5-Search-Xapian OR p5-Xapian
+- Xapian(.pm) (or Search::Xapian)  deb: libsearch-xapian-perl
+                                   pkg: p5-Xapian (FreeBSD, NetBSD)
+                                        xapian-bindings-perl (OpenBSD)
                                    rpm: perl-Search-Xapian
-                                   (HTTP and IMAP search)
+                                   (required for lei; HTTP and IMAP search)
+
+Other modules might be useful as well, depending on your use case and
+preferences:
+
+- Plack                            deb: libplack-perl
+                                   pkg: p5-Plack
+                                   rpm: perl-Plack, perl-Plack-Test,
+                                   (for WWW interface, public-inbox-httpd(1))
 
 - Inline::C                        deb: libinline-c-perl
-                                   pkg: pkg-Inline-C
+                                   pkg: p5-Inline-C
                                    rpm: perl-Inline (or perl-Inline-C)
-                                   (speeds up process spawning on Linux,
+                                   (required for lei on *BSD;
+                                    speeds up process spawning on Linux,
                                     see public-inbox-daemon(8))
 
 - Email::Address::XS               deb: libemail-address-xs-perl
-                                   pkg: pkg-Email-Address-XS
+                                   pkg: p5-Email-Address-XS
                                    (correct parsing of tricky email
-                                    addresses, phrases and comments,
-                                    required for IMAP)
+                                    addresses, phrases and comments)
 
 - Parse::RecDescent                deb: libparse-recdescent-perl
                                    pkg: p5-Parse-RecDescent
                                    rpm: perl-ParseRecDescent
-                                   (optional, for public-inbox-imapd(1))
+                                   (for public-inbox-imapd(1))
 
 - Mail::IMAPClient                 deb: libmail-imapclient-perl
                                    pkg: p5-Mail-IMAPClient
                                    rpm: perl-Mail-IMAPClient
-                                   (optional for lei and public-inbox-watch)
+                                   (only for lei and public-inbox-watch
+                                    when reading from IMAP)
 
 - BSD::Resource                    deb: libbsd-resource-perl
                                    pkg: p5-BSD-Resource
                                    rpm: perl-BSD-Resource
-                                   (optional, for PSGI limiters
+                                   (only for PSGI limiters,
                                     see public-inbox-config(5))
 
 - Plack::Middleware::ReverseProxy  deb: libplack-middleware-reverseproxy-perl
@@ -109,32 +117,43 @@ Numerous optional modules are likely to be useful as well:
 * highlight                        deb: libhighlight-perl
                                    (for syntax highlighting with coderepo)
 
-* xapian-compact (tool)            deb: xapian-tools
+* xapian-tools                     deb: xapian-tools
                                    pkg: xapian-core
+                                   pkgin: xapian
                                    rpm: xapian-core
-                                   (optional, for public-inbox-compact(1))
+                                   (for public-inbox-compact(1) and
+                                    public-inbox-cindex(1))
+
+* Xapian development files         deb: libxapian-dev
+                                   pkg: xapian-core
+                                   pkgin: xapian
+                                   apk: xapian-core-dev
+                                   rpm: xapian-core-devel / xapian14-core-libs
+                                   (for public-inbox-cindex(1) and future
+                                    performance enhancements)
 
 * curl (tool)                      deb, pkg, rpm: curl
-                                   (for HTTP(S) externals with curl)
+                                   (for lei HTTP(S) externals with curl and
+                                    public-inbox-clone(1))
 
 - Linux::Inotify2                  deb: liblinux-inotify2-perl
                                    rpm: perl-Linux-Inotify2
                                    (for lei, public-inbox-watch and -imapd
-                                    on Linux)
+                                    on Linux; not required as of 2.0))
 
 - IO::KQueue                       pkg: p5-IO-KQueue
                                    (for lei, public-inbox-watch and -imapd
-                                    on *BSDs)
+                                    on *BSDs only)
 
 - Net::Server                      deb: libnet-server-perl
-                                   pkg: pkg-Net-Server
+                                   pkg: p5-Net-Server
                                    rpm: perl-Net-Server
                                    (for HTTP/IMAP/NNTP background daemons,
                                     not needed as systemd services or
                                     foreground servers)
 
 The following module is typically pulled in by dependencies listed
-above, so there is no need to explicitly install them:
+above, so there is no need to explicitly install it:
 
 - DBI                              deb: libdbi-perl
                                    pkg: p5-DBI
@@ -155,6 +174,11 @@ Uncommonly needed modules (see HACKING for development-only modules):
                                    pkg: p5-Crypt-CBC
                                    (for PublicInbox::Unsubscribe (rarely used))
 
+- Date::Parse                      deb: libtimedate-perl
+                                   pkg: p5-TimeDate
+                                   rpm: perl-TimeDate
+                                   (for broken, mostly historical emails)
+
 standard MakeMaker installation (Perl)
 --------------------------------------
 
@@ -200,15 +224,15 @@ library.  Debian-based distros put them in "libperl5.$MINOR" or
 "perl-modules-5.$MINOR"; and FreeBSD puts them in "perl5".
 RPM-based distros split them out into separate packages:
 
+* autodie                          rpm: perl-autodie
 * Digest::SHA                      rpm: perl-Digest-SHA
 * Data::Dumper                     rpm: perl-Data-Dumper
-* Encode                           rpm: perl-Encode
 * IO::Compress                     rpm: perl-IO-Compress
-* Storable                         rpm: perl-Storable
+* Sys::Syslog                      rpm: perl-Sys-Syslog
 * Text::ParseWords                 rpm: perl-Text-Parsewords
 
 Copyright
 ---------
 
-Copyright 2013-2021 all contributors <meta@public-inbox.org>
+Copyright all contributors <meta@public-inbox.org>
 License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
diff --git a/MANIFEST b/MANIFEST
index 607a4c5b..4c974338 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -14,6 +14,8 @@ Documentation/RelNotes/v1.6.0.eml
 Documentation/RelNotes/v1.6.1.eml
 Documentation/RelNotes/v1.7.0.eml
 Documentation/RelNotes/v1.8.0.eml
+Documentation/RelNotes/v1.9.0.eml
+Documentation/RelNotes/v2.0.0.wip
 Documentation/clients.txt
 Documentation/common.perl
 Documentation/dc-dlvr-spam-flow.txt
@@ -21,7 +23,6 @@ Documentation/design_notes.txt
 Documentation/design_www.txt
 Documentation/flow.ge
 Documentation/flow.txt
-Documentation/hosted.txt
 Documentation/include.mk
 Documentation/lei-add-external.pod
 Documentation/lei-add-watch.pod
@@ -55,6 +56,7 @@ Documentation/lei-p2q.pod
 Documentation/lei-q.pod
 Documentation/lei-rediff.pod
 Documentation/lei-refresh-mail-sync.pod
+Documentation/lei-reindex.pod
 Documentation/lei-rm-watch.pod
 Documentation/lei-rm.pod
 Documentation/lei-security.pod
@@ -65,6 +67,7 @@ Documentation/lei.pod
 Documentation/lei_design_notes.txt
 Documentation/marketing.txt
 Documentation/mknews.perl
+Documentation/public-inbox-cindex.pod
 Documentation/public-inbox-clone.pod
 Documentation/public-inbox-compact.pod
 Documentation/public-inbox-config.pod
@@ -84,6 +87,7 @@ Documentation/public-inbox-mda.pod
 Documentation/public-inbox-netd.pod
 Documentation/public-inbox-nntpd.pod
 Documentation/public-inbox-overview.pod
+Documentation/public-inbox-pop3d.pod
 Documentation/public-inbox-purge.pod
 Documentation/public-inbox-tuning.pod
 Documentation/public-inbox-v1-format.pod
@@ -96,6 +100,7 @@ Documentation/standards.perl
 Documentation/technical/data_structures.txt
 Documentation/technical/ds.txt
 Documentation/technical/memory.txt
+Documentation/technical/weird-stuff.txt
 Documentation/technical/whyperl.txt
 Documentation/txt2pre
 HACKING
@@ -107,8 +112,7 @@ TODO
 certs/.gitignore
 certs/create-certs.perl
 ci/README
-ci/deps.perl
-ci/profiles.sh
+ci/profiles.perl
 ci/run.sh
 contrib/completion/lei-completion.bash
 contrib/css/216dark.css
@@ -117,13 +121,10 @@ contrib/css/README
 contrib/selinux/el7/publicinbox.fc
 contrib/selinux/el7/publicinbox.te
 devel/README
-devel/syscall-list
+devel/longest-tests
+devel/sysdefs-list
 examples/README
 examples/README.unsubscribe
-examples/apache2_cgi.conf
-examples/apache2_perl.conf
-examples/apache2_perl_old.conf
-examples/cgi-webrick.rb
 examples/cgit-commit-filter.lua
 examples/cgit-wwwhighlight-filter.lua
 examples/cgit.psgi
@@ -136,13 +137,12 @@ examples/nginx_proxy
 examples/public-inbox-config
 examples/public-inbox-httpd.socket
 examples/public-inbox-httpd@.service
-examples/public-inbox-imap-onion.socket
 examples/public-inbox-imapd.socket
 examples/public-inbox-imapd@.service
-examples/public-inbox-imaps.socket
+examples/public-inbox-netd.socket
+examples/public-inbox-netd@.service
 examples/public-inbox-nntpd.socket
 examples/public-inbox-nntpd@.service
-examples/public-inbox-nntps.socket
 examples/public-inbox-watch.service
 examples/public-inbox.psgi
 examples/unsubscribe-milter.socket
@@ -152,22 +152,34 @@ examples/unsubscribe-psgi@.service
 examples/unsubscribe.milter
 examples/unsubscribe.psgi
 examples/varnish-4.vcl
+install/README
+install/deps.perl
+install/os.perl
 lei.sh
 lib/PublicInbox/Address.pm
 lib/PublicInbox/AddressPP.pm
 lib/PublicInbox/Admin.pm
 lib/PublicInbox/AdminEdit.pm
 lib/PublicInbox/AltId.pm
+lib/PublicInbox/Aspawn.pm
 lib/PublicInbox/AutoReap.pm
 lib/PublicInbox/Cgit.pm
+lib/PublicInbox/CidxComm.pm
+lib/PublicInbox/CidxLogP.pm
+lib/PublicInbox/CidxXapHelperAux.pm
 lib/PublicInbox/CmdIPC4.pm
+lib/PublicInbox/CodeSearch.pm
+lib/PublicInbox/CodeSearchIdx.pm
+lib/PublicInbox/Compat.pm
 lib/PublicInbox/CompressNoop.pm
 lib/PublicInbox/Config.pm
 lib/PublicInbox/ConfigIter.pm
+lib/PublicInbox/ContentDigestDbg.pm
 lib/PublicInbox/ContentHash.pm
 lib/PublicInbox/DS.pm
 lib/PublicInbox/DSKQXS.pm
 lib/PublicInbox/DSPoll.pm
+lib/PublicInbox/DSdeflate.pm
 lib/PublicInbox/Daemon.pm
 lib/PublicInbox/DirIdle.pm
 lib/PublicInbox/DummyInbox.pm
@@ -175,6 +187,7 @@ lib/PublicInbox/EOFpipe.pm
 lib/PublicInbox/Emergency.pm
 lib/PublicInbox/Eml.pm
 lib/PublicInbox/EmlContentFoo.pm
+lib/PublicInbox/Epoll.pm
 lib/PublicInbox/ExtMsg.pm
 lib/PublicInbox/ExtSearch.pm
 lib/PublicInbox/ExtSearchIdx.pm
@@ -190,7 +203,7 @@ lib/PublicInbox/Filter/SubjectTag.pm
 lib/PublicInbox/Filter/Vger.pm
 lib/PublicInbox/Gcf2.pm
 lib/PublicInbox/Gcf2Client.pm
-lib/PublicInbox/GetlineBody.pm
+lib/PublicInbox/GetlineResponse.pm
 lib/PublicInbox/Git.pm
 lib/PublicInbox/GitAsyncCat.pm
 lib/PublicInbox/GitCredential.pm
@@ -198,22 +211,25 @@ lib/PublicInbox/GitHTTPBackend.pm
 lib/PublicInbox/GzipFilter.pm
 lib/PublicInbox/HTTP.pm
 lib/PublicInbox/HTTPD.pm
-lib/PublicInbox/HTTPD/Async.pm
 lib/PublicInbox/HlMod.pm
 lib/PublicInbox/Hval.pm
 lib/PublicInbox/IMAP.pm
 lib/PublicInbox/IMAPClient.pm
 lib/PublicInbox/IMAPD.pm
 lib/PublicInbox/IMAPTracker.pm
-lib/PublicInbox/IMAPdeflate.pm
 lib/PublicInbox/IMAPsearchqp.pm
+lib/PublicInbox/IO.pm
 lib/PublicInbox/IPC.pm
 lib/PublicInbox/IdxStack.pm
 lib/PublicInbox/Import.pm
 lib/PublicInbox/In2Tie.pm
+lib/PublicInbox/In3Event.pm
+lib/PublicInbox/In3Watch.pm
 lib/PublicInbox/Inbox.pm
 lib/PublicInbox/InboxIdle.pm
 lib/PublicInbox/InboxWritable.pm
+lib/PublicInbox/Inotify.pm
+lib/PublicInbox/Inotify3.pm
 lib/PublicInbox/InputPipe.pm
 lib/PublicInbox/Isearch.pm
 lib/PublicInbox/KQNotify.pm
@@ -259,6 +275,7 @@ lib/PublicInbox/LeiPmdir.pm
 lib/PublicInbox/LeiQuery.pm
 lib/PublicInbox/LeiRediff.pm
 lib/PublicInbox/LeiRefreshMailSync.pm
+lib/PublicInbox/LeiReindex.pm
 lib/PublicInbox/LeiRemote.pm
 lib/PublicInbox/LeiRm.pm
 lib/PublicInbox/LeiRmWatch.pm
@@ -274,18 +291,22 @@ lib/PublicInbox/LeiUp.pm
 lib/PublicInbox/LeiViewText.pm
 lib/PublicInbox/LeiWatch.pm
 lib/PublicInbox/LeiXSearch.pm
+lib/PublicInbox/Limiter.pm
 lib/PublicInbox/Linkify.pm
 lib/PublicInbox/Listener.pm
 lib/PublicInbox/Lock.pm
 lib/PublicInbox/MDA.pm
+lib/PublicInbox/MHreader.pm
 lib/PublicInbox/MID.pm
 lib/PublicInbox/MIME.pm
+lib/PublicInbox/MailDiff.pm
 lib/PublicInbox/ManifestJsGz.pm
 lib/PublicInbox/Mbox.pm
 lib/PublicInbox/MboxGz.pm
 lib/PublicInbox/MboxLock.pm
 lib/PublicInbox/MboxReader.pm
 lib/PublicInbox/MdirReader.pm
+lib/PublicInbox/MdirSort.pm
 lib/PublicInbox/MiscIdx.pm
 lib/PublicInbox/MiscSearch.pm
 lib/PublicInbox/MsgIter.pm
@@ -294,7 +315,6 @@ lib/PublicInbox/Msgmap.pm
 lib/PublicInbox/MultiGit.pm
 lib/PublicInbox/NNTP.pm
 lib/PublicInbox/NNTPD.pm
-lib/PublicInbox/NNTPdeflate.pm
 lib/PublicInbox/NetNNTPSocks.pm
 lib/PublicInbox/NetReader.pm
 lib/PublicInbox/NetWriter.pm
@@ -302,10 +322,16 @@ lib/PublicInbox/NewsWWW.pm
 lib/PublicInbox/OnDestroy.pm
 lib/PublicInbox/Over.pm
 lib/PublicInbox/OverIdx.pm
+lib/PublicInbox/POP3.pm
+lib/PublicInbox/POP3D.pm
 lib/PublicInbox/PktOp.pm
-lib/PublicInbox/ProcessPipe.pm
 lib/PublicInbox/Qspawn.pm
 lib/PublicInbox/Reply.pm
+lib/PublicInbox/RepoAtom.pm
+lib/PublicInbox/RepoList.pm
+lib/PublicInbox/RepoSnapshot.pm
+lib/PublicInbox/RepoTree.pm
+lib/PublicInbox/SHA.pm
 lib/PublicInbox/SaPlugin/ListMirror.pm
 lib/PublicInbox/SaPlugin/ListMirror.pod
 lib/PublicInbox/Search.pm
@@ -314,6 +340,7 @@ lib/PublicInbox/SearchIdxShard.pm
 lib/PublicInbox/SearchQuery.pm
 lib/PublicInbox/SearchThread.pm
 lib/PublicInbox/SearchView.pm
+lib/PublicInbox/Select.pm
 lib/PublicInbox/SharedKV.pm
 lib/PublicInbox/Sigfd.pm
 lib/PublicInbox/Smsg.pm
@@ -324,16 +351,19 @@ lib/PublicInbox/Spawn.pm
 lib/PublicInbox/SpawnPP.pm
 lib/PublicInbox/Syscall.pm
 lib/PublicInbox/TLS.pm
+lib/PublicInbox/TailNotify.pm
 lib/PublicInbox/TestCommon.pm
 lib/PublicInbox/Tmpfile.pm
 lib/PublicInbox/URIimap.pm
 lib/PublicInbox/URInntps.pm
+lib/PublicInbox/Umask.pm
 lib/PublicInbox/Unsubscribe.pm
 lib/PublicInbox/UserContent.pm
 lib/PublicInbox/V2Writable.pm
 lib/PublicInbox/View.pm
 lib/PublicInbox/ViewDiff.pm
 lib/PublicInbox/ViewVCS.pm
+lib/PublicInbox/WQBlocked.pm
 lib/PublicInbox/WQWorker.pm
 lib/PublicInbox/WWW.pm
 lib/PublicInbox/WWW.pod
@@ -341,18 +371,27 @@ lib/PublicInbox/Watch.pm
 lib/PublicInbox/WwwAltId.pm
 lib/PublicInbox/WwwAtomStream.pm
 lib/PublicInbox/WwwAttach.pm
+lib/PublicInbox/WwwCoderepo.pm
 lib/PublicInbox/WwwHighlight.pm
 lib/PublicInbox/WwwListing.pm
 lib/PublicInbox/WwwStatic.pm
 lib/PublicInbox/WwwStream.pm
 lib/PublicInbox/WwwText.pm
+lib/PublicInbox/WwwTopics.pm
+lib/PublicInbox/XapClient.pm
+lib/PublicInbox/XapHelper.pm
+lib/PublicInbox/XapHelperCxx.pm
 lib/PublicInbox/Xapcmd.pm
 lib/PublicInbox/gcf2_libgit2.h
+lib/PublicInbox/xap_helper.h
+lib/PublicInbox/xh_cidx.h
+lib/PublicInbox/xh_mset.h
 sa_config/Makefile
 sa_config/README
 sa_config/root/etc/spamassassin/public-inbox.pre
 sa_config/user/.spamassassin/user_prefs
 script/lei
+script/public-inbox-cindex
 script/public-inbox-clone
 script/public-inbox-compact
 script/public-inbox-convert
@@ -367,6 +406,7 @@ script/public-inbox-learn
 script/public-inbox-mda
 script/public-inbox-netd
 script/public-inbox-nntpd
+script/public-inbox-pop3d
 script/public-inbox-purge
 script/public-inbox-watch
 script/public-inbox-xcpdb
@@ -386,10 +426,17 @@ scripts/xhdr-num2mid
 t/.gitconfig
 t/address.t
 t/admin.t
+t/alt.psgi
 t/altid.t
 t/altid_v2.t
 t/cgi.t
 t/check-www-inbox.perl
+t/cindex-join.t
+t/cindex.t
+t/clone-coderepo-puh1.sh
+t/clone-coderepo-puh2.sh
+t/clone-coderepo.psgi
+t/clone-coderepo.t
 t/cmd_ipc.t
 t/config.t
 t/config_limiter.t
@@ -397,6 +444,7 @@ t/content_hash.t
 t/convert-compact.t
 t/data-gen/.gitignore
 t/data/0001.patch
+t/data/attached-mbox-with-utf8.eml
 t/data/binary.patch
 t/data/message_embed.eml
 t/dir_idle.t
@@ -450,6 +498,8 @@ t/index-git-times.t
 t/indexlevels-mirror-v1.t
 t/indexlevels-mirror.t
 t/init.t
+t/inotify3.t
+t/io.t
 t/ipc.t
 t/iso-2202-jp.eml
 t/kqnotify.t
@@ -466,6 +516,7 @@ t/lei-import.t
 t/lei-index.t
 t/lei-inspect.t
 t/lei-lcat.t
+t/lei-mail-diff.t
 t/lei-mirror.psgi
 t/lei-mirror.t
 t/lei-p2q.t
@@ -474,7 +525,9 @@ t/lei-q-remote-import.t
 t/lei-q-save.t
 t/lei-q-thread.t
 t/lei-refresh-mail-sync.t
+t/lei-reindex.t
 t/lei-sigpipe.t
+t/lei-store-fail.t
 t/lei-tag.t
 t/lei-up.t
 t/lei-watch.t
@@ -496,6 +549,7 @@ t/mda-mime.eml
 t/mda.t
 t/mda_filter_rubylang.t
 t/mdir_reader.t
+t/mh_reader.t
 t/mid.t
 t/mime.t
 t/miscsearch.t
@@ -519,6 +573,9 @@ t/plack-2-txt-bodies.eml
 t/plack-attached-patch.eml
 t/plack-qp.eml
 t/plack.t
+t/pop3d-limit.t
+t/pop3d.t
+t/pop3d_lock.t
 t/precheck.t
 t/psgi_attach.eml
 t/psgi_attach.t
@@ -537,10 +594,11 @@ t/reindex-time-range.t
 t/rename_noreplace.t
 t/replace.t
 t/reply.t
-t/run.perl
 t/search-amsg.eml
 t/search-thr-index.t
 t/search.t
+t/select.t
+t/sha.t
 t/shared_kv.t
 t/sigfd.t
 t/solve/0001-simple-mod.patch
@@ -549,6 +607,7 @@ t/solve/bare.patch
 t/solver_git.t
 t/spamcheck_spamc.t
 t/spawn.t
+t/tail_notify.t
 t/thread-cycle.t
 t/thread-index-gap.t
 t/time.t
@@ -569,15 +628,18 @@ t/watch_filter_rubylang.t
 t/watch_imap.t
 t/watch_maildir.t
 t/watch_maildir_v2.t
+t/watch_mh.t
 t/watch_multiple_headers.t
 t/www_altid.t
 t/www_listing.t
 t/www_static.t
 t/x-unknown-alpine.eml
+t/xap_helper.t
 t/xcpdb-reshard.t
 version-gen.perl
+xt/check-debris.t
+xt/check-run.t
 xt/cmp-msgstr.t
-xt/cmp-msgview.t
 xt/create-many-inboxes.t
 xt/eml_check_limits.t
 xt/eml_octet-stream.t
@@ -590,6 +652,7 @@ xt/lei-auth-fail.t
 xt/lei-onion-convert.t
 xt/mem-imapd-tls.t
 xt/mem-msgview.t
+xt/mem-nntpd-tls.t
 xt/msgtime_cmp.t
 xt/net_nntp_socks.t
 xt/net_writer-imap.t
@@ -597,7 +660,7 @@ xt/nntpd-validate.t
 xt/over-fsck.perl
 xt/perf-msgview.t
 xt/perf-nntpd.t
-xt/perf-obfuscate.t
 xt/perf-threading.t
+xt/pop3d-mpop.t
 xt/solver.t
 xt/stress-sharedkv.t
diff --git a/Makefile.PL b/Makefile.PL
index 848eb702..2b2e6b18 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -11,7 +11,8 @@ my $v = {};
 my $t = {};
 
 # do not sort
-my @RELEASES = qw(v1.8.0 v1.7.0 v1.6.1 v1.6.0 v1.5.0 v1.4.0 v1.3.0 v1.2.0
+my @RELEASES = qw(v1.9.0
+        v1.8.0 v1.7.0 v1.6.1 v1.6.0 v1.5.0 v1.4.0 v1.3.0 v1.2.0
         v1.1.0-pre1 v1.0.0);
 
 $v->{news_deps} = [ map { "Documentation/RelNotes/$_.eml" } @RELEASES ];
@@ -33,6 +34,19 @@ my @syn = (@EXE_FILES, grep(m!^lib/.*\.pm$!, @manifest), @scripts);
 @syn = grep(!/SaPlugin/, @syn) if !eval { require Mail::SpamAssasin };
 $v->{syn_files} = \@syn;
 $v->{my_syntax} = [map { "$_.syntax" } @syn];
+my %native = (
+        XapHelperCxx => [ qw(xh_cidx.h xh_mset.h xap_helper.h) ],
+);
+my @ck_build;
+for my $m (sort keys %native) {
+        my $hdr = $native{$m};
+        my @dep = map { "lib/PublicInbox/$_" } ("$m.pm", @$hdr);
+        $t->{"$m.check_build: @dep"} = [ "\$(PERL) -w -I lib ".
+                "-MPublicInbox::$m -e PublicInbox::${m}::check_build" ];
+        push @ck_build, "$m.check_build";
+}
+$t->{"check-build: @ck_build"} = [];
+
 my @no_pod;
 $v->{-m1} = [ map {
                 my $x = (split('/'))[-1];
@@ -52,7 +66,8 @@ $v->{-m1} = [ map {
         lei-import lei-index lei-init lei-inspect lei-lcat
         lei-ls-external lei-ls-label lei-ls-mail-source lei-ls-mail-sync
         lei-ls-search lei-ls-watch lei-mail-diff lei-p2q lei-q
-        lei-rediff lei-refresh-mail-sync lei-rm lei-rm-watch lei-tag
+        lei-rediff lei-refresh-mail-sync lei-reindex
+        lei-rm lei-rm-watch lei-tag
         lei-up)];
 $v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
                 public-inbox-v2-format public-inbox-extindex-format
@@ -79,7 +94,7 @@ for my $i (@sections) {
                 $t->{"Documentation/$m.html : $txt"} = [ "\$(txt2pre) <$txt",
                                                         "touch -r $txt \$@" ];
                 $t->{".$m.cols : $m.$i"} = [
-                        "\@echo CHECK80 $m.$i;".
+                        "\@echo CHECK80 $m.$i; LC_ALL=C LANG=C ".
                         "COLUMNS=80 \$(MAN) ./$m.$i | \$(check_man)",
                         '>$@' ];
                 $t->{".$m.lexgrog: $m.$i"} = [
@@ -124,14 +139,14 @@ my %man3 = map {; # semi-colon tells Perl this is a BLOCK (and not EXPR)
         "lib/PublicInbox/$_" => "blib/man3/PublicInbox::$mod.\$(MAN3EXT)"
 } qw(Git.pm Import.pm WWW.pod SaPlugin/ListMirror.pod);
 my $warn_no_pod = @no_pod ? "\n\t\@echo W: missing .pod: @no_pod\n" : '';
-chomp(my $lexgrog = `which lexgrog 2>/dev/null`);
+chomp(my $lexgrog = `command -v lexgrog 2>/dev/null`);
 my $check_lexgrog = $lexgrog ? 'check-lexgrog' : '';
 
 WriteMakefile(
         NAME => 'PublicInbox', # n.b. camel-case is not our choice
 
         # XXX drop "PENDING" in .pod before updating this!
-        VERSION => '1.9.0.PENDING',
+        VERSION => '2.0.0.PENDING',
 
         AUTHOR => 'public-inbox hackers <meta@public-inbox.org>',
         ABSTRACT => 'an "archives first" approach to mailing lists',
@@ -148,14 +163,14 @@ WriteMakefile(
 
                 # perl-modules-5.xx or libperl5.xx in Debian-based
                 # part of "perl5" on FreeBSD
-                'Compress::Raw::Zlib' => 0,
-                'Compress::Zlib' => 0,
-                'Data::Dumper' => 0,
+                'autodie' => 0, # rpm: perl-autodie
+                'Compress::Raw::Zlib' => 0, # rpm: perl-Compress-Raw-Zlib
+                'Compress::Zlib' => 0, # rpm: perl-IO-Compress
+                'Data::Dumper' => 0, # rpm: perl-Data-Dumper
                 'Digest::SHA' => 0, # rpm: perl-Digest-SHA
-                'Encode' => 2.35, # 2.35 shipped with 5.10.1
-                'IO::Compress::Gzip' => 0,
-                'IO::Uncompress::Gunzip' => 0,
-                'Storable' => 0, # rpm: perl-Storable
+                'IO::Compress::Gzip' => 0, # rpm: perl-IO-Compress
+                'IO::Uncompress::Gunzip' => 0, # rpm: perl-IO-Compress
+                'Sys::Syslog' => 0, # rpm: perl-Sys-Syslog
                 'Text::ParseWords' => 0, # rpm: perl-Text-ParseWords
 
                 # Plack is needed for public-inbox-httpd and PublicInbox::WWW
@@ -167,11 +182,16 @@ WriteMakefile(
                 # users to install them.  See INSTALL
 
                 # All Perl installs I know about have these, but RH-based
-                # distros make them separate even though 'perl' pulls them in
+                # distros can separate these even if `perl' depends on them:
+                'constant' => 0, # rpm: perl-constant
+                'Encode' => 2.35, # rpm: perl-Encode # 2.35 shipped with 5.10.1
                 'File::Path' => 0,
                 'File::Temp' => '0.19', # for ->tmpdir support
-                'Getopt::Long' => 0,
-                'Exporter' => 0,
+                'Getopt::Long' => 0, # rpm: perl-Getopt-Long
+                'Exporter' => 0, # rpm: perl-Exporter
+                'IO::Poll' => 0,
+                'Storable' => 0, # rpm: perl-Storable
+                'Time::HiRes' => 0, # rpm: perl-Time-HiRes
                 # ExtUtils::MakeMaker # this file won't run w/o it...
         },
         MAN3PODS => \%man3,
@@ -192,7 +212,8 @@ WriteMakefile(
 );
 
 sub MY::postamble {
-        my $N = (`{ getconf _NPROCESSORS_ONLN || nproc; } 2>/dev/null` || 1);
+        my $N = (`{ getconf _NPROCESSORS_ONLN ||
+                getconf NPROCESSORS_ONLN; } 2>/dev/null` || 1);
         $N += 1; # account for sleeps in some tests (and makes an IV)
         <<EOF;
 PROVE = prove
@@ -205,7 +226,7 @@ $VARS
 -include Documentation/include.mk
 $TGTS
 
-check-man :: $check_lexgrog$warn_no_pod
+check-man : $check_lexgrog$warn_no_pod
 
 # syntax checks are currently GNU make only:
 %.syntax :: %
@@ -223,19 +244,24 @@ check-manifest : MANIFEST
         \$(check_manifest)
 
 # the traditional way running per-*.t processes:
-check-each :: pure_all
+check-each : pure_all
         \$(EATMYDATA) \$(PROVE) --state=save -bvw -j\$(N)
         -@\$(check_manifest)
 
-# lightly-tested way to run tests, relies "--state=save" in check-each
-# for best performance
-check-run :: pure_all check-man
-        \$(EATMYDATA) \$(PROVE) -bvw t/run.perl :: -j\$(N)
+# check-run relies "--state=save" in check-each for best performance
+check-run : check-man
+
+# n.b. while `-' isn't specified as an allowed make(1posix) macro name,
+# GNU and *BSD both allow it.
+check-run_T_ARGS = -j\$(N)
+
+check-debris check-run : pure_all
+        \$(EATMYDATA) \$(PROVE) -bvw xt/\$@.t :: \$(\$\@_T_ARGS)
         -@\$(check_manifest)
 
-check :: check-each
+check : check-each
 
-lib/PublicInbox/UserContent.pm :: contrib/css/216dark.css
+lib/PublicInbox/UserContent.pm : contrib/css/216dark.css
         \$(PERL) -I lib \$@ \$?
 
 # Ensure new .pm files will always be installed by updating
@@ -251,19 +277,25 @@ prefix = \$(HOME)
 bindir = \$(prefix)/bin
 symlink-install : lib/PublicInbox.pm
         mkdir -p \$(bindir)
-        lei=\$\$(realpath lei.sh) && cd \$(bindir) && \\
+        lei="\$(PWD)/lei.sh" && cd \$(bindir) && \\
         for x in \$(EXE_FILES); do \\
                 ln -sf "\$\$lei" \$\$(basename "\$\$x"); \\
         done
 
-pure_all :: lib/PublicInbox.pm
+pm_to_blib : lib/PublicInbox.pm
 lib/PublicInbox.pm : FORCE
         VERSION=\$(VERSION) \$(PERL) -w ./version-gen.perl
 
-update-copyrights :
-        \@case '\$(GNULIB_PATH)' in '') echo >&2 GNULIB_PATH unset; false;; esac
-        git ls-files | UPDATE_COPYRIGHT_HOLDER='all contributors' \\
-                UPDATE_COPYRIGHT_USE_INTERVALS=2 \\
-                xargs \$(GNULIB_PATH)/build-aux/update-copyright
+XH_TESTS = t/xap_helper.t t/cindex.t
+
+test-asan : pure_all
+        TEST_XH_CXX_ONLY=1 CXXFLAGS='-Wall -ggdb3 -fsanitize=address' \\
+                prove -bvw \$(XH_TESTS)
+
+VG_OPT = -v --trace-children=yes --track-fds=yes
+VG_OPT += --leak-check=yes --track-origins=yes
+test-valgrind : pure_all
+        TEST_XH_CXX_ONLY=1 VALGRIND="valgrind \$(VG_OPT)" \\
+                prove -bvw \$(XH_TESTS)
 EOF
 }
diff --git a/README b/README
index 364ef7e0..a9aa0e86 100644
--- a/README
+++ b/README
@@ -3,7 +3,7 @@ public-inbox - an "archives first" approach to mailing lists
 
 public-inbox implements the sharing of an email inbox via git to
 complement or replace traditional mailing lists.  Readers may
-read via NNTP, IMAP, Atom feeds or HTML archives.
+read via NNTP, IMAP, POP3, Atom feeds or HTML archives.
 
 public-inbox spawned around three main ideas:
 
@@ -17,7 +17,7 @@ public-inbox spawned around three main ideas:
   communication.  Users may have broken graphics drivers, limited
   eyesight, or be unable to afford modern hardware.
 
-public-inbox aims to be easy-to-deploy and manage; encouraging projects
+public-inbox aims to be easy to deploy and manage, encouraging projects
 to run their own instances with minimal overhead.
 
 Implementation
@@ -27,7 +27,7 @@ public-inbox stores mail in git repositories as documented
 in https://public-inbox.org/public-inbox-v2-format.txt and
 https://public-inbox.org/public-inbox-v1-format.txt
 
-By storing (and optionally) exposing an inbox via git, it is
+By storing and (optionally) exposing an inbox via git, it is
 fast and efficient to host and mirror public-inboxes.
 
 Traditional mailing lists use the "push" model.  For readers,
@@ -38,15 +38,15 @@ headers.  List server admins are also burdened with delivery
 failures.
 
 public-inbox uses the "pull" model.  Casual readers may
-follow the list via NNTP, IMAP, Atom feed or HTML archives.
+follow the list via NNTP, IMAP, POP3, Atom feed or HTML archives.
 
 If a reader loses interest, they simply stop following.
 
-Since we use git, mirrors are easy-to-setup, and lists are
-easy-to-relocate to different mail addresses without losing
+Since we use git, mirrors are easy to set up, and lists are
+easy to relocate to different mail addresses without losing
 or splitting archives.
 
-_Anybody_ may also setup a delivery-only mailing list server to
+_Anybody_ may also set up a delivery-only mailing list server to
 replay a public-inbox git archive to subscribers via SMTP.
 
 Features
@@ -56,7 +56,7 @@ Features
 
 * stores email in git, readers may have a complete archive of the inbox
 
-* Atom feed, IMAP, NNTP allows casual readers to follow via local tools
+* Atom feed, IMAP, NNTP, POP3 allows casual readers to follow via local tools
 
 * uses only well-documented and easy-to-implement data formats
 
@@ -64,7 +64,7 @@ Try it out now, see https://try.public-inbox.org/
 
 Requirements for reading:
 
-* any software capable of IMAP, NNTP or following Atom feeds
+* any software capable of IMAP, NNTP, POP3 or following Atom feeds
 
 Any basic web browser will do for the HTML archives.
 We primarily develop on w3m to maximize accessibility.
@@ -111,28 +111,32 @@ and pull requests to our public-inbox address at:
 
 Please Cc: all recipients when replying as we do not require
 subscription.  This also makes it easier to rope in folks of
-tangentially related projects we depend on (e.g. git developers
+tangentially related projects we depend on (e.g., git developers
 on git@vger.kernel.org).
 
 The archives are readable via IMAP, NNTP or HTTP:
 
         nntps://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
-        imaps://news.public-inbox.org/inbox.comp.mail.public-inbox.meta.0
+        imaps://;AUTH=ANONYMOUS@public-inbox.org/inbox.comp.mail.public-inbox.meta.0
         https://public-inbox.org/meta/
 
-AUTH=ANONYMOUS is supported for IMAP, but any username + password works
+AUTH=ANONYMOUS is recommended for IMAP, but any username + password works
 
 And as Tor hidden services:
 
         http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/
         nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta
-        imap://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta.0
+        imap://;AUTH=ANONYMOUS@4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta.0
 
 You may also clone all messages via git:
 
         git clone --mirror https://public-inbox.org/meta/
         torsocks git clone --mirror http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/
 
+POP3 access instructions are at:
+
+        https://public-inbox.org/meta/_/text/help/#pop3
+
 Anti-Spam
 ---------
 
@@ -151,13 +155,13 @@ This improves accessibility, and saves bandwidth and storage
 as mail is archived forever.
 
 As of the 2010s, successful online social networks and forums are the
-ones which heavily restrict users formatting options; so public-inbox
-aims to preserve the focus on content, and not presentation.
+ones which heavily restrict users' formatting options; public-inbox
+aims to preserve the focus on content, not presentation.
 
 Copyright
 ---------
 
-Copyright 2013-2021 all contributors <meta@public-inbox.org>
+Copyright all contributors <meta@public-inbox.org>
 License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 This program is free software: you can redistribute it and/or modify
diff --git a/TODO b/TODO
index 7a27fdd2..36a7f0cf 100644
--- a/TODO
+++ b/TODO
@@ -1,8 +1,8 @@
 TODO items for public-inbox
 
 (Not in any particular order, and
-performance, ease-of-setup, installation, maintainability, etc
-all need to be considered for everything we introduce)
+performance, ease of setup, installation, maintainability, etc.
+all need to be considered for everything we introduce.)
 
 * general performance improvements, but without relying on
   XS or pre-built modules any more than we currently do.
@@ -13,40 +13,31 @@ all need to be considered for everything we introduce)
 * support remapping of expired URLs similar to mailmap
   (coordinate with git.git with this?)
 
-* POP3 server, since some webmail providers support external POP3:
-  https://public-inbox.org/meta/20160411034104.GA7817@dcvr.yhbt.net/
-  Perhaps make this depend solely the NNTP server and work as a proxy.
-  Meaning users can run this without needing a full copy of the
-  archives in git repositories.
-
-* HTTP, IMAP and NNTP proxy support.  Allow us to be a frontend for
+* HTTP, IMAP, NNTP, POP3 proxy support.  Allow us to be a frontend for
   firewalled off (or Tor-exclusive) instances.  The use case is
   for offering a publicly accessible IP with a cheap VPS,
   yet storing large amounts of data on computers without a
   public IP behind a home Internet connection.
 
-* support HTTP(S) CONNECT proxying to NNTP for users with
+* support HTTP(S) CONNECT proxying to IMAP/NNTP/POP3 for users with
   firewall problems
 
 * DHT (distributed hash table) for mapping Message-IDs to various
   archive locations to avoid SPOF.
 
 * optional Cache::FastMmap support so production deployments won't
-  need Varnish (Varnish doesn't protect NNTP nor IMAP, either)
+  need Varnish (Varnish doesn't protect NNTP or IMAP, either)
 
 * dogfood and take advantage of new kernel APIs (while maintaining
   portability to older Linux, free BSDs and maybe Hurd).
 
 * dogfood latest Xapian, Perl5, SQLite, git and various modules to
-  ensure things continue working as they should (or more better)
+  ensure things continue working as they should (or better)
   while retaining compatibility with old versions.
 
 * Support more of RFC 3977 (NNTP)
   Is there anything left for read-only support?
 
-* Combined "super server" for NNTP/HTTP/POP3/IMAP to reduce memory,
-  process, and FD overhead
-
 * Configurable linkification for per-inbox shorthands:
   "$gmane/123456" could be configured to expand to the
   appropriate link pointing to the gmane.io list archives,
@@ -84,6 +75,9 @@ all need to be considered for everything we introduce)
 * use REQUEST_URI properly for CGI / mod_perl2 compatibility
   with Message-IDs which include '%' (done?)
 
+* handle unencoded slashes from user-generated URLs properly
+  https://public-inbox.org/meta/20230217085255.xcsaoozloz2yuxil@pengutronix.de/
+
 * better test cases, make faster by reusing more setup
   code across tests
 
@@ -120,6 +114,13 @@ all need to be considered for everything we introduce)
 * improve performance and avoid head-of-line blocking on slow storage
   (done for most git blob retrievals, Xapian needs work)
 
+* allow optional use of separate Xapian worker process to implement
+  timeouts and avoid head-of-line blocking problems.  Consider
+  just-ahead-of-time builds to take advantage of custom date parsers
+  (approxidate) and other features not available to Perl bindings.
+
+* integrate git approxidate parsing into Xapian w/o spawning git
+
 * HTTP(S) search API (likely JMAP, but GraphQL could be an option)
   It should support git-specific prefixes (dfpre:, dfpost:, dfn:, etc)
   as extensions.  If JMAP, it should have HTTP(S) analogues to
@@ -155,4 +156,8 @@ all need to be considered for everything we introduce)
 
 * support pipelining as an IMAP/NNTP client for -watch + lei
 
-* auto-detect and reload on TLS cert+key changes in daemons
+* expose lei contents via read/write IMAP/JMAP server for personal use
+
+* git SHA-256 migration/coexistence path
+
+* decode RFC 3676 format=flowed + DelSp properly (see mflow (mblaze), mutt, ...)
diff --git a/ci/README b/ci/README
index 4687fbc5..c57c510c 100644
--- a/ci/README
+++ b/ci/README
@@ -2,9 +2,10 @@ various scripts for automated testing in chroots/VMs/jails
 
 TL;DR: ./ci/run.sh
 
-By default, `sudo' is used to install/uninstall packages.  It may be
-overridden with the `SUDO' environment variable.  These scripts should
-run in the top-level source tree, that is, as `./ci/run.sh'.
+By default, `sudo' is used to run install/deps.perl to install/uninstall
+packages.  It may be overridden with the `SUDO' environment variable.
+These scripts should run in the top-level source tree, that is, as
+`./ci/run.sh'.
 
 * ci/run.sh - runs tests against all profiles for the current OS
 
@@ -19,15 +20,11 @@ run in the top-level source tree, that is, as `./ci/run.sh'.
         * PERL - default: "perl"
         * SUDO - default: "sudo"
 
-* ci/deps.perl - script to mass-install/remove packages (requires root/sudo)
+* install/deps.perl - see install/README
 
         Called automatically by ci/run.sh
 
-        There is no need to run this manually unless you are debugging
-        or doing development.  However, it can be convenient to for
-        users to mass-install several packages.
-
-* ci/profiles.sh - prints to-be tested package profile for the current OS
+* ci/profiles.perl - prints to-be-tested package profile for the current OS
 
         Called automatically by ci/run.sh
         The output is read by ci/run.sh
diff --git a/ci/deps.perl b/ci/deps.perl
deleted file mode 100755
index ae85986d..00000000
--- a/ci/deps.perl
+++ /dev/null
@@ -1,267 +0,0 @@
-#!/usr/bin/perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# Helper script for installing/uninstalling packages for CI use
-# Intended for use on non-production chroots or VMs since it
-# changes installed packages
-use strict;
-my $usage = "$0 PKG_FMT PROFILE [PROFILE_MOD]";
-my $pkg_fmt = shift;
-@ARGV or die $usage, "\n";
-
-my @test_essential = qw(Test::Simple); # we actually use Test::More
-
-# package profiles
-my $profiles = {
-        # the smallest possible profile for testing
-        essential => [ qw(
-                git
-                perl
-                Digest::SHA
-                Encode
-                ExtUtils::MakeMaker
-                IO::Compress::Gzip
-                URI
-                ), @test_essential ],
-
-        # everything optional for normal use
-        optional => [ qw(
-                Date::Parse
-                BSD::Resource
-                DBD::SQLite
-                DBI
-                Inline::C
-                Net::Server
-                Plack
-                Plack::Test
-                Plack::Middleware::ReverseProxy
-                Search::Xapian
-                Socket6
-                highlight.pm
-                xapian-compact
-                ) ],
-
-        # optional developer stuff
-        devtest => [ qw(
-                XML::TreePP
-                curl
-                w3m
-                Plack::Test::ExternalServer
-                ) ],
-};
-
-# account for granularity differences between package systems and OSes
-my @precious;
-if ($^O eq 'freebsd') {
-        @precious = qw(perl curl Socket6 IO::Compress::Gzip);
-} elsif ($pkg_fmt eq 'rpm') {
-        @precious = qw(perl curl);
-}
-
-if (@precious) {
-        my $re = join('|', map { quotemeta($_) } @precious);
-        for my $list (values %$profiles) {
-                @$list = grep(!/\A(?:$re)\z/, @$list);
-        }
-        push @{$profiles->{essential}}, @precious;
-}
-
-
-# bare minimum for v2
-$profiles->{v2essential} = [ @{$profiles->{essential}}, qw(DBD::SQLite DBI) ];
-
-# package names which can't be mapped automatically:
-my $non_auto = {
-        'perl' => { pkg => 'perl5' },
-        'Date::Parse' => {
-                deb => 'libtimedate-perl',
-                pkg => 'p5-TimeDate',
-                rpm => 'perl-TimeDate',
-        },
-        'Digest::SHA' => {
-                deb => 'perl', # libperl5.XX, but the XX varies
-                pkg => 'perl5',
-        },
-        'Encode' => {
-                deb => 'perl', # libperl5.XX, but the XX varies
-                pkg => 'perl5',
-                rpm => 'perl-Encode',
-        },
-        'ExtUtils::MakeMaker' => {
-                deb => 'perl', # perl-modules-5.xx
-                pkg => 'perl5',
-                rpm => 'perl-ExtUtils-MakeMaker',
-        },
-        'IO::Compress::Gzip' => {
-                deb => 'perl', # perl-modules-5.xx
-                pkg => 'perl5',
-                rpm => 'perl-IO-Compress',
-        },
-        'DBD::SQLite' => { deb => 'libdbd-sqlite3-perl' },
-        'Plack::Test' => {
-                deb => 'libplack-perl',
-                pkg => 'p5-Plack',
-                rpm => 'perl-Plack-Test',
-        },
-        'URI' => {
-                deb => 'liburi-perl',
-                pkg => 'p5-URI',
-                rpm => 'perl-URI',
-        },
-        'Test::Simple' => {
-                deb => 'perl', # perl-modules-5.XX, but the XX varies
-                pkg => 'perl5',
-                rpm => 'perl-Test-Simple',
-        },
-        'highlight.pm' => {
-                deb => 'libhighlight-perl',
-                pkg => [],
-                rpm => [],
-        },
-
-        # we call xapian-compact(1) in public-inbox-compact(1)
-        'xapian-compact' => {
-                deb => 'xapian-tools',
-                pkg => 'xapian-core',
-                rpm => 'xapian-core', # ???
-        },
-
-        # OS-specific
-        'IO::KQueue' => {
-                deb => [],
-                pkg => 'p5-IO-KQueue',
-                rpm => [],
-        },
-};
-
-my (@pkg_install, @pkg_remove, %all);
-for my $ary (values %$profiles) {
-        $all{$_} = \@pkg_remove for @$ary;
-}
-if ($^O eq 'freebsd') {
-        $all{'IO::KQueue'} = \@pkg_remove;
-}
-$profiles->{all} = [ keys %all ]; # pseudo-profile for all packages
-
-# parse the profile list from the command-line
-for my $profile (@ARGV) {
-        if ($profile =~ s/-\z//) {
-                # like apt-get, trailing "-" means remove
-                profile2dst($profile, \@pkg_remove);
-        } else {
-                profile2dst($profile, \@pkg_install);
-        }
-}
-
-# fill in @pkg_install and @pkg_remove:
-while (my ($pkg, $dst_pkg_list) = each %all) {
-        push @$dst_pkg_list, list(pkg2ospkg($pkg, $pkg_fmt));
-}
-
-my @apt_opts =
-        qw(-o APT::Install-Recommends=false -o APT::Install-Suggests=false);
-
-# OS-specific cleanups appreciated
-
-if ($pkg_fmt eq 'deb') {
-        my @quiet = $ENV{V} ? () : ('-q');
-        root('apt-get', @apt_opts, qw(install --purge -y), @quiet,
-                @pkg_install,
-                # apt-get lets you suffix a package with "-" to
-                # remove it in an "install" sub-command:
-                map { "$_-" } @pkg_remove);
-        root('apt-get', @apt_opts, qw(autoremove --purge -y), @quiet);
-} elsif ($pkg_fmt eq 'pkg') {
-        my @quiet = $ENV{V} ? () : ('-q');
-        # FreeBSD, maybe other *BSDs are similar?
-
-        # don't remove stuff that isn't installed:
-        exclude_uninstalled(\@pkg_remove);
-        root(qw(pkg remove -y), @quiet, @pkg_remove) if @pkg_remove;
-        root(qw(pkg install -y), @quiet, @pkg_install) if @pkg_install;
-        root(qw(pkg autoremove -y), @quiet);
-# TODO: yum / rpm support
-} elsif ($pkg_fmt eq 'rpm') {
-        my @quiet = $ENV{V} ? () : ('-q');
-        exclude_uninstalled(\@pkg_remove);
-        root(qw(yum remove -y), @quiet, @pkg_remove) if @pkg_remove;
-        root(qw(yum install -y), @quiet, @pkg_install) if @pkg_install;
-} else {
-        die "unsupported package format: $pkg_fmt\n";
-}
-exit 0;
-
-
-# map a generic package name to an OS package name
-sub pkg2ospkg {
-        my ($pkg, $fmt) = @_;
-
-        # check explicit overrides, first:
-        if (my $ospkg = $non_auto->{$pkg}->{$fmt}) {
-                return $ospkg;
-        }
-
-        # check common Perl module name patterns:
-        if ($pkg =~ /::/ || $pkg =~ /\A[A-Z]/) {
-                if ($fmt eq 'deb') {
-                        $pkg =~ s/::/-/g;
-                        $pkg =~ tr/A-Z/a-z/;
-                        return "lib$pkg-perl";
-                } elsif ($fmt eq 'rpm') {
-                        $pkg =~ s/::/-/g;
-                        return "perl-$pkg"
-                } elsif ($fmt eq 'pkg') {
-                        $pkg =~ s/::/-/g;
-                        return "p5-$pkg"
-                } else {
-                        die "unsupported package format: $fmt for $pkg\n"
-                }
-        }
-
-        # use package name as-is (e.g. 'curl' or 'w3m')
-        $pkg;
-}
-
-# maps a install profile to a package list (@pkg_remove or @pkg_install)
-sub profile2dst {
-        my ($profile, $dst_pkg_list) = @_;
-        if (my $pkg_list = $profiles->{$profile}) {
-                $all{$_} = $dst_pkg_list for @$pkg_list;
-        } elsif ($all{$profile}) { # $profile is just a package name
-                $all{$profile} = $dst_pkg_list;
-        } else {
-                die "unrecognized profile or package: $profile\n";
-        }
-}
-
-sub exclude_uninstalled {
-        my ($list) = @_;
-        my %inst_check = (
-                pkg => sub { system(qw(pkg info -q), $_[0]) == 0 },
-                deb => sub { system("dpkg -s $_[0] >/dev/null 2>&1") == 0 },
-                rpm => sub { system("rpm -qs $_[0] >/dev/null 2>&1") == 0 },
-        );
-
-        my $cb = $inst_check{$pkg_fmt} || die <<"";
-don't know how to check install status for $pkg_fmt
-
-        my @tmp;
-        for my $pkg (@$list) {
-                push @tmp, $pkg if $cb->($pkg);
-        }
-        @$list = @tmp;
-}
-
-sub root {
-        print join(' ', @_), "\n";
-        return if $ENV{DRY_RUN};
-        return if system(@_) == 0;
-        warn 'command failed: ', join(' ', @_), "\n";
-        exit($? >> 8);
-}
-
-# ensure result can be pushed into an array:
-sub list {
-        my ($pkg) = @_;
-        ref($pkg) eq 'ARRAY' ? @$pkg : $pkg;
-}
diff --git a/ci/profiles.perl b/ci/profiles.perl
new file mode 100755
index 00000000..6f90a0e4
--- /dev/null
+++ b/ci/profiles.perl
@@ -0,0 +1,34 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Prints OS-specific package profiles to stdout (one per line) to use
+# as command-line args for ci/deps.perl.  Called automatically by ci/run.sh
+eval 'exec perl -wS $0 ${1+"$@"}' # no shebang
+if 0; # running under some shell
+use v5.12;
+BEGIN { require './install/os.perl' }
+my $TASKS = do {
+        if ($ID =~ /\A(?:free|net|open)bsd\z/) { <<EOM
+all devtest Xapian-
+all devtest IO::KQueue-
+all devtest IO::KQueue
+all devtest Inline::C-
+all devtest Inline::C
+EOM
+        } elsif ($ID eq 'debian') { <<EOM
+all devtest
+all devtest Xapian-
+all devtest-
+v2essential
+essential
+essential devtest-
+EOM
+        } elsif ($ID eq 'centos') { <<EOM
+v2essential devtest
+essential devtest
+all Xapian-
+EOM
+        } else { die "TODO: support ID=$ID VERSION_ID=$VERSION_ID" }
+};
+
+# this output is read by ci/run.sh and fed to install/deps.perl:
+print $TASKS;
diff --git a/ci/profiles.sh b/ci/profiles.sh
deleted file mode 100755
index 3cd8fa38..00000000
--- a/ci/profiles.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/bin/sh
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-
-# Prints OS-specific package profiles to stdout (one per-newline) to use
-# as command-line args for ci/deps.perl.  Called automatically by ci/run.sh
-
-# set by os-release(5) or similar
-ID= VERSION_ID=
-case $(uname -o) in
-GNU/Linux)
-        for f in /etc/os-release /usr/lib/os-release
-        do
-                test -f $f || continue
-                . $f
-
-                # Debian sid (and testing) have no VERSION_ID
-                case $ID--$VERSION_ID in
-                debian--)
-                        case $PRETTY_NAME in
-                        */sid) VERSION_ID=sid ;;
-                        *)
-                                echo >&2 "$ID, but no VERSION_ID"
-                                echo >&2 "==> $f <=="
-                                cat >&2 $f
-                                exit 1
-                                ;;
-                        esac
-                        ;;
-                esac
-
-                case $ID--$VERSION_ID in
-                -|*--|--*) continue ;;
-                *--*) break ;;
-                esac
-        done
-        ;;
-FreeBSD)
-        ID=freebsd
-        VERSION_ID=$(uname -r | cut -d . -f 1)
-        test "$VERSION_ID" -lt 11 && {
-                echo >&2 "ID=$ID $(uname -r) too old to support";
-                exit 1
-        }
-esac
-
-case $ID in
-freebsd) PKG_FMT=pkg ;;
-debian|ubuntu) PKG_FMT=deb ;;
-centos|redhat|fedora) PKG_FMT=rpm ;;
-*) echo >&2 "PKG_FMT undefined for ID=$ID in $0"
-esac
-
-case $ID-$VERSION_ID in
-freebsd-11|freebsd-12) sed "s/^/$PKG_FMT /" <<EOF
-all devtest-
-all devtest IO::KQueue-
-all devtest IO::KQueue
-v2essential
-essential
-essential devtest-
-EOF
-        ;;
-debian-sid|debian-9|debian-10) sed "s/^/$PKG_FMT /" <<EOF
-all devtest
-all devtest Search::Xapian-
-all devtest-
-v2essential
-essential
-essential devtest-
-EOF
-        ;;
-centos-7) sed "s/^/$PKG_FMT /" <<EOF
-v2essential devtest
-essential devtest
-all Search::Xapian-
-EOF
-        ;;
-esac
diff --git a/ci/run.sh b/ci/run.sh
index 9613943b..bd1d8a4d 100755
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -1,6 +1,7 @@
 #!/bin/sh
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Beware, this alters system-wide package installation.
 set -e
 SUDO=${SUDO-'sudo'} PERL=${PERL-'perl'} MAKE=${MAKE-'make'}
 DO=${DO-''}
@@ -8,14 +9,17 @@ DO=${DO-''}
 set -x
 if test -f Makefile
 then
-        $DO $MAKE clean
+        $DO $MAKE clean >/dev/null
 fi
+NPROC=${NPROC-$({ getconf _NPROCESSORS_ONLN || getconf NPROCESSORS_ONLN ||
+                echo 2; } 2>/dev/null)}
 
-./ci/profiles.sh | while read args
+TEST_JOBS=${TEST_JOBS-1}
+$PERL -w ci/profiles.perl | while read args
 do
-        $DO $SUDO $PERL -w ci/deps.perl $args
+        $DO $SUDO $PERL -w install/deps.perl -y --allow-remove $args
         $DO $PERL Makefile.PL
-        $DO $MAKE
-        $DO $MAKE check
-        $DO $MAKE clean
+        $DO $MAKE -j${BUILD_JOBS-$NPROC}
+        $DO $MAKE ${TEST_TARGET-check} N=${N-$TEST_JOBS}
+        $DO $MAKE clean >/dev/null
 done
diff --git a/contrib/completion/lei-completion.bash b/contrib/completion/lei-completion.bash
index 5c137e68..b86afa2c 100644
--- a/contrib/completion/lei-completion.bash
+++ b/contrib/completion/lei-completion.bash
@@ -1,16 +1,19 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # preliminary bash completion support for lei (Local Email Interface)
 # Needs a lot of work, see `lei__complete' in lib/PublicInbox::LEI.pm
 _lei() {
         local wordlist="$(lei _complete ${COMP_WORDS[@]})"
-        case $wordlist in
-        *':'* | *'='* | '//'*) compopt -o nospace ;;
-        *) compopt +o nospace ;; # the default
-        esac
         wordlist="${wordlist//;/\\\\;}" # escape ';' for ';UIDVALIDITY' and such
-        COMPREPLY=($(compgen -W "$wordlist" -- "${COMP_WORDS[COMP_CWORD]}"))
+
+        local word="${COMP_WORDS[COMP_CWORD]}"
+        if test "$word" = ':' && test $COMP_CWORD -ge 1
+        then
+                COMPREPLY=($(compgen -W "$wordlist" --))
+        else
+                COMPREPLY=($(compgen -W "$wordlist" -- "$word"))
+        fi
         return 0
 }
 complete -o default -o bashdefault -F _lei lei
diff --git a/devel/README b/devel/README
index 8f9a0485..c4be5141 100644
--- a/devel/README
+++ b/devel/README
@@ -1 +1 @@
-scripts use for public-inbox development that don't belong in t/
+scripts used for public-inbox development that don't belong in t/
diff --git a/devel/longest-tests b/devel/longest-tests
new file mode 100755
index 00000000..bf46e166
--- /dev/null
+++ b/devel/longest-tests
@@ -0,0 +1,7 @@
+eval 'exec perl -wS $0 ${1+"$@"}' # this script is too short to copyright
+if 0; # running under some shell
+use v5.12; use autodie; use YAML::XS qw(Load);
+open(my $fh, '<', shift // '.prove');
+my $t = Load(do { local $/; <$fh> })->{tests};
+my @t = sort { $t->{$b}->{elapsed} <=> $t->{$a}->{elapsed} } keys %$t;
+printf "%0.6f %s\n", $t->{$_}->{elapsed}, $_ for @t;
diff --git a/devel/syscall-list b/devel/syscall-list
deleted file mode 100755
index d33a8a78..00000000
--- a/devel/syscall-list
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <http://www.gnu.org/licenses/agpl-3.0.txt>
-# Dump syscall numbers under Linux and any other kernel which
-# promises stable syscall numbers.  This is to maintain
-# PublicInbox::Syscall
-# DO NOT USE this for *BSDs, none of the current BSD kernels
-# we know about promise stable syscall numbers, we'll use
-# Inline::C to support them.
-eval 'exec perl -S $0 ${1+"$@"}' # no shebang
-        if 0; # running under some shell
-use strict;
-use v5.10.1;
-use File::Temp 0.19;
-use POSIX qw(uname);
-say '$machine='.(POSIX::uname())[-1];
-my $cc = $ENV{CC} // 'cc';
-my @cflags = split(/\s+/, $ENV{CFLAGS} // '-Wall');
-my $str = do { local $/; <DATA> };
-my $tmp = File::Temp->newdir('syscall-list-XXXX', TMPDIR => 1);
-my $f = "$tmp/sc.c";
-my $x = "$tmp/sc";
-open my $fh, '>', $f or die "open $f $!";
-print $fh $str or die "print $f $!";
-close $fh or die "close $f $!";
-system($cc, '-o', $x, $f, @cflags) == 0 or die "cc failed \$?=$?";
-exec($x);
-__DATA__
-#define _GNU_SOURCE
-#include <sys/syscall.h>
-#include <sys/ioctl.h>
-#include <linux/fs.h>
-#include <unistd.h>
-#include <stdio.h>
-
-#define D(x) printf("$" #x " = %ld;\n", (long)x)
-
-int main(void)
-{
-#ifdef __linux__
-        D(SYS_epoll_create1);
-        D(SYS_epoll_ctl);
-#ifdef SYS_epoll_wait
-        D(SYS_epoll_wait);
-#endif
-        D(SYS_epoll_pwait);
-        D(SYS_signalfd4);
-        D(SYS_inotify_init1);
-        D(SYS_inotify_add_watch);
-        D(SYS_inotify_rm_watch);
-        D(SYS_prctl);
-        D(SYS_fstatfs);
-        D(SYS_sendmsg);
-        D(SYS_recvmsg);
-#ifdef FS_IOC_GETFLAGS
-        printf("FS_IOC_GETFLAGS=%#lx\nFS_IOC_SETFLAGS=%#lx\n",
-                (unsigned long)FS_IOC_GETFLAGS, (unsigned long)FS_IOC_SETFLAGS);
-#endif
-
-#ifdef SYS_renameat2
-        D(SYS_renameat2);
-#endif
-#endif /* Linux, any other OSes with stable syscalls? */
-        printf("size_t=%zu off_t=%zu\n", sizeof(size_t), sizeof(off_t));
-        return 0;
-}
diff --git a/devel/sysdefs-list b/devel/sysdefs-list
new file mode 100755
index 00000000..ba51de6c
--- /dev/null
+++ b/devel/sysdefs-list
@@ -0,0 +1,188 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <http://www.gnu.org/licenses/agpl-3.0.txt>
+# Dump system-specific constant numbers this is to maintain
+# PublicInbox::Syscall and any other system-specific pieces.
+# However, sysconf(3) constants are stable ABI on all safe to dump.
+eval 'exec perl -S $0 ${1+"$@"}' # no shebang
+        if 0; # running under some shell
+use v5.12;
+use File::Temp 0.19;
+use POSIX qw(uname);
+use Config;
+print STDERR '# $machine='.(POSIX::uname())[-1]."\n";
+my $cc = $ENV{CC} // $Config{cc} // 'cc';
+my @cflags = split(/\s+/, $ENV{CFLAGS} // $Config{ccflags} // '-Wall');
+my $str = do { local $/; <DATA> };
+$str =~ s/^\s*MAYBE\s*([DX])\((\w+)\)/
+#ifdef $2
+        $1($2);
+#endif
+/sgxm;
+my $tmp = File::Temp->newdir('sysdefs-list-XXXX', TMPDIR => 1);
+my $f = "$tmp/sysdefs.c";
+my $x = "$tmp/sysdefs";
+open my $fh, '>', $f or die "open $f $!";
+print $fh $str or die "print $f $!";
+close $fh or die "close $f $!";
+system($cc, '-o', $x, $f, @cflags) == 0 or die "$cc failed \$?=$?";
+print STDERR '# %Config',
+        (map { " $_=$Config{$_}" } qw(ptrsize sizesize lseeksize)), "\n";
+exit(system($x)); # exit is to ensure File::Temp::Dir->DESTROY fires
+__DATA__
+#ifndef _GNU_SOURCE
+#  define _GNU_SOURCE
+#endif
+#include <assert.h>
+#include <signal.h>
+#include <stddef.h>
+#include <sys/socket.h>
+#include <sys/syscall.h>
+#include <sys/ioctl.h>
+#ifdef __linux__
+#        include <linux/fs.h>
+#        include <sys/epoll.h>
+#        include <sys/inotify.h>
+#        include <sys/vfs.h>
+#endif
+#include <sys/types.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdio.h>
+
+#define STRUCT_BEGIN(t) do { t x; printf("'"#t"' => '%zu bytes\n", sizeof(x))
+#define STRUCT_END puts("',"); } while (0)
+
+// prints the struct field name, @offset, and signed/unsigned bit size
+#define PR_NUM(f) do { \
+        x.f = ~0; \
+        printf("\t.%s @%zu %c%zu\n", #f, \
+                offsetof(typeof(x),f), \
+                x.f > 0 ? 'u' : 's', \
+                sizeof(x.f) * 8); \
+} while (0)
+
+#define PR_PTR(f) do { \
+        assert(sizeof(x.f) == sizeof(void *)); \
+        printf("\t.%s @%zu ptr\n", #f, offsetof(typeof(x),f)); \
+} while (0)
+
+#define PR_OFF(f) do { \
+        printf("\t.%s @%zu\n", #f, offsetof(typeof(x),f)); \
+} while (0)
+
+#define D(x) printf(#x " => %ld,\n", (long)x)
+#define X(x) printf(#x " => 0x%lx,\n", (unsigned long)x)
+
+int main(void)
+{
+        // verify Config{(ptr|size|lseek)size} entries match:
+        printf("'sizeof(ptr)' => %zu,\n", sizeof(void *));
+        printf("'sizeof(size_t)' => %zu,\n", sizeof(size_t));
+        printf("'sizeof(off_t)' => %zu,\n", sizeof(off_t));
+
+#ifdef __linux__
+        D(SYS_epoll_create1);
+        D(SYS_epoll_ctl);
+        MAYBE D(SYS_epoll_wait);
+        D(SYS_epoll_pwait);
+        D(SYS_signalfd4);
+
+        X(IN_CLOEXEC);
+        X(IN_ACCESS);
+        X(IN_ALL_EVENTS);
+        X(IN_ATTRIB);
+        X(IN_CLOSE);
+        X(IN_CLOSE_NOWRITE);
+        X(IN_CLOSE_WRITE);
+        X(IN_CREATE);
+        X(IN_DELETE);
+        X(IN_DELETE_SELF);
+        X(IN_DONT_FOLLOW);
+        X(IN_EXCL_UNLINK);
+        X(IN_IGNORED);
+        X(IN_ISDIR);
+        X(IN_MASK_ADD);
+        X(IN_MODIFY);
+        X(IN_MOVE);
+        X(IN_MOVED_FROM);
+        X(IN_MOVED_TO);
+        X(IN_MOVE_SELF);
+        X(IN_ONESHOT);
+        X(IN_ONLYDIR);
+        X(IN_OPEN);
+        X(IN_Q_OVERFLOW);
+        X(IN_UNMOUNT);
+
+        D(SYS_inotify_init1);
+        D(SYS_inotify_add_watch);
+        D(SYS_inotify_rm_watch);
+
+        D(SYS_prctl);
+        D(SYS_fstatfs);
+
+        MAYBE X(FS_IOC_GETFLAGS);
+        MAYBE X(FS_IOC_SETFLAGS);
+
+        MAYBE D(SYS_renameat2);
+
+        STRUCT_BEGIN(struct epoll_event);
+                PR_NUM(events);
+                PR_NUM(data.u64);
+        STRUCT_END;
+
+        STRUCT_BEGIN(struct inotify_event);
+                PR_NUM(wd);
+                PR_NUM(mask);
+                PR_NUM(cookie);
+                PR_NUM(len);
+                PR_OFF(name);
+        STRUCT_END;
+
+        STRUCT_BEGIN(struct statfs);
+                PR_NUM(f_type);
+        STRUCT_END;
+#endif /* Linux, any other OSes with stable syscalls? */
+
+        D(SIGWINCH);
+        MAYBE D(SO_ACCEPTFILTER);
+        MAYBE D(_SC_NPROCESSORS_ONLN);
+        MAYBE D(_SC_AVPHYS_PAGES);
+        MAYBE D(_SC_PAGE_SIZE);
+        MAYBE D(_SC_PAGESIZE);
+
+        D(SYS_sendmsg);
+        D(SYS_recvmsg);
+
+        STRUCT_BEGIN(struct flock);
+                PR_NUM(l_start);
+                PR_NUM(l_len);
+                PR_NUM(l_pid);
+                PR_NUM(l_type);
+                PR_NUM(l_whence);
+        STRUCT_END;
+
+        STRUCT_BEGIN(struct msghdr);
+                PR_PTR(msg_name);
+                PR_NUM(msg_namelen);
+                PR_PTR(msg_iov);
+                PR_NUM(msg_iovlen);
+                PR_PTR(msg_control);
+                PR_NUM(msg_controllen);
+                PR_NUM(msg_flags);
+        STRUCT_END;
+
+        STRUCT_BEGIN(struct cmsghdr);
+                PR_NUM(cmsg_len);
+                PR_NUM(cmsg_level);
+                PR_NUM(cmsg_type);
+        STRUCT_END;
+
+        {
+                struct cmsghdr cmsg;
+                uintptr_t cmsg_data_off;
+                cmsg_data_off = (uintptr_t)CMSG_DATA(&cmsg) - (uintptr_t)&cmsg;
+                D(cmsg_data_off);
+        }
+
+        return 0;
+}
diff --git a/examples/README b/examples/README
index 1d5dcd34..5674d7ed 100644
--- a/examples/README
+++ b/examples/README
@@ -9,10 +9,6 @@ For PSGI/Plack (HTTP) servers
 -----------------------------
 public-inbox.psgi - starting point for PSGI/Plack users in production and dev
 
-For Apache2 users
------------------
-apache2_perl.conf - intended to be the basis of a production config
-
 Contact
 -------
 Please send any related feedback to public-inbox: meta@public-inbox.org
diff --git a/examples/README.unsubscribe b/examples/README.unsubscribe
index 3e80e838..3b407960 100644
--- a/examples/README.unsubscribe
+++ b/examples/README.unsubscribe
@@ -3,10 +3,9 @@ Unsubscribe endpoints for mlmmj users (and possibly Mailman, too)
 * examples/unsubscribe.milter filters outgoing messages
   and appends an HTTPS URL to the List-Unsubscribe header.
   This List-Unsubscribe header should point to the PSGI
-  described below.
-  Currently, this is only active for a whitelist of test
-  addresses in /etc/unsubscribe-milter.whitelist
-  with one email address per line.
+  described below.  You may edit the archive_addr sub
+  to disable List-Unsubscribe headers for well-known archiver
+  addresses to prevent saboteurs from stopping archival.
 
 * examples/unsubscribe.psgi is a PSGI which needs to run
   as the mlmmj user with permission to run mlmmj-unsub.
@@ -36,5 +35,5 @@ in /etc/postfix/main.cf:
   # This is not needed for mlmmj since mlmmj uses SMTP:
   # non_smtpd_milters = local:/var/spool/postfix/unsubscribe/unsubscribe.sock
 
-Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+Copyright (C) all contributors <meta@public-inbox.org>
 License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
diff --git a/examples/apache2_cgi.conf b/examples/apache2_cgi.conf
deleted file mode 100644
index 5ec64d72..00000000
--- a/examples/apache2_cgi.conf
+++ /dev/null
@@ -1,34 +0,0 @@
-# Example Apache2 configuration using CGI mod_cgi
-# If possible, use mod_perl (see apache2_perl.conf) or
-# a standalone PSGI/Plack # server instead of this.
-# Adjust paths to your installation.
-
-ServerName "public-inbox"
-ServerRoot "/var/www/cgi-bin"
-DocumentRoot "/var/www/cgi-bin"
-ErrorLog "/tmp/public-inbox-error.log"
-PidFile "/tmp/public-inbox.pid"
-Listen 127.0.0.1:8080
-LoadModule cgi_module /usr/lib/apache2/modules/mod_cgi.so
-LoadModule env_module /usr/lib/apache2/modules/mod_env.so
-LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so
-LoadModule dir_module /usr/lib/apache2/modules/mod_dir.so
-LoadModule mime_module /usr/lib/apache2/modules/mod_mime.so
-TypesConfig "/dev/null"
-
-<Directory /var/www/cgi-bin>
-        Options +ExecCGI
-        AddHandler cgi-script .cgi
-
-        # we use this hack to ensure "public-inbox.cgi" doesn't show up
-        # in any of our redirects:
-        SetEnv NO_SCRIPT_NAME 1
-
-        # our public-inbox.cgi requires PATH_INFO-based URLs with minimal
-        # use of query parameters
-        DirectoryIndex public-inbox.cgi
-        RewriteEngine On
-        RewriteCond %{REQUEST_FILENAME} !-f
-        RewriteCond %{REQUEST_FILENAME} !-d
-        RewriteRule ^.* /public-inbox.cgi/$0 [L,PT]
-</Directory>
diff --git a/examples/apache2_perl.conf b/examples/apache2_perl.conf
deleted file mode 100644
index a4721b5b..00000000
--- a/examples/apache2_perl.conf
+++ /dev/null
@@ -1,25 +0,0 @@
-# Example Apache2 configuration using Plack::Handler::Apache2
-# Adjust paths to your installation
-
-ServerName "public-inbox"
-ServerRoot "/var/www"
-DocumentRoot "/var/www"
-ErrorLog "/tmp/public-inbox-error.log"
-PidFile "/tmp/public-inbox.pid"
-Listen 127.0.0.1:8080
-LoadModule perl_module /usr/lib/apache2/modules/mod_perl.so
-
-# no need to set no rely on HOME if using this:
-PerlSetEnv PI_CONFIG /home/pi/.public-inbox/config
-
-<Location />
-        SetHandler perl-script
-        PerlResponseHandler Plack::Handler::Apache2
-        PerlSetVar psgi_app /path/to/public-inbox.psgi
-</Location>
-
-# Optional, preload the application in the parent like startup.pl
-<Perl>
-        use Plack::Handler::Apache2;
-        Plack::Handler::Apache2->preload("/path/to/public-inbox.psgi");
-</Perl>
diff --git a/examples/apache2_perl_old.conf b/examples/apache2_perl_old.conf
deleted file mode 100644
index a6de2304..00000000
--- a/examples/apache2_perl_old.conf
+++ /dev/null
@@ -1,38 +0,0 @@
-# Example legacy Apache2 configuration using CGI + mod_perl2
-# Consider using Plack::Handler::Apache2 instead (see apache2_perl.conf)
-# Adjust paths to your installation
-
-ServerName "public-inbox"
-ServerRoot "/var/www/cgi-bin"
-DocumentRoot "/var/www/cgi-bin"
-ErrorLog "/tmp/public-inbox-error.log"
-PidFile "/tmp/public-inbox.pid"
-Listen 127.0.0.1:8080
-LoadModule perl_module /usr/lib/apache2/modules/mod_perl.so
-LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so
-LoadModule dir_module /usr/lib/apache2/modules/mod_dir.so
-LoadModule mime_module /usr/lib/apache2/modules/mod_mime.so
-TypesConfig "/dev/null"
-
-# PerlPassEnv PATH # this is implicit
-<Directory /var/www/cgi-bin>
-        Options +ExecCGI
-        AddHandler perl-script .cgi
-        PerlResponseHandler ModPerl::Registry
-        PerlOptions +ParseHeaders
-
-        # we use this hack to ensure "public-inbox.cgi" doesn't show up
-        # in any of our redirects:
-        PerlSetEnv NO_SCRIPT_NAME 1
-
-        # no need to set no rely on HOME if using this:
-        PerlSetEnv PI_CONFIG /home/pi/.public-inbox/config
-
-        # our public-inbox.cgi requires PATH_INFO-based URLs with minimal
-        # use of query parameters
-        DirectoryIndex public-inbox.cgi
-        RewriteEngine On
-        RewriteCond %{REQUEST_FILENAME} !-f
-        RewriteCond %{REQUEST_FILENAME} !-d
-        RewriteRule ^.* /public-inbox.cgi/$0 [L,PT]
-</Directory>
diff --git a/examples/cgi-webrick.rb b/examples/cgi-webrick.rb
deleted file mode 100644
index 5554a012..00000000
--- a/examples/cgi-webrick.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env ruby
-# Sample configuration using WEBrick, mainly intended dev/testing
-# for folks familiar with Ruby and not various Perl webserver
-# deployment options.  For those familiar with Perl web servers,
-# plackup(1) is recommended for development and public-inbox-httpd(1)
-# is our production deployment server.
-require 'webrick'
-require 'logger'
-options = {
-  :BindAddress => '127.0.0.1',
-  :Port => 8080,
-  :Logger => Logger.new($stderr),
-  :CGIPathEnv => ENV['PATH'], # need to run 'git' commands
-  :AccessLog => [
-    [ Logger.new($stdout), WEBrick::AccessLog::COMBINED_LOG_FORMAT ]
-  ],
-}
-server = WEBrick::HTTPServer.new(options)
-server.mount("/",
-             WEBrick::HTTPServlet::CGIHandler,
-            "/var/www/cgi-bin/public-inbox.cgi")
-['INT', 'TERM'].each do |signal|
-  trap(signal) {exit!(0)}
-end
-server.start
diff --git a/examples/grok-pull.post_update_hook.sh b/examples/grok-pull.post_update_hook.sh
index 77489472..4d303c03 100755
--- a/examples/grok-pull.post_update_hook.sh
+++ b/examples/grok-pull.post_update_hook.sh
@@ -111,7 +111,7 @@ case $cfg_dir in
                         "publicinbox.$inbox_name.infourl" "$url"
         done
         curl -sSfv "$remote_inbox_url"/description >"$inbox_dir"/description
-        echo "I: $inbox_name at $inbox_dir ($addresses) $local_url"
+        echo "# $inbox_name at $inbox_dir ($addresses) $local_url"
         ;;
 esac
 
diff --git a/examples/logrotate.conf b/examples/logrotate.conf
index 4ce08843..fad40cfc 100644
--- a/examples/logrotate.conf
+++ b/examples/logrotate.conf
@@ -18,7 +18,7 @@
                 # systemd users do not need PID files,
                 # only signal the @1 process since the @2 is short-lived
                 # For systemd users, assuming you use two services
-                systemctl kill -s SIGUSR1 public-inbox-httpd@1.service
-                systemctl kill -s SIGUSR1 public-inbox-nntpd@1.service
+                systemctl kill -s SIGUSR1 --kill-who=main \
+                                public-inbox-netd@1.service
         endscript
 }
diff --git a/examples/nginx_proxy b/examples/nginx_proxy
index d8d1e6df..754a4931 100644
--- a/examples/nginx_proxy
+++ b/examples/nginx_proxy
@@ -1,8 +1,14 @@
 # Example NGINX configuration to proxy-pass requests
-# to public-inbox-httpd or to a standalone PSGI/Plack server.
+# to varnish, public-inbox-(httpd|netd) or any PSGI/Plack server.
 # The daemon is assumed to be running locally on port 8001.
 # Adjust ssl certificate paths if you use any, or remove
 # the ssl configuration directives if you don't.
+#
+# Note: public-inbox-httpd and -netd both support HTTPS, but they
+# don't support caching which Varnish provides.  The recommended
+# setup is currently:
+#
+#   (nginx|any-HTTPS-proxy) <-> varnish <-> public-inbox-(httpd|netd)
 server {
         server_name _;
         listen 80;
@@ -14,6 +20,7 @@ server {
                 proxy_set_header    HOST $host;
                 proxy_set_header    X-Real-IP $remote_addr;
                 proxy_set_header    X-Forwarded-Proto $scheme;
+                proxy_buffering off; # lowers response latency
                 proxy_pass          http://127.0.0.1:8001$request_uri;
         }
 
diff --git a/examples/public-inbox-httpd.socket b/examples/public-inbox-httpd.socket
index 1a1ed735..3a6e4432 100644
--- a/examples/public-inbox-httpd.socket
+++ b/examples/public-inbox-httpd.socket
@@ -1,4 +1,7 @@
 # ==> /etc/systemd/system/public-inbox-httpd.socket <==
+# Consider looking at public-inbox-netd.socket instead of this file
+# to simplify management when serving multiple protocols.
+
 [Unit]
 Description = public-inbox-httpd socket
 
diff --git a/examples/public-inbox-httpd@.service b/examples/public-inbox-httpd@.service
index 147f7c6d..11859198 100644
--- a/examples/public-inbox-httpd@.service
+++ b/examples/public-inbox-httpd@.service
@@ -1,4 +1,7 @@
 # ==> /etc/systemd/system/public-inbox-httpd@.service <==
+# Consider looking at public-inbox-netd@.service instead of this file
+# to simplify management when serving multiple protocols.
+#
 # Since SIGUSR2 upgrades do not work under systemd, this service file
 # allows starting two simultaneous services during upgrade time
 # (e.g. public-inbox-httpd@1 public-inbox-httpd@2) with the intention
@@ -22,7 +25,6 @@ LimitNOFILE = 30000
 ExecStartPre = /bin/mkdir -p -m 1777 /tmp/.pub-inline
 ExecStart = /usr/local/bin/public-inbox-httpd \
 -1 /var/log/public-inbox/httpd.out.log
-StandardError = syslog
 
 # NonBlocking is REQUIRED to avoid a race condition if running
 # simultaneous services
@@ -30,8 +32,8 @@ NonBlocking = true
 Sockets = public-inbox-httpd.socket
 
 KillSignal = SIGQUIT
-User = nobody
-Group = nogroup
+User = news
+Group = ssl-cert
 ExecReload = /bin/kill -HUP $MAINPID
 TimeoutStopSec = 86400
 KillMode = process
diff --git a/examples/public-inbox-imap-onion.socket b/examples/public-inbox-imap-onion.socket
deleted file mode 100644
index 76b4e7ca..00000000
--- a/examples/public-inbox-imap-onion.socket
+++ /dev/null
@@ -1,12 +0,0 @@
-# ==> /etc/systemd/system/public-inbox-imap-onion.socket <==
-# This unit is for the corresponding line in torrc(5):
-# HiddenServicePort 143 unix:/run/imapd.onion.sock
-[Unit]
-Description = public-inbox-imap .onion socket
-
-[Socket]
-ListenStream = /run/imapd.onion.sock
-Service = public-inbox-imapd@1.service
-
-[Install]
-WantedBy = sockets.target
diff --git a/examples/public-inbox-imapd.socket b/examples/public-inbox-imapd.socket
index fcd924fd..22ce16fb 100644
--- a/examples/public-inbox-imapd.socket
+++ b/examples/public-inbox-imapd.socket
@@ -1,11 +1,26 @@
 # ==> /etc/systemd/system/public-inbox-imapd.socket <==
+# Consider looking at public-inbox-netd.socket instead of this file
+# to simplify management when serving multiple protocols.
+#
+# This contains 5 sockets for an public-inbox-imapd instance.
+# The TCP ports are well-known ports registered in /etc/services.
+# The /run/imapd.onion.sock entry is meant for the Tor hidden service
+# enabled by the following line in the torrc(5) file:
+#   HiddenServicePort 143 unix:/run/imapd.onion.sock
 [Unit]
-Description = public-inbox-imapd socket
+Description = public-inbox-imapd sockets
 
 [Socket]
 ListenStream = 0.0.0.0:143
+ListenStream = 0.0.0.0:993
+ListenStream = /run/imapd.onion.sock
+
+# Separating IPv4 from IPv6 listeners makes for nicer output
+# of IPv4 addresses in various reporting/monitoring tools
 BindIPv6Only = ipv6-only
 ListenStream = [::]:143
+ListenStream = [::]:993
+
 Service = public-inbox-imapd@1.service
 
 [Install]
diff --git a/examples/public-inbox-imapd@.service b/examples/public-inbox-imapd@.service
index e0446ed3..80104605 100644
--- a/examples/public-inbox-imapd@.service
+++ b/examples/public-inbox-imapd@.service
@@ -1,4 +1,7 @@
 # ==> /etc/systemd/system/public-inbox-imapd@.service <==
+# Consider looking at public-inbox-netd@.service instead of this file
+# to simplify management when serving multiple protocols.
+#
 # Since SIGUSR2 upgrades do not work under systemd, this service file
 # allows starting two simultaneous services during upgrade time
 # (e.g. public-inbox-imapd@1 public-inbox-imapd@2) with the intention
@@ -7,10 +10,8 @@
 
 [Unit]
 Description = public-inbox-imapd IMAP server %i
-Wants = public-inbox-imapd.socket public-inbox-imaps.socket \
-public-inbox-imap-onion.socket
-After = public-inbox-imapd.socket public-inbox-imaps.socket \
-public-inbox-imap-onion.socket
+Wants = public-inbox-imapd.socket
+After = public-inbox-imapd.socket
 
 [Service]
 Environment = PI_CONFIG=/home/pi/.public-inbox/config \
@@ -23,17 +24,15 @@ ExecStart = /usr/local/bin/public-inbox-imapd -W0 \
 -1 /var/log/public-inbox/imapd.out.log \
 --cert /etc/ssl/certs/news.example.com.pem \
 --key /etc/ssl/private/news.example.com.key
-StandardError = syslog
 
 # NonBlocking is REQUIRED to avoid a race condition if running
 # simultaneous services
 NonBlocking = true
 
-Sockets = public-inbox-imapd.socket public-inbox-imaps.socket \
-public-inbox-imap-onion.socket
+Sockets = public-inbox-imapd.socket
 
 KillSignal = SIGQUIT
-User = nobody
+User = news
 Group = ssl-cert
 ExecReload = /bin/kill -HUP $MAINPID
 TimeoutStopSec = 86400
diff --git a/examples/public-inbox-imaps.socket b/examples/public-inbox-imaps.socket
deleted file mode 100644
index b61cc742..00000000
--- a/examples/public-inbox-imaps.socket
+++ /dev/null
@@ -1,12 +0,0 @@
-# ==> /etc/systemd/system/public-inbox-imaps.socket <==
-[Unit]
-Description = public-inbox-imaps socket
-
-[Socket]
-ListenStream = 0.0.0.0:993
-BindIPv6Only = ipv6-only
-ListenStream = [::]:993
-Service = public-inbox-imapd@1.service
-
-[Install]
-WantedBy = sockets.target
diff --git a/examples/public-inbox-netd.socket b/examples/public-inbox-netd.socket
new file mode 100644
index 00000000..9a19602e
--- /dev/null
+++ b/examples/public-inbox-netd.socket
@@ -0,0 +1,45 @@
+# ==> /etc/systemd/system/public-inbox-netd.socket <==
+# This contains all the services that public-inbox-netd can run;
+# allowing it to replace (or run in parallel to) any existing -httpd,
+# -imapd, -nntpd, or -pop3d instances.
+#
+# The TCP ports are well-known ports registered in /etc/services.
+# The /run/*.sock entries are meant for the Tor hidden service
+# enabled by the following lines in the torrc(5) file:
+#   HiddenServicePort 110 unix:/run/pop3.sock
+#   HiddenServicePort 119 unix:/run/nntp.sock
+#   HiddenServicePort 143 unix:/run/imap.sock
+[Unit]
+Description = public-inbox-netd sockets
+
+[Socket]
+# for tor (see torrc(5))
+ListenStream = /run/imap.sock
+ListenStream = /run/pop3.sock
+ListenStream = /run/nntp.sock
+
+# this is for varnish:
+ListenStream = 127.0.0.1:280
+
+# public facing
+ListenStream = 0.0.0.0:110
+ListenStream = 0.0.0.0:119
+ListenStream = 0.0.0.0:143
+ListenStream = 0.0.0.0:563
+ListenStream = 0.0.0.0:993
+ListenStream = 0.0.0.0:995
+
+# Separating IPv4 from IPv6 listeners makes for nicer output
+# of IPv4 addresses in various reporting/monitoring tools
+BindIPv6Only = ipv6-only
+ListenStream = [::]:110
+ListenStream = [::]:119
+ListenStream = [::]:143
+ListenStream = [::]:563
+ListenStream = [::]:993
+ListenStream = [::]:995
+
+Service = public-inbox-netd@1.service
+
+[Install]
+WantedBy = sockets.target
diff --git a/examples/public-inbox-netd@.service b/examples/public-inbox-netd@.service
new file mode 100644
index 00000000..83d2e995
--- /dev/null
+++ b/examples/public-inbox-netd@.service
@@ -0,0 +1,62 @@
+# ==> /etc/systemd/system/public-inbox-netd@.service <==
+# Since SIGUSR2 upgrades do not work under systemd, this service file
+# allows starting two simultaneous services during upgrade time
+# (e.g. public-inbox-netd@1 public-inbox-netd@2) with the intention
+# that they take turns running in-between upgrades.  This should
+# allow upgrading without downtime.
+# For servers expecting visitors from multiple timezones, TZ=UTC
+# is needed to ensure a consistent approxidate experience with search.
+[Unit]
+Description = public-inbox-netd server %i
+Wants = public-inbox-netd.socket
+After = public-inbox-netd.socket
+
+[Service]
+# An LD_PRELOAD for libjemalloc can be added here.  It currently seems
+# more resistant to fragmentation in long-lived daemons than glibc.
+Environment = PI_CONFIG=/home/pi/.public-inbox/config \
+PATH=/usr/local/bin:/usr/bin:/bin \
+TZ=UTC \
+PERL_INLINE_DIRECTORY=/tmp/.netd-inline
+
+LimitNOFILE = 30000
+LimitCORE = infinity
+ExecStartPre = /bin/mkdir -p -m 1777 /tmp/.netd-inline
+
+# The '-l' args below map each socket in public-inbox-netd.socket to
+# the appropriate IANA service name:
+ExecStart = /usr/local/bin/public-inbox-netd -W0 \
+-1 /var/log/netd/stdout.out.log \
+--cert /etc/ssl/certs/news.example.com.pem \
+--key /etc/ssl/private/news.example.com.key
+-l imap:///run/imap.sock?out=/var/log/netd/imap.out,err=/var/log/netd/imap.err \
+-l nntp:///run/nntp.sock?out=/var/log/netd/nntp.out,err=/var/log/netd/nntp.err \
+-l pop3:///run/pop3.sock?out=/var/log/netd/pop3.out,err=/var/log/netd/pop3.err \
+-l imap://0.0.0.0/?out=/var/log/netd/imap.out,err=/var/log/netd/imap.err \
+-l nntp://0.0.0.0/?out=/var/log/netd/nntp.out,err=/var/log/netd/nntp.err \
+-l pop3://0.0.0.0/?out=/var/log/netd/pop3.out,err=/var/log/netd/pop3.err \
+-l imap://[::]/?out=/var/log/netd/imap.out,err=/var/log/netd/imap.err \
+-l nntp://[::]/?out=/var/log/netd/nntp.out,err=/var/log/netd/nntp.err \
+-l pop3://[::]/?out=/var/log/netd/pop3.out,err=/var/log/netd/pop3.err \
+-l imaps://0.0.0.0/?out=/var/log/netd/imap.out,err=/var/log/netd/imap.err \
+-l nntps://0.0.0.0/?out=/var/log/netd/nntp.out,err=/var/log/netd/nntp.err \
+-l pop3s://0.0.0.0/?out=/var/log/netd/pop3.out,err=/var/log/netd/pop3.err \
+-l imaps://[::]/?out=/var/log/netd/imap.out,err=/var/log/netd/imap.err \
+-l nntps://[::]/?out=/var/log/netd/nntp.out,err=/var/log/netd/nntp.err \
+-l pop3s://[::]/?out=/var/log/netd/pop3.out,err=/var/log/netd/pop3.err \
+-l http://127.0.0.1:280/?psgi=/etc/public.psgi,err=/var/log/netd/http.err
+
+# NonBlocking is REQUIRED to avoid a race condition if running
+# simultaneous services
+NonBlocking = true
+
+Sockets = public-inbox-netd.socket
+KillSignal = SIGQUIT
+User = news
+Group = ssl-cert
+ExecReload = /bin/kill -HUP $MAINPID
+TimeoutStopSec = 30
+KillMode = process
+
+[Install]
+WantedBy = multi-user.target
diff --git a/examples/public-inbox-nntpd.socket b/examples/public-inbox-nntpd.socket
index eeddf343..10335d8d 100644
--- a/examples/public-inbox-nntpd.socket
+++ b/examples/public-inbox-nntpd.socket
@@ -1,9 +1,26 @@
 # ==> /etc/systemd/system/public-inbox-nntpd.socket <==
+# Consider looking at public-inbox-netd.socket instead of this file
+# to simplify management when serving multiple protocols.
+#
+# This contains 5 sockets for an public-inbox-nntpd instance.
+# The TCP ports are well-known ports registered in /etc/services.
+# The /run/nntpd.onion.sock entry is meant for the Tor hidden service
+# enabled by the following line in the torrc(5) file:
+#   HiddenServicePort 119 unix:/run/nntpd.onion.sock
 [Unit]
-Description = public-inbox-nntpd socket
+Description = public-inbox-nntpd sockets
 
 [Socket]
-ListenStream = 119
+ListenStream = 0.0.0.0:119
+ListenStream = 0.0.0.0:563
+ListenStream = /run/nntpd.onion.sock
+
+# Separating IPv4 from IPv6 listeners makes for nicer output
+# of IPv4 addresses in various reporting/monitoring tools
+BindIPv6Only = ipv6-only
+ListenStream = [::]:119
+ListenStream = [::]:563
+
 Service = public-inbox-nntpd@1.service
 
 [Install]
diff --git a/examples/public-inbox-nntpd@.service b/examples/public-inbox-nntpd@.service
index 4dd2f5d7..24f9ca73 100644
--- a/examples/public-inbox-nntpd@.service
+++ b/examples/public-inbox-nntpd@.service
@@ -1,4 +1,7 @@
 # ==> /etc/systemd/system/public-inbox-nntpd@.service <==
+# Consider looking at public-inbox-netd@.service instead of this file
+# to simplify management when serving multiple protocols.
+#
 # Since SIGUSR2 upgrades do not work under systemd, this service file
 # allows starting two simultaneous services during upgrade time
 # (e.g. public-inbox-nntpd@1 public-inbox-nntpd@2) with the intention
@@ -7,8 +10,8 @@
 
 [Unit]
 Description = public-inbox NNTP server %i
-Wants = public-inbox-nntpd.socket public-inbox-nntps.socket
-After = public-inbox-nntpd.socket public-inbox-nntps.socket
+Wants = public-inbox-nntpd.socket
+After = public-inbox-nntpd.socket
 
 [Service]
 Environment = PI_CONFIG=/home/pi/.public-inbox/config \
@@ -21,16 +24,15 @@ ExecStart = /usr/local/bin/public-inbox-nntpd \
 -1 /var/log/public-inbox/nntpd.out.log \
 --cert /etc/ssl/certs/news.example.com.pem \
 --key /etc/ssl/private/news.example.com.key
-StandardError = syslog
 
 # NonBlocking is REQUIRED to avoid a race condition if running
 # simultaneous services
 NonBlocking = true
 
-Sockets = public-inbox-nntpd.socket public-inbox-nntps.socket
+Sockets = public-inbox-nntpd.socket
 
 KillSignal = SIGQUIT
-User = nobody
+User = news
 Group = ssl-cert
 ExecReload = /bin/kill -HUP $MAINPID
 TimeoutStopSec = 86400
diff --git a/examples/public-inbox-nntps.socket b/examples/public-inbox-nntps.socket
deleted file mode 100644
index fa678196..00000000
--- a/examples/public-inbox-nntps.socket
+++ /dev/null
@@ -1,12 +0,0 @@
-# ==> /etc/systemd/system/public-inbox-nntps.socket <==
-[Unit]
-Description = public-inbox-nntps socket
-
-[Socket]
-ListenStream = 0.0.0.0:563
-BindIPv6Only = ipv6-only
-ListenStream = [::]:563
-Service = public-inbox-nntpd@1.service
-
-[Install]
-WantedBy = sockets.target
diff --git a/examples/public-inbox-watch.service b/examples/public-inbox-watch.service
index abb41469..0e4860f7 100644
--- a/examples/public-inbox-watch.service
+++ b/examples/public-inbox-watch.service
@@ -9,8 +9,6 @@ Environment = PI_CONFIG=/home/pi/.public-inbox/config \
 PATH=/usr/local/bin:/usr/bin:/bin
 ExecStart = /usr/local/bin/public-inbox-watch
 
-StandardOutput = syslog
-StandardError = syslog
 ExecReload = /bin/kill -HUP $MAINPID
 # this user must have read access to Maildirs it watches
 User = pi
diff --git a/examples/unsubscribe-milter@.service b/examples/unsubscribe-milter@.service
index eb5dcbe4..a68e6e81 100644
--- a/examples/unsubscribe-milter@.service
+++ b/examples/unsubscribe-milter@.service
@@ -24,7 +24,13 @@ Sockets = unsubscribe-milter.socket
 
 # the corresponding PSGI app needs permissions to modify the
 # mlmmj spool, so we might as well use the same user since
+# they both need to read /home/mlmmj/.unsubscribe.key
 User = mlmmj
 
+# only kill the parent process when using the default Sendmail::PMilter
+# postfork dispatcher, children will die naturally when they're done
+# with a given message.
+KillMode = process
+
 [Install]
 WantedBy = multi-user.target
diff --git a/examples/unsubscribe.milter b/examples/unsubscribe.milter
index 216b0ddd..8c682012 100644
--- a/examples/unsubscribe.milter
+++ b/examples/unsubscribe.milter
@@ -27,6 +27,28 @@ my $crypt = Crypt::CBC->new(-key => $key,
                         -cipher => 'Blowfish');
 $fh = $iv = $key = undef;
 
+my $allow_domains = '/etc/unsubscribe-milter.allow_domains';
+my $ALLOW_DOMAINS;
+if (open my $fh, '<', $allow_domains) {
+        local $/ = "\n";
+        chomp(my @l = <$fh>);
+        die "close: $!" unless eof($fh) && close($fh);
+        my %l = map { lc($_) => 1 } @l;
+        $ALLOW_DOMAINS = \%l;
+} else {
+        warn <<EOM;
+W: open $allow_domains: $! (all domains allowed)
+W: all mlmmj-looking messages will have List-Unsubscribe added,
+W: this is probably not what you want.
+EOM
+}
+
+# only allow users hitting SMTP server locally:
+# Is a config file necessary?  Regexps are ugly for IP addresses
+# but Net::Patricia (or similar) seems like overkill.  Ugly it is:
+my @ALLOW_ADDR = (qr/\A::1\z/, qr/\A127\./);
+my $ALLOW_ADDR = join('|', @ALLOW_ADDR);
+
 my %cbs;
 $cbs{connect} = sub {
         my ($ctx) = @_;
@@ -88,10 +110,24 @@ $cbs{eom} = sub {
         eval {
                 my $priv = $ctx->getpriv;
                 $ctx->setpriv({ header => {}, envrcpt => {} });
-                my @rcpt = keys %{$priv->{envrcpt}};
+
+                # XXX my postfix (3.5.18-0+deb11u1) + Sendmail::PMilter
+                # instance doesn't seem to get {client_addr}, but
+                # {daemon_addr} seems to make sense since I only want it
+                # to apply to users connecting to postfix locally:
+                if ($ALLOW_ADDR) {
+                        my $x = $ctx->getsymval('{daemon_addr}');
+                        return SMFIS_CONTINUE if $x && $x !~ /$ALLOW_ADDR/;
+                }
 
                 # one recipient, one unique HTTP(S) URL
+                my @rcpt = keys %{$priv->{envrcpt}};
                 return SMFIS_CONTINUE if @rcpt != 1;
+                if ($ALLOW_DOMAINS) {
+                        my $addr = $ctx->getsymval('{mail_addr}');
+                        my (undef, $d) = split /\@/, $addr;
+                        return SMFIS_CONTINUE if !$ALLOW_DOMAINS->{$d};
+                }
                 return SMFIS_CONTINUE if archive_addr(lc($rcpt[0]));
 
                 my $unsub = $priv->{header}->{'list-unsubscribe'} || [];
diff --git a/examples/varnish-4.vcl b/examples/varnish-4.vcl
index 5fc202ed..624f6013 100644
--- a/examples/varnish-4.vcl
+++ b/examples/varnish-4.vcl
@@ -28,7 +28,7 @@ sub vcl_recv {
 }
 
 sub vcl_pipe {
-        # By default Connection: close is set on all piped requests by varnish,
+        # By default, Connection: close is set on all piped requests by varnish,
         # but public-inbox-httpd supports persistent connections well :)
         unset bereq.http.connection;
         return (pipe);
diff --git a/install/README b/install/README
new file mode 100644
index 00000000..2cdac4d2
--- /dev/null
+++ b/install/README
@@ -0,0 +1,16 @@
+tooling for mass package installation
+-------------------------------------
+
+TL;DR (as root or with sudo):
+
+        ./install/deps.perl all                # install everything
+        ./install/deps.perl lei                # only what's needed for lei
+        ./install/deps.perl www-search        # for hosting WWW search
+
+Files in this directory are designed for:
+
+* users running our code from git or tarballs (and not the OS package manager)
+
+* lazy users who can't be bothered to read all of INSTALL
+
+* automated testing scripts (see ci/README)
diff --git a/install/deps.perl b/install/deps.perl
new file mode 100755
index 00000000..6563c3ce
--- /dev/null
+++ b/install/deps.perl
@@ -0,0 +1,414 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Helper script for mass installing/uninstalling with the OS package manager
+# TODO: figure out how to handle 3rd-party repo packages for CentOS 7.x
+eval 'exec perl -S $0 ${1+"$@"}' # no shebang
+if 0; # running under some shell
+use v5.12;
+my $help = <<EOM; # make sure this fits in 80x24 terminals
+usage: $^X $0 [-f PKG_FMT] [--allow-remove] PROFILE [PROFILE_MOD]
+
+  -f PKG_FMT      package format (`deb', `pkg', `pkg_add', `pkgin' or `rpm')
+  --allow-remove  allow removing packages (DANGEROUS, non-production use only)
+  --dry-run | -n  show commands that would be run
+  --yes | -y      non-interactive mode / assume yes to package manager
+
+PROFILE is typically `www-search', `lei', or `nntpd'
+Some profile names are intended for developer use only and subject to change.
+PROFILE_MOD is only for developers checking dependencies
+
+OS package installation typically requires administrative privileges
+EOM
+use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
+BEGIN { require './install/os.perl' };
+my $opt = {};
+GetOptions($opt, qw(pkg-fmt|f=s allow-remove dry-run|n yes|y help|h))
+        or die $help;
+if ($opt->{help}) { print $help; exit }
+my $pkg_fmt = $opt->{'pkg-fmt'} // do {
+        my $fmt = pkg_fmt;
+        warn "# using detected --pkg-fmt=$fmt on $ID/$VERSION_ID\n";
+        $fmt;
+};
+@ARGV or die $help;
+my @test_essential = qw(Test::Simple); # we actually use Test::More
+
+# package profiles.  Note we specify packages at maximum granularity,
+# which is typically deb for most things, but rpm seems to have the
+# highest granularity for things in the Perl standard library.
+my $profiles = {
+        # the smallest possible profile for testing
+        essential => [ qw(
+                autodie
+                git
+                perl
+                Digest::SHA
+                ExtUtils::MakeMaker
+                IO::Compress
+                Text::ParseWords
+                URI
+                ), @test_essential ],
+
+        # Everything else is optional for normal use.  Only specify
+        # the minimum to pull in dependencies here:
+        optional => [ qw(
+                Date::Parse
+                BSD::Resource
+                DBD::SQLite
+                Inline::C
+                Mail::IMAPClient
+                Net::Server
+                Parse::RecDescent
+                Plack
+                Plack::Test
+                Plack::Middleware::ReverseProxy
+                Xapian
+                curl
+                highlight.pm
+                libgit2-dev
+                libxapian-dev
+                sqlite3
+                xapian-tools
+                ) ],
+                # no pkg-config, libsqlite3, libxapian, libz, etc. since
+                # they'll get pulled in lib*-dev, DBD::SQlite and
+                # Xapian(.pm)  respectively
+
+        # optional developer stuff
+        devtest => [ qw(
+                XML::TreePP
+                w3m
+                Plack::Test::ExternalServer
+                ) ],
+};
+
+# only for distro-agnostic dependencies which are always true:
+my $always_deps = {
+        # we only load DBI explicitly
+        'DBD::SQLite' => [ qw(DBI libsqlite3) ],
+        'Mail::IMAPClient' => 'Parse::RecDescent',
+        'Plack::Middleware::ReverseProxy' => 'Plack',
+        'Xapian' => 'libxapian',
+        'xapian-tools' => 'libxapian',
+        'libxapian-dev' => [ qw(pkg-config libxapian) ],
+        'libgit2-dev' => 'pkg-config',
+};
+
+# bare minimum for v2
+$profiles->{v2essential} = [ @{$profiles->{essential}}, qw(DBD::SQLite) ];
+
+# for old v1 installs
+$profiles->{'www-v1'} = [ @{$profiles->{essential}}, qw(Plack) ];
+$profiles->{'www-thread'} = [ @{$profiles->{v2essential}}, qw(Plack) ];
+
+# common profile for PublicInbox::WWW
+$profiles->{'www-search'} = [ @{$profiles->{'www-thread'}}, qw(Xapian) ];
+
+# bare mininum for lei
+$profiles->{'lei-core'} = [ @{$profiles->{v2essential}}, qw(Xapian) ];
+push @{$profiles->{'lei-core'}}, 'Inline::C' if $^O ne 'linux';
+
+# common profile for lei:
+$profiles->{lei} = [ @{$profiles->{'lei-core'}}, qw(Mail::IMAPClient curl) ];
+
+$profiles->{nntpd} = [ @{$profiles->{v2essential}} ];
+$profiles->{pop3d} = [ @{$profiles->{v2essential}} ];
+$profiles->{'imapd-bare'} = [ @{$profiles->{v2essential}},
+                                qw(Parse::RecDescent) ];
+$profiles->{imapd} = [ @{$profiles->{'imapd-bare'}}, qw(Xapian) ];
+$profiles->{pop3d} = [ @{$profiles->{v2essential}} ];
+$profiles->{watch} = [ @{$profiles->{v2essential}}, qw(Mail::IMAPClient) ];
+$profiles->{'watch-v1'} = [ @{$profiles->{essential}} ];
+$profiles->{'watch-maildir'} = [ @{$profiles->{v2essential}} ];
+
+# package names which can't be mapped automatically and explicit
+# dependencies to prevent essential package removal:
+my $non_auto = { # git and perl (+autodie) are essential
+        git => {
+                pkg => [ qw(curl p5-TimeDate git) ],
+                rpm => [ qw(curl git) ],
+                pkg_add => [ qw(curl p5-Time-TimeDate git) ],
+        },
+        perl => {
+                apk => [ qw(perl perl-utils) ],
+                pkg => 'perl5',
+                pkgin => 'perl',
+                pkg_add => [], # Perl is part of OpenBSD base
+        },
+        # optional stuff:
+        'BSD::Resource' => {
+                apk => [], # not packaged for Alpine 3.19
+        },
+        'Date::Parse' => {
+                apk => 'perl-timedate',
+                deb => 'libtimedate-perl',
+                pkg => 'p5-TimeDate',
+                rpm => 'perl-TimeDate',
+                pkg_add => 'p5-Time-TimeDate',
+        },
+        'Inline::C' => {
+                apk => [ qw(perl-inline-c perl-dev) ],
+                pkg_add => 'p5-Inline', # tested OpenBSD 7.3
+                rpm => 'perl-Inline', # for CentOS 7.x, at least
+        },
+        'DBD::SQLite' => { deb => 'libdbd-sqlite3-perl' },
+        'Plack::Middleware::ReverseProxy' => {
+                apk => [], # not packaged for Alpine 3.19.0
+        },
+        'Plack::Test' => {
+                apk => 'perl-plack',
+                deb => 'libplack-perl',
+                pkg => 'p5-Plack',
+        },
+        'Plack::Test::ExternalServer' => {
+                apk => [], # not packaged for Alpine 3.19.0
+        },
+        'Xapian' => {
+                apk => 'xapian-bindings-perl',
+                deb => 'libsearch-xapian-perl',
+                pkg => 'p5-Xapian',
+                pkg_add => 'xapian-bindings-perl',
+                rpm => [], # xapian14-bindings-perl in 3rd-party repo
+        },
+        'highlight.pm' => {
+                apk => [],
+                deb => 'libhighlight-perl',
+                pkg => [],
+                pkgin => 'p5-highlight',
+                rpm => [],
+        },
+
+        # `libgit2' is the project name (since git has libgit)
+        'libgit2-dev' => {
+                pkg => 'libgit2',
+                rpm => 'libgit2-devel',
+        },
+
+        # some distros have both sqlite 2 and 3, we've only ever used 3
+        'libsqlite3' => {
+                apk => [], # handled by apk w/ perl-dbd-sqlite
+                pkg => 'sqlite3',
+                rpm => [], # `sqlite' is not removable due to yum/systemd
+                deb => [], # libsqlite3-0, but no need to specify
+        },
+
+        # only one version of Xapian distros
+        'libxapian' => { # avoid .so version numbers in our deps
+                deb => [], # libxapian30 atm, but no need to specify
+                pkg => 'xapian-core',
+                pkgin => 'xapian',
+                rpm => 'xapian-core',
+        },
+        'libxapian-dev' => {
+                apk => 'xapian-core-dev',
+                pkg => 'xapian-core',
+                pkgin => 'xapian',
+                rpm => 'xapian-core-devel',
+        },
+        'pkg-config' => {
+                apk => [], # handled by apk w/ xapian-core-dev
+                pkg_add => [], # part of the OpenBSD base system
+                pkg => 'pkgconf', # pkg-config is a symlink to pkgconf
+                pkgin => 'pkg-config',
+        },
+        'sqlite3' => { # this is just the executable binary on deb
+                apk => 'sqlite',
+                rpm => [], # `sqlite' is not removable due to yum/systemd
+        },
+
+        # we call xapian-compact(1) in public-inbox-compact(1) and
+        # xapian-delve(1) in public-inbox-cindex(1)
+        'xapian-tools' => {
+                apk => 'xapian-core',
+                pkg => 'xapian-core',
+                pkgin => 'xapian',
+                rpm => 'xapian-core', # ???
+        },
+
+        # OS-specific
+        'IO::KQueue' => {
+                apk => [],
+                deb => [],
+                rpm => [],
+        },
+};
+
+# standard library stuff that CentOS 7.x (and presumably other RPM)
+# split out and can be removed without removing the `perl' RPM:
+for (qw(autodie Digest::SHA ExtUtils::MakeMaker IO::Compress Sys::Syslog
+                Test::Simple Text::ParseWords)) {
+        # n.b.: Compress::Raw::Zlib is pulled in by IO::Compress
+        # qw(constant Encode Getopt::Long Exporter Storable Time::HiRes)
+        # don't need to be here since it's impossible to have `perl'
+        # on CentOS 7.x without them.
+        my $rpm = $_;
+        $rpm =~ s/::/-/g;
+        $non_auto->{$_} = {
+                deb => 'perl', # libperl5.XX, but the XX varies
+                pkg => 'perl5',
+                pkg_add => [], # perl is in the OpenBSD base system
+                apk => 'perl',
+                pkgin => 'perl',
+                rpm => "perl-$rpm",
+        };
+}
+
+# NetBSD and OpenBSD package names are similar to FreeBSD in most cases
+if ($pkg_fmt =~ /\A(?:pkg_add|pkgin)\z/) {
+        for my $name (keys %$non_auto) {
+                my $fbsd_pkg = $non_auto->{$name}->{pkg};
+                $non_auto->{$name}->{$pkg_fmt} //= $fbsd_pkg if $fbsd_pkg;
+        }
+}
+
+my %inst_check = ( # subs which return true if a package is intalled
+        apk => sub { system(qw(apk info -q -e), $_[0]) == 0 },
+        deb => sub { system("dpkg -s $_[0] >/dev/null 2>&1") == 0 },
+        pkg => sub { system(qw(pkg info -q), $_[0]) == 0 },
+        pkg_add => sub { system(qw(pkg_info -q -e), "$_[0]->=0") == 0 },
+        pkgin => sub { system(qw(pkg_info -q -e), $_[0]) == 0 },
+        rpm => sub { system("rpm -qs $_[0] >/dev/null 2>&1") == 0 },
+);
+
+our $INST_CHECK = $inst_check{$pkg_fmt} || die <<"";
+don't know how to check install status for $pkg_fmt
+
+my (@pkg_install, @pkg_remove, %all);
+for my $ary (values %$profiles) {
+        my @extra;
+        for my $pkg (@$ary) {
+                my $deps = $always_deps->{$pkg} // next;
+                push @extra, list($deps);
+        }
+        push @$ary, @extra;
+        $all{$_} = \@pkg_remove for @$ary;
+}
+if ($^O =~ /\A(?:free|net|open)bsd\z/) {
+        $all{'IO::KQueue'} = \@pkg_remove;
+}
+$profiles->{all} = [ keys %all ]; # pseudo-profile for all packages
+
+# parse the profile list from the command-line
+my @profiles = @ARGV;
+while (defined(my $profile = shift @profiles)) {
+        if ($profile =~ s/-\z//) {
+                # like apt-get, trailing "-" means remove
+                profile2dst($profile, \@pkg_remove);
+        } else {
+                profile2dst($profile, \@pkg_install);
+        }
+}
+
+# fill in @pkg_install and @pkg_remove:
+while (my ($pkg, $dst_pkg_list) = each %all) {
+        push @$dst_pkg_list, list(pkg2ospkg($pkg, $pkg_fmt));
+}
+
+my (%add, %rm); # uniquify lists
+@pkg_install = grep { !$add{$_}++ && !$INST_CHECK->($_) } @pkg_install;
+@pkg_remove = $opt->{'allow-remove'} ? grep {
+                !$add{$_} && !$rm{$_}++ && $INST_CHECK->($_)
+        } @pkg_remove : ();
+
+(@pkg_remove || @pkg_install) or warn "# no packages to install nor remove\n";
+
+# OS-specific cleanups appreciated
+if ($pkg_fmt eq 'apk') {
+        root('apk', 'add', @pkg_install) if @pkg_install;
+        root('apk', 'del', @pkg_remove) if @pkg_remove;
+} elsif ($pkg_fmt eq 'deb') {
+        my @apt_opt = qw(-o APT::Install-Recommends=false
+                        -o APT::Install-Suggests=false);
+        push @apt_opt, '-y' if $opt->{yes};
+        root('apt-get', @apt_opt, qw(install),
+                @pkg_install,
+                # apt-get lets you suffix a package with "-" to
+                # remove it in an "install" sub-command:
+                map { "$_-" } @pkg_remove) if (@pkg_remove || @pkg_install);
+        root('apt-get', @apt_opt, qw(autoremove)) if $opt->{'allow-remove'};
+} elsif ($pkg_fmt eq 'pkg') { # FreeBSD
+        my @pkg_opt = $opt->{yes} ? qw(-y) : ();
+
+        # don't remove stuff that isn't installed:
+        root(qw(pkg remove), @pkg_opt, @pkg_remove) if @pkg_remove;
+        root(qw(pkg install), @pkg_opt, @pkg_install) if @pkg_install;
+        root(qw(pkg autoremove), @pkg_opt) if $opt->{'allow-remove'};
+} elsif ($pkg_fmt eq 'pkgin') { # NetBSD
+        my @pkg_opt = $opt->{yes} ? qw(-y) : ();
+        root(qw(pkgin), @pkg_opt, 'remove', @pkg_remove) if @pkg_remove;
+        root(qw(pkgin), @pkg_opt, 'install', @pkg_install) if @pkg_install;
+        root(qw(pkgin), @pkg_opt, 'autoremove') if $opt->{'allow-remove'};
+# TODO: yum / rpm support
+} elsif ($pkg_fmt eq 'rpm') {
+        my @pkg_opt = $opt->{yes} ? qw(-y) : ();
+        root(qw(yum remove), @pkg_opt, @pkg_remove) if @pkg_remove;
+        root(qw(yum install), @pkg_opt, @pkg_install) if @pkg_install;
+} elsif ($pkg_fmt eq 'pkg_add') { # OpenBSD
+        my @pkg_opt = $opt->{yes} ? qw(-I) : (); # -I means non-interactive
+        root(qw(pkg_delete), @pkg_opt, @pkg_remove) if @pkg_remove;
+        @pkg_install = map { "$_--" } @pkg_install; # disambiguate w3m
+        root(qw(pkg_add), @pkg_opt, @pkg_install) if @pkg_install;
+        root(qw(pkg_delete -a), @pkg_opt) if $opt->{'allow-remove'};
+} else {
+        die "unsupported package format: $pkg_fmt\n";
+}
+exit 0;
+
+
+# map a generic package name to an OS package name
+sub pkg2ospkg {
+        my ($pkg, $fmt) = @_;
+
+        # check explicit overrides, first:
+        if (my $ospkg = $non_auto->{$pkg}->{$fmt}) {
+                return $ospkg;
+        }
+
+        # check common Perl module name patterns:
+        if ($pkg =~ /::/ || $pkg =~ /\A[A-Z]/) {
+                if ($fmt eq 'apk') {
+                        $pkg =~ s/::/-/g;
+                        return "perl-\L$pkg"
+                } elsif ($fmt eq 'deb') {
+                        $pkg =~ s/::/-/g;
+                        return "lib\L$pkg-perl";
+                } elsif ($fmt eq 'rpm') {
+                        $pkg =~ s/::/-/g;
+                        return "perl-$pkg"
+                } elsif ($fmt =~ /\Apkg(?:_add|in)?\z/) {
+                        $pkg =~ s/::/-/g;
+                        return "p5-$pkg"
+                } else {
+                        die "unsupported package format: $fmt for $pkg\n"
+                }
+        }
+
+        # use package name as-is (e.g. 'curl' or 'w3m')
+        $pkg;
+}
+
+# maps a install profile to a package list (@pkg_remove or @pkg_install)
+sub profile2dst {
+        my ($profile, $dst_pkg_list) = @_;
+        if (my $pkg_list = $profiles->{$profile}) {
+                $all{$_} = $dst_pkg_list for @$pkg_list;
+        } elsif ($all{$profile}) { # $profile is just a package name
+                $all{$profile} = $dst_pkg_list;
+        } else {
+                die "unrecognized profile or package: $profile\n";
+        }
+}
+
+sub root {
+        warn "# @_\n";
+        return if $opt->{'dry-run'};
+        return if system(@_) == 0;
+        warn "E: command failed: @_\n";
+        exit($? >> 8);
+}
+
+# ensure result can be pushed into an array:
+sub list {
+        my ($pkg) = @_;
+        ref($pkg) eq 'ARRAY' ? @$pkg : $pkg;
+}
diff --git a/install/os.perl b/install/os.perl
new file mode 100644
index 00000000..00edbadf
--- /dev/null
+++ b/install/os.perl
@@ -0,0 +1,85 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Helper library for detecting distro info and mapping to package manager.
+# This should NOT be installed via `make install'.
+# This is used by install/deps.perl and ci/profiles.perl
+package PublicInbox::InstallOS;
+use v5.12;
+use parent qw(Exporter);
+our ($ID, $PRETTY_NAME, $VERSION_ID); # same vars as os-release(5)
+our @EXPORT = qw($ID $VERSION_ID pkg_fmt);
+
+my ($release, $version); # from uname
+if ($^O eq 'linux') { # try using os-release(5)
+        for my $f (qw(/etc/os-release /usr/lib/os-release)) {
+                next unless -f $f;
+                my @echo = map {
+                        qq{echo "\$"$_" = qq[\$$_];"; }
+                } qw(ID PRETTY_NAME VERSION_ID);
+                # rely on sh(1) to handle interpolation and such:
+                my $vars = `sh -c '. $f; @echo'`;
+                die "sh \$?=$?" if $?;
+                eval $vars;
+                die $@ if $@;
+                $VERSION_ID //= '';
+                $ID //= '';
+                if ($ID eq 'debian' && $VERSION_ID eq '') {
+                        if ($PRETTY_NAME =~ m!/sid\z!) {
+                                $VERSION_ID = 'sid';
+                        } else {
+                                open my $fh, '<', $f or die "open($f): $!";
+                                my $msg = do { local $/; <$fh> };
+                                die <<EOM;
+ID=$ID, but no VERSION_ID
+==> $f <==
+$msg
+EOM
+                        }
+                }
+                last if $ID ne '' && $VERSION_ID ne '';
+        }
+        $ID = 'linux' if $ID eq ''; # cf. os-release(5)
+} elsif ($^O =~ m!\A(?:free|net|open)bsd\z! || $^O eq 'dragonfly') {
+        $ID = $^O;
+        require POSIX;
+        (undef, undef, $release, $version) = POSIX::uname();
+        $VERSION_ID = lc $release;
+        $VERSION_ID =~ s/[^0-9a-z\.\_\-]//sg; # cf. os-release(5)
+} else { # only support POSIX-like and Free systems:
+        die "$^O unsupported";
+}
+$VERSION_ID //= 0; # numeric? could be 'sid', actually...
+my %MIN_VER = ( # likely older versions work for many of these...
+        alpine => v3.19,
+        dragonfly => v6.4,
+        freebsd => v11,
+        netbsd => v9.3,
+        openbsd => v7.3,
+);
+
+if (defined(my $min_ver = $MIN_VER{$^O})) {
+        my $vid = $VERSION_ID;
+        $vid =~ s/-.*\z//s; # no dashes in v-strings
+        my $vstr = eval "v$vid";
+        die "can't convert VERSION_ID=$VERSION_ID to v-string" if $@;
+        die <<EOM if $vstr lt $min_ver;
+ID=$ID VERSION_ID=$VERSION_ID release=$release ($version) too old to support
+EOM
+}
+
+sub pkg_fmt () {
+        if ($ID eq 'alpine') { 'apk' }
+        elsif ($ID =~ /\A(?:freebsd|dragonfly)\z/) { 'pkg' }
+        # *shrug*, as long as the (Net|Open)BSD names don't conflict w/ FreeBSD
+        elsif ($ID eq 'netbsd') { 'pkgin' }
+        elsif ($ID eq 'openbsd') { 'pkg_add' }
+        elsif ($ID =~ m!\A(?:debian|ubuntu)\z!) { 'deb' }
+        elsif ($ID =~ m!\A(?:centos|redhat|fedora)\z!) { 'rpm' }
+        else { warn "PKG_FMT undefined for ID=$ID"; undef }
+}
+
+package main;
+PublicInbox::InstallOS->import;
+
+1;
diff --git a/lei.sh b/lei.sh
index f1510a73..afb03f3e 100755
--- a/lei.sh
+++ b/lei.sh
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 # symlink this file to a directory in PATH to run lei (or anything in script/*)
 # without needing perms to install globally.  Used by "make symlink-install"
-p=$(realpath "$0" || readlink "$0") # neither is POSIX, but common
+p=$(realpath "$0" 2>/dev/null || readlink "$0") # neither is POSIX, but common
 p=$(dirname "$p") c=$(basename "$0") # both are POSIX
 exec ${PERL-perl} -w -I"$p"/lib "$p"/script/"${c%.sh}" "$@"
 : this script is too short to copyright
diff --git a/lib/PublicInbox/Address.pm b/lib/PublicInbox/Address.pm
index 2c9c4395..3a59945c 100644
--- a/lib/PublicInbox/Address.pm
+++ b/lib/PublicInbox/Address.pm
@@ -1,9 +1,8 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 package PublicInbox::Address;
-use strict;
-use v5.10.1;
-use parent 'Exporter';
+use v5.12;
+use parent qw(Exporter);
 our @EXPORT_OK = qw(pairs);
 
 sub xs_emails {
@@ -20,8 +19,11 @@ sub xs_names {
 }
 
 sub xs_pairs { # for JMAP, RFC 8621 section 4.1.2.3
-        [ map { # LHS (name) may be undef
-                [ $_->phrase // $_->comment, $_->address ]
+        [ map { # LHS (name) may be undef if there's an address
+                my @p = ($_->phrase // $_->comment, $_->address);
+                # show original if totally bogus:
+                $p[0] = $_->original unless defined $p[1];
+                \@p;
         } parse_email_addresses($_[0]) ];
 }
 
@@ -31,6 +33,7 @@ eval {
         *emails = \&xs_emails;
         *names = \&xs_names;
         *pairs = \&xs_pairs;
+        *objects = sub { Email::Address::XS->parse(@_) };
 };
 
 if ($@) {
@@ -38,6 +41,7 @@ if ($@) {
         *emails = \&PublicInbox::AddressPP::emails;
         *names = \&PublicInbox::AddressPP::names;
         *pairs = \&PublicInbox::AddressPP::pairs;
+        *objects = \&PublicInbox::AddressPP::objects;
 }
 
 1;
diff --git a/lib/PublicInbox/AddressPP.pm b/lib/PublicInbox/AddressPP.pm
index 6a3ae4fe..65ba36a9 100644
--- a/lib/PublicInbox/AddressPP.pm
+++ b/lib/PublicInbox/AddressPP.pm
@@ -1,7 +1,8 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 package PublicInbox::AddressPP;
 use strict;
+use v5.10.1; # TODO check regexps for unicode_strings compat
 
 # very loose regexes, here.  We don't need RFC-compliance,
 # just enough to make thing sanely displayable and pass to git
@@ -56,4 +57,13 @@ sub pairs { # for JMAP, RFC 8621 section 4.1.2.3
         } emails($s) ];
 }
 
+# Mail::Address->name is inconsistent with Email::Address::XS, so we're
+# doing our own thing, here:
+sub objects { map { bless $_, __PACKAGE__ } @{pairs($_[0])} }
+
+# OO API for objects() results
+sub user { (split(/@/, $_[0]->[1]))[0] }
+sub host { (split(/@/, $_[0]->[1]))[1] }
+sub name { $_[0]->[0] // user($_[0]) }
+
 1;
diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index 11ea8f83..a1b1fc07 100644
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -1,15 +1,15 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # common stuff for administrative command-line tools
 # Unstable internal API
 package PublicInbox::Admin;
-use strict;
+use v5.12;
 use parent qw(Exporter);
-our @EXPORT_OK = qw(setup_signals);
+our @EXPORT_OK = qw(setup_signals fmt_localtime);
 use PublicInbox::Config;
 use PublicInbox::Inbox;
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Spawn qw(run_qx);
 use PublicInbox::Eml;
 *rel2abs_collapsed = \&PublicInbox::Config::rel2abs_collapsed;
 
@@ -28,12 +28,12 @@ sub setup_signals {
         };
 }
 
-sub resolve_eidxdir {
-        my ($cd) = @_;
+sub resolve_any_idxdir ($$) {
+        my ($cd, $lock_bn) = @_;
         my $try = $cd // '.';
         my $root_dev_ino;
-        while (1) { # favor v2, first
-                if (-f "$try/ei.lock") {
+        while (1) {
+                if (-f "$try/$lock_bn") { # inbox.lock, ei.lock, cidx.lock
                         return rel2abs_collapsed($try);
                 } elsif (-d $try) {
                         my @try = stat _;
@@ -49,61 +49,47 @@ sub resolve_eidxdir {
         }
 }
 
+sub resolve_eidxdir ($) { resolve_any_idxdir($_[0], 'ei.lock') }
+sub resolve_cidxdir ($) { resolve_any_idxdir($_[0], 'cidx.lock') }
+
 sub resolve_inboxdir {
         my ($cd, $ver) = @_;
-        my $try = $cd // '.';
-        my $root_dev_ino;
-        while (1) { # favor v2, first
-                if (-f "$try/inbox.lock") {
-                        $$ver = 2 if $ver;
-                        return rel2abs_collapsed($try);
-                } elsif (-d $try) {
-                        my @try = stat _;
-                        $root_dev_ino //= do {
-                                my @root = stat('/') or die "stat /: $!\n";
-                                "$root[0]\0$root[1]";
-                        };
-                        last if "$try[0]\0$try[1]" eq $root_dev_ino;
-                        $try .= '/..'; # continue, cd up
-                } else {
-                        die "`$try' is not a directory\n";
-                }
-        }
-        # try v1 bare git dirs
-        my $cmd = [ qw(git rev-parse --git-dir) ];
-        my $fh = popen_rd($cmd, undef, {-C => $cd});
-        my $dir = do { local $/; <$fh> };
-        close $fh or die "error in @$cmd (cwd:${\($cd // '.')}): $!\n";
-        chomp $dir;
-        $$ver = 1 if $ver;
-        rel2abs_collapsed($dir eq '.' ? ($cd // $dir) : $dir);
+        my $dir;
+        if (defined($dir = resolve_any_idxdir($cd, 'inbox.lock'))) { # try v2
+                $$ver = 2 if $ver;
+        } elsif (defined($dir = resolve_git_dir($cd))) { # try v1
+                $$ver = 1 if $ver;
+        } # else: not an inbox at all
+        $dir;
 }
 
-# for unconfigured inboxes
-sub detect_indexlevel ($) {
-        my ($ibx) = @_;
-
-        my $over = $ibx->over;
-        my $srch = $ibx->search;
-        delete @$ibx{qw(over search)}; # don't leave open FDs lying around
-
-        # brand new or never before indexed inboxes default to full
-        return 'full' unless $over;
-        my $l = 'basic';
-        return $l unless $srch;
-        if (my $xdb = $srch->xdb) {
-                $l = 'full';
-                my $m = $xdb->get_metadata('indexlevel');
-                if ($m eq 'medium') {
-                        $l = $m;
-                } elsif ($m ne '') {
-                        warn <<"";
-$ibx->{inboxdir} has unexpected indexlevel in Xapian: $m
+sub valid_pwd {
+        my $pwd = $ENV{PWD} // return;
+        my @st_pwd = stat $pwd or return;
+        my @st_cwd = stat '.' or die "stat(.): $!";
+        "@st_pwd[1,0]" eq "@st_cwd[1,0]" ? $pwd : undef;
+}
 
-                }
-                $ibx->{-skip_docdata} = 1 if $xdb->get_metadata('skip_docdata');
+sub resolve_git_dir {
+        my ($cd) = @_; # cd may be `undef' for cwd
+        # try v1 bare git dirs
+        my $pwd = valid_pwd();
+        my $env;
+        defined($pwd) && substr($cd // '/', 0, 1) ne '/' and
+                $env->{PWD} = "$pwd/$cd";
+        my $cmd = [ qw(git rev-parse --git-dir) ];
+        my $dir = run_qx($cmd, $env, { -C => $cd });
+        die "error in @$cmd (cwd:${\($cd // '.')}): $?\n" if $?;
+        chomp $dir;
+        # --absolute-git-dir requires git v2.13.0+, and we want to
+        # respect symlinks when $ENV{PWD} if $ENV{PWD} ne abs_path('.')
+        # since we store absolute GIT_DIR paths in cindex.
+        if (substr($dir, 0, 1) ne '/') {
+                substr($cd // '/', 0, 1) eq '/' or
+                        $cd = File::Spec->rel2abs($cd, $pwd);
+                $dir = rel2abs_collapsed($dir, $cd);
         }
-        $l;
+        $dir;
 }
 
 sub unconfigured_ibx ($$) {
@@ -128,12 +114,22 @@ sub resolve_inboxes ($;$$) {
                 $cfg or die "--all specified, but $cfgfile not readable\n";
                 @$argv and die "--all specified, but directories specified\n";
         }
-        my (@old, @ibxs, @eidx);
+        my (@old, @ibxs, @eidx, @cidx);
+        if ($opt->{-cidx_ok}) {
+                require PublicInbox::CodeSearchIdx;
+                @$argv = grep {
+                        if (defined(my $d = resolve_cidxdir($_))) {
+                                push @cidx, PublicInbox::CodeSearchIdx->new(
+                                                        $d, $opt);
+                                undef;
+                        } else {
+                                1;
+                        }
+                } @$argv;
+        }
         if ($opt->{-eidx_ok}) {
                 require PublicInbox::ExtSearchIdx;
-                my $i = -1;
                 @$argv = grep {
-                        $i++;
                         if (defined(my $ei = resolve_eidxdir($_))) {
                                 $ei = PublicInbox::ExtSearchIdx->new($ei, $opt);
                                 push @eidx, $ei;
@@ -155,6 +151,7 @@ sub resolve_inboxes ($;$$) {
                                 warn "W: $ibx->{name} $ibx->{inboxdir}: $!\n";
                         }
                 });
+                # TODO: no way to configure cindex in config file, yet
         } else { # directories specified on the command-line
                 my @dirs = @$argv;
                 push @dirs, '.' if !@dirs && $opt->{-use_cwd};
@@ -195,7 +192,8 @@ sub resolve_inboxes ($;$$) {
                 die "-V$min_ver inboxes not supported by $0\n\t",
                     join("\n\t", @old), "\n";
         }
-        $opt->{-eidx_ok} ? (\@ibxs, \@eidx) : @ibxs;
+        ($opt->{-eidx_ok} || $opt->{-cidx_ok}) ? (\@ibxs, \@eidx, \@cidx)
+                                                : @ibxs;
 }
 
 my @base_mod = ();
@@ -203,13 +201,13 @@ my @over_mod = qw(DBD::SQLite DBI);
 my %mod_groups = (
         -index => [ @base_mod, @over_mod ],
         -base => \@base_mod,
-        -search => [ @base_mod, @over_mod, 'Search::Xapian' ],
+        -search => [ @base_mod, @over_mod, 'Xapian' ],
 );
 
 sub scan_ibx_modules ($$) {
         my ($mods, $ibx) = @_;
         if (!$ibx->{indexlevel} || $ibx->{indexlevel} ne 'basic') {
-                $mods->{'Search::Xapian'} = 1;
+                $mods->{'Xapian'} = 1;
         } else {
                 $mods->{$_} = 1 foreach @over_mod;
         }
@@ -221,10 +219,10 @@ sub check_require {
         while (my $mod = shift @mods) {
                 if (my $groups = $mod_groups{$mod}) {
                         push @mods, @$groups;
-                } elsif ($mod eq 'Search::Xapian') {
+                } elsif ($mod eq 'Xapian') {
                         require PublicInbox::Search;
                         PublicInbox::Search::load_xapian() or
-                                $err->{'Search::Xapian || Xapian'} = $@;
+                                $err->{'Xapian || Search::Xapian'} = $@;
                 } else {
                         eval "require $mod";
                         $err->{$mod} = $@ if $@;
@@ -383,4 +381,12 @@ sub do_chdir ($) {
         }
 }
 
+sub fmt_localtime ($) {
+        require POSIX;
+        my @lt = localtime $_[0];
+        my (undef, $M, $H, $d, $m, $Y) = @lt;
+        sprintf('%u-%02u-%02u % 2u:%02u ', $Y + 1900, $m + 1, $d, $H, $M)
+                .POSIX::strftime('%z', @lt);
+}
+
 1;
diff --git a/lib/PublicInbox/AdminEdit.pm b/lib/PublicInbox/AdminEdit.pm
index c8c3d3e8..654141a7 100644
--- a/lib/PublicInbox/AdminEdit.pm
+++ b/lib/PublicInbox/AdminEdit.pm
@@ -19,11 +19,11 @@ sub check_editable ($) {
                 }
 
                 # Undefined indexlevel, so `full'...
-                # Search::Xapian exists and the DB can be read, at least, fine
+                # Xapian exists and the DB can be read, at least, fine
                 $ibx->search and next;
 
                 # it's possible for a Xapian directory to exist,
-                # but Search::Xapian to go missing/broken.
+                # but Xapian to go missing/broken.
                 # Make sure it's purged in that case:
                 $ibx->over or die "no over.sqlite3 in $ibx->{inboxdir}\n";
 
diff --git a/lib/PublicInbox/Aspawn.pm b/lib/PublicInbox/Aspawn.pm
new file mode 100644
index 00000000..49f8651a
--- /dev/null
+++ b/lib/PublicInbox/Aspawn.pm
@@ -0,0 +1,34 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# async system()/qx() which takes callback
+package PublicInbox::Aspawn;
+use v5.12;
+use parent qw(Exporter);
+use PublicInbox::DS qw(awaitpid);
+use PublicInbox::Spawn qw(spawn);
+our @EXPORT_OK = qw(run_await);
+
+sub _await_cb { # awaitpid cb
+        my ($pid, $cmd, $env, $opt, $cb, @args) = @_;
+        PublicInbox::Spawn::read_out_err($opt);
+        if ($? && !$opt->{quiet}) {
+                my ($status, $sig) = ($? >> 8, $? & 127);
+                my $msg = '';
+                $msg .= " (-C=$opt->{-C})" if defined $opt->{-C};
+                $msg .= " status=$status" if $status;
+                $msg .= " signal=$sig" if $sig;
+                warn "E: @$cmd", $msg, "\n";
+        }
+        $cb->($pid, $cmd, $env, $opt, @args) if $cb;
+}
+
+sub run_await {
+        my ($cmd, $env, $opt, $cb, @args) = @_;
+        $opt->{1} //= \(my $out);
+        my $pid = spawn($cmd, $env, $opt);
+        awaitpid($pid, \&_await_cb, $cmd, $env, $opt, $cb, @args);
+        awaitpid($pid); # synchronous for non-$in_loop
+}
+
+1;
diff --git a/lib/PublicInbox/AutoReap.pm b/lib/PublicInbox/AutoReap.pm
index 23ecce77..ae4984b8 100644
--- a/lib/PublicInbox/AutoReap.pm
+++ b/lib/PublicInbox/AutoReap.pm
@@ -3,8 +3,7 @@
 
 # automatically kill + reap children when this goes out-of-scope
 package PublicInbox::AutoReap;
-use v5.10.1;
-use strict;
+use v5.12;
 
 sub new {
         my (undef, $pid, $cb) = @_;
@@ -21,8 +20,8 @@ sub join {
         my $pid = delete $self->{pid} or return;
         $self->{cb}->() if defined $self->{cb};
         CORE::kill($sig, $pid) if defined $sig;
-        my $ret = waitpid($pid, 0) // die "waitpid($pid): $!";
-        $ret == $pid or die "BUG: waitpid($pid) != $ret";
+        my $r = waitpid($pid, 0);
+        $r == $pid or die "BUG? waitpid($pid) => $r (\$?=$? \$!=$!)";
 }
 
 sub DESTROY {
diff --git a/lib/PublicInbox/Cgit.pm b/lib/PublicInbox/Cgit.pm
index cc729aa2..78fc9ca0 100644
--- a/lib/PublicInbox/Cgit.pm
+++ b/lib/PublicInbox/Cgit.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # wrapper for cgit(1) and git-http-backend(1) for browsing and
@@ -6,7 +6,8 @@
 # directive to be set in the public-inbox config file.
 
 package PublicInbox::Cgit;
-use strict;
+use v5.12;
+use parent qw(PublicInbox::WwwCoderepo);
 use PublicInbox::GitHTTPBackend;
 use PublicInbox::Git;
 # not bothering with Exporter for a one-off
@@ -40,10 +41,9 @@ sub locate_cgit ($) {
                 if (defined($cgit_bin) && $cgit_bin =~ m!\A(.+?)/[^/]+\z!) {
                         unshift @dirs, $1 if -d $1;
                 }
-                foreach my $d (@dirs) {
-                        my $f = "$d/cgit.css";
-                        next unless -f $f;
-                        $cgit_data = $d;
+                for (@dirs) {
+                        next unless -f "$_/cgit.css";
+                        $cgit_data = $_;
                         last;
                 }
         }
@@ -53,29 +53,18 @@ sub locate_cgit ($) {
 sub new {
         my ($class, $pi_cfg) = @_;
         my ($cgit_bin, $cgit_data) = locate_cgit($pi_cfg);
-        # TODO: support gitweb and other repository viewers?
-        if (defined(my $cgitrc = $pi_cfg->{-cgitrc_unparsed})) {
-                $pi_cfg->parse_cgitrc($cgitrc, 0);
-        }
+        $cgit_bin // return; # fall back in WWW->cgit
         my $self = bless {
                 cmd => [ $cgit_bin ],
                 cgit_data => $cgit_data,
                 pi_cfg => $pi_cfg,
+                cgitrc => $pi_cfg->{'publicinbox.cgitrc'} // $ENV{CGIT_CONFIG},
         }, $class;
 
         # some cgit repos may not be mapped to inboxes, so ensure those exist:
-        my $code_repos = $pi_cfg->{-code_repos};
-        foreach my $k (keys %$pi_cfg) {
-                $k =~ /\Acoderepo\.(.+)\.dir\z/ or next;
-                my $dir = $pi_cfg->{$k};
-                $code_repos->{$1} ||= $pi_cfg->fill_code_repo($1);
-        }
-        while (my ($nick, $repo) = each %$code_repos) {
-                $self->{"\0$nick"} = $repo;
-        }
-        my $cgit_static = $pi_cfg->{-cgit_static};
-        my $static = join('|', map { quotemeta $_ } keys %$cgit_static);
-        $self->{static} = qr/\A($static)\z/;
+        PublicInbox::WwwCoderepo::prepare_coderepos($self);
+        my $s = join('|', map { quotemeta } keys %{$pi_cfg->{-cgit_static}});
+        $self->{static} = qr/\A($s)\z/;
         $self;
 }
 
@@ -96,14 +85,14 @@ my @PASS_ENV = qw(
 my $parse_cgi_headers = \&PublicInbox::GitHTTPBackend::parse_cgi_headers;
 
 sub call {
-        my ($self, $env) = @_;
+        my ($self, $env, $ctx) = @_; # $ctx is optional, used by WWW
         my $path_info = $env->{PATH_INFO};
         my $cgit_data;
 
         # handle requests without spawning cgit iff possible:
         if ($path_info =~ m!\A/(.+?)/($PublicInbox::GitHTTPBackend::ANY)\z!ox) {
                 my ($nick, $path) = ($1, $2);
-                if (my PublicInbox::Git $git = $self->{"\0$nick"}) {
+                if (my $git = $self->{pi_cfg}->get_coderepo($nick)) {
                         return serve($env, $git, $path);
                 }
         } elsif ($path_info =~ m!$self->{static}! &&
@@ -112,17 +101,14 @@ sub call {
                 return PublicInbox::WwwStatic::response($env, [], $f);
         }
 
-        my $cgi_env = { PATH_INFO => $path_info };
-        foreach (@PASS_ENV) {
-                defined(my $v = $env->{$_}) or next;
-                $cgi_env->{$_} = $v;
-        }
-        $cgi_env->{'HTTPS'} = 'on' if $env->{'psgi.url_scheme'} eq 'https';
+        my %cgi_env = (CGIT_CONFIG => $self->{cgitrc}, PATH_INFO => $path_info);
+        @cgi_env{@PASS_ENV} = @$env{@PASS_ENV}; # spawn ignores undef vals
+        $cgi_env{HTTPS} = 'on' if $env->{'psgi.url_scheme'} eq 'https';
 
         my $rdr = input_prepare($env) or return r(500);
-        my $qsp = PublicInbox::Qspawn->new($self->{cmd}, $cgi_env, $rdr);
+        my $qsp = PublicInbox::Qspawn->new($self->{cmd}, \%cgi_env, $rdr);
         my $limiter = $self->{pi_cfg}->limiter('-cgit');
-        $qsp->psgi_return($env, $limiter, $parse_cgi_headers);
+        $qsp->psgi_yield($env, $limiter, $parse_cgi_headers, $ctx);
 }
 
 1;
diff --git a/lib/PublicInbox/CidxComm.pm b/lib/PublicInbox/CidxComm.pm
new file mode 100644
index 00000000..80a235e9
--- /dev/null
+++ b/lib/PublicInbox/CidxComm.pm
@@ -0,0 +1,28 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# Waits for initial comm(1) output for PublicInbox::CodeSearchIdx.
+# The initial output from `comm' can take a while to generate because
+# it needs to wait on:
+# `git cat-file --batch-all-objects --batch-check --unordered | sort'
+# We still rely on blocking reads, here, since comm should be fast once
+# it's seeing input.  (`--unordered | sort' is intentional for HDDs)
+package PublicInbox::CidxComm;
+use v5.12;
+use parent qw(PublicInbox::DS);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+
+sub new {
+        my ($cls, $rd, $cidx, $drs) = @_;
+        my $self = bless { cidx => $cidx, drs => $drs }, $cls;
+        $self->SUPER::new($rd, EPOLLIN|EPOLLONESHOT);
+}
+
+sub event_step {
+        my ($self) = @_;
+        my $rd = $self->{sock} // return warn('BUG?: no {sock}');
+        $self->close; # EPOLL_CTL_DEL
+        delete($self->{cidx})->cidx_read_comm($rd, delete $self->{drs});
+}
+
+1;
diff --git a/lib/PublicInbox/CidxLogP.pm b/lib/PublicInbox/CidxLogP.pm
new file mode 100644
index 00000000..5ea675bf
--- /dev/null
+++ b/lib/PublicInbox/CidxLogP.pm
@@ -0,0 +1,28 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# Waits for initial `git log -p' output for PublicInbox::CodeSearchIdx.
+# The initial output from `git log -p' can take a while to generate,
+# CodeSearchIdx can process prune work while it's happening.  Once
+# `git log -p' starts generating output, it should be able to keep
+# up with Xapian indexing, so we still rely on blocking reads to simplify
+# cidx_read_log_p
+package PublicInbox::CidxLogP;
+use v5.12;
+use parent qw(PublicInbox::DS);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+
+sub new {
+        my ($cls, $rd, $cidx, $git, $roots) = @_;
+        my $self = bless { cidx => $cidx, git => $git, roots => $roots }, $cls;
+        $self->SUPER::new($rd, EPOLLIN|EPOLLONESHOT);
+}
+
+sub event_step {
+        my ($self) = @_;
+        my $rd = $self->{sock} // return warn('BUG?: no {sock}');
+        $self->close; # EPOLL_CTL_DEL
+        delete($self->{cidx})->cidx_read_log_p($self, $rd);
+}
+
+1;
diff --git a/lib/PublicInbox/CidxXapHelperAux.pm b/lib/PublicInbox/CidxXapHelperAux.pm
new file mode 100644
index 00000000..91c9b021
--- /dev/null
+++ b/lib/PublicInbox/CidxXapHelperAux.pm
@@ -0,0 +1,48 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Intended for PublicInbox::DS::event_loop for -cindex --associate,
+# this reports auxilliary status while dumping
+package PublicInbox::CidxXapHelperAux;
+use v5.12;
+use parent qw(PublicInbox::DS);
+use PublicInbox::Syscall qw(EPOLLIN);
+
+# rpipe connects to req->fp[1] in xap_helper.h
+sub new {
+        my ($cls, $rpipe, $cidx, $pfx) = @_;
+        my $self = bless { cidx => $cidx, pfx => $pfx }, $cls;
+        $rpipe->blocking(0);
+        $self->SUPER::new($rpipe, EPOLLIN);
+}
+
+sub event_step {
+        my ($self) = @_; # xap_helper.h is line-buffered
+        my $buf = delete($self->{buf}) // '';
+        my $n = sysread($self->{sock}, $buf, 65536, length($buf));
+        if (!defined($n)) {
+                return if $!{EAGAIN};
+                die "sysread: $!";
+        }
+        my $pfx = $self->{pfx};
+        if ($n == 0) {
+                warn "BUG? $pfx buf=$buf" if $buf ne '';
+                if (delete $self->{cidx}->{PENDING}->{$pfx}) {
+                        warn "BUG? $pfx did not get mset.size";
+                        $self->{cidx}->index_next;
+                }
+                return $self->close;
+        }
+        my @lines = split(/^/m, $buf);
+        $self->{buf} = pop @lines if substr($lines[-1], -1) ne "\n";
+        for my $l (@lines) {
+                if ($l =~ /\Amset\.size=[0-9]+ nr_out=[0-9]+\n\z/) {
+                        delete $self->{cidx}->{PENDING}->{$pfx};
+                        $self->{cidx}->index_next;
+                }
+                chomp $l;
+                $self->{cidx}->progress("$pfx $l");
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/CmdIPC4.pm b/lib/PublicInbox/CmdIPC4.pm
index e368d032..2f102ec6 100644
--- a/lib/PublicInbox/CmdIPC4.pm
+++ b/lib/PublicInbox/CmdIPC4.pm
@@ -7,6 +7,16 @@
 package PublicInbox::CmdIPC4;
 use v5.12;
 use Socket qw(SOL_SOCKET SCM_RIGHTS);
+
+sub sendmsg_retry ($) {
+        return 1 if $!{EINTR};
+        return unless ($!{ENOMEM} || $!{ENOBUFS} || $!{ETOOMANYREFS});
+        return if ++$_[0] >= 50;
+        warn "# sleeping on sendmsg: $! (#$_[0])\n";
+        select(undef, undef, undef, 0.1);
+        1;
+}
+
 BEGIN { eval {
 require Socket::MsgHdr; # XS
 no warnings 'once';
@@ -20,18 +30,21 @@ no warnings 'once';
         my $try = 0;
         do {
                 $s = Socket::MsgHdr::sendmsg($sock, $mh, $flags);
-        } while (!defined($s) &&
-                        ($!{ENOBUFS} || $!{ENOMEM} || $!{ETOOMANYREFS}) &&
-                        (++$try < 50) &&
-                        warn "sleeping on sendmsg: $! (#$try)\n" &&
-                        select(undef, undef, undef, 0.1) == 0);
+        } while (!defined($s) && sendmsg_retry($try));
         $s;
 };
 
 *recv_cmd4 = sub ($$$) {
         my ($s, undef, $len) = @_; # $_[1] = destination buffer
         my $mh = Socket::MsgHdr->new(buflen => $len, controllen => 256);
-        my $r = Socket::MsgHdr::recvmsg($s, $mh, 0) // return (undef);
+        my $r;
+        do {
+                $r = Socket::MsgHdr::recvmsg($s, $mh, 0);
+        } while (!defined($r) && $!{EINTR});
+        if (!defined($r)) {
+                $_[1] = '';
+                return (undef);
+        }
         $_[1] = $mh->buf;
         return () if $r == 0;
         my (undef, undef, $data) = $mh->cmsghdr;
diff --git a/lib/PublicInbox/CodeSearch.pm b/lib/PublicInbox/CodeSearch.pm
new file mode 100644
index 00000000..e5fa4480
--- /dev/null
+++ b/lib/PublicInbox/CodeSearch.pm
@@ -0,0 +1,372 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# read-only external index for coderepos
+# currently, it only indexes commits and repository metadata
+# (pathname, root commits); not blob contents
+package PublicInbox::CodeSearch;
+use v5.12;
+use parent qw(PublicInbox::Search);
+use PublicInbox::Config;
+use PublicInbox::Search qw(retry_reopen int_val xap_terms);
+use PublicInbox::Compat qw(uniqstr);
+use Compress::Zlib qw(uncompress);
+use constant {
+        AT => 0, # author time YYYYMMDDHHMMSS, dt: for mail)
+        CT => 1, # commit time (Unix time stamp, like TS/rt: in mail)
+        CIDX_SCHEMA_VER => 1, # brand new schema for code search
+        # for repos (`Tr'), CT(col=1) is used for the latest tip commit time
+        # in refs/{heads,tags}.  AT(col=0) may be used to store disk usage
+        # in the future, but disk usage calculation is espensive w/ alternates
+};
+our @CODE_NRP;
+our @CODE_VMAP = (
+        [ AT, 'd:' ], # mairix compat
+        [ AT, 'dt:' ], # public-inbox mail compat
+        [ CT, 'ct:' ],
+);
+
+# note: the non-X term prefix allocations are shared with Xapian omega,
+# see xapian-applications/omega/docs/termprefixes.rst
+# bool_pfx_internal:
+#        type => 'T', # 'c' - commit, 'r' - repo GIT_DIR
+#        tags are not indexed, only normal branches (refs/heads/*), not hidden
+#        'P' # (pathname) GIT_DIR # uniq
+#        'G' # (group) root commit (may have multiple roots)
+my %bool_pfx_external = (
+        oid => 'Q', # type:commit - git OID hex (40|64)-byte SHA-(1|256)
+                # type:repo - rel2abs_collapsed(GIT_DIR)
+        parent => 'XP',
+        %PublicInbox::Search::PATCH_BOOL_COMMON,
+);
+
+my %prob_prefix = ( # copied from PublicInbox::Search
+        # do we care about committer? or partial commit OID via Xapian?
+        # o => 'XQ', # 'oid:' (bool) is exact, 'o:' (prob) can do partial
+        %PublicInbox::Search::PATCH_PROB_COMMON,
+
+        # default:
+        '' => 'S A XQUOT XFN ' . $PublicInbox::Search::NON_QUOTED_BODY
+);
+
+sub new {
+        my ($cls, $dir, $cfg) = @_;
+        # can't have a PublicInbox::Config here due to circular refs
+        bless { topdir => $dir, xpfx => "$dir/cidx".CIDX_SCHEMA_VER,
+                -cfg_f => $cfg->{-f} }, $cls;
+}
+
+sub join_data_key ($) { "join:$_[0]->{-cfg_f}" }
+
+sub join_data {
+        my ($self) = @_;
+        my $key = join_data_key($self);
+        my $cur = $self->xdb->get_metadata($key) or return;
+        $cur = eval { PublicInbox::Config::json()->decode(uncompress($cur)) };
+        warn "E: $@ (corrupt metadata in `$key' key?)" if $@;
+        my @m = grep { ref($cur->{$_}) ne 'ARRAY' } qw(ekeys roots ibx2root);
+        if (@m) {
+                warn <<EOM;
+W: $self->{topdir} join data for $self->{-cfg_f} missing: @m
+EOM
+                undef;
+        } elsif (@{$cur->{ekeys}} < @{$cur->{ibx2root}}) {
+                warn <<EOM;
+W: $self->{topdir} join data for $self->{-cfg_f} mismatched ekeys and ibx2root
+EOM
+                undef;
+        } else {
+                $cur;
+        }
+}
+
+sub qparse_new ($) {
+        my ($self) = @_;
+        my $qp = $self->qp_init_common;
+        my $cb = $qp->can('add_valuerangeprocessor') //
+                $qp->can('add_rangeprocessor'); # Xapian 1.5.0+
+        if (!@CODE_NRP) {
+                @CODE_NRP = map {
+                        $PublicInbox::Search::NVRP->new(@$_)
+                } @CODE_VMAP;
+        }
+        $cb->($qp, $_) for @CODE_NRP;
+        while (my ($name, $pfx) = each %bool_pfx_external) {
+                $qp->add_boolean_prefix($name, $_) for split(/ /, $pfx);
+        }
+        while (my ($name, $pfx) = each %prob_prefix) {
+                $qp->add_prefix($name, $_) for split(/ /, $pfx);
+        }
+        $qp;
+}
+
+sub generate_cxx () { # generates snippet for xap_helper.h
+        my $ret = <<EOM;
+# line ${\__LINE__} "${\__FILE__}"
+static NRP *code_nrp[${\scalar(@CODE_VMAP)}];
+static void code_nrp_init(void)
+{
+EOM
+        for (0..$#CODE_VMAP) {
+                my $x = $CODE_VMAP[$_];
+                $ret .= qq{\tcode_nrp[$_] = new NRP($x->[0], "$x->[1]");\n}
+        }
+$ret .= <<EOM;
+}
+
+# line ${\__LINE__} "${\__FILE__}"
+static void qp_init_code_search(Xapian::QueryParser *qp)
+{
+        for (size_t i = 0; i < MY_ARRAY_SIZE(code_nrp); i++)
+                qp->ADD_RP(code_nrp[i]);
+EOM
+        for my $name (sort keys %bool_pfx_external) {
+                for (split(/ /, $bool_pfx_external{$name})) {
+                        $ret .= qq{\tqp->add_boolean_prefix("$name", "$_");\n}
+                }
+        }
+        for my $name (sort keys %prob_prefix) {
+                for (split(/ /, $prob_prefix{$name})) {
+                        $ret .= qq{\tqp->add_prefix("$name", "$_");\n}
+                }
+        }
+        $ret .= "}\n";
+}
+
+# returns a Xapian::Query to filter by roots
+sub roots_filter { # retry_reopen callback
+        my ($self, $git_dir) = @_;
+        my $xdb = $self->xdb;
+        my $P = 'P'.$git_dir;
+        my ($cur, $end) = ($xdb->postlist_begin($P), $xdb->postlist_end($P));
+        if ($cur == $end) {
+                warn "W: $git_dir not indexed?\n";
+                return;
+        }
+        my @roots = xap_terms('G', $xdb, $cur->get_docid);
+        if (!@roots) {
+                warn "W: $git_dir has no root commits?\n";
+                return;
+        }
+        my $q = $PublicInbox::Search::X{Query}->new('G'.shift(@roots));
+        for my $r (@roots) {
+                $q = $PublicInbox::Search::X{Query}->new(
+                                        PublicInbox::Search::OP_OR(),
+                                        $q, 'G'.$r);
+        }
+        $q;
+}
+
+sub mset {
+        my ($self, $qry_str, $opt) = @_;
+        my $qp = $self->{qp} //= qparse_new($self);
+        my $qry = $qp->parse_query($qry_str, $self->{qp_flags});
+
+        # limit to commits with shared roots
+        if (defined(my $git_dir = $opt->{git_dir})) {
+                my $rf = retry_reopen($self, \&roots_filter, $git_dir)
+                        or return;
+
+                $qry = $PublicInbox::Search::X{Query}->new(
+                                PublicInbox::Search::OP_FILTER(),
+                                $qry, $rf);
+        }
+
+        # we only want commits:
+        $qry = $PublicInbox::Search::X{Query}->new(
+                                PublicInbox::Search::OP_FILTER(),
+                                $qry, 'T'.'c');
+        $self->do_enquire($qry, $opt, CT);
+}
+
+sub roots2paths { # for diagnostics
+        my ($self) = @_;
+        my $cur = $self->xdb->allterms_begin('G');
+        my $end = $self->{xdb}->allterms_end('G');
+        my $qrepo = $PublicInbox::Search::X{Query}->new('T'.'r');
+        my $enq = $PublicInbox::Search::X{Enquire}->new($self->{xdb});
+        $enq->set_weighting_scheme($PublicInbox::Search::X{BoolWeight}->new);
+        $enq->set_docid_order($PublicInbox::Search::ENQ_ASCENDING);
+        my %ret;
+        for (; $cur != $end; $cur++) {
+                my $G_oidhex = $cur->get_termname;
+                my $qry = $PublicInbox::Search::X{Query}->new(
+                                PublicInbox::Search::OP_FILTER(),
+                                $qrepo, $G_oidhex);
+                $enq->set_query($qry);
+                my ($size, $off, $lim) = (0, 0, 100000);
+                my $dirs = $ret{substr($G_oidhex, 1)} = [];
+                do {
+                        my $mset = $enq->get_mset($off += $size, $lim);
+                        for my $x ($mset->items) {
+                                push @$dirs, xap_terms('P', $x->get_document);
+                        }
+                        $size = $mset->size;
+                } while ($size);
+                @$dirs = sort(uniqstr(@$dirs));
+        }
+        \%ret;
+}
+
+sub docids_of_git_dir ($$) {
+        my ($self, $git_dir) = @_;
+        my @ids = $self->docids_by_postlist('P'.$git_dir);
+        warn <<"" if @ids > 1;
+BUG: (non-fatal) $git_dir indexed multiple times in $self->{topdir}
+
+        @ids;
+}
+
+sub root_oids ($$) {
+        my ($self, $git_dir) = @_;
+        my @ids = docids_of_git_dir $self, $git_dir or warn <<"";
+BUG? (non-fatal) `$git_dir' not indexed in $self->{topdir}
+
+        my @ret = map { xap_terms('G', $self->xdb, $_) } @ids;
+        @ret = uniqstr(@ret) if @ids > 1;
+        @ret;
+}
+
+sub paths2roots {
+        my ($self, $paths) = @_;
+        my %ret;
+        if ($paths) {
+                for my $p (keys %$paths) { @{$ret{$p}} = root_oids($self, $p) }
+        } else {
+                my $tmp = roots2paths($self);
+                for my $root_oidhex (keys %$tmp) {
+                        my $paths = delete $tmp->{$root_oidhex};
+                        push @{$ret{$_}}, $root_oidhex for @$paths;
+                }
+                @$_ = sort(@$_) for values %ret;
+        }
+        \%ret;
+}
+
+sub load_ct { # retry_reopen cb
+        my ($self, $git_dir) = @_;
+        my @ids = docids_of_git_dir $self, $git_dir or return;
+        for (@ids) {
+                my $doc = $self->get_doc($_) // next;
+                return int_val($doc, CT);
+        }
+}
+
+sub load_commit_times { # each_cindex callback
+        my ($self, $todo) = @_; # todo = [ [ time, git ], [ time, git ] ...]
+        my (@pending, $rec, $ct);
+        while ($rec = shift @$todo) {
+                $ct = $self->retry_reopen(\&load_ct, $rec->[1]->{git_dir});
+                if (defined $ct) {
+                        $rec->[0] = $ct;
+                } else { # may be in another cindex:
+                        push @pending, $rec;
+                }
+        }
+        @$todo = @pending;
+}
+
+sub load_coderepos { # each_cindex callback
+        my ($self, $pi_cfg) = @_;
+        my $name = $self->{name};
+        my $cfg_f = $pi_cfg->{-f};
+        my $lpfx = $self->{localprefix} or return warn <<EOM;
+W: cindex.$name.localprefix unset in $cfg_f, ignoring cindex.$name
+EOM
+        my $lre = join('|', map { $_ .= '/'; tr!/!/!s; quotemeta } @$lpfx);
+        $lre = qr!\A(?:$lre)!;
+        my $coderepos = $pi_cfg->{-coderepos};
+        my $nick_pfx = $name eq '' ? '' : "$name/";
+        my %dir2cr;
+        for my $p ($self->all_terms('P')) {
+                my $nick = $p;
+                $nick =~ s!$lre!$nick_pfx!s or next;
+                $dir2cr{$p} = $coderepos->{$nick} //= do {
+                        my $git = PublicInbox::Git->new($p);
+                        my %dedupe = ($nick => undef);
+                        ($git->{nick}) = keys %dedupe; # for git->pub_urls
+                        $git;
+                };
+        }
+        my $jd = $self->retry_reopen(\&join_data, $self) or return warn <<EOM;
+W: cindex.$name.topdir=$self->{topdir} has no usable join data for $cfg_f
+EOM
+        my ($ekeys, $roots, $ibx2root) = @$jd{qw(ekeys roots ibx2root)};
+        my $roots2paths = roots2paths($self);
+        my %dedupe; # 50x alloc reduction w/ lore + gko mirror (Mar 2024)
+        for my $root_offs (@$ibx2root) {
+                my $ekey = shift(@$ekeys) // die 'BUG: {ekeys} empty';
+                scalar(@$root_offs) or next;
+                my $ibx = $pi_cfg->lookup_eidx_key($ekey) // do {
+                        warn "W: `$ekey' gone from $cfg_f\n";
+                        next;
+                };
+                my $gits = $ibx->{-repo_objs} //= [];
+                my $cr_score = $ibx->{-cr_score} //= {};
+                my %ibx_p2g = map { $_->{git_dir} => $_ } @$gits;
+                my $ibx2self; # cindex has an association w/ inbox?
+                for (@$root_offs) { # sorted by $nr descending
+                        my ($nr, $root_off) = @$_;
+                        my $root_oid = $roots->[$root_off] // do {
+                                warn <<EOM;
+BUG: root #$root_off invalid in join data for `$ekey' with $cfg_f
+EOM
+                                next;
+                        };
+                        my $git_dirs = $roots2paths->{$root_oid};
+                        my @gits = map { $dir2cr{$_} // () } @$git_dirs;
+                        $cr_score->{$_->{nick}} //= $nr for @gits;
+                        @$git_dirs = grep { !$ibx_p2g{$_} } @$git_dirs;
+                        # @$git_dirs or warn "W: no matches for $root_oid\n";
+                        for (@$git_dirs) {
+                                if (my $git = $dir2cr{$_}) {
+                                        $ibx_p2g{$_} = $git;
+                                        $ibx2self = 1;
+                                        if (!$ibx->{-hide_www}) {
+                                                # don't stringify $nr directly
+                                                # to avoid long-lived PV
+                                                my $k = ($nr + 0)."\0".
+                                                        ($ibx + 0);
+                                                my $s = $dedupe{$k} //=
+                                                        [ $nr, $ibx->{name} ];
+                                                push @{$git->{ibx_score}}, $s;
+                                        }
+                                        push @$gits, $git;
+                                } else {
+                                        warn <<EOM;
+W: no coderepo available for $_ (localprefix=@$lpfx)
+EOM
+                                }
+                        }
+                }
+                if (@$gits) {
+                        push @{$ibx->{-csrch}}, $self if $ibx2self;
+                } else {
+                        delete $ibx->{-repo_objs};
+                        delete $ibx->{-cr_score};
+                }
+        }
+        for my $git (values %dir2cr) {
+                my $s = $git->{ibx_score};
+                @$s = sort { $b->[0] <=> $a->[0] } @$s if $s;
+        }
+        my $ALL = $pi_cfg->ALL or return;
+        my @alls_gits = sort {
+                scalar @{$b->{ibx_score} // []} <=>
+                        scalar @{$a->{ibx_score} // []}
+        } values %$coderepos;
+        my $gits = $ALL->{-repo_objs} //= [];
+        push @$gits, @alls_gits;
+        my $cr_score = $ALL->{-cr_score} //= {};
+        $cr_score->{$_->{nick}} //= scalar(@{$_->{ibx_score}//[]}) for @$gits;
+}
+
+sub repos_sorted {
+        my $pi_cfg = shift;
+        my @recs = map { [ 0, $_ ] } @_; # PublicInbox::Git objects
+        my @todo = @recs;
+        $pi_cfg->each_cindex(\&load_commit_times, \@todo);
+        @recs = sort { $b->[0] <=> $a->[0] } @recs; # sort by commit time
+}
+
+1;
diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
new file mode 100644
index 00000000..570ff64f
--- /dev/null
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -0,0 +1,1394 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# indexer for git coderepos, just commits and repo paths for now
+# this stores normalized absolute paths of indexed GIT_DIR inside
+# the DB itself and is designed to handle forks by designating roots
+# At minimum, it needs to have the pathnames of all git repos in
+# memory at runtime.  --join also requires all inbox pathnames to
+# be in memory (as it happens when loaded from ~/.public-inbox/config).
+#
+# Unlike mail search, docid isn't tied to NNTP artnum or IMAP UID,
+# there's no serial number dependency at all.  The first 32-bits of
+# the commit SHA-(1|256) is used to select a shard.
+#
+# We shard repos using the first 32-bits of sha256($ABS_GIT_DIR)
+#
+# --join associates root commits of coderepos to inboxes based on prefixes.
+#
+# Internally, each inbox is assigned a non-negative integer index ($IBX_OFF),
+# and each root commit object ID (SHA-1/SHA-256 hex) is also assigned
+# a non-negative integer index ($ROOT_COMMIT_OID_ID).
+#
+# join dumps to 2 intermediate files in $TMPDIR:
+#
+# * to_root_off - each line is of the format:
+#
+#        $PFX @ROOT_COMMIT_OID_OFFS
+#
+# * to_ibx_off - each line is of the format:
+#
+#        $PFX @IBX_OFFS
+#
+# $IBX_OFFS is a comma-delimited list of integers ($IBX_ID)
+# The $IBX_OFF here is ephemeral (per-join_data) and NOT related to
+# the `ibx_off' column of `over.sqlite3' for extindex.
+# @ROOT_COMMIT_OID_OFFS is space-delimited
+# In both cases, $PFX is typically the value of the 7-(hex)char dfpost
+# XDFPOST but it can be configured to use any combination of patchid,
+# dfpre, dfpost or dfblob.
+#
+# WARNING: this is vulnerable to arbitrary memory usage attacks if we
+# attempt to index or join against malicious coderepos with
+# thousands/millions of root commits.  Most coderepos have only one
+# root commit, some have several: git.git currently has 7,
+# torvalds/linux.git has 4.
+# --max-size= is required to keep memory usage reasonable for gigantic
+# commits.
+#
+# See PublicInbox::CodeSearch (read-only API) for more
+package PublicInbox::CodeSearchIdx;
+use v5.12;
+# parent order matters, we want ->DESTROY from IPC, not SearchIdx
+use parent qw(PublicInbox::CodeSearch PublicInbox::IPC PublicInbox::SearchIdx);
+use PublicInbox::DS qw(awaitpid);
+use PublicInbox::PktOp;
+use PublicInbox::IPC qw(nproc_shards);
+use POSIX qw(WNOHANG SEEK_SET strftime);
+use File::Path ();
+use File::Spec ();
+use List::Util qw(max);
+use PublicInbox::SHA qw(sha256_hex sha_all);
+use PublicInbox::Search qw(xap_terms);
+use PublicInbox::SearchIdx qw(add_val);
+use PublicInbox::Config qw(glob2re rel2abs_collapsed);
+use PublicInbox::Spawn qw(which spawn popen_rd);
+use PublicInbox::OnDestroy;
+use PublicInbox::CidxLogP;
+use PublicInbox::CidxComm;
+use PublicInbox::Git qw(%OFMT2HEXLEN);
+use PublicInbox::Compat qw(uniqstr);
+use PublicInbox::Aspawn qw(run_await);
+use Compress::Zlib qw(compress);
+use Carp qw(croak);
+use Time::Local qw(timegm);
+use autodie qw(close pipe open sysread seek sysseek send);
+our $DO_QUIT = 15; # signal number
+our (
+        $LIVE_JOBS, # integer
+        $GITS_NR, # number of coderepos
+        $MY_SIG, # like %SIG
+        $SIGSET,
+        $TXN_BYTES, # number of bytes in current shard transaction
+        $BATCH_BYTES,
+        @RDONLY_XDB, # Xapian::Database
+        @IDX_SHARDS, # clones of self
+        $MAX_SIZE,
+        $REINDEX, # PublicInbox::SharedKV
+        @GIT_DIR_GONE, # [ git_dir1, git_dir2 ]
+        $PRUNE_DONE, # marks off prune completions
+        $NCHANGE, # current number of changes
+        $NPROC,
+        $XHC, # XapClient
+        $REPO_CTX, # current repo being indexed in shards
+        $IDXQ, # PublicInbox::Git object arrayref
+        $SCANQ, # PublicInbox::Git object arrayref
+        %ALT_FH, # hexlen => tmp IO for TMPDIR git alternates
+        $TMPDIR, # File::Temp->newdir object for prune
+        @PRUNEQ, # GIT_DIRs to prepare for pruning
+        %TODO, @IBXQ, @IBX,
+        @JOIN, # join(1) command for --join
+        $CMD_ENV, # env for awk(1), comm(1), sort(1) commands during prune
+        @AWK, @COMM, @SORT, # awk(1), comm(1), sort(1) commands
+        %JOIN, # CLI --join= suboptions
+        @JOIN_PFX, # any combination of XDFID, XDFPRE, XDFPOST
+        @JOIN_DT, # YYYYmmddHHMMSS for dt:
+        $QRY_STR, # common query string for both code and inbox associations
+        $DUMP_IBX_WPIPE, # goes to sort(1)
+        $ANY_SHARD, # shard round-robin for scan fingerprinting
+        @OFF2ROOT,
+        $GIT_VER,
+        @NO_ABBREV,
+);
+
+# stop walking history if we see >$SEEN_MAX existing commits, this assumes
+# branches don't diverge by more than this number of commits...
+# git walks commits quickly if it doesn't have to read trees
+our $SEEN_MAX = 100000;
+
+# window for commits/emails to determine a inbox <-> coderepo association
+my $JOIN_WINDOW = 50000;
+
+our @PRUNE_BATCH = qw(cat-file --batch-all-objects --batch-check);
+
+# TODO: do we care about committer name + email? or tree OID?
+my @FMT = qw(H P ct an ae at s b); # (b)ody must be last
+
+# git log --stdin buffers all commits before emitting, thus --reverse
+# doesn't incur extra overhead.  We use --reverse to keep Xapian docids
+# increasing so we may be able to avoid sorting results in some cases
+my @LOG_STDIN = (qw(log --no-decorate --no-color --no-notes -p --stat -M
+        --reverse --stdin --no-walk=unsorted), '--pretty=format:%n%x00'.
+        join('%n', map { "%$_" } @FMT));
+
+sub new {
+        my (undef, $dir, $opt) = @_;
+        my $l = $opt->{indexlevel} // 'full';
+        $l !~ $PublicInbox::SearchIdx::INDEXLEVELS and
+                die "invalid indexlevel=$l\n";
+        $l eq 'basic' and die "E: indexlevel=basic not supported\n";
+        my $self = bless {
+                xpfx => "$dir/cidx".  PublicInbox::CodeSearch::CIDX_SCHEMA_VER,
+                cidx_dir => $dir,
+                creat => 1, # TODO: get rid of this, should be implicit
+                transact_bytes => 0, # for checkpoint
+                total_bytes => 0, # for lock_release
+                current_info => '',
+                parallel => 1,
+                -opt => $opt,
+                lock_path => "$dir/cidx.lock",
+        }, __PACKAGE__;
+        $self->{nshard} = count_shards($self) ||
+                nproc_shards({nproc => $opt->{jobs}});
+        $self->{-no_fsync} = 1 if !$opt->{fsync};
+        $self->{-dangerous} = 1 if $opt->{dangerous};
+        $self;
+}
+
+# This is similar to uniq(1) on the first column, but combines the
+# contents of subsequent columns using $OFS.
+our @UNIQ_FOLD = ($^X, $^W ? ('-w') : (), qw(-MList::Util=uniq -ane), <<'EOM');
+BEGIN { $ofs = $ENV{OFS} // ','; $apfx = '' }
+if ($F[0] eq $apfx) {
+        shift @F;
+        push @ids, @F;
+} else {
+        print $apfx.' '.join($ofs, uniq(@ids))."\n" if @ids;
+        ($apfx, @ids) = @F;
+}
+END { print $apfx.' '.join($ofs, uniq(@ids))."\n" if @ids }
+EOM
+
+# TODO: may be used for reshard/compact
+sub count_shards { scalar($_[0]->xdb_shards_flat) }
+
+sub update_commit ($$$) {
+        my ($self, $cmt, $roots) = @_; # fields from @FMT
+        my $x = 'Q'.$cmt->{H};
+        my ($docid, @extra) = sort { $a <=> $b } $self->docids_by_postlist($x);
+        @extra and warn "W: $cmt->{H} indexed multiple times, pruning ",
+                        join(', ', map { "#$_" } @extra), "\n";
+        $self->{xdb}->delete_document($_) for @extra;
+        my $doc = $PublicInbox::Search::X{Document}->new;
+        $doc->add_boolean_term($x);
+        $doc->add_boolean_term('G'.$_) for @$roots;
+        $doc->add_boolean_term('XP'.$_) for split(/ /, $cmt->{P});
+        $doc->add_boolean_term('T'.'c');
+
+        # Author-Time is compatible with dt: for mail search schema_version=15
+        add_val($doc, PublicInbox::CodeSearch::AT,
+                POSIX::strftime('%Y%m%d%H%M%S', gmtime($cmt->{at})));
+
+        # Commit-Time is the fallback used by rt: (TS) for mail search:
+        add_val($doc, PublicInbox::CodeSearch::CT, $cmt->{ct});
+
+        $self->term_generator->set_document($doc);
+
+        # email address is always indexed with positional data for usability
+        $self->index_phrase("$cmt->{an} <$cmt->{ae}>", 1, 'A');
+
+        $x = $cmt->{'s'};
+        $self->index_text($x, 1, 'S') if $x =~ /\S/s;
+        $doc->set_data($x); # subject is the first (and currently only) line
+
+        $x = delete $cmt->{b};
+        $self->index_body_text($doc, \$x) if $x =~ /\S/s;
+        defined($docid) ? $self->{xdb}->replace_document($docid, $doc) :
+                        $self->{xdb}->add_document($doc);
+}
+
+sub progress {
+        my ($self, @msg) = @_;
+        my $pr = $self->{-opt}->{-progress} or return;
+        $pr->($self->{git} ? ("$self->{git}->{git_dir}: ") : (), @msg, "\n");
+}
+
+sub check_objfmt_status ($$$) {
+        my ($git, $chld_err, $fmt) = @_;
+        my ($status, $sig) = ($chld_err >> 8, $chld_err & 127);
+        if (!$sig && $status == 1) { # unset, default is '' (SHA-1)
+                $fmt = 'sha1';
+        } elsif (!$sig && $status == 0) {
+                chomp($fmt ||= 'sha1');
+        }
+        $fmt // warn("git --git-dir=$git->{git_dir} config \$?=$chld_err");
+        $fmt;
+}
+
+sub store_repo { # wq_io_do, sends docid back
+        my ($self, $repo) = @_;
+        my $op_p = delete($self->{0}) // die 'BUG: no {0} op_p';
+        my $git = bless $repo, 'PublicInbox::Git';
+        my $rd = $git->popen(qw(config extensions.objectFormat));
+        $self->begin_txn_lazy;
+        $self->{xdb}->delete_document($_) for @{$repo->{to_delete}};
+        my $doc = $PublicInbox::Search::X{Document}->new;
+        add_val($doc, PublicInbox::CodeSearch::CT, $repo->{ct});
+        $doc->add_boolean_term("P$repo->{git_dir}");
+        $doc->add_boolean_term('T'.'r');
+        $doc->add_boolean_term('G'.$_) for @{$repo->{roots}};
+        $doc->set_data($repo->{fp}); # \n delimited
+        my $fmt = readline($rd);
+        $rd->close;
+        $fmt = check_objfmt_status $git, $?, $fmt;
+        $OFMT2HEXLEN{$fmt} // warn <<EOM; # store unknown formats anyways
+E: unknown extensions.objectFormat=$fmt in $repo->{git_dir}
+EOM
+        $doc->add_boolean_term('H'.$fmt);
+        my $did = $repo->{docid};
+        $did ? $self->{xdb}->replace_document($did, $doc)
+                : ($did = $self->{xdb}->add_document($doc));
+        send($op_p, "repo_stored $did", 0);
+}
+
+sub cidx_ckpoint ($;$) {
+        my ($self, $msg) = @_;
+        progress($self, $msg) if defined($msg);
+        $TXN_BYTES = $BATCH_BYTES; # reset
+        return if $PublicInbox::Search::X{CLOEXEC_UNSET};
+        $self->commit_txn_lazy;
+        $self->begin_txn_lazy;
+}
+
+sub truncate_cmt ($$) {
+        my ($cmt) = @_; # _[1] is $buf (giant)
+        my ($orig_len, $len);
+        $len = $orig_len = length($_[1]);
+        @$cmt{@FMT} = split(/\n/, $_[1], scalar(@FMT));
+        undef $_[1];
+        $len -= length($cmt->{b});
+
+        # try to keep the commit message body.
+        # n.b. this diffstat split may be unreliable but it's not worth
+        # perfection for giant commits:
+        my ($bdy) = split(/^---\n/sm, delete($cmt->{b}), 2);
+        if (($len + length($bdy)) <= $MAX_SIZE) {
+                $len += length($bdy);
+                $cmt->{b} = $bdy;
+                warn <<EOM;
+W: $cmt->{H}: truncated body ($orig_len => $len bytes)
+W: to be under --max-size=$MAX_SIZE
+EOM
+        } else {
+                $cmt->{b} = '';
+                warn <<EOM;
+W: $cmt->{H}: deleted body ($orig_len => $len bytes)
+W: to be under --max-size=$MAX_SIZE
+EOM
+        }
+        $len;
+}
+
+sub cidx_reap_log { # awaitpid cb
+        my ($pid, $cmd, $self, $op_p) = @_;
+        if (!$? || ($DO_QUIT && (($? & 127) == $DO_QUIT ||
+                                ($? & 127) == POSIX::SIGPIPE))) {
+                send($op_p, "shard_done $self->{shard}", 0);
+        } else {
+                warn "W: @$cmd (\$?=$?)\n";
+                $self->{xdb}->cancel_transaction;
+        }
+}
+
+sub shard_index { # via wq_io_do in IDX_SHARDS
+        my ($self, $git, $roots) = @_;
+
+        my $in = delete($self->{0}) // die 'BUG: no {0} input';
+        my $op_p = delete($self->{1}) // die 'BUG: no {1} op_p';
+        sysseek($in, 0, SEEK_SET);
+        my $cmd = $git->cmd(@NO_ABBREV, @LOG_STDIN);
+        my $rd = popen_rd($cmd, undef, { 0 => $in },
+                                \&cidx_reap_log, $cmd, $self, $op_p);
+        PublicInbox::CidxLogP->new($rd, $self, $git, $roots);
+        # CidxLogP->event_step will call cidx_read_log_p once there's input
+}
+
+# sharded reader for `git log --pretty=format: --stdin'
+sub cidx_read_log_p {
+        my ($self, $log_p, $rd) = @_;
+        my $git = delete $log_p->{git} // die 'BUG: no {git}';
+        local $self->{current_info} = "$git->{git_dir} [$self->{shard}]";
+        my $roots = delete $log_p->{roots} // die 'BUG: no {roots}';
+        # local-ized in parent before fork
+        $TXN_BYTES = $BATCH_BYTES;
+        local $self->{git} = $git; # for patchid
+        return if $DO_QUIT;
+        my $nr = 0;
+
+        # a patch may have \0, see c4201214cbf10636e2c1ab9131573f735b42c8d4
+        # in linux.git, so we use $/ = "\n\0" to check end-of-patch
+        my $FS = "\n\0";
+        my $len;
+        my $cmt = {};
+        local $/ = $FS;
+        my $buf = <$rd> // return; # leading $FS
+        $buf eq $FS or die "BUG: not LF-NUL: $buf\n";
+        $self->begin_txn_lazy;
+        while (!$DO_QUIT && defined($buf = <$rd>)) {
+                chomp($buf);
+                $/ = "\n";
+                $len = length($buf);
+                if (defined($MAX_SIZE) && $len > $MAX_SIZE) {
+                        $len = truncate_cmt($cmt, $buf);
+                } else {
+                        @$cmt{@FMT} = split(/\n/, $buf, scalar(@FMT));
+                }
+                if (($TXN_BYTES -= $len) <= 0) {
+                        cidx_ckpoint($self, "[$self->{shard}] $nr");
+                        $TXN_BYTES -= $len; # len may be huge, >TXN_BYTES;
+                }
+                update_commit($self, $cmt, $roots);
+                ++$nr;
+                cidx_ckpoint($self, "[$self->{shard}] $nr") if $TXN_BYTES <= 0;
+                $/ = $FS;
+        }
+        # return and wait for cidx_reap_log
+}
+
+sub shard_done { # called via PktOp on shard_index completion
+        my ($self, $repo_ctx, $on_destroy, $n) = @_;
+        $repo_ctx->{shard_ok}->{$n} = 1;
+}
+
+sub repo_stored {
+        my ($self, $repo_ctx, $drs, $did) = @_;
+        # check @IDX_SHARDS instead of DO_QUIT to avoid wasting prior work
+        # because shard_commit is fast
+        return unless @IDX_SHARDS;
+        $did > 0 or die "BUG: $repo_ctx->{repo}->{git_dir}: docid=$did";
+        my ($c, $p) = PublicInbox::PktOp->pair;
+        $c->{ops}->{shard_done} = [ $self, $repo_ctx,
+                PublicInbox::OnDestroy->new(\&next_repos, $repo_ctx, $drs)];
+        # shard_done fires when all shards are committed
+        my @active = keys %{$repo_ctx->{active}};
+        $IDX_SHARDS[$_]->wq_io_do('shard_commit', [ $p->{op_p} ]) for @active;
+}
+
+sub prune_done { # called via prune_do completion
+        my ($self, $drs, $n) = @_;
+        return if $DO_QUIT || !$PRUNE_DONE;
+        die "BUG: \$PRUNE_DONE->[$n] already defined" if $PRUNE_DONE->[$n];
+        $PRUNE_DONE->[$n] = 1;
+        if (grep(defined, @$PRUNE_DONE) == @IDX_SHARDS) {
+                progress($self, 'prune done');
+                index_next($self); # may kick dump_roots_start
+        }
+}
+
+sub seen ($$) {
+        my ($xdb, $q) = @_; # $q = "Q$COMMIT_HASH"
+        for (1..100) {
+                my $ret = eval {
+                        $xdb->postlist_begin($q) != $xdb->postlist_end($q);
+                };
+                return $ret unless $@;
+                if (ref($@) =~ /\bDatabaseModifiedError\b/) {
+                        $xdb->reopen;
+                } else {
+                        Carp::croak($@);
+                }
+        }
+        Carp::croak('too many Xapian DB modifications in progress');
+}
+
+# used to select the shard for a GIT_DIR
+sub git_dir_hash ($) { hex(substr(sha256_hex($_[0]), 0, 8)) }
+
+sub _cb { # run_await cb
+        my ($pid, $cmd, undef, $opt, $cb, $self, $git, @arg) = @_;
+        return if $DO_QUIT;
+        return $cb->($opt, $self, $git, @arg) if $opt->{quiet};
+        $? ? ($git->{-cidx_err} = warn("W: @$cmd (\$?=$?)\n")) :
+                        $cb->($opt, $self, $git, @arg);
+}
+
+sub run_git {
+        my ($cmd, $opt, $cb, $self, $git, @arg) = @_;
+        run_await($git->cmd(@$cmd), undef, $opt, \&_cb, $cb, $self, $git, @arg)
+}
+
+# this is different from the grokmirror-compatible fingerprint since we
+# only care about --heads (branches) and --tags, and not even their names
+sub fp_start ($$) {
+        my ($self, $git) = @_;
+        return if $DO_QUIT;
+        open my $refs, '+>', undef;
+        $git->{-repo}->{refs} = $refs;
+        my ($c, $p) = PublicInbox::PktOp->pair;
+        my $next_on_err = PublicInbox::OnDestroy->new(\&index_next, $self);
+        $c->{ops}->{fp_done} = [ $self, $git, $next_on_err ];
+        $IDX_SHARDS[++$ANY_SHARD % scalar(@IDX_SHARDS)]->wq_io_do('fp_async',
+                                        [ $p->{op_p}, $refs ], $git->{git_dir})
+}
+
+sub fp_async { # via wq_io_do in worker
+        my ($self, $git_dir) = @_;
+        my $op_p = delete $self->{0} // die 'BUG: no {0} op_p';
+        my $refs = delete $self->{1} // die 'BUG: no {1} refs';
+        my $git = PublicInbox::Git->new($git_dir);
+        run_git([qw(show-ref --heads --tags --hash)], { 1 => $refs },
+                \&fp_async_done, $self, $git, $op_p);
+}
+
+sub fp_async_done { # run_git cb from worker
+        my ($opt, $self, $git, $op_p) = @_;
+        my $refs = delete $opt->{1} // 'BUG: no {-repo}->{refs}';
+        sysseek($refs, 0, SEEK_SET);
+        send($op_p, 'fp_done '.sha_all(256, $refs)->hexdigest, 0);
+}
+
+sub fp_done { # called parent via PktOp by fp_async_done
+        my ($self, $git, $next_on_err, $hex) = @_;
+        $next_on_err->cancel;
+        return if $DO_QUIT;
+        $git->{-repo}->{fp} = $hex;
+        my $n = git_dir_hash($git->{git_dir}) % scalar(@RDONLY_XDB);
+        my $shard = bless { %$self, shard => $n }, ref($self);
+        $git->{-repo}->{shard_n} = $n;
+        delete @$shard{qw(lockfh lock_path)};
+        local $shard->{xdb} = $RDONLY_XDB[$n] // die "BUG: shard[$n] undef";
+        $shard->retry_reopen(\&check_existing, $self, $git);
+}
+
+sub check_existing { # retry_reopen callback
+        my ($shard, $self, $git) = @_;
+        my @docids = $shard->docids_of_git_dir($git->{git_dir});
+        my $docid = shift(@docids) // return prep_repo($self, $git); # new repo
+        my $doc = $shard->get_doc($docid) //
+                        die "BUG: no #$docid ($git->{git_dir})";
+        my $old_fp = $REINDEX ? "\0invalid" : $doc->get_data;
+        if ($old_fp eq $git->{-repo}->{fp}) { # no change
+                delete $git->{-repo};
+                return index_next($self);
+        }
+        $git->{-repo}->{docid} = $docid;
+        if (@docids) {
+                warn "BUG: $git->{git_dir} indexed multiple times, culling\n";
+                $git->{-repo}->{to_delete} = \@docids; # XXX needed?
+        }
+        prep_repo($self, $git);
+}
+
+sub partition_refs ($$$) {
+        my ($self, $git, $refs) = @_; # show-ref --heads --tags --hash output
+        sysseek($refs, 0, SEEK_SET);
+        my $rfh = $git->popen(qw(rev-list --stdin), undef, { 0 => $refs });
+        my $seen = 0;
+        my @shard_in = map {
+                $_->reopen;
+                open my $fh, '+>', undef;
+                $fh;
+        } @RDONLY_XDB;
+
+        my $n0 = $NCHANGE;
+        while (defined(my $cmt = <$rfh>)) {
+                chomp $cmt;
+                my $n = hex(substr($cmt, 0, 8)) % scalar(@RDONLY_XDB);
+                if ($REINDEX && $REINDEX->set_maybe(pack('H*', $cmt), '')) {
+                        say { $shard_in[$n] } $cmt;
+                        ++$NCHANGE;
+                } elsif (seen($RDONLY_XDB[$n], 'Q'.$cmt)) {
+                        last if ++$seen > $SEEN_MAX;
+                } else {
+                        say { $shard_in[$n] } $cmt;
+                        ++$NCHANGE;
+                        $seen = 0;
+                }
+                if ($DO_QUIT) {
+                        $rfh->close;
+                        return ();
+                }
+        }
+        $rfh->close;
+        return () if $DO_QUIT;
+        if (!$? || (($? & 127) == POSIX::SIGPIPE && $seen > $SEEN_MAX)) {
+                my $n = $NCHANGE - $n0;
+                progress($self, "$git->{git_dir}: $n commits") if $n;
+                return @shard_in;
+        }
+        die "git --git-dir=$git->{git_dir} rev-list: \$?=$?\n";
+}
+
+sub shard_commit { # via wq_io_do
+        my ($self) = @_;
+        my $op_p = delete($self->{0}) // die 'BUG: no {0} op_p';
+        $self->commit_txn_lazy;
+        send($op_p, "shard_done $self->{shard}", 0);
+}
+
+sub dump_roots_start {
+        my ($self, $do_join) = @_;
+        return if $DO_QUIT;
+        $XHC //= PublicInbox::XapClient::start_helper("-j$NPROC");
+        $do_join // die 'BUG: no $do_join';
+        progress($self, 'dumping IDs from coderepos');
+        local $self->{xdb};
+        @OFF2ROOT = $self->all_terms('G');
+        my $root2id = "$TMPDIR/root2id";
+        open my $fh, '>', $root2id;
+        my $nr = -1;
+        for (@OFF2ROOT) { print $fh $_, "\0", ++$nr, "\0" } # mmap-friendly
+        close $fh;
+        # dump_roots | sort -k1,1 | OFS=' ' uniq_fold >to_root_off
+        my ($sort_opt, $fold_opt);
+        pipe(local $sort_opt->{0}, my $sort_w);
+        pipe(local $fold_opt->{0}, local $sort_opt->{1});
+        my @sort = (@SORT, '-k1,1');
+        my $dst = "$TMPDIR/to_root_off";
+        open $fold_opt->{1}, '>', $dst;
+        my $fold_env = { %$CMD_ENV, OFS => ' ' };
+        run_await(\@sort, $CMD_ENV, $sort_opt, \&cmd_done, $do_join);
+        run_await(\@UNIQ_FOLD, $fold_env, $fold_opt, \&cmd_done, $do_join);
+        my $window = $JOIN{window} // $JOIN_WINDOW;
+        my @m = $window <= 0 ? () : ('-m', $window);
+        my @arg = ((map { ('-A', $_) } @JOIN_PFX), '-c',
+                @m, $root2id, $QRY_STR);
+        for my $d ($self->shard_dirs) {
+                pipe(my $err_r, my $err_w);
+                $XHC->mkreq([$sort_w, $err_w], qw(dump_roots -d), $d, @arg);
+                my $desc = "dump_roots $d";
+                $self->{PENDING}->{$desc} = $do_join;
+                PublicInbox::CidxXapHelperAux->new($err_r, $self, $desc);
+        }
+        progress($self, 'waiting on dump_roots sort');
+}
+
+sub dump_ibx { # sends to xap_helper.h
+        my ($self, $ibx_off) = @_;
+        my $ibx = $IBX[$ibx_off] // die "BUG: no IBX[$ibx_off]";
+        my $ekey = $ibx->eidx_key;
+        my $srch = $ibx->isrch or return warn <<EOM;
+W: $ekey not indexed for search
+EOM
+        # note: we don't send `-m MAX' to dump_ibx since we have to
+        # post-filter non-patch messages for now...
+        my @cmd = ('dump_ibx', $srch->xh_args,
+                        (map { ('-A', $_) } @JOIN_PFX), $ibx_off, $QRY_STR);
+        pipe(my $r, my $w);
+        $XHC->mkreq([$DUMP_IBX_WPIPE, $w], @cmd);
+        $self->{PENDING}->{$ekey} = $TODO{do_join};
+        PublicInbox::CidxXapHelperAux->new($r, $self, $ekey);
+}
+
+sub dump_ibx_start {
+        my ($self, $do_join) = @_;
+        return if $DO_QUIT;
+        $XHC //= PublicInbox::XapClient::start_helper("-j$NPROC");
+        my ($sort_opt, $fold_opt);
+        pipe(local $sort_opt->{0}, $DUMP_IBX_WPIPE);
+        pipe(local $fold_opt->{0}, local $sort_opt->{1});
+        my @sort = (@SORT, '-k1,1'); # sort only on JOIN_PFX
+        # pipeline: dump_ibx | sort -k1,1 | uniq_fold >to_ibx_off
+        open $fold_opt->{1}, '>', "$TMPDIR/to_ibx_off";
+        run_await(\@sort, $CMD_ENV, $sort_opt, \&cmd_done, $do_join);
+        run_await(\@UNIQ_FOLD, $CMD_ENV, $fold_opt, \&cmd_done, $do_join);
+}
+
+sub index_next ($) {
+        my ($self) = @_;
+        return if $DO_QUIT;
+        if ($IDXQ && @$IDXQ) {
+                index_repo($self, shift @$IDXQ);
+        } elsif ($SCANQ && @$SCANQ) {
+                fp_start $self, shift @$SCANQ;
+        } elsif ($TMPDIR) {
+                delete $TODO{dump_roots_start};
+                delete $TODO{dump_ibx_start}; # runs OnDestroy once
+                return dump_ibx($self, shift @IBXQ) if @IBXQ;
+                undef $DUMP_IBX_WPIPE; # done dumping inboxes
+                delete $TODO{do_join};
+        }
+        # else: wait for shards_active (post_loop_do) callback
+}
+
+sub next_repos { # OnDestroy cb
+        my ($repo_ctx, $drs) = @_;
+        my ($self, $repo, $active) = @$repo_ctx{qw(self repo active)};
+        progress($self, "$repo->{git_dir}: done");
+        return if $DO_QUIT || !$REPO_CTX;
+        my $n = grep { ! $repo_ctx->{shard_ok}->{$_} } keys %$active;
+        die "E: $repo->{git_dir} $n shards failed" if $n;
+        $REPO_CTX == $repo_ctx or die "BUG: $REPO_CTX != $repo_ctx";
+        $REPO_CTX = undef;
+        index_next($self);
+}
+
+sub index_done { # OnDestroy cb called when done indexing each code repo
+        my ($repo_ctx, $drs) = @_;
+        return if $DO_QUIT;
+        my ($self, $repo, $active) = @$repo_ctx{qw(self repo active)};
+        # $active may be undef here, but it's fine to vivify
+        my $n = grep { ! $repo_ctx->{shard_ok}->{$_} } keys %$active;
+        die "E: $repo->{git_dir} $n shards failed" if $n;
+        $repo_ctx->{shard_ok} = {}; # reset for future shard_done
+        $n = $repo->{shard_n};
+        $repo_ctx->{active}->{$n} = undef; # may vivify $repo_ctx->{active}
+        my ($c, $p) = PublicInbox::PktOp->pair;
+        $c->{ops}->{repo_stored} = [ $self, $repo_ctx, $drs ];
+        $IDX_SHARDS[$n]->wq_io_do('store_repo', [ $p->{op_p} ], $repo);
+        # repo_stored will fire once store_repo is done
+}
+
+sub index_repo {
+        my ($self, $git) = @_;
+        return if $DO_QUIT;
+        my $repo = $git->{-repo} // die 'BUG: no {-repo}';
+        return index_next($self) if $git->{-cidx_err};
+        if (!defined($repo->{ct})) {
+                warn "W: $git->{git_dir} has no commits, skipping\n";
+                return index_next($self);
+        }
+        return push(@$IDXQ, $git) if $REPO_CTX; # busy
+        delete $git->{-repo};
+        my $roots_fh = delete $repo->{roots_fh} // die 'BUG: no {roots_fh}';
+        seek($roots_fh, 0, SEEK_SET);
+        chomp(my @roots = PublicInbox::IO::read_all $roots_fh);
+        if (!@roots) {
+                warn("E: $git->{git_dir} has no root commits\n");
+                return index_next($self);
+        }
+        $repo->{roots} = \@roots;
+        local $self->{current_info} = $git->{git_dir};
+        my @shard_in = partition_refs($self, $git, delete($repo->{refs}));
+        $repo->{git_dir} = $git->{git_dir};
+        my $repo_ctx = $REPO_CTX = { self => $self, repo => $repo };
+        delete $git->{-cidx_gits_fini}; # may fire gits_fini
+        my $drs = delete $git->{-cidx_dump_roots_start};
+        my $index_done = PublicInbox::OnDestroy->new(\&index_done,
+                                                        $repo_ctx, $drs);
+        my ($c, $p) = PublicInbox::PktOp->pair;
+        $c->{ops}->{shard_done} = [ $self, $repo_ctx, $index_done ];
+        for my $n (0..$#shard_in) {
+                $shard_in[$n]->flush or die "flush shard[$n]: $!";
+                -s $shard_in[$n] or next;
+                last if $DO_QUIT;
+                $IDX_SHARDS[$n]->wq_io_do('shard_index',
+                                        [ $shard_in[$n], $p->{op_p} ],
+                                        $git, \@roots);
+                $repo_ctx->{active}->{$n} = undef;
+        }
+        # shard_done fires when shard_index is done
+}
+
+sub ct_fini { # run_git cb
+        my ($opt, $self, $git, $index_repo) = @_;
+        my ($ct) = split(/\s+/, ${$opt->{1}}); # drop TZ + LF
+        $git->{-repo}->{ct} = $ct + 0;
+}
+
+# TODO: also index gitweb.owner and the full fingerprint for grokmirror?
+sub prep_repo ($$) {
+        my ($self, $git) = @_;
+        return if $DO_QUIT;
+        my $index_repo = PublicInbox::OnDestroy->new(\&index_repo, $self, $git);
+        my $refs = $git->{-repo}->{refs} // die 'BUG: no {-repo}->{refs}';
+        sysseek($refs, 0, SEEK_SET);
+        open my $roots_fh, '+>', undef;
+        $git->{-repo}->{roots_fh} = $roots_fh;
+        run_git([ qw(rev-list --stdin --max-parents=0) ],
+                { 0 => $refs, 1 => $roots_fh }, \&PublicInbox::Config::noop,
+                $self, $git, $index_repo);
+        run_git([ qw[for-each-ref --sort=-committerdate
+                --format=%(committerdate:raw) --count=1
+                refs/heads/ refs/tags/] ], undef, # capture like qx
+                \&ct_fini, $self, $git, $index_repo);
+}
+
+# for PublicInbox::SearchIdx `git patch-id' call and with_umask
+sub git { $_[0]->{git} }
+
+sub load_existing ($) { # for -u/--update
+        my ($self) = @_;
+        my $dirs = $self->{git_dirs} //= [];
+        if ($self->{-opt}->{update} || $self->{-opt}->{prune}) {
+                local $self->{xdb};
+                $self->xdb or
+                        die "E: $self->{cidx_dir} non-existent for --update\n";
+                my @cur = grep {
+                        if (-e $_) {
+                                1;
+                        } else {
+                                push @GIT_DIR_GONE, $_;
+                                undef;
+                        }
+                } $self->all_terms('P');
+                if (@GIT_DIR_GONE && !$self->{-opt}->{prune}) {
+                        warn "W: the following repos no longer exist:\n",
+                                (map { "W:\t$_\n" } @GIT_DIR_GONE),
+                                "W: use --prune to remove them from ",
+                                $self->{cidx_dir}, "\n";
+                }
+                push @$dirs, @cur;
+        }
+        @$dirs = uniqstr @$dirs;
+}
+
+# SIG handlers:
+sub shard_quit { $DO_QUIT = POSIX->can("SIG$_[0]")->() }
+sub shard_usr1 { $TXN_BYTES = -1 }
+
+sub cidx_init ($) {
+        my ($self) = @_;
+        my $dir = $self->{cidx_dir};
+        unless (-d $dir) {
+                warn "# creating $dir\n" if !$self->{-opt}->{quiet};
+                File::Path::mkpath($dir);
+        }
+        $self->lock_acquire;
+        my @shards;
+        my $l = $self->{indexlevel} //= $self->{-opt}->{indexlevel};
+
+        for my $n (0..($self->{nshard} - 1)) {
+                my $shard = bless { %$self, shard => $n }, ref($self);
+                delete @$shard{qw(lockfh lock_path)};
+                my $xdb = $shard->idx_acquire;
+                if (!$n) {
+                        if (($l // '') eq 'medium') {
+                                $xdb->set_metadata('indexlevel', $l);
+                        } elsif (($l // '') eq 'full') {
+                                $xdb->set_metadata('indexlevel', ''); # unset
+                        }
+                        $l ||= $xdb->get_metadata('indexlevel') || 'full';
+                }
+                $shard->{indexlevel} = $l;
+                $shard->idx_release;
+                $shard->wq_workers_start("cidx shard[$n]", 1, $SIGSET, {
+                        siblings => \@shards, # for ipc_atfork_child
+                }, \&shard_done_wait, $self);
+                push @shards, $shard;
+        }
+        $self->{indexlevel} //= $l;
+        # this warning needs to happen after idx_acquire
+        state $once;
+        warn <<EOM if $PublicInbox::Search::X{CLOEXEC_UNSET} && !$once++;
+W: Xapian v1.2.21..v1.2.24 were missing close-on-exec on OFD locks,
+W: memory usage may be high for large indexing runs
+EOM
+        @shards;
+}
+
+# called when all git coderepos are done
+sub gits_fini {
+        undef $GITS_NR;
+        PublicInbox::DS::enqueue_reap(); # kick @post_loop_do
+}
+
+sub scan_git_dirs ($) {
+        my ($self) = @_;
+        @$SCANQ = () unless $self->{-opt}->{scan};
+        $GITS_NR = @$SCANQ or return;
+        my $gits_fini = PublicInbox::OnDestroy->new(\&gits_fini);
+        $_->{-cidx_gits_fini} = $gits_fini for @$SCANQ;
+        if (my $drs = $TODO{dump_roots_start}) {
+                $_->{-cidx_dump_roots_start} = $drs for @$SCANQ;
+        }
+        progress($self, "scanning $GITS_NR code repositories...");
+}
+
+sub prune_init { # via wq_io_do in IDX_SHARDS
+        my ($self) = @_;
+        $self->{nr_prune} = 0;
+        $TXN_BYTES = $BATCH_BYTES;
+        $self->begin_txn_lazy;
+}
+
+sub prune_one { # via wq_io_do in IDX_SHARDS
+        my ($self, $term) = @_;
+        my @docids = $self->docids_by_postlist($term);
+        for (@docids) {
+                $TXN_BYTES -= $self->{xdb}->get_doclength($_) * 42;
+                $self->{xdb}->delete_document($_);
+        }
+        ++$self->{nr_prune};
+        $TXN_BYTES < 0 and
+                cidx_ckpoint($self, "prune [$self->{shard}] $self->{nr_prune}");
+}
+
+sub prune_commit { # via wq_io_do in IDX_SHARDS
+        my ($self) = @_;
+        my $prune_op_p = delete $self->{0} // die 'BUG: no {0} op_p';
+        my $nr = delete $self->{nr_prune} // die 'BUG: nr_prune undef';
+        cidx_ckpoint($self, "prune [$self->{shard}] $nr done") if $nr;
+        send($prune_op_p, "prune_done $self->{shard}", 0);
+}
+
+sub shards_active { # post_loop_do
+        return if $DO_QUIT;
+        return if grep(defined, $PRUNE_DONE, $SCANQ, $IDXQ) != 3;
+        return 1 if grep(defined, @$PRUNE_DONE) != @IDX_SHARDS;
+        return 1 if $GITS_NR || scalar(@$IDXQ) || $REPO_CTX;
+        return 1 if @IBXQ || keys(%TODO);
+        for my $s (grep { $_->{-wq_s1} } @IDX_SHARDS) {
+                $s->{-cidx_quit} = 1 if defined($s->{-wq_s1});
+                $s->wq_close; # may recurse via awaitpid outside of event_loop
+        }
+        scalar(grep { $_->{-cidx_quit} } @IDX_SHARDS);
+}
+
+# signal handlers
+sub kill_shards { $_->wq_kill(@_) for (@IDX_SHARDS) }
+
+sub parent_quit {
+        $DO_QUIT = POSIX->can("SIG$_[0]")->();
+        $XHC = 0; # stops the process
+        kill_shards(@_);
+        warn "# SIG$_[0] received, quitting...\n";
+}
+
+sub prep_umask ($) {
+        my ($self) = @_;
+        if ($self->{-cidx_internal}) { # respect core.sharedRepository
+                @{$self->{git_dirs}} == 1 or die 'BUG: only for GIT_DIR';
+                local $self->{git} =
+                        PublicInbox::Git->new($self->{git_dirs}->[0]);
+                $self->with_umask;
+        } elsif (-d $self->{cidx_dir}) { # respect existing perms
+                my @st = stat(_);
+                my $um = (~$st[2] & 0777);
+                $self->{umask} = $um; # for SearchIdx->with_umask
+                umask == $um or progress($self, 'using umask from ',
+                                                $self->{cidx_dir}, ': ',
+                                                sprintf('0%03o', $um));
+                PublicInbox::OnDestroy->new(\&CORE::umask, umask($um));
+        } else {
+                $self->{umask} = umask; # for SearchIdx->with_umask
+                undef;
+        }
+}
+
+sub prep_alternate_end ($$) {
+        my ($objdir, $fmt) = @_;
+        my $hexlen = $OFMT2HEXLEN{$fmt} // return warn <<EOM;
+E: ignoring objdir=$objdir, unknown extensions.objectFormat=$fmt
+EOM
+        unless ($ALT_FH{$hexlen}) {
+                require PublicInbox::Import;
+                my $git_dir = "$TMPDIR/hexlen$hexlen.git";
+                PublicInbox::Import::init_bare($git_dir, 'cidx-all', $fmt);
+                open $ALT_FH{$hexlen}, '>', "$git_dir/objects/info/alternates";
+        }
+        say { $ALT_FH{$hexlen} } $objdir;
+}
+
+sub store_objfmt { # via wq_do - make early cidx users happy
+        my ($self, $docid, $git_dir, $fmt) = @_;
+        $self->begin_txn_lazy;
+        my $doc = $self->get_doc($docid) // return
+                warn "BUG? #$docid for $git_dir missing";
+        my @p = xap_terms('P', $doc) or return
+                warn "BUG? #$docid for $git_dir has no P(ath)";
+        @p == 1 or return warn "BUG? #$docid $git_dir multi: @p";
+        $p[0] eq $git_dir or return warn "BUG? #$docid $git_dir != @p";
+        $doc->add_boolean_term('H'.$fmt);
+        $self->{xdb}->replace_document($docid, $doc);
+        # wait for prune_commit to commit...
+}
+
+# TODO: remove prep_alternate_read and store_objfmt 1-2 years after 2.0 is out
+# they are for compatibility with pre-release indices
+sub prep_alternate_read { # run_git cb for config extensions.objectFormat
+        my ($opt, $self, $git, $objdir, $docid, $shard_n, $run_prune) = @_;
+        return if $DO_QUIT;
+        my $chld_err = $?;
+        prep_alternate_start($self, shift(@PRUNEQ), $run_prune) if @PRUNEQ;
+        my $fmt = check_objfmt_status $git, $chld_err, ${$opt->{1}};
+        $IDX_SHARDS[$shard_n]->wq_do('store_objfmt', # async
+                                        $docid, $git->{git_dir}, $fmt);
+        prep_alternate_end $objdir, $fmt;
+}
+
+sub prep_alternate_start {
+        my ($self, $git, $run_prune) = @_;
+        local $self->{xdb};
+        my ($o, $n, @ids, @fmt);
+start:
+        $o = $git->git_path('objects');
+        while (!-d $o) {
+                $git = shift(@PRUNEQ) // return;
+                $o = $git->git_path('objects');
+        }
+        $n = git_dir_hash($git->{git_dir}) % scalar(@RDONLY_XDB);
+        $self->{xdb} = $RDONLY_XDB[$n] // croak("BUG: no shard[$n]");
+        @ids = $self->docids_by_postlist('P'.$git->{git_dir});
+        @fmt = @ids ? xap_terms('H', $self->{xdb}, $ids[0]) : ();
+        @fmt > 1 and warn "BUG? multi `H' for shard[$n] #$ids[0]: @fmt";
+
+        if (@fmt) { # cache hit
+                prep_alternate_end $o, $fmt[0];
+                $git = shift(@PRUNEQ) and goto start;
+        } else { # compatibility w/ early cidx format
+                run_git([qw(config extensions.objectFormat)], { quiet => 1 },
+                        \&prep_alternate_read, $self, $git, $o, $ids[0], $n,
+                        $run_prune);
+        }
+}
+
+sub cmd_done { # run_await cb for sort, xapian-delve, sed failures
+        my ($pid, $cmd, undef, undef, $run_on_destroy) = @_;
+        $? and die "fatal: @$cmd (\$?=$?)\n";
+        # $run_on_destroy calls do_join() or run_prune()
+}
+
+sub current_join_data ($) {
+        my ($self) = @_;
+        local $self->{xdb} = $RDONLY_XDB[0] // die 'BUG: shard[0] undef';
+        # we support multiple PI_CONFIG files for a cindex:
+        $self->join_data;
+}
+
+# combined previously stored stats with new
+sub score_old_join_data ($$$) {
+        my ($self, $score, $ekeys_new) = @_;
+        my $old = ($JOIN{reset} ? undef : current_join_data($self)) or return;
+        progress($self, 'merging old join data...');
+        my ($ekeys_old, $roots_old, $ibx2root_old) =
+                                        @$old{qw(ekeys roots ibx2root)};
+        # score: "ibx_off root_off" => nr
+        my $i = -1;
+        my %root2id_new = map { $_ => ++$i } @OFF2ROOT;
+        $i = -1;
+        my %ekey2id_new = map { $_ => ++$i } @$ekeys_new;
+        for my $ibx_off_old (0..$#$ibx2root_old) {
+                my $root_offs_old = $ibx2root_old->[$ibx_off_old];
+                my $ekey = $ekeys_old->[$ibx_off_old] // do {
+                        warn "W: no ibx #$ibx_off_old in old join data\n";
+                        next;
+                };
+                my $ibx_off_new = $ekey2id_new{$ekey} // do {
+                        warn "W: `$ekey' no longer exists\n";
+                        next;
+                };
+                for (@$root_offs_old) {
+                        my ($nr, $rid_old) = @$_;
+                        my $root_old = $roots_old->[$rid_old] // do {
+                                warn "W: no root #$rid_old in old data\n";
+                                next;
+                        };
+                        my $rid_new = $root2id_new{$root_old} // do {
+                                warn "W: root `$root_old' no longer exists\n";
+                                next;
+                        };
+                        $score->{"$ibx_off_new $rid_new"} += $nr;
+                }
+        }
+}
+
+sub metadata_set { # via wq_do
+        my ($self, $key, $val, $commit) = @_;
+        $self->begin_txn_lazy;
+        $self->{xdb}->set_metadata($key, $val);
+        $self->commit_txn_lazy if $commit || defined(wantarray);
+}
+
+# runs once all inboxes and shards are dumped via OnDestroy
+sub do_join {
+        my ($self) = @_;
+        return if $DO_QUIT;
+        $XHC = 0; # should not be recreated again
+        @IDX_SHARDS or return warn("# aborting on no shards\n");
+        unlink("$TMPDIR/root2id");
+        my @pending = keys %{$self->{PENDING}};
+        die "BUG: pending=@pending jobs not done\n" if @pending;
+        progress($self, 'joining...');
+        my @join = (@JOIN, 'to_ibx_off', 'to_root_off');
+        if (my $time = which('time')) { unshift @join, $time };
+        my $rd = popen_rd(\@join, $CMD_ENV, { -C => "$TMPDIR" });
+        my %score;
+        while (<$rd>) { # PFX ibx_offs root_off
+                chop eq "\n" or die "no newline from @join: <$_>";
+                my (undef, $ibx_offs, @root_offs) = split / /, $_;
+                for my $ibx_off (split(/,/, $ibx_offs)) {
+                        ++$score{"$ibx_off $_"} for @root_offs;
+                }
+        }
+        $rd->close or die "fatal: @join failed: \$?=$?";
+        my $nr = scalar(keys %score) or do {
+                delete $TODO{joining};
+                return progress($self, 'no potential new pairings');
+        };
+        progress($self, "$nr potential new pairings...");
+        my @ekeys = map { $_->eidx_key } @IBX;
+        score_old_join_data($self, \%score, \@ekeys);
+        my $new;
+        while (my ($k, $nr) = each %score) {
+                my ($ibx_off, $root_off) = split(/ /, $k);
+                my ($ekey, $root) = ($ekeys[$ibx_off], $OFF2ROOT[$root_off]);
+                progress($self, "$ekey => $root has $nr matches");
+                push @{$new->{ibx2root}->[$ibx_off]}, [ $nr, $root_off ];
+        }
+        for my $ary (values %$new) { # sort by nr (largest first)
+                for (@$ary) { @$_ = sort { $b->[0] <=> $a->[0] } @$_ }
+        }
+        $new->{ekeys} = \@ekeys;
+        $new->{roots} = \@OFF2ROOT;
+        $new->{dt} = \@JOIN_DT;
+        $new = compress(PublicInbox::Config::json()->encode($new));
+        my $key = $self->join_data_key;
+        my $wait = $IDX_SHARDS[0]->wq_do('metadata_set', $key, $new);
+        delete $TODO{joining};
+}
+
+sub require_progs {
+        my $op = shift;
+        while (my ($x, $argv) = splice(@_, 0, 2)) {
+                my $e = $x;
+                $e =~ tr/a-z-/A-Z_/;
+                my $c = $ENV{$e} // $x;
+                $argv->[0] //= which($c) // die "E: `$x' required for --$op\n";
+        }
+}
+
+sub init_join_postfork ($) {
+        my ($self) = @_;
+        return unless $self->{-opt}->{join};
+        require_progs('join', join => \@JOIN);
+        my $d2 = '([0-9]{2})';
+        my $dt_re = qr!([0-9]{4})$d2$d2$d2$d2$d2!;
+        if (my $cur = $JOIN{reset} ? undef : current_join_data($self)) {
+                if (($cur->{dt}->[1] // '') =~ m!\A$dt_re\z!o) {
+                        my ($Y, $m, $d, $H, $M, $S) = ($1, $2, $3, $4, $5, $6);
+                        my $t = timegm($S, $M, $H, $d, $m - 1, $Y);
+                        $t = strftime('%Y%m%d%H%M%S', gmtime($t + 1));
+                        $JOIN{dt} //= "$t..";
+                } else {
+                        warn <<EOM;
+BUG?: previous --join invocation did not store usable `dt' key
+EOM
+                }
+        }
+        if ($JOIN{aggressive}) {
+                $JOIN{window} //= -1;
+                $JOIN{dt} //= '..1.month.ago';
+        }
+        $QRY_STR = $JOIN{dt} // '1.year.ago..';
+        index($QRY_STR, '..') >= 0 or die "E: dt:$QRY_STR is not a range\n";
+        # Account for send->apply delay (torvalds/linux.git mean is ~20 days
+        # from Author to CommitDate in cases where CommitDate > AuthorDate
+        $QRY_STR .= '1.month.ago' if $QRY_STR =~ /\.\.\z/;
+        @{$self->{git_dirs} // []} or die "E: no coderepos to join\n";
+        @IBX or die "E: no inboxes to join\n";
+        my $approx_git = PublicInbox::Git->new($self->{git_dirs}->[0]); # ugh
+        substr($QRY_STR, 0, 0) = 'dt:';
+        $self->query_approxidate($approx_git, $QRY_STR); # in-place
+        ($JOIN_DT[1]) = ($QRY_STR =~ /\.\.([0-9]{14})\z/); # YYYYmmddHHMMSS
+        ($JOIN_DT[0]) = ($QRY_STR =~ /\Adt:([0-9]{14})/); # YYYYmmddHHMMSS
+        $JOIN_DT[0] //= '19700101'.'000000'; # git uses unsigned times
+        $TODO{do_join} = PublicInbox::OnDestroy->new(\&do_join, $self);
+        $TODO{joining} = 1; # keep shards_active() happy
+        $TODO{dump_ibx_start} = PublicInbox::OnDestroy->new(\&dump_ibx_start,
+                                                        $self, $TODO{do_join});
+        $TODO{dump_roots_start} = PublicInbox::OnDestroy->new(
+                                \&dump_roots_start, $self, $TODO{do_join});
+        progress($self, "will join in $QRY_STR date range...");
+        my $id = -1;
+        @IBXQ = map { ++$id } @IBX;
+}
+
+sub init_prune ($) {
+        my ($self) = @_;
+        return (@$PRUNE_DONE = map { 1 } @IDX_SHARDS) if !$self->{-opt}->{prune};
+
+        # Dealing with millions of commits here at once, so use faster tools.
+        # xapian-delve is nearly an order-of-magnitude faster than Xapian Perl
+        # bindings.  sed/awk are faster than Perl for simple stream ops, and
+        # sort+comm are more memory-efficient with gigantic lists.
+        # pipeline: delve | sed | sort >indexed_commits
+        my @delve = (undef, qw(-A Q -1));
+        my @sed = (undef, '-ne', 's/^Q//p');
+        @COMM = (undef, qw(-2 -3 indexed_commits -));
+        @AWK = (undef, '$2 == "commit" { print $1 }'); # --batch-check output
+        require_progs('prune', 'xapian-delve' => \@delve, sed => \@sed,
+                        comm => \@COMM, awk => \@AWK);
+        for (0..$#IDX_SHARDS) { push @delve, "$self->{xpfx}/$_" }
+        my $run_prune = PublicInbox::OnDestroy->new(\&run_prune, $self,
+                                                $TODO{dump_roots_start});
+        my ($sort_opt, $sed_opt, $delve_opt);
+        pipe(local $sed_opt->{0}, local $delve_opt->{1});
+        pipe(local $sort_opt->{0}, local $sed_opt->{1});
+        open($sort_opt->{1}, '+>', "$TMPDIR/indexed_commits");
+        run_await([@SORT, '-u'], $CMD_ENV, $sort_opt, \&cmd_done, $run_prune);
+        run_await(\@sed, $CMD_ENV, $sed_opt, \&cmd_done, $run_prune);
+        run_await(\@delve, undef, $delve_opt, \&cmd_done, $run_prune);
+        @PRUNEQ = @$SCANQ;
+        for (1..$LIVE_JOBS) {
+                prep_alternate_start($self, shift(@PRUNEQ) // last, $run_prune);
+        }
+}
+
+sub dump_git_commits { # run_await cb
+        my ($pid, $cmd, undef, $batch_opt, $self) = @_;
+        (defined($pid) && $?) and die "E: @$cmd \$?=$?";
+        return if $DO_QUIT;
+        my ($hexlen) = keys(%ALT_FH) or return; # done, DESTROY batch_opt->{1}
+        close(delete $ALT_FH{$hexlen}); # flushes `say' buffer
+        progress($self, "preparing $hexlen-byte hex OID commits for prune...");
+        my $g = PublicInbox::Git->new("$TMPDIR/hexlen$hexlen.git");
+        run_await($g->cmd(@PRUNE_BATCH), undef, $batch_opt,
+                        \&dump_git_commits, $self);
+}
+
+sub run_prune { # OnDestroy when `git config extensions.objectFormat' are done
+        my ($self, $drs) = @_;
+        return if $DO_QUIT;
+        # setup the following pipeline: (
+        #        git --git-dir=hexlen40.git cat-file \
+        #                --batch-all-objects --batch-check &&
+        #        git --git-dir=hexlen64.git cat-file \
+        #                --batch-all-objects --batch-check
+        # ) | awk | sort | comm | cidx_read_comm()
+        my ($awk_opt, $sort_opt, $batch_opt);
+        my $comm_opt = { -C => "$TMPDIR" };
+        pipe(local $awk_opt->{0}, $batch_opt->{1});
+        pipe(local $sort_opt->{0}, local $awk_opt->{1});
+        pipe(local $comm_opt->{0}, local $sort_opt->{1});
+        run_await(\@AWK, $CMD_ENV, $awk_opt, \&cmd_done);
+        run_await([@SORT, '-u'], $CMD_ENV, $sort_opt, \&cmd_done);
+        my $comm_rd = popen_rd(\@COMM, $CMD_ENV, $comm_opt, \&cmd_done, \@COMM);
+        PublicInbox::CidxComm->new($comm_rd, $self, $drs); # ->cidx_read_comm
+        push @PRUNE_BATCH, '--buffer' if $GIT_VER ge v2.6;
+
+        # Yes, we pipe --unordered git output to sort(1) because sorting
+        # inside git leads to orders-of-magnitude slowdowns on rotational
+        # storage.  GNU sort(1) also works well on larger-than-memory
+        # datasets, and it's not worth eliding sort(1) for old git.
+        push @PRUNE_BATCH, '--unordered' if $GIT_VER ge v2.19;
+        warn(sprintf(<<EOM, $GIT_VER)) if $GIT_VER lt v2.19;
+W: git v2.19+ recommended for high-latency storage (have git v%vd)
+EOM
+        dump_git_commits(undef, undef, undef, $batch_opt, $self);
+}
+
+sub cidx_read_comm { # via PublicInbox::CidxComm::event_step
+        my ($self, $comm_rd, $drs) = @_;
+        return if $DO_QUIT;
+        progress($self, 'starting prune...');
+        $_->wq_do('prune_init') for @IDX_SHARDS;
+        while (defined(my $cmt = <$comm_rd>)) {
+                chop($cmt) eq "\n" or die "BUG: no LF in comm output ($cmt)";
+                my $n = hex(substr($cmt, 0, 8)) % scalar(@IDX_SHARDS);
+                $IDX_SHARDS[$n]->wq_do('prune_one', 'Q'.$cmt);
+                last if $DO_QUIT;
+        }
+        for my $git_dir (@GIT_DIR_GONE) {
+                my $n = git_dir_hash($git_dir) % scalar(@IDX_SHARDS);
+                $IDX_SHARDS[$n]->wq_do('prune_one', 'P'.$git_dir);
+                last if $DO_QUIT;
+        }
+        my ($c, $p) = PublicInbox::PktOp->pair;
+        $c->{ops}->{prune_done} = [ $self, $drs ];
+        $_->wq_io_do('prune_commit', [ $p->{op_p} ]) for @IDX_SHARDS;
+}
+
+sub init_join_prefork ($) {
+        my ($self) = @_;
+        my $subopt = $self->{-opt}->{join} // return;
+        %JOIN = map {
+                my ($k, $v) = split /:/, $_, 2;
+                $k => $v // 1;
+        } split(/,/, join(',', @$subopt));
+        require PublicInbox::CidxXapHelperAux;
+        require PublicInbox::XapClient;
+        my @unknown;
+        my $pfx = $JOIN{prefixes} // 'dfpost7';
+        for my $p (split /\+/, $pfx) {
+                my $n = '';
+                $p =~ s/([0-9]+)\z// and $n = $1;
+                my $v = $PublicInbox::Search::PATCH_BOOL_COMMON{$p} //
+                        push(@unknown, $p);
+                push(@JOIN_PFX, map { $_.$n } split(/ /, $v));
+        }
+        @unknown and die <<EOM;
+E: --join=prefixes= contains unsupported prefixes: @unknown
+EOM
+        @JOIN_PFX = uniqstr @JOIN_PFX;
+        my %incl = map {
+                if (-f "$_/inbox.lock" || -d "$_/public-inbox") {
+                        rel2abs_collapsed($_) => undef;
+                } else {
+                        warn "W: `$_' is not a public inbox, skipping\n";
+                        ();
+                }
+        } (@{$self->{-opt}->{include} // []});
+        my $all = $self->{-opt}->{all};
+        if (my $only = $self->{-opt}->{only}) {
+                die <<'' if $all;
+E: --all is incompatible with --only
+
+                $incl{rel2abs_collapsed($_)} = undef for @$only;
+        }
+        unless (keys(%incl)) {
+                $all = 1;
+                warn <<EOM unless $self->{opt}->{quiet};
+# --all implied since no inboxes were specified with --only or --include
+EOM
+        }
+        $self->{-opt}->{-pi_cfg}->each_inbox(\&_prep_ibx, $self, \%incl, $all);
+        my $nr = scalar(@IBX) or die "E: no inboxes to join with\n";
+        progress($self, "will join with $nr inboxes in ",
+                        $self->{-opt}->{-pi_cfg}->{-f}, " using: $pfx");
+}
+
+sub _prep_ibx { # each_inbox callback
+        my ($ibx, $self, $incl, $all) = @_;
+        ($all || exists($incl->{$ibx->{inboxdir}})) and push @IBX, $ibx;
+}
+
+sub show_json { # for diagnostics (unstable output)
+        my ($self) = @_;
+        my $s = $self->{-opt}->{show} or return; # for diagnostics
+        local $self->{xdb};
+        my %ret;
+        my @todo = @$s;
+        while (defined(my $f = shift @todo)) {
+                if ($f =~ /\A(?:roots2paths|paths2roots|join_data)\z/) {
+                        $ret{$f} = $self->$f;
+                } elsif ($f eq '') { # default --show (no args)
+                        push @todo, qw(roots2paths join_data);
+                } else {
+                        warn "E: cannot show `$f'\n";
+                }
+        }
+        my $json = ref(PublicInbox::Config::json())->new;
+        $json->utf8->canonical->pretty; # n.b. FS pathnames may not be UTF-8...
+        say $json->encode(\%ret);
+}
+
+sub do_inits { # called via PublicInbox::DS::add_timer
+        my ($self) = @_;
+        grep !!$_, @{$self->{-opt}}{qw(scan prune)} and
+                @$SCANQ = map PublicInbox::Git->new($_), @{$self->{git_dirs}};
+        init_join_postfork $self;
+        init_prune $self;
+        scan_git_dirs $self;
+        my $max = $TODO{do_join} ? max($LIVE_JOBS, $NPROC) : $LIVE_JOBS;
+        index_next($self) for (1..$max);
+}
+
+sub cidx_run { # main entry point
+        my ($self) = @_;
+        my $restore_umask = prep_umask($self);
+        local $SIGSET = PublicInbox::DS::block_signals(
+                                        POSIX::SIGTSTP, POSIX::SIGCONT);
+        my $restore = PublicInbox::OnDestroy->new($$,
+                \&PublicInbox::DS::sig_setmask, $SIGSET);
+        local $PRUNE_DONE = [];
+        local $IDXQ = [];
+        local $SCANQ = [];
+        local ($DO_QUIT, $REINDEX, $TXN_BYTES, @GIT_DIR_GONE, @PRUNEQ,
+                $REPO_CTX, %ALT_FH, $TMPDIR, @AWK, @COMM, $CMD_ENV,
+                %TODO, @IBXQ, @IBX, @JOIN, %JOIN, @JOIN_PFX, @NO_ABBREV,
+                @JOIN_DT, $DUMP_IBX_WPIPE, @OFF2ROOT, $XHC, @SORT, $GITS_NR);
+        local $BATCH_BYTES = $self->{-opt}->{batch_size} //
+                                $PublicInbox::SearchIdx::BATCH_BYTES;
+        local $MAX_SIZE = $self->{-opt}->{max_size};
+        local $self->{PENDING} = {}; # used by PublicInbox::CidxXapHelperAux
+        my $cfg = $self->{-opt}->{-pi_cfg} // die 'BUG: -pi_cfg unset';
+        $self->{-cfg_f} = $cfg->{-f} = rel2abs_collapsed($cfg->{-f});
+        local $GIT_VER = PublicInbox::Git::git_version();
+        @NO_ABBREV = ('-c', 'core.abbrev='.($GIT_VER lt v2.31.0 ? 40 : 'no'));
+        if (grep { $_ } @{$self->{-opt}}{qw(prune join)}) {
+                require File::Temp;
+                $TMPDIR = File::Temp->newdir('cidx-all-git-XXXX', TMPDIR => 1);
+                $CMD_ENV = { TMPDIR => "$TMPDIR", LC_ALL => 'C', LANG => 'C' };
+                require_progs('(prune|join)', sort => \@SORT);
+                for (qw(parallel compress-program buffer-size)) { # GNU sort
+                        my $v = $self->{-opt}->{"sort-$_"};
+                        push @SORT, "--$_=$v" if defined $v;
+                }
+                ($self->{-opt}->{prune} && $GIT_VER le v2.6) and
+                        die "W: --prune requires git v2.6+\n";
+                init_join_prefork($self)
+        }
+        local @IDX_SHARDS = cidx_init($self); # forks workers
+        local $ANY_SHARD = -1;
+        local $self->{current_info} = '';
+        local $MY_SIG = {
+                CHLD => \&PublicInbox::DS::enqueue_reap,
+                USR1 => \&kill_shards,
+        };
+        local @PRUNE_BATCH = @PRUNE_BATCH;
+        $MY_SIG->{$_} = \&parent_quit for qw(TERM QUIT INT);
+        my $cb = $SIG{__WARN__} || \&CORE::warn;
+        local $SIG{__WARN__} = sub {
+                my $m = shift @_;
+                $self->{current_info} eq '' or
+                        $m =~ s/\A(#?\s*)/$1$self->{current_info}: /;
+                $cb->($m, @_);
+        };
+        load_existing($self) unless $self->{-cidx_internal};
+        if ($self->{-opt}->{reindex}) {
+                require PublicInbox::SharedKV;
+                $REINDEX = PublicInbox::SharedKV->new;
+                delete $REINDEX->{lock_path};
+                $REINDEX->dbh;
+        }
+        my @nc = grep { File::Spec->canonpath($_) ne $_ } @{$self->{git_dirs}};
+        if (@nc) {
+                warn "E: BUG? paths in $self->{cidx_dir} not canonicalized:\n";
+                for my $d (@{$self->{git_dirs}}) {
+                        my $c = File::Spec->canonpath($_);
+                        warn "E: $d => $c\n";
+                        $d = $c;
+                }
+                warn "E: canonicalized and attempting to continue\n";
+        }
+        if (defined(my $excl = $self->{-opt}->{exclude})) {
+                my $re = '(?:'.join('\\z|', map {
+                                glob2re($_) // qr/\A\Q$_\E/
+                        } @$excl).'\\z)';
+                my @excl;
+                @{$self->{git_dirs}} = grep {
+                        $_ =~ /$re/ ? (push(@excl, $_), 0) : 1;
+                } @{$self->{git_dirs}};
+                warn("# excluding $_\n") for @excl;
+                @GIT_DIR_GONE = uniqstr @GIT_DIR_GONE, @excl;
+        }
+        local $NCHANGE = 0;
+        local $NPROC = PublicInbox::IPC::detect_nproc();
+        local $LIVE_JOBS = $self->{-opt}->{jobs} || $NPROC || 2;
+        local @RDONLY_XDB = $self->xdb_shards_flat;
+        PublicInbox::DS::add_timer(0, \&do_inits, $self);
+
+        # FreeBSD ignores/discards SIGCHLD while signals are blocked and
+        # EVFILT_SIGNAL is inactive, so we pretend we have a SIGCHLD pending
+        PublicInbox::DS::enqueue_reap();
+
+        local @PublicInbox::DS::post_loop_do = (\&shards_active);
+        PublicInbox::DS::event_loop($MY_SIG, $SIGSET);
+        $self->lock_release(!!$NCHANGE);
+        show_json($self);
+}
+
+sub ipc_atfork_child { # @IDX_SHARDS
+        my ($self) = @_;
+        $self->SUPER::ipc_atfork_child;
+        $SIG{USR1} = \&shard_usr1;
+        $SIG{$_} = \&shard_quit for qw(INT TERM QUIT);
+        my $x = delete $self->{siblings} // die 'BUG: no {siblings}';
+        $_->wq_close for @$x;
+        undef;
+}
+
+sub shard_done_wait { # awaitpid cb via ipc_worker_reap
+        my ($pid, $shard, $self) = @_;
+        my $quit_req = delete($shard->{-cidx_quit});
+        return if $DO_QUIT;
+        if ($? == 0) { # success
+                $quit_req // warn 'BUG: {-cidx_quit} unset';
+        } else {
+                warn "PID:$pid $shard->{shard} exited with \$?=$?\n";
+                ++$self->{shard_err} if defined($self->{shard_err});
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/Compat.pm b/lib/PublicInbox/Compat.pm
new file mode 100644
index 00000000..78cba90e
--- /dev/null
+++ b/lib/PublicInbox/Compat.pm
@@ -0,0 +1,24 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# compatibility code for old Perl and standard modules, mainly
+# List::Util but maybe other stuff
+package PublicInbox::Compat;
+use v5.12;
+use parent qw(Exporter);
+require List::Util;
+
+our @EXPORT_OK = qw(uniqstr);
+
+# uniqstr is in List::Util 1.45+, which means Perl 5.26+;
+# so maybe 2030 for us since we need to support enterprise distros.
+# We can use uniqstr everywhere in our codebase and don't need
+# to account for special cases of `uniqnum' nor `uniq' in List::Util
+# even if they make more sense in some contexts
+no warnings 'once';
+*uniqstr = List::Util->can('uniqstr') // sub (@) {
+        my %seen;
+        grep { !$seen{$_}++ } @_;
+};
+
+1;
diff --git a/lib/PublicInbox/CompressNoop.pm b/lib/PublicInbox/CompressNoop.pm
index e3301473..5135299f 100644
--- a/lib/PublicInbox/CompressNoop.pm
+++ b/lib/PublicInbox/CompressNoop.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Provide the same methods as Compress::Raw::Zlib::Deflate but
@@ -10,7 +10,7 @@ use Compress::Raw::Zlib qw(Z_OK);
 sub new { bless \(my $self), __PACKAGE__ }
 
 sub deflate { # ($self, $input, $output)
-        $_[2] .= $_[1];
+        $_[2] .= ref($_[1]) ? ${$_[1]} : $_[1];
         Z_OK;
 }
 
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 0f002e5e..d6300610 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Used throughout the project for reading configuration
@@ -10,26 +10,28 @@
 package PublicInbox::Config;
 use strict;
 use v5.10.1;
+use parent qw(Exporter);
+our @EXPORT_OK = qw(glob2re rel2abs_collapsed);
 use PublicInbox::Inbox;
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Spawn qw(popen_rd run_qx);
 our $LD_PRELOAD = $ENV{LD_PRELOAD}; # only valid at startup
+our $DEDUPE; # set to {} to dedupe or clear cache
 
 sub _array ($) { ref($_[0]) eq 'ARRAY' ? $_[0] : [ $_[0] ] }
 
 # returns key-value pairs of config directives in a hash
 # if keys may be multi-value, the value is an array ref containing all values
 sub new {
-        my ($class, $file, $errfh) = @_;
+        my ($class, $file, $lei) = @_;
         $file //= default_file();
-        my $self;
-        if (ref($file) eq 'SCALAR') { # used by some tests
-                open my $fh, '<', $file or die;  # PerlIO::scalar
-                $self = config_fh_parse($fh, "\n", '=');
-                bless $self, $class;
-        } else {
-                $self = git_config_dump($class, $file, $errfh);
-                $self->{'-f'} = $file;
-        }
+        my ($self, $set_dedupe);
+        if (-f $file && $DEDUPE) {
+                $file = rel2abs_collapsed($file);
+                $self = $DEDUPE->{$file} and return $self;
+                $set_dedupe = 1;
+        }
+        $self = git_config_dump($class, $file, $lei);
+        $self->{-f} = $file;
         # caches
         $self->{-by_addr} = {};
         $self->{-by_list_id} = {};
@@ -38,8 +40,7 @@ sub new {
         $self->{-by_eidx_key} = {};
         $self->{-no_obfuscate} = {};
         $self->{-limiters} = {};
-        $self->{-code_repos} = {}; # nick => PublicInbox::Git object
-        $self->{-cgitrc_unparsed} = $self->{'publicinbox.cgitrc'};
+        $self->{-coderepos} = {}; # nick => PublicInbox::Git object
 
         if (my $no = delete $self->{'publicinbox.noobfuscate'}) {
                 $no = _array($no);
@@ -62,7 +63,7 @@ sub new {
         if (my $css = delete $self->{'publicinbox.css'}) {
                 $self->{css} = _array($css);
         }
-
+        $DEDUPE->{$file} = $self if $set_dedupe;
         $self;
 }
 
@@ -123,9 +124,9 @@ sub lookup_newsgroup {
 sub limiter {
         my ($self, $name) = @_;
         $self->{-limiters}->{$name} //= do {
-                require PublicInbox::Qspawn;
+                require PublicInbox::Limiter;
                 my $max = $self->{"publicinboxlimiter.$name.max"} || 1;
-                my $limiter = PublicInbox::Qspawn::Limiter->new($max);
+                my $limiter = PublicInbox::Limiter->new($max);
                 $limiter->setup_rlimit($name, $self);
                 $limiter;
         };
@@ -143,8 +144,11 @@ sub config_fh_parse ($$$) {
         local $/ = $rs;
         while (defined($line = <$fh>)) { # perf critical with giant configs
                 $i = index($line, $fs);
+                # $i may be -1 if $fs not found and it's a key-only entry
+                # (meaning boolean true).  Either way the -1 will drop the
+                # $rs either from $k or $v.
                 $k = substr($line, 0, $i);
-                $v = substr($line, $i + 1, -1); # chop off $fs
+                $v = $i >= 0 ? substr($line, $i + 1, -1) : 1;
                 $section = substr($k, 0, rindex($k, '.'));
                 $seen{$section} //= push(@section_order, $section);
 
@@ -163,13 +167,34 @@ sub config_fh_parse ($$$) {
         \%rv;
 }
 
+sub tmp_cmd_opt ($$) {
+        my ($env, $opt) = @_;
+        # quiet global and system gitconfig if supported by installed git,
+        # but normally harmless if too noisy (NOGLOBAL no longer exists)
+        $env->{GIT_CONFIG_NOSYSTEM} = 1;
+        $env->{GIT_CONFIG_GLOBAL} = '/dev/null'; # git v2.32+
+        $opt->{-C} = '/'; # avoid $worktree/.git/config on MOST systems :P
+}
+
 sub git_config_dump {
-        my ($class, $file, $errfh) = @_;
-        return bless {}, $class unless -e $file;
-        my $cmd = [ qw(git config -z -l --includes), "--file=$file" ];
-        my $fh = popen_rd($cmd, undef, { 2 => $errfh // 2 });
+        my ($class, $file, $lei) = @_;
+        my @opt_c = map { ('-c', $_) } @{$lei->{opt}->{c} // []};
+        $file = undef if !-e $file;
+        # XXX should we set {-f} if !-e $file?
+        return bless {}, $class if (!@opt_c && !defined($file));
+        my %env;
+        my $opt = { 2 => $lei->{2} // 2 };
+        if (@opt_c) {
+                unshift(@opt_c, '-c', "include.path=$file") if defined($file);
+                tmp_cmd_opt(\%env, $opt);
+        }
+        my @cmd = ('git', @opt_c, qw(config -z -l --includes));
+        push(@cmd, '-f', $file) if !@opt_c && defined($file);
+        my $fh = popen_rd(\@cmd, \%env, $opt);
         my $rv = config_fh_parse($fh, "\0", "\n");
-        close $fh or die "@$cmd failed: \$?=$?\n";
+        $fh->close or die "@cmd failed: \$?=$?\n";
+        $rv->{-opt_c} = \@opt_c if @opt_c; # for ->urlmatch
+        $rv->{-f} = $file;
         bless $rv, $class;
 }
 
@@ -220,7 +245,6 @@ sub cgit_repo_merge ($$$) {
                         $rel =~ s!/?\.git\z!!;
         }
         $self->{"coderepo.$rel.dir"} //= $path;
-        $self->{"coderepo.$rel.cgiturl"} //= _array($rel);
 }
 
 sub is_git_dir ($) {
@@ -256,10 +280,11 @@ sub scan_tree_coderepo ($$) {
         scan_path_coderepo($self, $path, $path);
 }
 
-sub scan_projects_coderepo ($$$) {
-        my ($self, $list, $path) = @_;
-        open my $fh, '<', $list or do {
-                warn "failed to open cgit projectlist=$list: $!\n";
+sub scan_projects_coderepo ($$) {
+        my ($self, $path) = @_;
+        my $l = $self->{-cgit_project_list} // die 'BUG: no cgit_project_list';
+        open my $fh, '<', $l or do {
+                warn "failed to open cgit project-list=$l: $!\n";
                 return;
         };
         while (<$fh>) {
@@ -268,8 +293,20 @@ sub scan_projects_coderepo ($$$) {
         }
 }
 
+sub apply_cgit_scan_path {
+        my ($self, @paths) = @_;
+        @paths or @paths = @{$self->{-cgit_scan_path}};
+        if (defined($self->{-cgit_project_list})) {
+                for my $p (@paths) { scan_projects_coderepo($self, $p) }
+        } else {
+                for my $p (@paths) { scan_tree_coderepo($self, $p) }
+        }
+}
+
 sub parse_cgitrc {
         my ($self, $cgitrc, $nesting) = @_;
+        $cgitrc //= $self->{'publicinbox.cgitrc'} //
+                        $ENV{CGIT_CONFIG} // return;
         if ($nesting == 0) {
                 # defaults:
                 my %s = map { $_ => 1 } qw(/cgit.css /cgit.png
@@ -309,34 +346,41 @@ sub parse_cgitrc {
                         my ($k, $v) = ($1, $2);
                         $k =~ tr/-/_/;
                         $self->{"-cgit_$k"} = $v;
+                        delete $self->{-cgit_scan_path} if $k eq 'project_list';
                 } elsif (m!\Ascan-path=(.+)\z!) {
-                        if (defined(my $list = $self->{-cgit_project_list})) {
-                                scan_projects_coderepo($self, $list, $1);
-                        } else {
-                                scan_tree_coderepo($self, $1);
-                        }
+                        # this depends on being after project-list in the
+                        # config file, just like cgit.c
+                        push @{$self->{-cgit_scan_path}}, $1;
+                        apply_cgit_scan_path($self, $1);
                 } elsif (m!\A(?:css|favicon|logo|repo\.logo)=(/.+)\z!) {
                         # absolute paths for static files via PublicInbox::Cgit
                         $self->{-cgit_static}->{$1} = 1;
+                } elsif (s!\Asnapshots=\s*!!) {
+                        $self->{'coderepo.snapshots'} = $_;
                 }
         }
         cgit_repo_merge($self, $repo->{dir}, $repo) if $repo;
 }
 
+sub valid_dir ($$) {
+        my $dir = get_1($_[0], $_[1]) // return;
+        index($dir, "\n") < 0 ? $dir : do {
+                warn "E: `$_[1]=$dir' must not contain `\\n'\n";
+                undef;
+        }
+}
+
 # parse a code repo, only git is supported at the moment
-sub fill_code_repo {
+sub fill_coderepo {
         my ($self, $nick) = @_;
         my $pfx = "coderepo.$nick";
-        my $dir = $self->{"$pfx.dir"} // do { # aka "GIT_DIR"
-                warn "$pfx.dir unset\n";
-                return;
-        };
-        my $git = PublicInbox::Git->new($dir);
+        my $git = PublicInbox::Git->new(valid_dir($self, "$pfx.dir") // return);
         if (defined(my $cgits = $self->{"$pfx.cgiturl"})) {
                 $git->{cgit_url} = $cgits = _array($cgits);
                 $self->{"$pfx.cgiturl"} = $cgits;
         }
-
+        my %dedupe = ($nick => undef);
+        ($git->{nick}) = keys %dedupe;
         $git;
 }
 
@@ -361,7 +405,7 @@ sub git_bool {
 # is sufficient and doesn't leave "/.." or "/../"
 sub rel2abs_collapsed {
         require File::Spec;
-        my $p = File::Spec->rel2abs($_[-1]);
+        my $p = File::Spec->rel2abs(@_);
         return $p if substr($p, -3, 3) ne '/..' && index($p, '/../') < 0;
         require Cwd;
         Cwd::abs_path($p);
@@ -377,11 +421,12 @@ sub get_1 {
 
 sub repo_objs {
         my ($self, $ibxish) = @_;
-        my $ibx_code_repos = $ibxish->{coderepo} // return;
         $ibxish->{-repo_objs} // do {
-                my $code_repos = $self->{-code_repos};
+                my $ibx_coderepos = $ibxish->{coderepo} // return;
+                parse_cgitrc($self, undef, 0);
+                my $coderepos = $self->{-coderepos};
                 my @repo_objs;
-                for my $nick (@$ibx_code_repos) {
+                for my $nick (@$ibx_coderepos) {
                         my @parts = split(m!/!, $nick);
                         for (@parts) {
                                 @parts = () unless valid_foo_name($_);
@@ -390,12 +435,16 @@ sub repo_objs {
                                 warn "invalid coderepo name: `$nick'\n";
                                 next;
                         }
-                        my $repo = $code_repos->{$nick} //=
-                                                fill_code_repo($self, $nick);
-                        push @repo_objs, $repo if $repo;
+                        my $repo = $coderepos->{$nick} //=
+                                                fill_coderepo($self, $nick);
+                        $repo ? push(@repo_objs, $repo) :
+                                warn("coderepo.$nick.dir unset\n");
                 }
                 if (scalar @repo_objs) {
-                        $ibxish ->{-repo_objs} = \@repo_objs;
+                        for (@repo_objs) {
+                                push @{$_->{ibx_names}}, $ibxish->{name};
+                        }
+                        $ibxish->{-repo_objs} = \@repo_objs;
                 } else {
                         delete $ibxish->{coderepo};
                 }
@@ -410,18 +459,15 @@ sub _fill_ibx {
                 my $v = $self->{"$pfx.$k"};
                 $ibx->{$k} = $v if defined $v;
         }
-        for my $k (qw(filter inboxdir newsgroup replyto httpbackendmax feedmax
+        for my $k (qw(filter newsgroup replyto httpbackendmax feedmax
                         indexlevel indexsequentialshard boost)) {
                 my $v = get_1($self, "$pfx.$k") // next;
                 $ibx->{$k} = $v;
         }
 
         # "mainrepo" is backwards compatibility:
-        my $dir = $ibx->{inboxdir} //= $self->{"$pfx.mainrepo"} // return;
-        if (index($dir, "\n") >= 0) {
-                warn "E: `$dir' must not contain `\\n'\n";
-                return;
-        }
+        my $dir = $ibx->{inboxdir} = valid_dir($self, "$pfx.inboxdir") //
+                                valid_dir($self, "$pfx.mainrepo") // return;
         for my $k (qw(obfuscate)) {
                 my $v = $self->{"$pfx.$k"} // next;
                 if (defined(my $bval = git_bool($v))) {
@@ -434,13 +480,15 @@ sub _fill_ibx {
         # more things to encourage decentralization
         for my $k (qw(address altid nntpmirror imapmirror
                         coderepo hide listid url
-                        infourl watchheader nntpserver imapserver)) {
+                        infourl watchheader
+                        nntpserver imapserver pop3server)) {
                 my $v = $self->{"$pfx.$k"} // next;
                 $ibx->{$k} = _array($v);
         }
 
         return unless valid_foo_name($name, 'publicinbox');
-        $ibx->{name} = $name;
+        my %dedupe = ($name => undef);
+        ($ibx->{name}) = keys %dedupe; # used as a key everywhere
         $ibx->{-pi_cfg} = $self;
         $ibx = PublicInbox::Inbox->new($ibx);
         foreach (@{$ibx->{address}}) {
@@ -468,9 +516,16 @@ sub _fill_ibx {
                         delete $ibx->{newsgroup};
                         warn "newsgroup name invalid: `$ngname'\n";
                 } else {
+                        my $lc = $ibx->{newsgroup} = lc $ngname;
+                        warn <<EOM if $lc ne $ngname;
+W: newsgroup=`$ngname' lowercased to `$lc'
+EOM
                         # PublicInbox::NNTPD does stricter ->nntp_usable
                         # checks, keep this lean for startup speed
-                        $self->{-by_newsgroup}->{$ngname} = $ibx;
+                        my $cur = $self->{-by_newsgroup}->{$lc} //= $ibx;
+                        warn <<EOM if $cur != $ibx;
+W: newsgroup=`$lc' is used by both `$cur->{name}' and `$ibx->{name}'
+EOM
                 }
         }
         unless (defined $ibx->{newsgroup}) { # for ->eidx_key
@@ -490,19 +545,18 @@ sub _fill_ibx {
                 require PublicInbox::Isearch;
                 $ibx->{isrch} = PublicInbox::Isearch->new($ibx, $es);
         }
-        $self->{-by_eidx_key}->{$ibx->eidx_key} = $ibx;
+        my $cur = $self->{-by_eidx_key}->{my $ekey = $ibx->eidx_key} //= $ibx;
+        $cur == $ibx or warn
+                "W: `$ekey' used by both `$cur->{name}' and `$ibx->{name}'\n";
+        $ibx;
 }
 
 sub _fill_ei ($$) {
         my ($self, $name) = @_;
         eval { require PublicInbox::ExtSearch } or return;
         my $pfx = "extindex.$name";
-        my $d = $self->{"$pfx.topdir"} // return;
+        my $d = valid_dir($self, "$pfx.topdir") // return;
         -d $d or return;
-        if (index($d, "\n") >= 0) {
-                warn "E: `$d' must not contain `\\n'\n";
-                return;
-        }
         my $es = PublicInbox::ExtSearch->new($d);
         for my $k (qw(indexlevel indexsequentialshard)) {
                 my $v = get_1($self, "$pfx.$k") // next;
@@ -517,23 +571,76 @@ sub _fill_ei ($$) {
         $es;
 }
 
+sub _fill_csrch ($$) {
+        my ($self, $name) = @_; # "" is a valid name for cindex
+        return if $name ne '' && !valid_foo_name($name, 'cindex');
+        eval { require PublicInbox::CodeSearch } or return;
+        my $pfx = "cindex.$name";
+        my $d = valid_dir($self, "$pfx.topdir") // return;
+        -d $d or return;
+        my $csrch = PublicInbox::CodeSearch->new($d, $self);
+        for my $k (qw(localprefix)) {
+                my $v = $self->{"$pfx.$k"} // next;
+                $csrch->{$k} = _array($v);
+        }
+        $csrch->{name} = $name;
+        $csrch;
+}
+
+sub lookup_cindex ($$) {
+        my ($self, $name) = @_;
+        $self->{-csrch_by_name}->{$name} //= _fill_csrch($self, $name);
+}
+
+sub each_cindex {
+        my ($self, $cb, @arg) = @_;
+        my @csrch = map {
+                lookup_cindex($self, substr($_, length('cindex.'))) // ()
+        } grep(m!\Acindex\.[^\./]*\z!, @{$self->{-section_order}});
+        if (ref($cb) eq 'CODE') {
+                $cb->($_, @arg) for @csrch;
+        } else { # string function
+                $_->$cb(@arg) for @csrch;
+        }
+}
+
+sub config_cmd {
+        my ($self, $env, $opt) = @_;
+        my $f = $self->{-f} // default_file();
+        my @opt_c = @{$self->{-opt_c} // []};
+        my @cmd = ('git', @opt_c, 'config');
+        @opt_c ? tmp_cmd_opt($env, $opt) : push(@cmd, '-f', $f);
+        \@cmd;
+}
+
 sub urlmatch {
-        my ($self, $key, $url) = @_;
+        my $self = shift;
+        my @bool = $_[0] eq '--bool' ? (shift) : ();
+        my ($key, $url, $try_git) = @_;
         state $urlmatch_broken; # requires git 1.8.5
         return if $urlmatch_broken;
-        my $file = $self->{'-f'} // default_file();
-        my $cmd = [qw/git config -z --includes --get-urlmatch/,
-                "--file=$file", $key, $url ];
-        my $fh = popen_rd($cmd);
-        local $/ = "\0";
-        my $val = <$fh>;
-        if (close($fh)) {
-                chomp($val);
-                $val;
-        } else {
-                $urlmatch_broken = 1 if (($? >> 8) != 1);
-                undef;
+        my (%env, %opt);
+        my $cmd = $self->config_cmd(\%env, \%opt);
+        push @$cmd, @bool, qw(--includes -z --get-urlmatch), $key, $url;
+        my $val = run_qx($cmd, \%env, \%opt);
+        if ($?) {
+                undef $val;
+                if (@bool && ($? >> 8) == 128) { # not boolean
+                } elsif (($? >> 8) != 1) {
+                        $urlmatch_broken = 1;
+                } elsif ($try_git) { # n.b. this takes cwd into account
+                        $val = run_qx([qw(git config), @bool,
+                                        qw(-z --get-urlmatch), $key, $url]);
+                        undef $val if $?;
+                }
+        }
+        $? = 0; # don't influence lei exit status
+        if (defined($val)) {
+                local $/ = "\0";
+                chomp $val;
+                $val = git_bool($val) if @bool;
         }
+        $val;
 }
 
 sub json {
@@ -557,4 +664,48 @@ sub squote_maybe ($) {
         $val;
 }
 
+my %re_map = ( '*' => '[^/]*?', '?' => '[^/]',
+                '/**' => '/.*', '**/' => '.*/', '/**/' => '(?:/.*?/|/)',
+                '[' => '[', ']' => ']', ',' => ',' );
+
+sub glob2re ($) {
+        my ($re) = @_;
+        my $p = '';
+        my $in_bracket = 0;
+        my $qm = 0;
+        my $schema_host_port = '';
+
+        # don't glob URL-looking things that look like IPv6
+        if ($re =~ s!\A([a-z0-9\+]+://\[[a-f0-9\:]+\](?::[0-9]+)?/)!!i) {
+                $schema_host_port = quotemeta $1; # "http://[::1]:1234"
+        }
+        my $changes = ($re =~ s!(/\*\*/|\A\*\*/|/\*\*\z|.)!
+                $re_map{$p eq '\\' ? '' : do {
+                        if ($1 eq '[') { ++$in_bracket }
+                        elsif ($1 eq ']') { --$in_bracket }
+                        elsif ($1 eq ',') { ++$qm } # no change
+                        $p = $1;
+                }} // do {
+                        $p = $1;
+                        ($p eq '-' && $in_bracket) ? $p : (++$qm, "\Q$p")
+                }!sge);
+        # bashism (also supported by curl): {a,b,c} => (a|b|c)
+        $changes += ($re =~ s/([^\\]*)\\\{([^,]*,[^\\]*)\\\}/
+                        (my $in_braces = $2) =~ tr!,!|!;
+                        $1."($in_braces)";
+                        /sge);
+        ($changes - $qm) ? $schema_host_port.$re : undef;
+}
+
+sub get_coderepo {
+        my ($self, $nick) = @_;
+        $self->{-coderepos}->{$nick} // do {
+                defined($self->{-cgit_scan_path}) ? do {
+                        apply_cgit_scan_path($self);
+                        my $cr = fill_coderepo($self, $nick);
+                        $cr ? ($self->{-coderepos}->{$nick} = $cr) : undef;
+                } : undef;
+        };
+}
+
 1;
diff --git a/lib/PublicInbox/ContentDigestDbg.pm b/lib/PublicInbox/ContentDigestDbg.pm
new file mode 100644
index 00000000..853624f1
--- /dev/null
+++ b/lib/PublicInbox/ContentDigestDbg.pm
@@ -0,0 +1,22 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+package PublicInbox::ContentDigestDbg; # cf. PublicInbox::ContentDigest
+use v5.12;
+use Data::Dumper;
+use PublicInbox::SHA;
+$Data::Dumper::Useqq = $Data::Dumper::Terse = 1;
+
+sub new { bless [ PublicInbox::SHA->new(256), $_[1] ], __PACKAGE__ }
+
+sub add {
+        $_[0]->[0]->add($_[1]);
+        my @dbg = split(/^/sm, $_[1]);
+        if (@dbg && $dbg[0] =~ /\A(To|Cc)\0/) { # fold excessively long lines
+                @dbg = map { split(/,/s, $_) } @dbg;
+        }
+        print { $_[0]->[1] } Dumper(\@dbg) or die "print $!";
+}
+
+sub hexdigest { $_[0]->[0]->hexdigest }
+
+1;
diff --git a/lib/PublicInbox/ContentHash.pm b/lib/PublicInbox/ContentHash.pm
index bacc9cdd..95ca2929 100644
--- a/lib/PublicInbox/ContentHash.pm
+++ b/lib/PublicInbox/ContentHash.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Unstable internal API.
@@ -15,7 +15,8 @@ use PublicInbox::MID qw(mids references);
 use PublicInbox::MsgIter;
 
 # not sure if less-widely supported hash families are worth bothering with
-use Digest::SHA;
+use PublicInbox::SHA; # faster, but no ->clone
+use Digest::SHA; # we still need this for ->clone
 
 sub digest_addr ($$$) {
         my ($dig, $h, $v) = @_;
@@ -44,7 +45,7 @@ sub content_dig_i {
         my $ct = $part->content_type || 'text/plain';
         my ($s, undef) = msg_part_text($part, $ct);
         if (defined $s) {
-                $s =~ s/\r\n/\n/gs;
+                $s =~ s/\r\n/\n/gs; # TODO: consider \r+\n to match View
                 $s =~ s/\s*\z//s;
                 utf8::encode($s);
         } else {
@@ -53,18 +54,26 @@ sub content_dig_i {
         $dig->add($s);
 }
 
-sub content_digest ($;$) {
-        my ($eml, $dig) = @_;
+sub content_digest ($;$$) {
+        my ($eml, $dig, $hash_mids) = @_;
         $dig //= Digest::SHA->new(256);
 
         # References: and In-Reply-To: get used interchangeably
         # in some "duplicates" in LKML.  We treat them the same
         # in SearchIdx, so treat them the same for this:
         # do NOT consider the Message-ID as part of the content_hash
-        # if we got here, we've already got Message-ID reuse
-        my %seen = map { $_ => 1 } @{mids($eml)};
-        foreach my $mid (@{references($eml)}) {
-                $dig->add("ref\0$mid\0") unless $seen{$mid}++;
+        # if we got here, we've already got Message-ID reuse for v2.
+        #
+        # However, `lei q --dedupe=content' does use $hash_mids since
+        # it doesn't have any other dedupe
+        my $mids = mids($eml);
+        if ($hash_mids) {
+                $dig->add("mid\0$_\0") for @$mids;
+        }
+        my %seen = map { $_ => 1 } @$mids;
+        for (grep { !$seen{$_}++ } @{references($eml)}) {
+                utf8::encode($_);
+                $dig->add("ref\0$_\0");
         }
 
         # Only use Sender: if From is not present
@@ -74,8 +83,7 @@ sub content_digest ($;$) {
                 last;
         }
         foreach my $h (qw(Subject Date)) {
-                my @v = $eml->header($h);
-                foreach my $v (@v) {
+                for my $v ($eml->header($h)) {
                         utf8::encode($v);
                         $dig->add("$h\0$v\0");
                 }
@@ -84,23 +92,22 @@ sub content_digest ($;$) {
         # not in the original message.  For the purposes of deduplication,
         # do not take it into account:
         foreach my $h (qw(To Cc)) {
-                my @v = $eml->header($h);
-                digest_addr($dig, $h, $_) foreach @v;
+                digest_addr($dig, $h, $_) for ($eml->header($h));
         }
         msg_iter($eml, \&content_dig_i, $dig);
         $dig;
 }
 
 sub content_hash ($) {
-        content_digest($_[0])->digest;
+        content_digest($_[0], PublicInbox::SHA->new(256))->digest;
 }
 
+# don't clone the result of this
 sub git_sha ($$) {
         my ($n, $eml) = @_;
-        my $dig = Digest::SHA->new($n);
+        my $dig = PublicInbox::SHA->new($n);
         my $bref = ref($eml) eq 'SCALAR' ? $eml : \($eml->as_string);
-        $dig->add('blob '.length($$bref)."\0");
-        $dig->add($$bref);
+        $dig->add('blob '.length($$bref)."\0", $$bref);
         $dig;
 }
 
diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm
index bf8c4466..8bc8cfb7 100644
--- a/lib/PublicInbox/DS.pm
+++ b/lib/PublicInbox/DS.pm
@@ -24,29 +24,30 @@ use strict;
 use v5.10.1;
 use parent qw(Exporter);
 use bytes qw(length substr); # FIXME(?): needed for PublicInbox::NNTP
-use POSIX qw(WNOHANG sigprocmask SIG_SETMASK);
+use POSIX qw(WNOHANG sigprocmask SIG_SETMASK SIG_UNBLOCK);
 use Fcntl qw(SEEK_SET :DEFAULT O_APPEND);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
 use Scalar::Util qw(blessed);
-use PublicInbox::Syscall qw(:epoll);
+use PublicInbox::Syscall qw(%SIGNUM
+        EPOLLIN EPOLLOUT EPOLLONESHOT EPOLLEXCLUSIVE);
 use PublicInbox::Tmpfile;
-use Errno qw(EAGAIN EINVAL);
+use PublicInbox::Select;
+use Errno qw(EAGAIN EINVAL ECHILD);
 use Carp qw(carp croak);
-our @EXPORT_OK = qw(now msg_more dwaitpid add_timer add_uniq_timer);
+use autodie qw(fork);
+our @EXPORT_OK = qw(now msg_more awaitpid add_timer add_uniq_timer);
 
-my %Stack;
 my $nextq; # queue for next_tick
-my $wait_pids; # list of [ pid, callback, callback_arg ]
 my $reap_armed;
-my $ToClose; # sockets to close when event loop is done
-our (
-     %DescriptorMap,             # fd (num) -> PublicInbox::DS object
-     $Epoll,                     # Global epoll fd (or DSKQXS ref)
-     $ep_io,                     # IO::Handle for Epoll
+my @active; # FDs (or objs) returned by epoll/kqueue
+our (%AWAIT_PIDS, # pid => [ $callback, @args ]
+        $cur_runq, # only set inside next_tick
+     @FD_MAP, # fd (num) -> PublicInbox::DS object
+     $Poller, # global Select, Epoll, DSPoll, or DSKQXS ref
 
-     $PostLoopCallback,          # subref to call at the end of each loop, if defined (global)
+     @post_loop_do,              # subref + args to call at the end of each loop
 
-     $LoopTimeout,               # timeout of event loop in milliseconds
+     $loop_timeout,               # timeout of event loop in milliseconds
      @Timers,                    # timers
      %UniqTimer,
      $in_loop,
@@ -54,6 +55,9 @@ our (
 
 Reset();
 
+# clobber everything explicitly to avoid DESTROY ordering problems w/ DBI
+END { Reset() }
+
 #####################################################################
 ### C L A S S   M E T H O D S
 #####################################################################
@@ -64,37 +68,32 @@ Reset all state
 
 =cut
 sub Reset {
+        $Poller = bless [], 'PublicInbox::DummyPoller';
         do {
                 $in_loop = undef; # first in case DESTROY callbacks use this
-                %DescriptorMap = ();
+                # clobbering $Poller may call DSKQXS::DESTROY,
+                # we must always have this set to something to avoid
+                # needing branches before ep_del/ep_mod calls (via ->close).
+                @FD_MAP = ();
                 @Timers = ();
                 %UniqTimer = ();
-                $PostLoopCallback = undef;
-
-                # we may be iterating inside one of these on our stack
-                my @q = delete @Stack{keys %Stack};
-                for my $q (@q) { @$q = () }
-                $wait_pids = $nextq = $ToClose = undef;
-                $ep_io = undef; # closes real $Epoll FD
-                $Epoll = undef; # may call DSKQXS::DESTROY
-        } while (@Timers || keys(%Stack) || $nextq || $wait_pids ||
-                $ToClose || keys(%DescriptorMap) ||
-                $PostLoopCallback || keys(%UniqTimer));
+                @post_loop_do = ();
+
+                # we may be called from an *atfork_child inside next_tick:
+                @$cur_runq = () if $cur_runq;
+                @active = ();
+                $nextq = undef; # may call ep_del
+                %AWAIT_PIDS = ();
+        } while (@Timers || $nextq || keys(%AWAIT_PIDS) ||
+                @active || @FD_MAP ||
+                @post_loop_do || keys(%UniqTimer) ||
+                scalar(@{$cur_runq // []})); # do not vivify cur_runq
 
         $reap_armed = undef;
-        $LoopTimeout = -1;  # no timeout by default
+        $loop_timeout = -1;  # no timeout by default
+        $Poller = PublicInbox::Select->new;
 }
 
-=head2 C<< CLASS->SetLoopTimeout( $timeout ) >>
-
-Set the loop timeout for the event loop to some value in milliseconds.
-
-A timeout of 0 (zero) means poll forever. A timeout of -1 means poll and return
-immediately.
-
-=cut
-sub SetLoopTimeout { $LoopTimeout = $_[1] + 0 }
-
 sub _add_named_timer {
         my ($name, $secs, $coderef, @args) = @_;
         my $fire_time = now() + $secs;
@@ -124,49 +123,35 @@ sub add_uniq_timer { # ($name, $secs, $coderef, @args) = @_;
         $UniqTimer{$_[0]} //= _add_named_timer(@_);
 }
 
-# caller sets return value to $Epoll
+# caller sets return value to $Poller
 sub _InitPoller () {
-        if (PublicInbox::Syscall::epoll_defined())  {
-                my $fd = epoll_create();
-                die "epoll_create: $!" if $fd < 0;
-                open($ep_io, '+<&=', $fd) or return;
-                my $fl = fcntl($ep_io, F_GETFD, 0);
-                fcntl($ep_io, F_SETFD, $fl | FD_CLOEXEC);
-                $fd;
-        } else {
-                my $cls;
-                for (qw(DSKQXS DSPoll)) {
-                        $cls = "PublicInbox::$_";
-                        last if eval "require $cls";
-                }
-                $cls->import(qw(epoll_ctl epoll_wait));
-                $cls->new;
+        my @try = ($^O eq 'linux' ? 'Epoll' : 'DSKQXS');
+        my $cls;
+        for (@try, 'DSPoll') {
+                $cls = "PublicInbox::$_";
+                last if eval "require $cls";
         }
+        $cls->new;
 }
 
 sub now () { clock_gettime(CLOCK_MONOTONIC) }
 
 sub next_tick () {
-        my $q = $nextq or return;
+        $cur_runq = $nextq or return;
         $nextq = undef;
-        $Stack{cur_runq} = $q;
-        for my $obj (@$q) {
+        while (my $obj = shift @$cur_runq) {
                 # avoid "ref" on blessed refs to workaround a Perl 5.16.3 leak:
                 # https://rt.perl.org/Public/Bug/Display.html?id=114340
-                if (blessed($obj)) {
-                        $obj->event_step;
-                } else {
-                        $obj->();
-                }
+                blessed($obj) ? $obj->event_step : $obj->();
         }
-        delete $Stack{cur_runq};
+        1;
 }
 
 # runs timers and returns milliseconds for next one, or next event loop
 sub RunTimers {
-        next_tick();
+        my $ran = next_tick();
 
-        return (($nextq || $ToClose) ? 0 : $LoopTimeout) unless @Timers;
+        return ($nextq || $ran ? 0 : $loop_timeout) unless @Timers;
 
         my $now = now();
 
@@ -175,60 +160,64 @@ sub RunTimers {
                 my $to_run = shift(@Timers);
                 delete $UniqTimer{$to_run->[1] // ''};
                 $to_run->[2]->(@$to_run[3..$#$to_run]);
+                $ran = 1;
         }
 
         # timers may enqueue into nextq:
-        return 0 if ($nextq || $ToClose);
+        return 0 if $nextq || $ran;
 
-        return $LoopTimeout unless @Timers;
+        return $loop_timeout unless @Timers;
 
         # convert time to an even number of milliseconds, adding 1
         # extra, otherwise floating point fun can occur and we'll
         # call RunTimers like 20-30 times, each returning a timeout
         # of 0.0000212 seconds
-        my $timeout = int(($Timers[0][0] - $now) * 1000) + 1;
+        my $t = int(($Timers[0][0] - $now) * 1000) + 1;
 
         # -1 is an infinite timeout, so prefer a real timeout
-        ($LoopTimeout < 0 || $LoopTimeout >= $timeout) ? $timeout : $LoopTimeout
+        ($loop_timeout < 0 || $loop_timeout >= $t) ? $t : $loop_timeout
 }
 
 sub sig_setmask { sigprocmask(SIG_SETMASK, @_) or die "sigprocmask: $!" }
 
-sub block_signals () {
-        my $oldset = POSIX::SigSet->new;
+# ensure we detect bugs, HW problems and user rlimits
+our @UNBLOCKABLE = (POSIX::SIGABRT, POSIX::SIGBUS, POSIX::SIGFPE,
+        POSIX::SIGILL, POSIX::SIGSEGV, POSIX::SIGXCPU, POSIX::SIGXFSZ);
+
+sub block_signals { # anything in @_ stays unblocked
         my $newset = POSIX::SigSet->new;
         $newset->fillset or die "fillset: $!";
+        for (@_, @UNBLOCKABLE) { $newset->delset($_) or die "delset($_): $!" }
+        my $oldset = POSIX::SigSet->new;
         sig_setmask($newset, $oldset);
         $oldset;
 }
 
-# We can't use waitpid(-1) safely here since it can hit ``, system(),
-# and other things.  So we scan the $wait_pids list, which is hopefully
-# not too big.  We keep $wait_pids small by not calling dwaitpid()
-# until we've hit EOF when reading the stdout of the child.
+sub await_cb ($;@) {
+        my ($pid, @cb_args) = @_;
+        my $cb = shift @cb_args or return;
+        eval { $cb->($pid, @cb_args) };
+        warn "E: awaitpid($pid): $@" if $@;
+}
 
+# This relies on our Perl process being single-threaded, or at least
+# no threads spawning and waiting on processes (``, system(), etc...)
+# Threads are officially discouraged by the Perl5 team, and I expect
+# that to remain the case.
 sub reap_pids {
         $reap_armed = undef;
-        my $tmp = $wait_pids or return;
-        $wait_pids = undef;
-        $Stack{reap_runq} = $tmp;
-        my $oldset = block_signals();
-        foreach my $ary (@$tmp) {
-                my ($pid, $cb, $arg) = @$ary;
-                my $ret = waitpid($pid, WNOHANG);
-                if ($ret == 0) {
-                        push @$wait_pids, $ary; # autovivifies @$wait_pids
-                } elsif ($ret == $pid) {
-                        if ($cb) {
-                                eval { $cb->($arg, $pid) };
-                                warn "E: dwaitpid($pid) in_loop: $@" if $@;
-                        }
-                } else {
-                        warn "waitpid($pid, WNOHANG) = $ret, \$!=$!, \$?=$?";
+        while (1) {
+                my $pid = waitpid(-1, WNOHANG) or return;
+                if (defined(my $cb_args = delete $AWAIT_PIDS{$pid})) {
+                        await_cb($pid, @$cb_args) if $cb_args;
+                } elsif ($pid == -1 && $! == ECHILD) {
+                        return requeue(\&dflush); # force @post_loop_do to run
+                } elsif ($pid > 0) {
+                        warn "W: reaped unknown PID=$pid: \$?=$?\n";
+                } else { # does this happen?
+                        return warn("W: waitpid(-1, WNOHANG) => $pid ($!)");
                 }
         }
-        sig_setmask($oldset);
-        delete $Stack{reap_runq};
 }
 
 # reentrant SIGCHLD handler (since reap_pids is not reentrant)
@@ -236,81 +225,82 @@ sub enqueue_reap () { $reap_armed //= requeue(\&reap_pids) }
 
 sub in_loop () { $in_loop }
 
+# use inside @post_loop_do, returns number of busy clients
+sub close_non_busy () {
+        my $n = 0;
+        for my $s (grep defined, @FD_MAP) {
+                # close as much as possible, early as possible
+                ($s->busy ? ++$n : $s->close) if $s->can('busy');
+        }
+        $n;
+}
+
 # Internal function: run the post-event callback, send read events
 # for pushed-back data, and close pending connections.  returns 1
 # if event loop should continue, or 0 to shut it all down.
 sub PostEventLoop () {
-        # now we can close sockets that wanted to close during our event
-        # processing.  (we didn't want to close them during the loop, as we
-        # didn't want fd numbers being reused and confused during the event
-        # loop)
-        if (my $close_now = $ToClose) {
-                $ToClose = undef; # will be autovivified on push
-                @$close_now = map { fileno($_) } @$close_now;
-
-                # ->DESTROY methods may populate ToClose
-                delete @DescriptorMap{@$close_now};
-        }
-
         # by default we keep running, unless a postloop callback cancels it
-        $PostLoopCallback ? $PostLoopCallback->(\%DescriptorMap) : 1;
+        @post_loop_do ? $post_loop_do[0]->(@post_loop_do[1..$#post_loop_do]) : 1
 }
 
+sub sigset_prep ($$$) {
+        my ($sig, $init, $each) = @_; # $sig: { signame => whatever }
+        my $ret = POSIX::SigSet->new;
+        $ret->$init or die "$init: $!";
+        for my $s (keys %$sig) {
+                my $num = $SIGNUM{$s} // POSIX->can("SIG$s")->();
+                $ret->$each($num) or die "$each ($s => $num): $!";
+        }
+        for (@UNBLOCKABLE) { $ret->$each($_) or die "$each ($_): $!" }
+        $ret;
+}
+
+sub allowset ($) { sigset_prep $_[0], 'fillset', 'delset' }
+sub unblockset ($) { sigset_prep $_[0], 'emptyset', 'addset' }
+
 # Start processing IO events. In most daemon programs this never exits. See
-# C<PostLoopCallback> for how to exit the loop.
+# C<post_loop_do> for how to exit the loop.
 sub event_loop (;$$) {
         my ($sig, $oldset) = @_;
-        $Epoll //= _InitPoller();
+        $Poller //= _InitPoller();
         require PublicInbox::Sigfd if $sig;
-        my $sigfd = PublicInbox::Sigfd->new($sig, 1) if $sig;
+        my $sigfd = $sig ? PublicInbox::Sigfd->new($sig) : undef;
+        if ($sigfd && $sigfd->{is_kq}) {
+                my $tmp = allowset($sig);
+                local @SIG{keys %$sig} = values(%$sig);
+                sig_setmask($tmp, my $old = POSIX::SigSet->new);
+                # Unlike Linux signalfd, EVFILT_SIGNAL can't handle
+                # signals received before the filter is created,
+                # so we peek at signals here.
+                sig_setmask($old);
+        }
         local @SIG{keys %$sig} = values(%$sig) if $sig && !$sigfd;
         local $SIG{PIPE} = 'IGNORE';
         if (!$sigfd && $sig) {
                 # wake up every second to accept signals if we don't
                 # have signalfd or IO::KQueue:
-                sig_setmask($oldset);
-                PublicInbox::DS->SetLoopTimeout(1000);
+                sig_setmask($oldset) if $oldset;
+                sigprocmask(SIG_UNBLOCK, unblockset($sig)) or
+                        die "SIG_UNBLOCK: $!";
+                $loop_timeout = 1000;
         }
         $_[0] = $sigfd = $sig = undef; # $_[0] == sig
         local $in_loop = 1;
-        my @events;
         do {
                 my $timeout = RunTimers();
 
-                # get up to 1000 events
-                epoll_wait($Epoll, 1000, $timeout, \@events);
-                for my $fd (@events) {
-                        # it's possible epoll_wait returned many events,
-                        # including some at the end that ones in the front
-                        # triggered unregister-interest actions.  if we can't
-                        # find the %sock entry, it's because we're no longer
-                        # interested in that event.
-
-                        # guard stack-not-refcounted w/ Carp + @DB::args
-                        my $obj = $DescriptorMap{$fd};
+                # grab whatever FDs are ready
+                $Poller->ep_wait($timeout, \@active);
+
+                # map all FDs to their associated Perl object
+                @active = @FD_MAP[@active];
+
+                while (my $obj = shift @active) {
                         $obj->event_step;
                 }
         } while (PostEventLoop());
 }
 
-=head2 C<< CLASS->SetPostLoopCallback( CODEREF ) >>
-
-Sets post loop callback function.  Pass a subref and it will be
-called every time the event loop finishes.
-
-Return 1 (or any true value) from the sub to make the loop continue, 0 or false
-and it will exit.
-
-The callback function will be passed two parameters: \%DescriptorMap
-
-=cut
-sub SetPostLoopCallback {
-    my ($class, $ref) = @_;
-
-    # global callback
-    $PostLoopCallback = (defined $ref && ref $ref eq 'CODE') ? $ref : undef;
-}
-
 #####################################################################
 ### PublicInbox::DS-the-object code
 #####################################################################
@@ -332,62 +322,54 @@ sub new {
     $self->{sock} = $sock;
     my $fd = fileno($sock);
 
-    $Epoll //= _InitPoller();
+    $Poller //= _InitPoller();
 retry:
-    if (epoll_ctl($Epoll, EPOLL_CTL_ADD, $fd, $ev)) {
+    if ($Poller->ep_add($sock, $ev)) {
         if ($! == EINVAL && ($ev & EPOLLEXCLUSIVE)) {
             $ev &= ~EPOLLEXCLUSIVE;
             goto retry;
         }
         die "EPOLL_CTL_ADD $self/$sock/$fd: $!";
     }
-    croak("FD:$fd in use by $DescriptorMap{$fd} (for $self/$sock)")
-        if defined($DescriptorMap{$fd});
+    defined($FD_MAP[$fd]) and
+                croak("BUG: FD:$fd in use by $FD_MAP[$fd] (for $self/$sock)");
 
-    $DescriptorMap{$fd} = $self;
+    $FD_MAP[$fd] = $self;
 }
 
-
-#####################################################################
-### I N S T A N C E   M E T H O D S
-#####################################################################
+# for IMAP, NNTP, and POP3 which greet clients upon connect
+sub greet {
+        my ($self, $sock) = @_;
+        my $ev = EPOLLIN;
+        my $wbuf;
+        if ($sock->can('accept_SSL') && !$sock->accept_SSL) {
+                return if $! != EAGAIN || !($ev = PublicInbox::TLS::epollbit());
+                $wbuf = [ \&accept_tls_step, $self->can('do_greet')];
+        }
+        new($self, $sock, $ev | EPOLLONESHOT);
+        if ($wbuf) {
+                $self->{wbuf} = $wbuf;
+        } else {
+                $self->do_greet;
+        }
+        $self;
+}
 
 sub requeue ($) { push @$nextq, $_[0] } # autovivifies
 
-=head2 C<< $obj->close >>
-
-Close the socket.
-
-=cut
+# drop the IO::Handle ref, true if successful, false if not (or already dropped)
+# (this is closer to CORE::close than Danga::Socket::close)
 sub close {
-    my ($self) = @_;
-    my $sock = delete $self->{sock} or return;
-
-    # we need to flush our write buffer, as there may
-    # be self-referential closures (sub { $client->close })
-    # preventing the object from being destroyed
-    delete $self->{wbuf};
-
-    # if we're using epoll, we have to remove this from our epoll fd so we stop getting
-    # notifications about it
-    my $fd = fileno($sock);
-    epoll_ctl($Epoll, EPOLL_CTL_DEL, $fd, 0) and
-        croak("EPOLL_CTL_DEL($self/$sock): $!");
-
-    # we explicitly don't delete from DescriptorMap here until we
-    # actually close the socket, as we might be in the middle of
-    # processing an epoll_wait/etc that returned hundreds of fds, one
-    # of which is not yet processed and is what we're closing.  if we
-    # keep it in DescriptorMap, then the event harnesses can just
-    # looked at $pob->{sock} == undef and ignore it.  but if it's an
-    # un-accounted for fd, then it (understandably) freak out a bit
-    # and emit warnings, thinking their state got off.
+        my ($self) = @_;
+        my $sock = delete $self->{sock} or return;
 
-    # defer closing the actual socket until the event loop is done
-    # processing this round of events.  (otherwise we might reuse fds)
-    push @$ToClose, $sock; # autovivifies $ToClose
+        # we need to clear our write buffer, as there may
+        # be self-referential closures (sub { $client->close })
+        # preventing the object from being destroyed
+        delete $self->{wbuf};
+        $FD_MAP[fileno($sock)] = undef;
 
-    return 0;
+        !$Poller->ep_del($sock); # stop getting notifications
 }
 
 # portable, non-thread-safe sendfile emulation (no pread, yet)
@@ -433,8 +415,7 @@ next_buf:
                         shift @$wbuf;
                         goto next_buf;
                     }
-                } elsif ($! == EAGAIN) {
-                    my $ev = epbit($sock, EPOLLOUT) or return $self->close;
+                } elsif ($! == EAGAIN && (my $ev = epbit($sock, EPOLLOUT))) {
                     epwait($sock, $ev | EPOLLONESHOT);
                     return 0;
                 } else {
@@ -464,28 +445,28 @@ sub rbuf_idle ($$) {
     }
 }
 
+# returns true if bytes are read, false otherwise
 sub do_read ($$$;$) {
-    my ($self, $rbuf, $len, $off) = @_;
-    my $r = sysread(my $sock = $self->{sock}, $$rbuf, $len, $off // 0);
-    return ($r == 0 ? $self->close : $r) if defined $r;
-    # common for clients to break connections without warning,
-    # would be too noisy to log here:
-    if ($! == EAGAIN) {
-        my $ev = epbit($sock, EPOLLIN) or return $self->close;
-        epwait($sock, $ev | EPOLLONESHOT);
-        rbuf_idle($self, $rbuf);
-        0;
-    } else {
-        $self->close;
-    }
+        my ($self, $rbuf, $len, $off) = @_;
+        my ($ev, $r, $s);
+        $r = sysread($s = $self->{sock}, $$rbuf, $len, $off // 0) and return $r;
+
+        if (!defined($r) && $! == EAGAIN && ($ev = epbit($s, EPOLLIN))) {
+                epwait($s, $ev | EPOLLONESHOT);
+                rbuf_idle($self, $rbuf);
+        } else {
+                $self->close;
+        }
+        undef;
 }
 
 # drop the socket if we hit unrecoverable errors on our system which
 # require BOFH attention: ENOSPC, EFBIG, EIO, EMFILE, ENFILE...
 sub drop {
-    my $self = shift;
-    carp(@_);
-    $self->close;
+        my $self = shift;
+        carp(@_);
+        $self->close;
+        undef;
 }
 
 sub tmpio ($$$) {
@@ -527,7 +508,8 @@ sub write {
             push @$wbuf, $bref;
         } else {
             my $tmpio = $wbuf->[-1];
-            if ($tmpio && !defined($tmpio->[2])) { # append to tmp file buffer
+            if (ref($tmpio) eq 'ARRAY' && !defined($tmpio->[2])) {
+                # append to tmp file buffer
                 $tmpio->[0]->print($$bref) or return drop($self, "print: $!");
             } else {
                 my $tmpio = tmpio($self, $bref, 0) or return 0;
@@ -589,9 +571,8 @@ sub msg_more ($$) {
 }
 
 sub epwait ($$) {
-    my ($sock, $ev) = @_;
-    epoll_ctl($Epoll, EPOLL_CTL_MOD, fileno($sock), $ev) and
-        croak("EPOLL_CTL_MOD($sock): $!");
+        my ($io, $ev) = @_;
+        $Poller->ep_mod($io, $ev) and croak("EPOLL_CTL_MOD($io): $!");
 }
 
 # return true if complete, false if incomplete (or failure)
@@ -606,7 +587,7 @@ sub accept_tls_step ($) {
     0;
 }
 
-# return true if complete, false if incomplete (or failure)
+# return value irrelevant
 sub shutdn_tls_step ($) {
     my ($self) = @_;
     my $sock = $self->{sock} or return;
@@ -615,40 +596,108 @@ sub shutdn_tls_step ($) {
     my $ev = PublicInbox::TLS::epollbit() or return $self->close;
     epwait($sock, $ev | EPOLLONESHOT);
     unshift(@{$self->{wbuf}}, \&shutdn_tls_step); # autovivifies
-    0;
 }
 
 # don't bother with shutdown($sock, 2), we don't fork+exec w/o CLOEXEC
 # or fork w/o exec, so no inadvertent socket sharing
 sub shutdn ($) {
-    my ($self) = @_;
-    my $sock = $self->{sock} or return;
-    if ($sock->can('stop_SSL')) {
-        shutdn_tls_step($self);
-    } else {
-        $self->close;
-    }
+        my ($self) = @_;
+        my $sock = $self->{sock} or return;
+        $sock->can('stop_SSL') ? shutdn_tls_step($self) : $self->close;
 }
 
-sub dwaitpid ($;$$) {
-        my ($pid, $cb, $arg) = @_;
-        if ($in_loop) {
-                push @$wait_pids, [ $pid, $cb, $arg ];
-                # We could've just missed our SIGCHLD, cover it, here:
-                enqueue_reap();
-        } else {
+sub dflush {} # overridden by DSdeflate
+sub compressed {} # overridden by DSdeflate
+sub long_response_done {} # overridden by Net::NNTP
+
+sub long_step {
+        my ($self) = @_;
+        # wbuf is unset or empty, here; {long} may add to it
+        my ($fd, $cb, $t0, @args) = @{$self->{long_cb}};
+        my $more = eval { $cb->($self, @args) };
+        if ($@ || !$self->{sock}) { # something bad happened...
+                delete $self->{long_cb};
+                my $elapsed = now() - $t0;
+                $@ and warn("$@ during long response[$fd] - ",
+                                sprintf('%0.6f', $elapsed),"\n");
+                $self->out(" deferred[$fd] aborted - %0.6f", $elapsed);
+                $self->close;
+        } elsif ($more) { # $self->{wbuf}:
+                # control passed to ibx_async_cat if $more == \undef
+                requeue_once($self) if !ref($more);
+        } else { # all done!
+                delete $self->{long_cb};
+                $self->long_response_done;
+                my $elapsed = now() - $t0;
+                $self->out(" deferred[$fd] done - %0.6f", $elapsed);
+                my $wbuf = $self->{wbuf}; # do NOT autovivify
+                requeue($self) unless $wbuf && @$wbuf;
+        }
+}
+
+sub requeue_once {
+        my ($self) = @_;
+        # COMPRESS users all share the same DEFLATE context.
+        # Flush it here to ensure clients don't see each other's data
+        $self->dflush;
+
+        # no recursion, schedule another call ASAP,
+        # but only after all pending writes are done.
+        # autovivify wbuf.  wbuf may be populated by $cb,
+        # no need to rearm if so: (push returns new size of array)
+        $self->requeue if push(@{$self->{wbuf}}, \&long_step) == 1;
+}
+
+sub long_response ($$;@) {
+        my ($self, $cb, @args) = @_; # cb returns true if more, false if done
+        my $sock = $self->{sock} or return;
+        # make sure we disable reading during a long response,
+        # clients should not be sending us stuff and making us do more
+        # work while we are stream a response to them
+        $self->{long_cb} = [ fileno($sock), $cb, now(), @args ];
+        long_step($self); # kick off!
+        undef;
+}
+
+sub awaitpid {
+        my ($pid, @cb_args) = @_; # @cb_args = ($cb, @args), $cb may be undef
+        $AWAIT_PIDS{$pid} = \@cb_args if @cb_args;
+        # provide synchronous API
+        if (defined(wantarray) || (!$in_loop && !@cb_args)) {
                 my $ret = waitpid($pid, 0);
                 if ($ret == $pid) {
-                        if ($cb) {
-                                eval { $cb->($arg, $pid) };
-                                carp "E: dwaitpid($pid) !in_loop: $@" if $@;
-                        }
+                        my $cb_args = delete $AWAIT_PIDS{$pid};
+                        @cb_args = @$cb_args if !@cb_args && $cb_args;
+                        await_cb($pid, @cb_args);
                 } else {
-                        carp "waitpid($pid, 0) = $ret, \$!=$!, \$?=$?";
+                        carp "waitpid($pid) => $ret ($!)";
+                        delete $AWAIT_PIDS{$pid};
                 }
+                return $ret;
+        } elsif ($in_loop) { # We could've just missed our SIGCHLD, cover it, here:
+                enqueue_reap();
         }
 }
 
+sub do_fork () {
+        my $seed = rand(0xffffffff);
+        my $pid = fork;
+        if ($pid == 0) {
+                srand($seed);
+                eval { Net::SSLeay::randomize() };
+                Reset();
+        }
+        $pid;
+}
+
+package PublicInbox::DummyPoller; # only used during Reset
+use v5.12;
+
+sub ep_del {}
+no warnings 'once';
+*ep_add = \&ep_del;
+*ep_mod = \&ep_del;
+
 1;
 
 =head1 AUTHORS (Danga::Socket)
diff --git a/lib/PublicInbox/DSKQXS.pm b/lib/PublicInbox/DSKQXS.pm
index eccfa56d..f84c196a 100644
--- a/lib/PublicInbox/DSKQXS.pm
+++ b/lib/PublicInbox/DSKQXS.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # Licensed the same as Danga::Socket (and Perl5)
 # License: GPL-1.0+ or Artistic-1.0-Perl
 #  <https://www.gnu.org/licenses/gpl-1.0.txt>
@@ -11,15 +11,11 @@
 #
 # It also implements signalfd(2) emulation via "tie".
 package PublicInbox::DSKQXS;
-use strict;
-use warnings;
-use parent qw(Exporter);
+use v5.12;
 use Symbol qw(gensym);
 use IO::KQueue;
 use Errno qw(EAGAIN);
-use PublicInbox::Syscall qw(EPOLLONESHOT EPOLLIN EPOLLOUT EPOLLET
-        EPOLL_CTL_ADD EPOLL_CTL_MOD EPOLL_CTL_DEL);
-our @EXPORT_OK = qw(epoll_ctl epoll_wait);
+use PublicInbox::Syscall qw(EPOLLONESHOT EPOLLIN EPOLLOUT EPOLLET);
 
 sub EV_DISPATCH () { 0x0080 }
 
@@ -48,16 +44,15 @@ sub new {
 # It's wasteful in that it uses another FD, but it simplifies
 # our epoll-oriented code.
 sub signalfd {
-        my ($class, $signo, $nonblock) = @_;
+        my ($class, $signo) = @_;
         my $sym = gensym;
-        tie *$sym, $class, $signo, $nonblock; # calls TIEHANDLE
+        tie *$sym, $class, $signo; # calls TIEHANDLE
         $sym
 }
 
 sub TIEHANDLE { # similar to signalfd()
-        my ($class, $signo, $nonblock) = @_;
+        my ($class, $signo) = @_;
         my $self = $class->new;
-        $self->{timeout} = $nonblock ? 0 : -1;
         my $kq = $self->{kq};
         $kq->EV_SET($_, EVFILT_SIGNAL, EV_ADD) for @$signo;
         $self;
@@ -66,12 +61,11 @@ sub TIEHANDLE { # similar to signalfd()
 sub READ { # called by sysread() for signalfd compatibility
         my ($self, undef, $len, $off) = @_; # $_[1] = buf
         die "bad args for signalfd read" if ($len % 128) // defined($off);
-        my $timeout = $self->{timeout};
         my $sigbuf = $self->{sigbuf} //= [];
         my $nr = $len / 128;
         my $r = 0;
         $_[1] = '';
-        do {
+        while (1) {
                 while ($nr--) {
                         my $signo = shift(@$sigbuf) or last;
                         # caller only cares about signalfd_siginfo.ssi_signo:
@@ -79,13 +73,13 @@ sub READ { # called by sysread() for signalfd compatibility
                         $r += 128;
                 }
                 return $r if $r;
-                my @events = eval { $self->{kq}->kevent($timeout) };
+                my @events = eval { $self->{kq}->kevent(0) };
                 # workaround https://rt.cpan.org/Ticket/Display.html?id=116615
                 if ($@) {
                         next if $@ =~ /Interrupted system call/;
                         die;
                 }
-                if (!scalar(@events) && $timeout == 0) {
+                if (!scalar(@events)) {
                         $! = EAGAIN;
                         return;
                 }
@@ -94,36 +88,37 @@ sub READ { # called by sysread() for signalfd compatibility
                 # field shows coalesced signals, and maybe we'll use it
                 # in the future...
                 @$sigbuf = map { $_->[0] } @events;
-        } while (1);
+        }
 }
 
 # for fileno() calls in PublicInbox::DS
 sub FILENO { ${$_[0]->{kq}} }
 
-sub epoll_ctl {
-        my ($self, $op, $fd, $ev) = @_;
-        my $kq = $self->{kq};
-        if ($op == EPOLL_CTL_MOD) {
-                $kq->EV_SET($fd, EVFILT_READ, kq_flag(EPOLLIN, $ev));
-                eval { $kq->EV_SET($fd, EVFILT_WRITE, kq_flag(EPOLLOUT, $ev)) };
-        } elsif ($op == EPOLL_CTL_DEL) {
-                $kq->EV_SET($fd, EVFILT_READ, EV_DISABLE);
-                eval { $kq->EV_SET($fd, EVFILT_WRITE, EV_DISABLE) };
-        } else { # EPOLL_CTL_ADD
-                $kq->EV_SET($fd, EVFILT_READ, EV_ADD|kq_flag(EPOLLIN, $ev));
-
-                # we call this blindly for read-only FDs such as tied
-                # DSKQXS (signalfd emulation) and Listeners
-                eval {
-                        $kq->EV_SET($fd, EVFILT_WRITE, EV_ADD |
-                                                        kq_flag(EPOLLOUT, $ev));
-                };
-        }
+sub _ep_mod_add ($$$$) {
+        my ($kq, $fd, $ev, $add) = @_;
+        $kq->EV_SET($fd, EVFILT_READ, $add|kq_flag(EPOLLIN, $ev));
+
+        # we call this blindly for read-only FDs such as tied
+        # DSKQXS (signalfd emulation) and Listeners
+        eval { $kq->EV_SET($fd, EVFILT_WRITE, $add|kq_flag(EPOLLOUT, $ev)) };
+        0;
+}
+
+sub ep_add { _ep_mod_add($_[0]->{kq}, fileno($_[1]), $_[2], EV_ADD) };
+sub ep_mod { _ep_mod_add($_[0]->{kq}, fileno($_[1]), $_[2], 0) };
+
+sub ep_del {
+        my ($self, $io, $ev) = @_;
+        my $kq = $_[0]->{kq} // return; # called in cleanup
+        my $fd = fileno($io);
+        $kq->EV_SET($fd, EVFILT_READ, EV_DISABLE);
+        eval { $kq->EV_SET($fd, EVFILT_WRITE, EV_DISABLE) };
         0;
 }
 
-sub epoll_wait {
-        my ($self, $maxevents, $timeout_msec, $events) = @_;
+sub ep_wait {
+        my ($self, $timeout_msec, $events) = @_;
+        # n.b.: IO::KQueue is hard-coded to return up to 1000 events
         @$events = eval { $self->{kq}->kevent($timeout_msec) };
         if (my $err = $@) {
                 # workaround https://rt.cpan.org/Ticket/Display.html?id=116615
diff --git a/lib/PublicInbox/DSPoll.pm b/lib/PublicInbox/DSPoll.pm
index 56a400c2..a7055ec9 100644
--- a/lib/PublicInbox/DSPoll.pm
+++ b/lib/PublicInbox/DSPoll.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # Licensed the same as Danga::Socket (and Perl5)
 # License: GPL-1.0+ or Artistic-1.0-Perl
 #  <https://www.gnu.org/licenses/gpl-1.0.txt>
@@ -9,49 +9,47 @@
 # an all encompassing emulation of epoll via IO::Poll, but just to
 # support cases public-inbox-nntpd/httpd care about.
 package PublicInbox::DSPoll;
-use strict;
-use warnings;
-use parent qw(Exporter);
+use v5.12;
 use IO::Poll;
-use PublicInbox::Syscall qw(EPOLLONESHOT EPOLLIN EPOLLOUT EPOLL_CTL_DEL);
-our @EXPORT_OK = qw(epoll_ctl epoll_wait);
+use PublicInbox::Syscall qw(EPOLLONESHOT EPOLLIN EPOLLOUT);
+use Carp qw(carp);
+use Errno ();
 
-sub new { bless {}, $_[0] } # fd => events
+sub new { bless {}, __PACKAGE__ } # fd => events
 
-sub epoll_ctl {
-        my ($self, $op, $fd, $ev) = @_;
-
-        # not wasting time on error checking
-        if ($op != EPOLL_CTL_DEL) {
-                $self->{$fd} = $ev;
-        } else {
-                delete $self->{$fd};
-        }
-        0;
-}
-
-sub epoll_wait {
-        my ($self, $maxevents, $timeout_msec, $events) = @_;
-        my @pset;
+sub ep_wait {
+        my ($self, $timeout_msec, $events) = @_;
+        my (@pset, $n, $fd, $revents, $nval);
         while (my ($fd, $events) = each %$self) {
                 my $pevents = $events & EPOLLIN ? POLLIN : 0;
                 $pevents |= $events & EPOLLOUT ? POLLOUT : 0;
                 push(@pset, $fd, $pevents);
         }
         @$events = ();
-        my $n = IO::Poll::_poll($timeout_msec, @pset);
-        if ($n >= 0) {
-                for (my $i = 0; $i < @pset; ) {
-                        my $fd = $pset[$i++];
-                        my $revents = $pset[$i++] or next;
-                        delete($self->{$fd}) if $self->{$fd} & EPOLLONESHOT;
+        $n = IO::Poll::_poll($timeout_msec, @pset) or return; # timeout expired
+        return if $n < 0 && $! == Errno::EINTR; # caller recalculates timeout
+        die "poll: $!" if $n < 0;
+        while (defined($fd = shift @pset)) {
+                $revents = shift @pset or next; # no event
+                if ($revents & POLLNVAL) {
+                        carp "E: FD=$fd invalid in poll";
+                        delete $self->{$fd};
+                        $nval = 1;
+                } else {
+                        delete $self->{$fd} if $self->{$fd} & EPOLLONESHOT;
                         push @$events, $fd;
                 }
-                my $nevents = scalar @$events;
-                if ($n != $nevents) {
-                        warn "BUG? poll() returned $n, but got $nevents";
-                }
+        }
+        if ($nval && !@$events) {
+                $! = Errno::EBADF;
+                die "poll: $!";
         }
 }
 
+sub ep_del { delete($_[0]->{fileno($_[1])}); 0 }
+sub ep_add { $_[0]->{fileno($_[1])} = $_[2]; 0 }
+
+no warnings 'once';
+*ep_mod = \&ep_add;
+
 1;
diff --git a/lib/PublicInbox/NNTPdeflate.pm b/lib/PublicInbox/DSdeflate.pm
index 06b4499c..539adf0f 100644
--- a/lib/PublicInbox/NNTPdeflate.pm
+++ b/lib/PublicInbox/DSdeflate.pm
@@ -1,7 +1,8 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # RFC 8054 NNTP COMPRESS DEFLATE implementation
+# RFC 4978 IMAP COMPRESS=DEFLATE extension
 #
 # RSS usage for 10K idle-but-did-something NNTP clients on 64-bit:
 #   TLS + DEFLATE[a] :  1.8 GB  (MemLevel=9, 1.2 GB with MemLevel=8)
@@ -14,23 +15,22 @@
 # [b] - memory-optimized implementation using a global deflate context.
 #       It's less efficient in terms of compression, but way more
 #       efficient in terms of server memory usage.
-package PublicInbox::NNTPdeflate;
+package PublicInbox::DSdeflate;
 use strict;
-use 5.010_001;
-use parent qw(PublicInbox::NNTP);
+use v5.10.1;
 use Compress::Raw::Zlib;
 
 my %IN_OPT = (
-        -Bufsize => PublicInbox::NNTP::LINE_MAX,
+        -Bufsize => 1024,
         -WindowBits => -15, # RFC 1951
         -AppendOutput => 1,
 );
 
 # global deflate context and buffer
-my $zbuf = \(my $buf = '');
-my $zout;
+my ($zout, $zbuf);
 {
         my $err;
+        $zbuf = \(my $initial = ''); # replaced by $next in dflush/write
         ($zout, $err) = Compress::Raw::Zlib::Deflate->new(
                 # nnrpd (INN) and Compress::Raw::Zlib favor MemLevel=9,
                 # the zlib C library and git use MemLevel=8 as the default
@@ -42,21 +42,18 @@ my $zout;
         $err == Z_OK or die "Failed to initialize zlib deflate stream: $err";
 }
 
-
 sub enable {
         my ($class, $self) = @_;
         my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT);
         if ($err != Z_OK) {
-                $self->err("Inflate->new failed: $err");
-                $self->res('403 Unable to activate compression');
+                warn("Inflate->new failed: $err\n");
                 return;
         }
-        $self->res('206 Compression active');
         bless $self, $class;
         $self->{zin} = $in;
 }
 
-# overrides PublicInbox::NNTP::compressed
+# overrides PublicInbox::DS::compressed
 sub compressed { 1 }
 
 sub do_read ($$$$) {
@@ -103,7 +100,7 @@ sub msg_more ($$) {
         1;
 }
 
-sub zflush ($) {
+sub dflush ($) {
         my ($self) = @_;
 
         my $deflated = $zbuf;
diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm
index d08ce0f9..e578f2e8 100644
--- a/lib/PublicInbox/Daemon.pm
+++ b/lib/PublicInbox/Daemon.pm
@@ -5,93 +5,156 @@
 # and designed for handling thousands of untrusted clients over slow
 # and/or lossy connections.
 package PublicInbox::Daemon;
-use strict;
-use v5.10.1;
+use v5.12;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
 use IO::Handle; # ->autoflush
 use IO::Socket;
-use POSIX qw(WNOHANG :signal_h);
+use File::Spec;
+use POSIX qw(WNOHANG :signal_h F_SETFD);
 use Socket qw(IPPROTO_TCP SOL_SOCKET);
 STDOUT->autoflush(1);
 STDERR->autoflush(1);
-use PublicInbox::DS qw(now);
+use PublicInbox::DS qw(now awaitpid);
 use PublicInbox::Listener;
 use PublicInbox::EOFpipe;
-use PublicInbox::Sigfd;
 use PublicInbox::Git;
 use PublicInbox::GitAsyncCat;
 use PublicInbox::Eml;
+use PublicInbox::Config;
 our $SO_ACCEPTFILTER = 0x1000;
 my @CMD;
 my ($set_user, $oldset);
 my (@cfg_listen, $stdout, $stderr, $group, $user, $pid_file, $daemonize);
-my $worker_processes = 1;
-my @listeners;
-my %pids;
-my %tls_opt; # scheme://sockname => args for IO::Socket::SSL->start_SSL
+my ($nworker, @listeners, %WORKERS, %logs);
+my %tls_opt; # scheme://sockname => args for IO::Socket::SSL::SSL_Context->new
 my $reexec_pid;
 my ($uid, $gid);
 my ($default_cert, $default_key);
-my %KNOWN_TLS = ( 443 => 'https', 563 => 'nntps', 993 => 'imaps' );
-my %KNOWN_STARTTLS = ( 119 => 'nntp', 143 => 'imap' );
-
-sub accept_tls_opt ($) {
-        my ($opt_str) = @_;
-        # opt_str: opt1=val1,opt2=val2 (opt may repeat for multi-value)
-        require PublicInbox::TLS;
+my %KNOWN_TLS = (443 => 'https', 563 => 'nntps', 993 => 'imaps', 995 =>'pop3s');
+my %KNOWN_STARTTLS = (110 => 'pop3', 119 => 'nntp', 143 => 'imap');
+my %SCHEME2PORT = map { $KNOWN_TLS{$_} => $_ + 0 } keys %KNOWN_TLS;
+for (keys %KNOWN_STARTTLS) { $SCHEME2PORT{$KNOWN_STARTTLS{$_}} = $_ + 0 }
+$SCHEME2PORT{http} = 80;
+
+our ($parent_pipe, %POST_ACCEPT, %XNETD);
+our %WORKER_SIG = (
+        INT => \&worker_quit,
+        QUIT => \&worker_quit,
+        TERM => \&worker_quit,
+        TTIN => 'IGNORE',
+        TTOU => 'IGNORE',
+        USR1 => \&reopen_logs,
+        USR2 => 'IGNORE',
+        WINCH => 'IGNORE',
+        CHLD => \&PublicInbox::DS::enqueue_reap,
+);
+
+sub listener_opt ($) {
+        my ($str) = @_; # opt1=val1,opt2=val2 (opt may repeat for multi-value)
         my $o = {};
         # allow ',' as delimiter since '&' is shell-unfriendly
-        foreach (split(/[,&]/, $opt_str)) {
+        for (split(/[,&]/, $str)) {
                 my ($k, $v) = split(/=/, $_, 2);
-                push @{$o->{$k} ||= []}, $v;
+                push @{$o->{$k}}, $v;
         }
 
         # key may be a part of cert.  At least
         # p5-io-socket-ssl/example/ssl_server.pl has this fallback:
-        $o->{cert} //= [ $default_cert ];
+        $o->{cert} //= [ $default_cert ] if defined($default_cert);
         $o->{key} //= defined($default_key) ? [ $default_key ] : $o->{cert};
-        my %ctx_opt = (SSL_server => 1);
+        $o;
+}
+
+sub check_absolute ($$) {
+        my ($var, $val) = @_;
+        die <<EOM if index($val // '/', '/') != 0;
+$var must be an absolute path when using --daemonize: $val
+EOM
+}
+
+sub accept_tls_opt ($) {
+        my ($opt) = @_;
+        my $o = ref($opt) eq 'HASH' ? $opt : listener_opt($opt);
+        return if !defined($o->{cert});
+        require PublicInbox::TLS;
+        my @ctx_opt;
         # parse out hostname:/path/to/ mappings:
-        foreach my $k (qw(cert key)) {
-                my $x = $ctx_opt{'SSL_'.$k.'_file'} = {};
+        for my $k (qw(cert key)) {
+                $o->{$k} // next;
+                push(@ctx_opt, "SSL_${k}_file", {});
                 foreach my $path (@{$o->{$k}}) {
                         my $host = '';
                         $path =~ s/\A([^:]+):// and $host = $1;
-                        $x->{$host} = $path;
+                        $ctx_opt[-1]->{$host} = $path;
+                        check_absolute($k, $path) if $daemonize;
                 }
         }
-        my $ctx = IO::Socket::SSL::SSL_Context->new(%ctx_opt) or
-                die 'SSL_Context->new: '.PublicInbox::TLS::err();
-
-        # save ~34K per idle connection (cf. SSL_CTX_set_mode(3ssl))
-        # RSS goes from 346MB to 171MB with 10K idle NNTPS clients on amd64
-        # cf. https://rt.cpan.org/Ticket/Display.html?id=129463
-        my $mode = eval { Net::SSLeay::MODE_RELEASE_BUFFERS() };
-        if ($mode && $ctx->{context}) {
-                eval { Net::SSLeay::CTX_set_mode($ctx->{context}, $mode) };
-                warn "W: $@ (setting SSL_MODE_RELEASE_BUFFERS)\n" if $@;
-        }
+        \@ctx_opt;
+}
 
-        { SSL_server => 1, SSL_startHandshake => 0, SSL_reuse_ctx => $ctx };
+sub do_chown ($) {
+        $uid // return;
+        my ($path) = @_;
+        chown($uid, $gid, $path) or warn "chown $path: $!\n";
+}
+
+sub open_log_path ($$) { # my ($fh, $path) = @_; # $_[0] is modified
+        open $_[0], '>>', $_[1] or die "open(>> $_[1]): $!";
+        $_[0]->autoflush(1);
+        do_chown($_[1]);
+        $_[0];
 }
 
-sub load_mod ($) {
-        my ($scheme) = @_;
-        my $modc = "PublicInbox::\U$1";
+sub load_mod ($;$$) {
+        my ($scheme, $opt, $addr) = @_;
+        my $modc = "PublicInbox::\U$scheme";
+        $modc =~ s/S\z//;
         my $mod = $modc.'D';
         eval "require $mod"; # IMAPD|HTTPD|NNTPD|POP3D
         die $@ if $@;
-        my %xn = map { $_ => $mod->can($_) } qw(refresh post_accept);
-        $xn{tlsd} = $mod->new if $mod->can('refresh_groups'); #!HTTPD
-        my $tlsd = $xn{tlsd};
-        $xn{refresh} //= sub { $tlsd->refresh_groups(@_) };
-        $xn{post_accept} //= sub { $modc->new($_[0], $tlsd) };
-        $xn{af_default} = 'httpready' if $modc eq 'PublicInbox::HTTP';
+        my %xn;
+        my $tlsd = $xn{tlsd} = $mod->new;
+        my %env = map {
+                substr($_, length('env.')) => $opt->{$_}->[-1];
+        } grep(/\Aenv\./, keys %$opt);
+        $xn{refresh} = sub {
+                my ($sig) = @_;
+                local @ENV{keys %env} = values %env;
+                $tlsd->refresh_groups($sig);
+        };
+        $xn{post_accept} = $tlsd->can('post_accept_cb') ?
+                        $tlsd->post_accept_cb : sub { $modc->new($_[0], $tlsd) };
+        my @paths = qw(out err);
+        if ($modc eq 'PublicInbox::HTTP') {
+                @paths = qw(err);
+                $xn{af_default} = 'httpready';
+                if (my $p = $opt->{psgi}) {
+                        die "multiple psgi= options specified\n" if @$p > 1;
+                        check_absolute('psgi=', $p->[0]) if $daemonize;
+                        $tlsd->{psgi} = $p->[0];
+                        warn "# $scheme://$addr psgi=$p->[0]\n";
+                }
+        }
+        for my $f (@paths) {
+                my $p = $opt->{$f} or next;
+                die "multiple $f= options specified\n" if @$p > 1;
+                check_absolute("$f=", $p->[0]) if $daemonize;
+                $p = File::Spec->canonpath($p->[0]);
+                $tlsd->{$f} = $logs{$p} //= open_log_path(my $fh, $p);
+                warn "# $scheme://$addr $f=$p\n";
+        }
+        # for per-listener $SIG{__WARN__}:
+        my $err = $tlsd->{err};
+        $tlsd->{warn_cb} = sub {
+                print $err @_ unless PublicInbox::Eml::warn_ignore(@_)
+        };
+        $opt->{'multi-accept'} and
+                $xn{'multi-accept'} = $opt->{'multi-accept'}->[-1];
         \%xn;
 }
 
-sub daemon_prepare ($$) {
-        my ($default_listen, $xnetd) = @_;
+sub daemon_prepare ($) {
+        my ($default_listen) = @_;
         my $listener_names = {}; # sockname => IO::Handle
         $oldset = PublicInbox::DS::block_signals();
         @CMD = ($0, @ARGV);
@@ -104,7 +167,7 @@ options:
 
   -l ADDRESS    address to listen on$dh
   --cert=FILE   default SSL/TLS certificate
-  --key=FILE    default SSL/TLS certificate
+  --key=FILE    default SSL/TLS certificate key
   -W WORKERS    number of worker processes to spawn (default: 1)
 
 See public-inbox-daemon(8) and $prog(1) man pages for more.
@@ -113,11 +176,12 @@ EOF
                 'l|listen=s' => \@cfg_listen,
                 '1|stdout=s' => \$stdout,
                 '2|stderr=s' => \$stderr,
-                'W|worker-processes=i' => \$worker_processes,
+                'W|worker-processes=i' => \$nworker,
                 'P|pid-file=s' => \$pid_file,
                 'u|user=s' => \$user,
                 'g|group=s' => \$group,
                 'D|daemonize' => \$daemonize,
+                'multi-accept=i' => \$PublicInbox::Listener::MULTI_ACCEPT,
                 'cert=s' => \$default_cert,
                 'key=s' => \$default_key,
                 'help|h' => \(my $show_help),
@@ -125,14 +189,12 @@ EOF
         GetOptions(%opt) or die $help;
         if ($show_help) { print $help; exit 0 };
 
+        $_ = File::Spec->canonpath($_ // next) for ($stdout, $stderr);
         if (defined $pid_file && $pid_file =~ /\.oldbin\z/) {
                 die "--pid-file cannot end with '.oldbin'\n";
         }
         @listeners = inherit($listener_names);
-
-        # allow socket-activation users to set certs once and not
-        # have to configure each socket:
-        my @inherited_names = keys(%$listener_names) if defined($default_cert);
+        my @inherited_names = keys(%$listener_names);
 
         # ignore daemonize when inheriting
         $daemonize = undef if scalar @listeners;
@@ -141,27 +203,36 @@ EOF
                 $default_listen // die "no listeners specified\n";
                 push @cfg_listen, $default_listen
         }
-
+        my ($default_scheme) = (($default_listen // '') =~ m!\A([^:]+)://!);
         foreach my $l (@cfg_listen) {
                 my $orig = $l;
-                my $scheme = '';
-                if ($l =~ s!\A([^:]+)://!!) {
-                        $scheme = $1;
-                } elsif ($l =~ /\A(?:\[[^\]]+\]|[^:]+):([0-9])+/) {
-                        my $s = $KNOWN_TLS{$1} // $KNOWN_STARTTLS{$1};
-                        $scheme = $s if defined $s;
+                my ($scheme, $port, $opt);
+                $l =~ s!\A([a-z0-9]+)://!! and $scheme = $1;
+                $scheme //= $default_scheme;
+                if ($l =~ /\A(?:\[[^\]]+\]|[^:]+):([0-9]+)/) {
+                        $port = $1 + 0;
+                        $scheme //= $KNOWN_TLS{$port} // $KNOWN_STARTTLS{$port};
+                }
+                $scheme // die "unable to determine URL scheme of $orig\n";
+                if (!defined($port) && index($l, '/') != 0) { # AF_UNIX socket
+                        $port = $SCHEME2PORT{$scheme} //
+                                die "no port in listen=$orig\n";
+                        $l =~ s!\A([^/]+)!$1:$port! or
+                                die "unable to add port=$port to $l\n";
                 }
+                $l =~ s!/\z!!; # chop one trailing slash
                 if ($l =~ s!/?\?(.+)\z!!) {
-                        $tls_opt{"$scheme://$l"} = accept_tls_opt($1);
+                        $opt = listener_opt($1);
+                        $tls_opt{"$scheme://$l"} = accept_tls_opt($opt);
                 } elsif (defined($default_cert)) {
                         $tls_opt{"$scheme://$l"} = accept_tls_opt('');
-                } elsif ($scheme =~ /\A(?:https|imaps|imaps)\z/) {
+                } elsif ($scheme =~ /\A(?:https|imaps|nntps|pop3s)\z/) {
                         die "$orig specified w/o cert=\n";
                 }
-                $scheme =~ /\A(http|imap|nntp|pop3)/ and
-                        $xnetd->{$l} = load_mod($1);
-
-                next if $listener_names->{$l}; # already inherited
+                if ($listener_names->{$l}) { # already inherited
+                        $XNETD{$l} = load_mod($scheme, $opt, $l);
+                        next;
+                }
                 my (%o, $sock_pkg);
                 if (index($l, '/') == 0) {
                         $sock_pkg = 'IO::Socket::UNIX';
@@ -188,38 +259,43 @@ EOF
                 }
                 $o{Listen} = 1024;
                 my $prev = umask 0000;
-                my $s = eval { $sock_pkg->new(%o) };
-                warn "error binding $l: $! ($@)\n" unless $s;
+                my $s = eval { $sock_pkg->new(%o) } or
+                        warn "error binding $l: $! ($@)\n";
                 umask $prev;
-                if ($s) {
-                        $listener_names->{sockname($s)} = $s;
-                        $s->blocking(0);
-                        push @listeners, $s;
-                }
+                $s // next;
+                $s->blocking(0);
+                my $sockname = sockname($s);
+                warn "# bound $scheme://$sockname\n";
+                $XNETD{$sockname} //= load_mod($scheme, $opt);
+                $listener_names->{$sockname} = $s;
+                push @listeners, $s;
         }
 
         # cert/key options in @cfg_listen takes precedence when inheriting,
         # but map well-known inherited ports if --listen isn't specified
-        # at all
-        for my $sockname (@inherited_names) {
-                $sockname =~ /:([0-9]+)\z/ or next;
-                if (my $scheme = $KNOWN_TLS{$1}) {
-                        $tls_opt{"$scheme://$sockname"} ||= accept_tls_opt('');
-                } elsif (($scheme = $KNOWN_STARTTLS{$1})) {
-                        next if $tls_opt{"$scheme://$sockname"};
-                        $tls_opt{''} ||= accept_tls_opt('');
+        # at all.  This allows socket-activation users to set certs once
+        # and not have to configure each socket:
+        if (defined $default_cert) {
+                my ($stls) = (($default_scheme // '') =~ /\A(pop3|nntp|imap)/);
+                for my $x (@inherited_names) {
+                        $x =~ /:([0-9]+)\z/ or next; # no TLS for AF_UNIX
+                        if (my $scheme = $KNOWN_TLS{$1}) {
+                                $XNETD{$x} //= load_mod($scheme);
+                                $tls_opt{"$scheme://$x"} ||= accept_tls_opt('');
+                        } elsif (($scheme = $KNOWN_STARTTLS{$1})) {
+                                $XNETD{$x} //= load_mod($scheme);
+                                $tls_opt{"$scheme://$x"} ||= accept_tls_opt('');
+                        } elsif (defined $stls) {
+                                $tls_opt{"$stls://$x"} ||= accept_tls_opt('');
+                        }
                 }
         }
-
-        die "No listeners bound\n" unless @listeners;
-}
-
-sub check_absolute ($$) {
-        my ($var, $val) = @_;
-        if (defined $val && index($val, '/') != 0) {
-                die
-"--$var must be an absolute path when using --daemonize: $val\n";
+        if (defined $default_scheme) {
+                for my $x (@inherited_names) {
+                        $XNETD{$x} //= load_mod($default_scheme);
+                }
         }
+        die "No listeners bound\n" unless @listeners;
 }
 
 sub daemonize () {
@@ -230,9 +306,11 @@ sub daemonize () {
                         next unless -e $arg;
                         $ARGV[$i] = Cwd::abs_path($arg);
                 }
-                check_absolute('stdout', $stdout);
-                check_absolute('stderr', $stderr);
-                check_absolute('pid-file', $pid_file);
+                check_absolute('--stdout', $stdout);
+                check_absolute('--stderr', $stderr);
+                check_absolute('--pid-file', $pid_file);
+                check_absolute('--cert', $default_cert);
+                check_absolute('--key', $default_key);
 
                 chdir '/' or die "chdir failed: $!";
         }
@@ -278,55 +356,39 @@ EOF
         bless { pid => $$, pid_file => \$pid_file }, __PACKAGE__;
 }
 
+sub has_busy_clients { # post_loop_do CB
+        my ($state) = @_;
+        my $now = now();
+        my $n = PublicInbox::DS::close_non_busy();
+        if ($n) {
+                if ($state->{-w} < now()) {
+                        warn "$$ quitting, $n client(s) left\n";
+                        $state->{-w} = now() + 5;
+                }
+                unless (defined $state->{0}) {
+                        $state->{0} = (split(/\s+/, $0))[0];
+                        $state->{0} =~ s!\A.*?([^/]+)\z!$1!;
+                }
+                $0 = "$state->{0} quitting, $n client(s) left";
+        }
+        $n; # true: loop continues, false: loop breaks
+}
+
 sub worker_quit { # $_[0] = signal name or number (unused)
         # killing again terminates immediately:
         exit unless @listeners;
 
         $_->close foreach @listeners; # call PublicInbox::DS::close
         @listeners = ();
-        my $proc_name;
-        my $warn = 0;
+
         # drop idle connections and try to quit gracefully
-        PublicInbox::DS->SetPostLoopCallback(sub {
-                my ($dmap, undef) = @_;
-                my $n = 0;
-                my $now = now();
-                for my $s (values %$dmap) {
-                        $s->can('busy') or next;
-                        if ($s->busy) {
-                                ++$n;
-                        } else { # close as much as possible, early as possible
-                                $s->close;
-                        }
-                }
-                if ($n) {
-                        if (($warn + 5) < now()) {
-                                warn "$$ quitting, $n client(s) left\n";
-                                $warn = now();
-                        }
-                        unless (defined $proc_name) {
-                                $proc_name = (split(/\s+/, $0))[0];
-                                $proc_name =~ s!\A.*?([^/]+)\z!$1!;
-                        }
-                        $0 = "$proc_name quitting, $n client(s) left";
-                }
-                $n; # true: loop continues, false: loop breaks
-        });
+        @PublicInbox::DS::post_loop_do = (\&has_busy_clients, { -w => 0 })
 }
 
 sub reopen_logs {
-        if ($stdout) {
-                open STDOUT, '>>', $stdout or
-                        warn "failed to redirect stdout to $stdout: $!\n";
-                STDOUT->autoflush(1);
-                do_chown($stdout);
-        }
-        if ($stderr) {
-                open STDERR, '>>', $stderr or
-                        warn "failed to redirect stderr to $stderr: $!\n";
-                STDERR->autoflush(1);
-                do_chown($stderr);
-        }
+        $logs{$stdout} //= \*STDOUT if defined $stdout;
+        $logs{$stderr} //= \*STDERR if defined $stderr;
+        while (my ($p, $fh) = each %logs) { open_log_path($fh, $p) }
 }
 
 sub sockname ($) {
@@ -390,11 +452,12 @@ sub inherit ($) {
                 if (my $k = sockname($s)) {
                         my $prev_was_blocking = $s->blocking(0);
                         warn <<"" if $prev_was_blocking;
-Inherited socket (fd=$fd) is blocking, making it non-blocking.
+Inherited socket ($k fd=$fd) is blocking, making it non-blocking.
 Set 'NonBlocking = true' in the systemd.service unit to avoid stalled
 processes when multiple service instances start.
 
                         $listener_names->{$k} = $s;
+                        warn "# inherited $k fd=$fd\n";
                         push @rv, $s;
                 } else {
                         warn "failed to inherit fd=$fd (LISTEN_FDS=$fds)";
@@ -418,35 +481,29 @@ sub upgrade { # $_[0] = signal name or number (unused)
                 write_pid($pid_file);
         }
         my $pid = fork;
-        unless (defined $pid) {
+        if (!defined($pid)) {
                 warn "fork failed: $!\n";
-                return;
-        }
-        if ($pid == 0) {
-                use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD);
+        } elsif ($pid == 0) {
                 $ENV{LISTEN_FDS} = scalar @listeners;
                 $ENV{LISTEN_PID} = $$;
                 foreach my $s (@listeners) {
                         # @listeners are globs with workers, PI::L w/o workers
                         $s = $s->{sock} if ref($s) eq 'PublicInbox::Listener';
-
-                        my $fl = fcntl($s, F_GETFD, 0);
-                        fcntl($s, F_SETFD, $fl &= ~FD_CLOEXEC);
+                        fcntl($s, F_SETFD, 0) // die "F_SETFD: $!";
                 }
                 exec @CMD;
                 die "Failed to exec: $!\n";
+        } else {
+                awaitpid($pid, \&upgrade_aborted);
+                $reexec_pid = $pid;
         }
-        $reexec_pid = $pid;
 }
 
-sub kill_workers ($) {
-        my ($sig) = @_;
-        kill $sig, keys(%pids);
-}
+sub kill_workers ($) { kill $_[0], values(%WORKERS) }
 
-sub upgrade_aborted ($) {
-        my ($p) = @_;
-        warn "reexec PID($p) died with: $?\n";
+sub upgrade_aborted {
+        my ($pid) = @_;
+        warn "reexec PID($pid) died with: $?\n";
         $reexec_pid = undef;
         return unless $pid_file;
 
@@ -458,21 +515,6 @@ sub upgrade_aborted ($) {
         warn $@, "\n" if $@;
 }
 
-sub reap_children { # $_[0] = 'CHLD' or POSIX::SIGCHLD()
-        while (1) {
-                my $p = waitpid(-1, WNOHANG) or return;
-                if (defined $reexec_pid && $p == $reexec_pid) {
-                        upgrade_aborted($p);
-                } elsif (defined(my $id = delete $pids{$p})) {
-                        warn "worker[$id] PID($p) died with: $?\n";
-                } elsif ($p > 0) {
-                        warn "unknown PID($p) reaped: $?\n";
-                } else {
-                        return;
-                }
-        }
-}
-
 sub unlink_pid_file_safe_ish ($$) {
         my ($unlink_pid, $file) = @_;
         return unless defined $unlink_pid && $unlink_pid == $$;
@@ -489,101 +531,93 @@ sub unlink_pid_file_safe_ish ($$) {
 sub master_quit ($) {
         exit unless @listeners;
         @listeners = ();
-        kill_workers($_[0]);
+        exit unless kill_workers($_[0]);
+}
+
+sub reap_worker { # awaitpid CB
+        my ($pid, $nr) = @_;
+        warn "worker[$nr] died \$?=$?\n" if $?;
+        delete $WORKERS{$nr};
+        exit if !@listeners && !keys(%WORKERS);
+        PublicInbox::DS::requeue(\&start_workers);
+}
+
+sub start_worker ($) {
+        my ($nr) = @_;
+        return unless @listeners;
+        my $pid = PublicInbox::DS::do_fork;
+        if ($pid == 0) {
+                undef %WORKERS;
+                local $PublicInbox::DS::Poller; # allow epoll/kqueue
+                $set_user->() if $set_user;
+                PublicInbox::EOFpipe->new($parent_pipe, \&worker_quit);
+                worker_loop();
+                exit 0;
+        } else {
+                $WORKERS{$nr} = $pid;
+                awaitpid($pid, \&reap_worker, $nr);
+        }
+}
+
+sub start_workers {
+        my @idx = grep { !defined($WORKERS{$_}) } (0..($nworker - 1)) or return;
+        eval { start_worker($_) for @idx };
+        warn "E: $@\n" if $@;
+}
+
+sub trim_workers {
+        my @nr = grep { $_ >= $nworker } keys %WORKERS;
+        kill('TERM', @WORKERS{@nr});
 }
 
 sub master_loop {
-        pipe(my ($p0, $p1)) or die "failed to create parent-pipe: $!";
-        my $set_workers = $worker_processes;
+        local $parent_pipe;
+        pipe($parent_pipe, my $p1) or die "failed to create parent-pipe: $!";
+        my $set_workers = $nworker; # for SIGWINCH
         reopen_logs();
-        my $ignore_winch;
-        my $sig = {
+        my $msig = {
                 USR1 => sub { reopen_logs(); kill_workers($_[0]); },
                 USR2 => \&upgrade,
                 QUIT => \&master_quit,
                 INT => \&master_quit,
                 TERM => \&master_quit,
                 WINCH => sub {
-                        return if $ignore_winch || !@listeners;
-                        if (-t STDIN || -t STDOUT || -t STDERR) {
-                                $ignore_winch = 1;
-                                warn <<EOF;
-ignoring SIGWINCH since we are not daemonized
-EOF
-                        } else {
-                                $worker_processes = 0;
-                        }
+                        $nworker = 0;
+                        trim_workers();
                 },
                 HUP => sub {
-                        return unless @listeners;
-                        $worker_processes = $set_workers;
+                        $nworker = $set_workers; # undo WINCH
                         kill_workers($_[0]);
+                        PublicInbox::DS::requeue(\&start_workers)
                 },
                 TTIN => sub {
-                        return unless @listeners;
-                        if ($set_workers > $worker_processes) {
-                                ++$worker_processes;
+                        if ($set_workers > $nworker) {
+                                ++$nworker;
                         } else {
-                                $worker_processes = ++$set_workers;
+                                $nworker = ++$set_workers;
                         }
+                        PublicInbox::DS::requeue(\&start_workers);
                 },
                 TTOU => sub {
-                        $worker_processes = --$set_workers if $set_workers > 0;
+                        return if $nworker <= 0;
+                        --$nworker;
+                        trim_workers();
                 },
-                CHLD => \&reap_children,
+                CHLD => \&PublicInbox::DS::enqueue_reap,
         };
-        my $sigfd = PublicInbox::Sigfd->new($sig);
-        local @SIG{keys %$sig} = values(%$sig) unless $sigfd;
-        PublicInbox::DS::sig_setmask($oldset) if !$sigfd;
-        while (1) { # main loop
-                my $n = scalar keys %pids;
-                unless (@listeners) {
-                        exit if $n == 0;
-                        $set_workers = $worker_processes = $n = 0;
-                }
-
-                if ($n > $worker_processes) {
-                        while (my ($k, $v) = each %pids) {
-                                kill('TERM', $k) if $v >= $worker_processes;
-                        }
-                        $n = $worker_processes;
-                }
-                my $want = $worker_processes - 1;
-                if ($n <= $want) {
-                        PublicInbox::DS::block_signals() if !$sigfd;
-                        for my $i ($n..$want) {
-                                my $seed = rand(0xffffffff);
-                                my $pid = fork;
-                                if (!defined $pid) {
-                                        warn "failed to fork worker[$i]: $!\n";
-                                } elsif ($pid == 0) {
-                                        srand($seed);
-                                        eval { Net::SSLeay::randomize() };
-                                        $set_user->() if $set_user;
-                                        return $p0; # run normal work code
-                                } else {
-                                        warn "PID=$pid is worker[$i]\n";
-                                        $pids{$pid} = $i;
-                                }
-                        }
-                        PublicInbox::DS::sig_setmask($oldset) if !$sigfd;
-                }
-
-                if ($sigfd) { # Linux and IO::KQueue users:
-                        $sigfd->wait_once;
-                } else { # wake up every second
-                        sleep(1);
-                }
-        }
+        $msig->{WINCH} = sub {
+                warn "ignoring SIGWINCH since we are not daemonized\n";
+        } if -t STDIN || -t STDOUT || -t STDERR;
+        start_workers();
+        PublicInbox::DS::event_loop($msig, $oldset);
         exit # never gets here, just for documentation
 }
 
-sub tls_start_cb ($$) {
-        my ($opt, $orig_post_accept) = @_;
+sub tls_cb {
+        my ($post_accept, $tlsd) = @_;
         sub {
                 my ($io, $addr, $srv) = @_;
-                my $ssl = IO::Socket::SSL->start_SSL($io, %$opt);
-                $orig_post_accept->($ssl, $addr, $srv);
+                $post_accept->(PublicInbox::TLS::start($io, $tlsd), $addr, $srv)
         }
 }
 
@@ -597,102 +631,84 @@ sub defer_accept ($$) {
                 my $sec = unpack('i', $x);
                 return if $sec > 0; # systemd users may set a higher value
                 setsockopt($s, IPPROTO_TCP, $TCP_DEFER_ACCEPT, 1);
-        } elsif ($^O eq 'freebsd') {
+        } elsif ($^O =~ /\A(?:freebsd|netbsd|dragonfly)\z/) {
                 my $x = getsockopt($s, SOL_SOCKET, $SO_ACCEPTFILTER);
-                return if defined $x; # don't change if set
+                return if ($x // "\0") =~ /[^\0]/s; # don't change if set
                 my $accf_arg = pack('a16a240', $af_name, '');
                 setsockopt($s, SOL_SOCKET, $SO_ACCEPTFILTER, $accf_arg);
         }
 }
 
-sub daemon_loop ($) {
-        my ($xnetd) = @_;
-        my $refresh = sub {
+sub daemon_loop () {
+        local $PublicInbox::Config::DEDUPE = {}; # enable dedupe cache
+        my $refresh = $WORKER_SIG{HUP} = sub {
                 my ($sig) = @_;
-                for my $xn (values %$xnetd) {
+                %$PublicInbox::Config::DEDUPE = (); # clear cache
+                for my $xn (values %XNETD) {
+                        delete $xn->{tlsd}->{ssl_ctx}; # PublicInbox::TLS::start
                         eval { $xn->{refresh}->($sig) };
                         warn "refresh $@\n" if $@;
                 }
         };
-        my %post_accept;
-        while (my ($k, $v) = each %tls_opt) {
-                my $l = $k;
-                $l =~ s!\A([^:]+)://!!;
-                my $scheme = $1 // '';
-                my $xn = $xnetd->{$l} // $xnetd->{''};
-                if ($scheme =~ m!\A(?:https|imaps|nntps)!) {
-                        $post_accept{$l} = tls_start_cb($v, $xn->{post_accept});
-                } elsif ($xn->{tlsd}) { # STARTTLS, $k eq '' is OK
-                        $xn->{tlsd}->{accept_tls} = $v;
-                }
+        while (my ($k, $ctx_opt) = each %tls_opt) {
+                $ctx_opt // next;
+                my ($scheme, $l) = split(m!://!, $k, 2);
+                my $xn = $XNETD{$l} // die "BUG: no xnetd for $k";
+                $xn->{tlsd}->{ssl_ctx_opt} //= $ctx_opt;
+                $scheme =~ m!\A(?:https|imaps|nntps|pop3s)! and
+                        $POST_ACCEPT{$l} = tls_cb(@$xn{qw(post_accept tlsd)});
         }
-        my $sig = {
-                HUP => $refresh,
-                INT => \&worker_quit,
-                QUIT => \&worker_quit,
-                TERM => \&worker_quit,
-                TTIN => 'IGNORE',
-                TTOU => 'IGNORE',
-                USR1 => \&reopen_logs,
-                USR2 => 'IGNORE',
-                WINCH => 'IGNORE',
-                CHLD => \&PublicInbox::DS::enqueue_reap,
-        };
-        if ($worker_processes > 0) {
+        undef %tls_opt;
+        if ($nworker > 0) {
                 $refresh->(); # preload by default
-                my $fh = master_loop(); # returns if in child process
-                PublicInbox::EOFpipe->new($fh, \&worker_quit, undef);
+                return master_loop();
         } else {
                 reopen_logs();
                 $set_user->() if $set_user;
-                $sig->{USR2} = sub { worker_quit() if upgrade() };
+                $WORKER_SIG{USR2} = sub { worker_quit() if upgrade() };
                 $refresh->();
         }
+        local $PublicInbox::DS::Poller; # allow epoll/kqueue
+        worker_loop();
+}
+
+sub worker_loop {
         $uid = $gid = undef;
         reopen_logs();
         @listeners = map {;
                 my $l = sockname($_);
-                my $tls_cb = $post_accept{$l};
-                my $xn = $xnetd->{$l} // $xnetd->{''};
+                my $tls_cb = $POST_ACCEPT{$l};
+                my $xn = $XNETD{$l} // die "BUG: no xnetd for $l";
 
                 # NNTPS, HTTPS, HTTP, IMAPS and POP3S are client-first traffic
                 # IMAP, NNTP and POP3 are server-first
                 defer_accept($_, $tls_cb ? 'dataready' : $xn->{af_default});
 
                 # this calls epoll_create:
-                PublicInbox::Listener->new($_, $tls_cb || $xn->{post_accept})
+                PublicInbox::Listener->new($_, $tls_cb || $xn->{post_accept},
+                                                $xn->{'multi-accept'})
         } @listeners;
-        PublicInbox::DS::event_loop($sig, $oldset);
+        PublicInbox::DS::event_loop(\%WORKER_SIG, $oldset);
 }
 
 sub run {
         my ($default_listen) = @_;
-        my $xnetd = {};
-        if ($default_listen) {
-                $default_listen =~ /\A(http|imap|nntp|pop3)/ or
-                        die "BUG: $default_listen";
-                $xnetd->{''} = load_mod($1);
-        }
-        daemon_prepare($default_listen, $xnetd);
+        $nworker = 1;
+        local (%XNETD, %POST_ACCEPT);
+        daemon_prepare($default_listen);
         my $for_destroy = daemonize();
 
         # localize GCF2C for tests:
         local $PublicInbox::GitAsyncCat::GCF2C;
         local $PublicInbox::Git::async_warn = 1;
         local $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
+        local %WORKER_SIG = %WORKER_SIG;
+        local %POST_ACCEPT;
 
-        daemon_loop($xnetd);
-        PublicInbox::DS->Reset;
+        daemon_loop();
         # ->DESTROY runs when $for_destroy goes out-of-scope
 }
 
-sub do_chown ($) {
-        my ($path) = @_;
-        if (defined $uid and !chown($uid, $gid, $path)) {
-                warn "could not chown $path: $!\n";
-        }
-}
-
 sub write_pid ($) {
         my ($path) = @_;
         Net::Server::Daemonize::create_pid_file($path);
diff --git a/lib/PublicInbox/DirIdle.pm b/lib/PublicInbox/DirIdle.pm
index 9206da9c..230df166 100644
--- a/lib/PublicInbox/DirIdle.pm
+++ b/lib/PublicInbox/DirIdle.pm
@@ -1,22 +1,22 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Used by public-inbox-watch for Maildir (and possibly MH in the future)
 package PublicInbox::DirIdle;
-use strict;
+use v5.12;
 use parent 'PublicInbox::DS';
 use PublicInbox::Syscall qw(EPOLLIN);
 use PublicInbox::In2Tie;
 
 my ($MAIL_IN, $MAIL_GONE, $ino_cls);
-if ($^O eq 'linux' && eval { require Linux::Inotify2; 1 }) {
-        $MAIL_IN = Linux::Inotify2::IN_MOVED_TO() |
-                Linux::Inotify2::IN_CREATE();
-        $MAIL_GONE = Linux::Inotify2::IN_DELETE() |
-                        Linux::Inotify2::IN_DELETE_SELF() |
-                        Linux::Inotify2::IN_MOVE_SELF() |
-                        Linux::Inotify2::IN_MOVED_FROM();
-        $ino_cls = 'Linux::Inotify2';
+if ($^O eq 'linux' && eval { require PublicInbox::Inotify; 1 }) {
+        $MAIL_IN = PublicInbox::Inotify::IN_MOVED_TO() |
+                PublicInbox::Inotify::IN_CREATE();
+        $MAIL_GONE = PublicInbox::Inotify::IN_DELETE() |
+                        PublicInbox::Inotify::IN_DELETE_SELF() |
+                        PublicInbox::Inotify::IN_MOVE_SELF() |
+                        PublicInbox::Inotify::IN_MOVED_FROM();
+        $ino_cls = 'PublicInbox::Inotify';
 # Perl 5.22+ is needed for fileno(DIRHANDLE) support:
 } elsif ($^V ge v5.22 && eval { require PublicInbox::KQNotify }) {
         $MAIL_IN = PublicInbox::KQNotify::MOVED_TO_OR_CREATE();
@@ -68,12 +68,18 @@ sub rm_watches {
         }
 }
 
+sub close {
+        my ($self) = @_;
+        delete $self->{cb};
+        $self->SUPER::close; # if using real kevent/inotify
+}
+
 sub event_step {
         my ($self) = @_;
-        my $cb = $self->{cb};
-        local $PublicInbox::DS::in_loop = 0; # waitpid() synchronously
+        my $cb = $self->{cb} or return;
+        local $PublicInbox::DS::in_loop = 0; # waitpid() synchronously (FIXME)
         eval {
-                my @events = $self->{inot}->read; # Linux::Inotify2->read
+                my @events = $self->{inot}->read; # Inotify3->read
                 $cb->($_) for @events;
         };
         warn "$self->{inot}->read err: $@\n" if $@;
@@ -82,8 +88,8 @@ sub event_step {
 sub force_close {
         my ($self) = @_;
         my $inot = delete $self->{inot} // return;
-        if ($inot->can('fh')) { # Linux::Inotify2 2.3+
-                close($inot->fh) or warn "CLOSE ERROR: $!";
+        if ($inot->can('fh')) { # Inotify3 or Linux::Inotify2 2.3+
+                $inot->fh->close or warn "CLOSE ERROR: $!";
         } elsif ($inot->isa('Linux::Inotify2')) {
                 require PublicInbox::LI2Wrap;
                 PublicInbox::LI2Wrap::wrapclose($inot);
diff --git a/lib/PublicInbox/EOFpipe.pm b/lib/PublicInbox/EOFpipe.pm
index e537e2aa..3474874f 100644
--- a/lib/PublicInbox/EOFpipe.pm
+++ b/lib/PublicInbox/EOFpipe.pm
@@ -1,23 +1,23 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 package PublicInbox::EOFpipe;
-use strict;
+use v5.12;
 use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT $F_SETPIPE_SZ);
 
 sub new {
-        my (undef, $rd, $cb, $arg) = @_;
-        my $self = bless {  cb => $cb, arg => $arg }, __PACKAGE__;
-        # 1031: F_SETPIPE_SZ, 4096: page size
-        fcntl($rd, 1031, 4096) if $^O eq 'linux';
+        my (undef, $rd, $cb) = @_;
+        my $self = bless { cb => $cb }, __PACKAGE__;
+        # 4096: page size
+        fcntl($rd, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
         $self->SUPER::new($rd, EPOLLIN|EPOLLONESHOT);
 }
 
 sub event_step {
         my ($self) = @_;
         if ($self->do_read(my $buf, 1) == 0) { # auto-closed
-                $self->{cb}->($self->{arg});
+                $self->{cb}->();
         }
 }
 
diff --git a/lib/PublicInbox/Emergency.pm b/lib/PublicInbox/Emergency.pm
index 5a1ed1d7..968d7d6f 100644
--- a/lib/PublicInbox/Emergency.pm
+++ b/lib/PublicInbox/Emergency.pm
@@ -1,32 +1,24 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Emergency Maildir delivery for MDA
 package PublicInbox::Emergency;
-use strict;
-use v5.10.1;
+use v5.12;
 use Fcntl qw(:DEFAULT SEEK_SET);
 use Sys::Hostname qw(hostname);
 use IO::Handle; # ->flush
 use Errno qw(EEXIST);
+use File::Path ();
 
 sub new {
         my ($class, $dir) = @_;
-
-        foreach (qw(new tmp cur)) {
-                my $d = "$dir/$_";
-                next if -d $d;
-                require File::Path;
-                if (!File::Path::mkpath($d) && !-d $d) {
-                        die "failed to mkpath($d): $!\n";
-                }
-        }
+        File::Path::make_path(map { $dir.$_ } qw(/tmp /new /cur));
         bless { dir => $dir, t => 0 }, $class;
 }
 
 sub _fn_in {
         my ($self, $pid, $dir) = @_;
-        my $host = $self->{short_host} //= (split(/\./, hostname))[0];
+        my $host = $self->{-host} //= (split(/\./, hostname))[0] // 'localhost';
         my $now = time;
         my $n;
         if ($self->{t} != $now) {
@@ -42,14 +34,14 @@ sub prepare {
         my ($self, $strref) = @_;
         my $pid = $$;
         my $tmp_key = "tmp.$pid";
-        die "already in transaction: $self->{$tmp_key}" if $self->{$tmp_key};
+        die "BUG: in transaction: $self->{$tmp_key}" if $self->{$tmp_key};
         my ($tmp, $fh);
         do {
                 $tmp = _fn_in($self, $pid, 'tmp');
                 $! = undef;
         } while (!sysopen($fh, $tmp, O_CREAT|O_EXCL|O_RDWR) and $! == EEXIST);
-        print $fh $$strref or die "write failed: $!";
-        $fh->flush or die "flush failed: $!";
+        print $fh $$strref or die "print: $!";
+        $fh->flush or die "flush: $!";
         $self->{fh} = $fh;
         $self->{$tmp_key} = $tmp;
 }
@@ -58,15 +50,15 @@ sub abort {
         my ($self) = @_;
         delete $self->{fh};
         my $tmp = delete $self->{"tmp.$$"} or return;
-        unlink($tmp) or warn "Failed to unlink $tmp: $!";
+        unlink($tmp) or warn "W: unlink($tmp): $!";
         undef;
 }
 
 sub fh {
         my ($self) = @_;
-        my $fh = $self->{fh} or die "{fh} not open!\n";
-        seek($fh, 0, SEEK_SET) or die "seek(fh) failed: $!";
-        sysseek($fh, 0, SEEK_SET) or die "sysseek(fh) failed: $!";
+        my $fh = $self->{fh} or die "BUG: {fh} not open";
+        seek($fh, 0, SEEK_SET) or die "seek: $!";
+        sysseek($fh, 0, SEEK_SET) or die "sysseek: $!";
         $fh;
 }
 
@@ -80,7 +72,7 @@ sub commit {
                 $new = _fn_in($self, $pid, 'new');
         } while (!($ok = link($tmp, $new)) && $! == EEXIST);
         die "link($tmp, $new): $!" unless $ok;
-        unlink($tmp) or warn "Failed to unlink $tmp: $!";
+        unlink($tmp) or warn "W: unlink($tmp): $!";
 }
 
 sub DESTROY { commit($_[0]) }
diff --git a/lib/PublicInbox/Eml.pm b/lib/PublicInbox/Eml.pm
index 485f637a..d59d7c3f 100644
--- a/lib/PublicInbox/Eml.pm
+++ b/lib/PublicInbox/Eml.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Lazy MIME parser, it still slurps the full message but keeps short
@@ -144,6 +144,7 @@ sub header_raw {
         my $re = re_memo($_[1]);
         my @v = (${ $_[0]->{hdr} } =~ /$re/g);
         for (@v) {
+                utf8::decode($_); # SMTPUTF8
                 # for compatibility w/ Email::Simple::Header,
                 s/\s+\z//s;
                 s/\A\s+//s;
@@ -333,6 +334,11 @@ sub body_set {
         undef;
 }
 
+# workaround https://rt.cpan.org/Public/Bug/Display.html?id=139622
+# Encode 2.87..3.12 leaks on croak, so we defer and croak ourselves
+our @enc_warn;
+my $enc_warn = sub { push @enc_warn, @_ };
+
 sub body_str_set {
         my ($self, $str) = @_;
         my $cs = ct($self)->{attributes}->{charset} //
@@ -340,10 +346,10 @@ sub body_str_set {
         my $enc = find_encoding($cs) // croak "unknown encoding `$cs'";
         my $tmp;
         {
-                my @w;
-                local $SIG{__WARN__} = sub { push @w, @_ };
+                local @enc_warn;
+                local $SIG{__WARN__} = $enc_warn;
                 $tmp = $enc->encode($str, Encode::FB_WARN);
-                croak(@w) if @w;
+                croak(@enc_warn) if @enc_warn;
         };
         body_set($self, \$tmp);
 }
@@ -359,14 +365,15 @@ sub header_set {
         $pfx .= ': ';
         my $len = 78 - length($pfx);
         @vals = map {;
+                utf8::encode(my $v = $_); # to bytes, support SMTPUTF8
                 # folding differs from Email::Simple::Header,
                 # we favor tabs for visibility (and space savings :P)
                 if (length($_) >= $len && (/\n[^ \t]/s || !/\n/s)) {
                         local $Text::Wrap::columns = $len;
                         local $Text::Wrap::huge = 'overflow';
-                        $pfx . wrap('', "\t", $_) . $self->{crlf};
+                        $pfx . wrap('', "\t", $v) . $self->{crlf};
                 } else {
-                        $pfx . $_ . $self->{crlf};
+                        $pfx . $v . $self->{crlf};
                 }
         } @vals;
         $$hdr =~ s!$re!shift(@vals) // ''!ge; # replace current headers, first
@@ -468,12 +475,11 @@ sub body_str {
                         join("\n\t", header_raw($self, 'Content-Type')));
         };
         my $enc = find_encoding($cs) or croak "unknown encoding `$cs'";
-        my $tmp = body($self);
-        # workaround https://rt.cpan.org/Public/Bug/Display.html?id=139622
-        my @w;
-        local $SIG{__WARN__} = sub { push @w, @_ };
-        my $ret = $enc->decode($tmp, Encode::FB_WARN);
-        croak(@w) if @w;
+        my $ret = body($self);
+        local @enc_warn;
+        local $SIG{__WARN__} = $enc_warn;
+        $ret = $enc->decode($ret, Encode::FB_WARN);
+        croak(@enc_warn) if @enc_warn;
         $ret;
 }
 
@@ -526,4 +532,10 @@ sub willneed { re_memo($_) for @_ }
 willneed(qw(From To Cc Date Subject Content-Type In-Reply-To References
                 Message-ID X-Alt-Message-ID));
 
+# This fixes an old bug from import (pre-a0c07cba0e5d8b6a)
+# mutt also pipes single RFC822 messages with a "From " line,
+# but no Content-Length or "From " escaping.
+# "git format-patch" also generates such files by default.
+sub strip_from { $_[0] =~ s/\A[\r\n]*From [^\n]*\n//s }
+
 1;
diff --git a/lib/PublicInbox/Epoll.pm b/lib/PublicInbox/Epoll.pm
new file mode 100644
index 00000000..7e0aa6e7
--- /dev/null
+++ b/lib/PublicInbox/Epoll.pm
@@ -0,0 +1,26 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# OO API for epoll
+package PublicInbox::Epoll;
+use v5.12;
+use PublicInbox::Syscall qw(epoll_create epoll_ctl epoll_wait
+        EPOLL_CTL_ADD EPOLL_CTL_MOD EPOLL_CTL_DEL);
+use Fcntl qw(F_SETFD FD_CLOEXEC);
+use autodie qw(open fcntl);
+
+sub new {
+        open(my $fh, '+<&=', epoll_create());
+        fcntl($fh, F_SETFD, FD_CLOEXEC);
+        bless \$fh, __PACKAGE__;
+}
+
+sub ep_add { epoll_ctl(fileno(${$_[0]}), EPOLL_CTL_ADD, fileno($_[1]), $_[2]) }
+sub ep_mod { epoll_ctl(fileno(${$_[0]}), EPOLL_CTL_MOD, fileno($_[1]), $_[2]) }
+sub ep_del { epoll_ctl(fileno(${$_[0]}), EPOLL_CTL_DEL, fileno($_[1]), 0) }
+
+# n.b. maxevents=1000 is the historical default.  maxevents=1 (yes, one)
+# is more fair under load with multiple worker processes sharing one listener
+sub ep_wait { epoll_wait(fileno(${$_[0]}), 1000, @_[1, 2]) }
+
+1;
diff --git a/lib/PublicInbox/ExtMsg.pm b/lib/PublicInbox/ExtMsg.pm
index 72cae005..95feb885 100644
--- a/lib/PublicInbox/ExtMsg.pm
+++ b/lib/PublicInbox/ExtMsg.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Used by the web interface to link to messages outside of the our
@@ -11,7 +11,7 @@ use warnings;
 use PublicInbox::Hval qw(ascii_html prurl mid_href);
 use PublicInbox::WwwStream qw(html_oneshot);
 use PublicInbox::Smsg;
-our $MIN_PARTIAL_LEN = 16;
+our $MIN_PARTIAL_LEN = 14; # for 'XXXXXXXXXX.fsf' msgids gnus generates
 
 # TODO: user-configurable
 our @EXT_URL = map { ascii_html($_) } (
diff --git a/lib/PublicInbox/ExtSearch.pm b/lib/PublicInbox/ExtSearch.pm
index 2460d74f..d43c23e6 100644
--- a/lib/PublicInbox/ExtSearch.pm
+++ b/lib/PublicInbox/ExtSearch.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Read-only external (detached) index for cross inbox search.
@@ -33,9 +33,11 @@ sub misc {
 # same as per-inbox ->over, for now...
 sub over {
         my ($self) = @_;
-        $self->{over} //= do {
+        $self->{over} // eval {
                 PublicInbox::Inbox::_cleanup_later($self);
-                PublicInbox::Over->new("$self->{xpfx}/over.sqlite3");
+                my $over = PublicInbox::Over->new("$self->{xpfx}/over.sqlite3");
+                $over->dbh; # may die
+                $self->{over} = $over;
         };
 }
 
@@ -108,7 +110,7 @@ sub altid_map { {} }
 sub description {
         my ($self) = @_;
         ($self->{description} //=
-                PublicInbox::Inbox::cat_desc("$self->{topdir}/description")) //
+                PublicInbox::Git::cat_desc("$self->{topdir}/description")) //
                 '$EXTINDEX_DIR/description missing';
 }
 
@@ -117,13 +119,14 @@ sub search {
         $_[0];
 }
 
+sub thing_type { 'external index' }
+
 no warnings 'once';
 *base_url = \&PublicInbox::Inbox::base_url;
 *smsg_eml = \&PublicInbox::Inbox::smsg_eml;
 *smsg_by_mid = \&PublicInbox::Inbox::smsg_by_mid;
 *msg_by_mid = \&PublicInbox::Inbox::msg_by_mid;
 *modified = \&PublicInbox::Inbox::modified;
-*recent = \&PublicInbox::Inbox::recent;
 
 *max_git_epoch = *nntp_usable = *msg_by_path = \&mm; # undef
 *isrch = \&search;
diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 7c44a1a4..ebbffffc 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -16,13 +16,13 @@
 package PublicInbox::ExtSearchIdx;
 use strict;
 use v5.10.1;
-use parent qw(PublicInbox::ExtSearch PublicInbox::Lock);
+use parent qw(PublicInbox::ExtSearch PublicInbox::Umask PublicInbox::Lock);
 use Carp qw(croak carp);
 use Scalar::Util qw(blessed);
 use Sys::Hostname qw(hostname);
-use POSIX qw(strftime);
 use File::Glob qw(bsd_glob GLOB_NOSORT);
 use PublicInbox::MultiGit;
+use PublicInbox::Spawn ();
 use PublicInbox::Search;
 use PublicInbox::SearchIdx qw(prepare_stack is_ancestor is_bad_blob);
 use PublicInbox::OverIdx;
@@ -34,6 +34,7 @@ use PublicInbox::ContentHash qw(content_hash);
 use PublicInbox::Eml;
 use PublicInbox::DS qw(now add_timer);
 use DBI qw(:sql_types); # SQL_BLOB
+use PublicInbox::Admin qw(fmt_localtime);
 
 sub new {
         my (undef, $dir, $opt) = @_;
@@ -113,11 +114,30 @@ sub check_batch_limit ($) {
         ${$req->{need_checkpoint}} = 1 if $n >= $self->{batch_bytes};
 }
 
+sub bad_ibx_id ($$;$) {
+        my ($self, $ibx_id, $cb) = @_;
+        my $msg = "E: bad/stale ibx_id=#$ibx_id encountered";
+        my $ekey = $self->{oidx}->dbh->selectrow_array(<<EOM, undef, $ibx_id);
+SELECT eidx_key FROM inboxes WHERE ibx_id = ? LIMIT 1
+EOM
+        $msg .= " (formerly `$ekey')" if defined $ekey;
+        $cb //= \&carp;
+        $cb->($msg, "\nE: running $0 --gc may be required");
+}
+
+sub check_xr3 ($$$) {
+        my ($self, $id2pos, $xr3) = @_;
+        @$xr3 = grep {
+                defined($id2pos->{$_->[0]}) ? 1 : bad_ibx_id($self, $_->[0])
+        } @$xr3;
+}
+
 sub apply_boost ($$) {
         my ($req, $smsg) = @_;
         my $id2pos = $req->{id2pos}; # index in ibx_sorted
         my $xr3 = $req->{self}->{oidx}->get_xref3($smsg->{num}, 1);
-        @$xr3 = sort {
+        check_xr3($req->{self}, $id2pos, $xr3);
+        @$xr3 = sort { # sort ascending
                 $id2pos->{$a->[0]} <=> $id2pos->{$b->[0]}
                                 ||
                 $a->[1] <=> $b->[1] # break ties with {xnum}
@@ -406,14 +426,14 @@ EOM
         while (my ($ibx_id, $eidx_key) = $ibx_ck->fetchrow_array) {
                 next if $self->{ibx_map}->{$eidx_key};
                 $self->{midx}->remove_eidx_key($eidx_key);
-                warn "I: deleting messages for $eidx_key...\n";
+                warn "# deleting messages for $eidx_key...\n";
                 $x3_doc->execute($ibx_id);
                 my $ibx = { -ibx_id => $ibx_id, -gc_eidx_key => $eidx_key };
                 while (my ($docid, $xnum, $oid) = $x3_doc->fetchrow_array) {
                         my $r = _unref_doc($sync, $docid, $ibx, $xnum, $oid);
                         $oid = unpack('H*', $oid);
                         $r = $r ? 'unref' : 'remove';
-                        warn "I: $r #$docid $eidx_key $oid\n";
+                        warn "# $r #$docid $eidx_key $oid\n";
                         if (checkpoint_due($sync)) {
                                 $x3_doc = $ibx_ck = undef;
                                 reindex_checkpoint($self, $sync);
@@ -433,12 +453,12 @@ SELECT key FROM eidx_meta WHERE key LIKE ? ESCAPE ?
                 $lc_i->execute("lc-%:$pat//%", '\\');
                 while (my ($key) = $lc_i->fetchrow_array) {
                         next if $key !~ m!\Alc-v[1-9]+:\Q$eidx_key\E//!;
-                        warn "I: removing $key\n";
+                        warn "# removing $key\n";
                         $self->{oidx}->dbh->do(<<'', undef, $key);
 DELETE FROM eidx_meta WHERE key = ?
 
                 }
-                warn "I: $eidx_key removed\n";
+                warn "# $eidx_key removed\n";
         }
 }
 
@@ -447,20 +467,20 @@ sub eidx_gc_scan_shards ($$) { # TODO: use for lei/store
         my $nr = $self->{oidx}->dbh->do(<<'');
 DELETE FROM xref3 WHERE docid NOT IN (SELECT num FROM over)
 
-        warn "I: eliminated $nr stale xref3 entries\n" if $nr != 0;
+        warn "# eliminated $nr stale xref3 entries\n" if $nr != 0;
         reindex_checkpoint($self, $sync) if checkpoint_due($sync);
 
         # fixup from old bugs:
         $nr = $self->{oidx}->dbh->do(<<'');
 DELETE FROM over WHERE num > 0 AND num NOT IN (SELECT docid FROM xref3)
 
-        warn "I: eliminated $nr stale over entries\n" if $nr != 0;
+        warn "# eliminated $nr stale over entries\n" if $nr != 0;
         reindex_checkpoint($self, $sync) if checkpoint_due($sync);
 
         $nr = $self->{oidx}->dbh->do(<<'');
 DELETE FROM eidxq WHERE docid NOT IN (SELECT num FROM over)
 
-        warn "I: eliminated $nr stale reindex queue entries\n" if $nr != 0;
+        warn "# eliminated $nr stale reindex queue entries\n" if $nr != 0;
         reindex_checkpoint($self, $sync) if checkpoint_due($sync);
 
         my ($cur) = $self->{oidx}->dbh->selectrow_array(<<EOM);
@@ -490,7 +510,7 @@ SELECT num FROM over WHERE num >= ? ORDER BY num ASC LIMIT 10000
                         reindex_checkpoint($self, $sync);
                 }
         }
-        warn "I: eliminated $nr stale Xapian documents\n" if $nr != 0;
+        warn "# eliminated $nr stale Xapian documents\n" if $nr != 0;
 }
 
 sub eidx_gc {
@@ -513,8 +533,9 @@ sub eidx_gc {
 
 sub _ibx_for ($$$) {
         my ($self, $sync, $smsg) = @_;
-        my $ibx_id = delete($smsg->{ibx_id}) // die '{ibx_id} unset';
-        my $pos = $sync->{id2pos}->{$ibx_id} // die "$ibx_id no pos";
+        my $ibx_id = delete($smsg->{ibx_id}) // die 'BUG: {ibx_id} unset';
+        my $pos = $sync->{id2pos}->{$ibx_id} //
+                bad_ibx_id($self, $ibx_id, \&croak);
         $self->{-ibx_ary_known}->[$pos] //
                 die "BUG: ibx for $smsg->{blob} not mapped"
 }
@@ -657,7 +678,8 @@ BUG? #$docid $smsg->{blob} is not referenced by inboxes during reindex
         # hit the common case in _reindex_finalize without rereading
         # from git (or holding multiple messages in memory).
         my $id2pos = $sync->{id2pos}; # index in ibx_sorted
-        @$xr3 = sort {
+        check_xr3($self, $id2pos, $xr3);
+        @$xr3 = sort { # sort descending
                 $id2pos->{$b->[0]} <=> $id2pos->{$a->[0]}
                                 ||
                 $b->[1] <=> $a->[1] # break ties with {xnum}
@@ -728,16 +750,14 @@ sub eidxq_lock_acquire ($) {
                 return $locked if $locked eq $cur;
         }
         my ($pid, $time, $euid, $ident) = split(/-/, $cur, 4);
-        my $t = strftime('%Y-%m-%d %k:%M %z', localtime($time));
+        my $t = fmt_localtime($time);
         local $self->{current_info} = 'eidxq';
         if ($euid == $> && $ident eq host_ident) {
-                if (kill(0, $pid)) {
-                        warn <<EOM; return;
-I: PID:$pid (re)indexing since $t, it will continue our work
+                kill(0, $pid) and warn <<EOM and return;
+# PID:$pid (re)indexing since $t, it will continue our work
 EOM
-                }
                 if ($!{ESRCH}) {
-                        warn "I: eidxq_lock is stale ($cur), clobbering\n";
+                        warn "# eidxq_lock is stale ($cur), clobbering\n";
                         return _eidxq_take($self);
                 }
                 warn "E: kill(0, $pid) failed: $!\n"; # fall-through:
@@ -837,7 +857,7 @@ sub reindex_unseen ($$$$) {
                 xnum => $xsmsg->{num},
                 # {mids} and {chash} will be filled in at _reindex_unseen
         };
-        warn "I: reindex_unseen ${\$ibx->eidx_key}:$req->{xnum}:$req->{oid}\n";
+        warn "# reindex_unseen ${\$ibx->eidx_key}:$req->{xnum}:$req->{oid}\n";
         $self->git->cat_async($xsmsg->{blob}, \&_reindex_unseen, $req);
 }
 
@@ -1181,12 +1201,6 @@ sub update_last_commit { # overrides V2Writable
         $self->{oidx}->eidx_meta($meta_key, $latest_cmt);
 }
 
-sub _idx_init { # with_umask callback
-        my ($self, $opt) = @_;
-        PublicInbox::V2Writable::_idx_init($self, $opt); # acquires ei.lock
-        $self->{midx} = PublicInbox::MiscIdx->new($self);
-}
-
 sub symlink_packs ($$) {
         my ($ibx, $pd) = @_;
         my $ret = 0;
@@ -1272,15 +1286,17 @@ sub idx_init { # similar to V2Writable
         }
         ($has_new || $prune_nr || $new ne '') and
                 $self->{mg}->write_alternates($mode, $alt, $new);
-        $git_midx and $self->with_umask(sub {
+        my $restore = $self->with_umask;
+        if ($git_midx) {
                 my @cmd = ('multi-pack-index');
                 push @cmd, '--no-progress' if ($opt->{quiet}//0) > 1;
                 my $lk = $self->lock_for_scope;
                 system('git', "--git-dir=$ALL", @cmd, 'write');
                 # ignore errors, fairly new command, may not exist
-        });
+        }
         $self->parallel_init($self->{indexlevel});
-        $self->with_umask(\&_idx_init, $self, $opt);
+        PublicInbox::V2Writable::_idx_init($self, $opt); # acquires ei.lock
+        $self->{midx} = PublicInbox::MiscIdx->new($self);
         $self->{oidx}->begin_lazy;
         $self->{oidx}->eidx_prep;
         $self->{midx}->create_xdb if $new ne '';
@@ -1390,7 +1406,7 @@ sub eidx_watch { # public-inbox-extindex --watch main loop
         my $quit = PublicInbox::SearchIdx::quit_cb($sync);
         $sig->{QUIT} = $sig->{INT} = $sig->{TERM} = $quit;
         local $self->{-watch_sync} = $sync; # for ->on_inbox_unlock
-        PublicInbox::DS->SetPostLoopCallback(sub { !$sync->{quit} });
+        local @PublicInbox::DS::post_loop_do = (sub { !$sync->{quit} });
         $pr->("initial scan complete, entering event loop\n") if $pr;
         # calls InboxIdle->event_step:
         PublicInbox::DS::event_loop($sig, $oldset);
@@ -1399,7 +1415,6 @@ sub eidx_watch { # public-inbox-extindex --watch main loop
 
 no warnings 'once';
 *done = \&PublicInbox::V2Writable::done;
-*with_umask = \&PublicInbox::InboxWritable::with_umask;
 *parallel_init = \&PublicInbox::V2Writable::parallel_init;
 *nproc_shards = \&PublicInbox::V2Writable::nproc_shards;
 *sync_prepare = \&PublicInbox::V2Writable::sync_prepare;
diff --git a/lib/PublicInbox/FakeInotify.pm b/lib/PublicInbox/FakeInotify.pm
index 6d269601..8be07135 100644
--- a/lib/PublicInbox/FakeInotify.pm
+++ b/lib/PublicInbox/FakeInotify.pm
@@ -1,14 +1,13 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # for systems lacking Linux::Inotify2 or IO::KQueue, just emulates
-# enough of Linux::Inotify2
+# enough of Linux::Inotify2 we use.
 package PublicInbox::FakeInotify;
-use strict;
-use v5.10.1;
-use parent qw(Exporter);
+use v5.12;
 use Time::HiRes qw(stat);
 use PublicInbox::DS qw(add_timer);
+use Errno qw(ENOTDIR ENOENT);
 sub IN_MODIFY () { 0x02 } # match Linux inotify
 # my $IN_MOVED_FROM         0x00000040        /* File was moved from X.  */
 # my $IN_MOVED_TO = 0x80;
@@ -18,98 +17,125 @@ sub IN_DELETE () { 0x200 }
 sub IN_DELETE_SELF () { 0x400 }
 sub IN_MOVE_SELF () { 0x800 }
 
-our @EXPORT_OK = qw(fill_dirlist on_dir_change);
-
 my $poll_intvl = 2; # same as Filesys::Notify::Simple
 
-sub new { bless { watch => {}, dirlist => {} }, __PACKAGE__ }
+sub new { bless {}, __PACKAGE__ }
 
-sub fill_dirlist ($$$) {
-        my ($self, $path, $dh) = @_;
-        my $dirlist = $self->{dirlist}->{$path} = {};
-        while (defined(my $n = readdir($dh))) {
-                $dirlist->{$n} = undef if $n !~ /\A\.\.?\z/;
-        }
-}
+sub on_dir_change ($$$$$) { # used by KQNotify subclass
+        my ($self, $events, $dh, $path, $dir_delete) = @_;
+        my $old = $self->{dirlist}->{$path};
+        my @cur = grep(!/\A\.\.?\z/, readdir($dh));
+        $self->{dirlist}->{$path} = \@cur;
 
-# behaves like Linux::Inotify2->watch
-sub watch {
-        my ($self, $path, $mask) = @_;
-        my @st = stat($path) or return;
-        my $k = "$path\0$mask";
-        $self->{watch}->{$k} = $st[10]; # 10 - ctime
-        if ($mask & IN_DELETE) {
-                opendir(my $dh, $path) or return;
-                fill_dirlist($self, $path, $dh);
+        # new files:
+        my %tmp = map { $_ => undef } @cur;
+        delete @tmp{@$old};
+        push(@$events, map {
+                bless \"$path/$_", 'PublicInbox::FakeInotify::Event'
+        } keys %tmp);
+
+        if ($dir_delete) {
+                %tmp = map { $_ => undef } @$old;
+                delete @tmp{@cur};
+                push(@$events, map {
+                        bless \"$path/$_", 'PublicInbox::FakeInotify::GoneEvent'
+                } keys %tmp);
         }
-        bless [ $self->{watch}, $k ], 'PublicInbox::FakeInotify::Watch';
 }
 
-# also used by KQNotify since it kevent requires readdir on st_nlink
-# count changes.
-sub on_dir_change ($$$$$) {
-        my ($events, $dh, $path, $old_ctime, $dirlist) = @_;
-        my $oldlist = $dirlist->{$path};
-        my $newlist = $oldlist ? {} : undef;
-        while (defined(my $base = readdir($dh))) {
-                next if $base =~ /\A\.\.?\z/;
-                my $full = "$path/$base";
-                my @st = stat($full);
-                if (@st && $st[10] > $old_ctime) {
-                        push @$events,
-                                bless(\$full, 'PublicInbox::FakeInotify::Event')
+sub watch_open ($$$) { # used by KQNotify subclass
+        my ($self, $path, $dir_delete) = @_;
+        my ($fh, @st, @st0, $tries);
+        do {
+again:
+                unless (@st0 = stat($path)) {
+                        warn "W: stat($path): $!" if $! != ENOENT;
+                        return;
                 }
-                if (!@st) {
-                        # ignore ENOENT due to race
-                        warn "unhandled stat($full) error: $!\n" if !$!{ENOENT};
-                } elsif ($newlist) {
-                        $newlist->{$base} = undef;
+                if (!(-d _ ? opendir($fh, $path) : open($fh, '<', $path))) {
+                        goto again if $! == ENOTDIR && ++$tries < 10;
+                        warn "W: open($path): $!" if $! != ENOENT;
+                        return;
                 }
+                @st = stat($fh) or die "fstat($path): $!";
+        } while ("@st[0,1]" ne "@st0[0,1]" &&
+                ((++$tries < 10) || (warn(<<EOM) && return)));
+E: $path switching inodes too frequently to watch
+EOM
+        if (-d _) {
+                $self->{dirlist}->{$path} = [];
+                on_dir_change($self, [], $fh, $path, $$dir_delete);
+        } else {
+                $$dir_delete = 0;
         }
-        return if !$newlist;
-        delete @$oldlist{keys %$newlist};
-        $dirlist->{$path} = $newlist;
-        push(@$events, map {
-                bless \"$path/$_", 'PublicInbox::FakeInotify::GoneEvent'
-        } keys %$oldlist);
+        bless [ @st[0, 1, 10], $path, $fh ], 'PublicInbox::FakeInotify::Watch'
+}
+
+# behaves like Linux::Inotify2->watch
+sub watch {
+        my ($self, $path, $mask) = @_; # mask is ignored
+        my $dir_delete = $mask & IN_DELETE ? 1 : 0;
+        my $w = watch_open($self, $path, \$dir_delete) or return;
+        pop @$w; # no need to keep $fh open for non-kqueue
+        $self->{watch}->{"$path\0$dir_delete"} = $w;
+}
+
+sub gone ($$$) { # used by KQNotify subclass
+        my ($self, $ident, $path) = @_;
+        delete $self->{watch}->{$ident};
+        delete $self->{dirlist}->{$path};
+        bless(\$path, 'PublicInbox::FakeInotify::SelfGoneEvent');
+}
+
+# fuzz the time for freshly modified directories for low-res VFS
+sub dir_adj ($) {
+        my ($old_ctime) = @_;
+        my $now = Time::HiRes::time;
+        my $diff = $now - $old_ctime;
+        my $adj = $poll_intvl + 1;
+        ($diff > -$adj && $diff < $adj) ? 1 : 0;
 }
 
 # behaves like non-blocking Linux::Inotify2->read
 sub read {
         my ($self) = @_;
-        my $watch = $self->{watch} or return ();
-        my $events = [];
-        my @watch_gone;
-        for my $x (keys %$watch) {
-                my ($path, $mask) = split(/\0/, $x, 2);
-                my @now = stat($path);
-                if (!@now && $!{ENOENT} && ($mask & IN_DELETE_SELF)) {
-                        push @$events, bless(\$path,
-                                'PublicInbox::FakeInotify::SelfGoneEvent');
-                        push @watch_gone, $x;
-                        delete $self->{dirlist}->{$path};
+        my $ret = [];
+        while (my ($ident, $w) = each(%{$self->{watch}})) {
+                if (!@$w) { # cancelled
+                        delete($self->{watch}->{$ident});
+                        next;
                 }
-                next if !@now;
-                my $old_ctime = $watch->{$x};
-                $watch->{$x} = $now[10];
-                next if $old_ctime == $now[10];
-                if ($mask & IN_MODIFY) {
-                        push @$events,
-                                bless(\$path, 'PublicInbox::FakeInotify::Event')
-                } elsif ($mask & (MOVED_TO_OR_CREATE | IN_DELETE)) {
-                        if (opendir(my $dh, $path)) {
-                                on_dir_change($events, $dh, $path, $old_ctime,
-                                                $self->{dirlist});
-                        } elsif ($!{ENOENT}) {
-                                push @watch_gone, $x;
-                                delete $self->{dirlist}->{$path};
-                        } else {
-                                warn "W: opendir $path: $!\n";
+                my $dir_delete = (split(/\0/, $ident, 2))[1];
+                my ($old_dev, $old_ino, $old_ctime, $path) = @$w;
+                my @new_st = stat($path);
+                warn "W: stat($path): $!\n" if !@new_st && $! != ENOENT;
+                if (!@new_st || "$old_dev $old_ino" ne "@new_st[0,1]") {
+                        push @$ret, gone($self, $ident, $path);
+                        next;
+                }
+                if (-d _ && $new_st[10] > ($old_ctime - dir_adj($old_ctime))) {
+                        opendir(my $fh, $path) or do {
+                                if ($! == ENOENT || $! == ENOTDIR) {
+                                        push @$ret, gone($self, $ident, $path);
+                                } else {
+                                        warn "W: opendir($path): $!";
+                                }
+                                next;
+                        };
+                        @new_st = stat($fh) or die "fstat($path): $!";
+                        if ("$old_dev $old_ino" ne "@new_st[0,1]") {
+                                push @$ret, gone($self, $ident, $path);
+                                next;
                         }
+                        $w->[2] = $new_st[10];
+                        on_dir_change($self, $ret, $fh, $path, $dir_delete);
+                } elsif ($new_st[10] > $old_ctime) { # regular files, etc
+                        $w->[2] = $new_st[10];
+                        push @$ret, bless(\$path,
+                                        'PublicInbox::FakeInotify::Event');
                 }
         }
-        delete @$watch{@watch_gone};
-        @$events;
+        @$ret;
 }
 
 sub poll_once {
@@ -119,20 +145,14 @@ sub poll_once {
 }
 
 package PublicInbox::FakeInotify::Watch;
-use strict;
+use v5.12;
 
-sub cancel {
-        my ($self) = @_;
-        delete $self->[0]->{$self->[1]};
-}
+sub cancel { @{$_[0]} = () }
 
-sub name {
-        my ($self) = @_;
-        (split(/\0/, $self->[1], 2))[0];
-}
+sub name { $_[0]->[3] }
 
 package PublicInbox::FakeInotify::Event;
-use strict;
+use v5.12;
 
 sub fullname { ${$_[0]} }
 
@@ -141,14 +161,14 @@ sub IN_MOVED_FROM { 0 }
 sub IN_DELETE_SELF { 0 }
 
 package PublicInbox::FakeInotify::GoneEvent;
-use strict;
+use v5.12;
 our @ISA = qw(PublicInbox::FakeInotify::Event);
 
 sub IN_DELETE { 1 }
 sub IN_MOVED_FROM { 0 }
 
 package PublicInbox::FakeInotify::SelfGoneEvent;
-use strict;
+use v5.12;
 our @ISA = qw(PublicInbox::FakeInotify::GoneEvent);
 
 sub IN_DELETE_SELF { 1 }
diff --git a/lib/PublicInbox/Feed.pm b/lib/PublicInbox/Feed.pm
index b2219dad..225565f4 100644
--- a/lib/PublicInbox/Feed.pm
+++ b/lib/PublicInbox/Feed.pm
@@ -1,13 +1,13 @@
-# Copyright (C) 2013-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Used for generating Atom feeds for web-accessible mailing list archives.
 package PublicInbox::Feed;
 use strict;
-use warnings;
+use v5.10.1;
 use PublicInbox::View;
 use PublicInbox::WwwAtomStream;
-use PublicInbox::Smsg; # this loads w/o Search::Xapian
+use PublicInbox::Smsg; # this loads w/o Xapian
 
 sub generate_i {
         my ($ctx) = @_;
@@ -19,29 +19,25 @@ sub generate {
         my ($ctx) = @_;
         my $msgs = $ctx->{msgs} = recent_msgs($ctx);
         return _no_thread() unless @$msgs;
-        PublicInbox::WwwAtomStream->response($ctx, 200, \&generate_i);
+        PublicInbox::WwwAtomStream->response($ctx, \&generate_i);
 }
 
 sub generate_thread_atom {
         my ($ctx) = @_;
         my $msgs = $ctx->{msgs} = $ctx->{ibx}->over->get_thread($ctx->{mid});
         return _no_thread() unless @$msgs;
-        PublicInbox::WwwAtomStream->response($ctx, 200, \&generate_i);
+        PublicInbox::WwwAtomStream->response($ctx, \&generate_i);
 }
 
 sub generate_html_index {
         my ($ctx) = @_;
         # if the 'r' query parameter is given, it is a legacy permalink
         # which we must continue supporting:
-        my $qp = $ctx->{qp};
-        my $ibx = $ctx->{ibx};
-        if ($qp && !$qp->{r} && $ibx->over) {
+        !$ctx->{qp}->{r} && $ctx->{ibx}->over and
                 return PublicInbox::View::index_topics($ctx);
-        }
 
-        my $env = $ctx->{env};
-        my $url = $ibx->base_url($env) . 'new.html';
-        my $qs = $env->{QUERY_STRING};
+        my $url = $ctx->{ibx}->base_url($ctx->{env}) . 'new.html';
+        my $qs = $ctx->{env}->{QUERY_STRING};
         $url .= "?$qs" if $qs ne '';
         [302, [ 'Location', $url, 'Content-Type', 'text/plain'],
                 [ "Redirecting to $url\n" ] ];
@@ -49,12 +45,15 @@ sub generate_html_index {
 
 sub new_html_i {
         my ($ctx, $eml) = @_;
-        $ctx->zmore($ctx->html_top) if exists $ctx->{-html_tip};
+        print { $ctx->zfh } $ctx->html_top if exists $ctx->{-html_tip};
 
-        $eml and return PublicInbox::View::eml_entry($ctx, $eml);
+        if ($eml) {
+                $ctx->{smsg}->populate($eml) if !$ctx->{ibx}->{over};
+                return PublicInbox::View::eml_entry($ctx, $eml);
+        }
         my $smsg = shift @{$ctx->{msgs}} or
-                $ctx->zmore(PublicInbox::View::pagination_footer(
-                                                $ctx, './new.html'));
+                print { $ctx->zfh } PublicInbox::View::pagination_footer(
+                                                $ctx, './new.html');
         $smsg;
 }
 
@@ -67,8 +66,9 @@ sub new_html {
         }
         $ctx->{-html_tip} = '<pre>';
         $ctx->{-upfx} = '';
+        $ctx->{-spfx} = '' if $ctx->{ibx}->{coderepo};
         $ctx->{-hr} = 1;
-        PublicInbox::WwwStream::aresponse($ctx, 200, \&new_html_i);
+        PublicInbox::WwwStream::aresponse($ctx, \&new_html_i);
 }
 
 # private subs
@@ -84,7 +84,6 @@ sub recent_msgs {
         return PublicInbox::View::paginate_recent($ctx, $max) if $ibx->over;
 
         # only for rare v1 inboxes which aren't indexed at all
-        my $qp = $ctx->{qp};
         my $hex = '[a-f0-9]';
         my $addmsg = qr!^:000000 100644 \S+ (\S+) A\t${hex}{2}/${hex}{38}$!;
         my $delmsg = qr!^:100644 000000 (\S+) \S+ D\t(${hex}{2}/${hex}{38})$!;
@@ -92,7 +91,7 @@ sub recent_msgs {
 
         # revision ranges may be specified
         my $range = 'HEAD';
-        my $r = $qp->{r} if $qp;
+        my $r = $ctx->{qp}->{r};
         if ($r && ($r =~ /\A(?:$refhex\.\.)?$refhex\z/o)) {
                 $range = $r;
         }
@@ -108,13 +107,13 @@ sub recent_msgs {
         my $last;
         my $last_commit;
         local $/ = "\n";
-        my @oids;
+        my @ret;
         while (defined(my $line = <$log>)) {
                 if ($line =~ /$addmsg/o) {
                         my $add = $1;
                         next if $deleted{$add}; # optimization-only
-                        push @oids, $add;
-                        if (scalar(@oids) >= $max) {
+                        push(@ret, bless { blob => $add }, 'PublicInbox::Smsg');
+                        if (scalar(@ret) >= $max) {
                                 $last = 1;
                                 last;
                         }
@@ -136,8 +135,7 @@ sub recent_msgs {
         $last_commit and
                 $ctx->{next_page} = qq[<a\nhref="?r=$last_commit"\nrel=next>] .
                                         'next (older)</a>';
-
-        [ map { bless {blob => $_ }, 'PublicInbox::Smsg' } @oids ];
+        \@ret;
 }
 
 1;
diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index 5261cad1..b0f1437c 100644
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -2,54 +2,52 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Wrapper to "git fetch" remote public-inboxes
 package PublicInbox::Fetch;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::IPC);
 use URI ();
-use PublicInbox::Spawn qw(popen_rd run_die spawn);
+use PublicInbox::Spawn qw(popen_rd run_qx run_wait);
 use PublicInbox::Admin;
 use PublicInbox::LEI;
 use PublicInbox::LeiCurl;
 use PublicInbox::LeiMirror;
+use PublicInbox::SHA qw(sha_all);
 use File::Temp ();
-use PublicInbox::Config;
-use IO::Compress::Gzip qw(gzip $GzipError);
 
 sub new { bless {}, __PACKAGE__ }
 
-sub fetch_args ($$) {
-        my ($lei, $opt) = @_;
-        my @cmd; # (git --git-dir=...) to be added by caller
-        $opt->{$_} = $lei->{$_} for (0..2);
-        # we support "-c $key=$val" for arbitrary git config options
-        # e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
-        push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
-        push @cmd, 'fetch';
-        push @cmd, '-q' if $lei->{opt}->{quiet};
-        push @cmd, '-v' if $lei->{opt}->{verbose};
-        @cmd;
-}
-
 sub remote_url ($$) {
         my ($lei, $dir) = @_;
         my $rn = $lei->{opt}->{'try-remote'} // [ 'origin', '_grokmirror' ];
         for my $r (@$rn) {
                 my $cmd = [ qw(git config), "remote.$r.url" ];
-                my $fh = popen_rd($cmd, undef, { -C => $dir, 2 => $lei->{2} });
-                my $url = <$fh>;
-                close $fh or next;
+                my $url = run_qx($cmd, undef, { -C => $dir, 2 => $lei->{2} });
+                next if $?;
                 $url =~ s!/*\n!!s;
                 return $url;
         }
         undef
 }
 
+# PSGI mount prefixes and manifest.js.gz prefixes don't always align...
+# TODO: remove, handle multi-inbox fetch
+sub deduce_epochs ($$) {
+        my ($m, $path) = @_;
+        my ($v1_ent, @v2_epochs);
+        my $path_pfx = '';
+        $path =~ s!/+\z!!;
+        do {
+                $v1_ent = $m->{$path};
+                @v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
+        } while (!defined($v1_ent) && !@v2_epochs &&
+                $path =~ s!\A(/[^/]+)/!/! and $path_pfx .= $1);
+        ($path_pfx, $v1_ent ? $path : undef, @v2_epochs);
+}
+
 sub do_manifest ($$$) {
         my ($lei, $dir, $ibx_uri) = @_;
         my $muri = URI->new("$ibx_uri/manifest.js.gz");
         my $ft = File::Temp->new(TEMPLATE => 'm-XXXX',
                                 UNLINK => 1, DIR => $dir, SUFFIX => '.tmp');
-        my $fn = $ft->filename;
         my $mf = "$dir/manifest.js.gz";
         my $m0; # current manifest.js.gz contents
         if (open my $fh, '<', $mf) {
@@ -58,7 +56,7 @@ sub do_manifest ($$$) {
                 };
                 warn($@) if $@;
         }
-        my ($bn) = ($fn =~ m!/([^/]+)\z!);
+        my ($bn) = ($ft->filename =~ m!/([^/]+)\z!);
         my $curl_cmd = $lei->{curl}->for_uri($lei, $muri, qw(-R -o), $bn);
         my $opt = { -C => $dir };
         $opt->{$_} = $lei->{$_} for (0..2);
@@ -69,7 +67,7 @@ sub do_manifest ($$$) {
                 return;
         }
         my $m1 = eval {
-                PublicInbox::LeiMirror::decode_manifest($ft, $fn, $muri);
+                PublicInbox::LeiMirror::decode_manifest($ft, $ft, $muri);
         } or return [ 404, $muri ];
         my $mdiff = { %$m1 };
 
@@ -88,15 +86,14 @@ sub do_manifest ($$$) {
                 return;
         }
         my (undef, $v1_path, @v2_epochs) =
-                PublicInbox::LeiMirror::deduce_epochs($mdiff, $ibx_uri->path);
+                deduce_epochs($mdiff, $ibx_uri->path);
         [ 200, $muri, $v1_path, \@v2_epochs, $ft, $mf, $m1 ];
 }
 
 sub get_fingerprint2 {
         my ($git_dir) = @_;
-        require Digest::SHA;
         my $rd = popen_rd([qw(git show-ref)], undef, { -C => $git_dir });
-        Digest::SHA::sha256(do { local $/; <$rd> });
+        sha_all(256, $rd)->digest; # ignore show-ref errors
 }
 
 sub writable_dir ($) {
@@ -133,10 +130,10 @@ sub do_fetch { # main entry point
                                 $epoch = $nr;
                         } else {
                                 warn "W: $edir missing remote.*.url\n";
-                                my $pid = spawn([qw(git config -l)], undef,
-                                        { 1 => $lei->{2}, 2 => $lei->{2} });
-                                waitpid($pid, 0);
-                                $lei->child_error($?) if $?;
+                                my $o = { -C => $edir };
+                                $o->{1} = $o->{2} = $lei->{2};
+                                run_wait([qw(git config -l)], undef, $o) and
+                                        $lei->child_error($?);
                         }
                 }
                 @epochs = grep { !$skip->{$_} } @epochs if $skip;
@@ -192,7 +189,7 @@ EOM
                 if (-d $d) {
                         $fp2->[0] = get_fingerprint2($d) if $fp2;
                         $cmd = [ @$torsocks, 'git', "--git-dir=$d",
-                                fetch_args($lei, $opt) ];
+                               PublicInbox::LeiMirror::fetch_args($lei, $opt)];
                 } else {
                         my $e_uri = $ibx_uri->clone;
                         my ($epath) = ($d =~ m!(/git/[0-9]+\.git)\z!);
@@ -218,11 +215,7 @@ EOM
         }
         for my $i (@new_epoch) { $mg->epoch_cfg_set($i) }
         if ($ft) {
-                if ($mculled) {
-                        my $json = PublicInbox::Config->json->encode($m1);
-                        my $fn = $ft->filename;
-                        gzip(\$json => $fn) or die "gzip: $GzipError";
-                }
+                PublicInbox::LeiMirror::dump_manifest($m1 => $ft) if $mculled;
                 PublicInbox::LeiMirror::ft_rename($ft, $mf, 0666);
         }
         $lei->child_error($xit << 8) if $fp2 && $xit;
diff --git a/lib/PublicInbox/Filter/RubyLang.pm b/lib/PublicInbox/Filter/RubyLang.pm
index 09aa6aa8..57ebbe78 100644
--- a/lib/PublicInbox/Filter/RubyLang.pm
+++ b/lib/PublicInbox/Filter/RubyLang.pm
@@ -1,11 +1,10 @@
-# Copyright (C) 2017-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Filter for lists.ruby-lang.org trailers
 package PublicInbox::Filter::RubyLang;
-use base qw(PublicInbox::Filter::Base);
-use strict;
-use warnings;
+use v5.10.1;
+use parent qw(PublicInbox::Filter::Base);
 use PublicInbox::MID qw(mids);
 
 my $l1 = qr/Unsubscribe:\s
@@ -56,16 +55,22 @@ sub scrub {
                 my $hdr = $mime->header_obj;
                 my $mids = mids($hdr);
                 return $self->REJECT('Message-ID missing') unless (@$mids);
-                my @v = $hdr->header_raw('X-Mail-Count');
                 my $n;
-                foreach (@v) {
-                        /\A\s*([0-9]+)\s*\z/ or next;
-                        $n = $1;
-                        last;
-                }
-                unless (defined $n) {
-                        return $self->REJECT('X-Mail-Count not numeric');
+                my @v = $hdr->header_raw('X-Mail-Count'); # old host only
+                if (@v) {
+                        for (@v) {
+                                /\A\s*([0-9]+)\s*\z/ or next;
+                                $n = $1;
+                                last;
+                        }
+                } else { # new host: nue.mailmanlists.eu
+                        for ($hdr->header_str('Subject')) {
+                                /\A\[ruby-[^:]+:([0-9]+)\]/ or next;
+                                $n = $1;
+                                last;
+                        }
                 }
+                $n // return $self->REJECT('could not get count not numeric');
                 foreach my $mid (@$mids) {
                         my $r = $altid->mm_alt->mid_set($n, $mid);
                         next if $r == 0;
diff --git a/lib/PublicInbox/Gcf2.pm b/lib/PublicInbox/Gcf2.pm
index f546208f..78392990 100644
--- a/lib/PublicInbox/Gcf2.pm
+++ b/lib/PublicInbox/Gcf2.pm
@@ -1,84 +1,74 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # backend for a git-cat-file-workalike based on libgit2,
 # other libgit2 stuff may go here, too.
 package PublicInbox::Gcf2;
-use strict;
-use v5.10.1;
-use PublicInbox::Spawn qw(which popen_rd); # may set PERL_INLINE_DIRECTORY
-use Fcntl qw(LOCK_EX SEEK_SET);
+use v5.12;
+use PublicInbox::Spawn qw(which run_qx); # may set PERL_INLINE_DIRECTORY
+use Fcntl qw(SEEK_SET);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
 use IO::Handle; # autoflush
+use PublicInbox::Git qw($ck_unlinked_packs);
+use PublicInbox::Lock;
+use autodie qw(close open seek truncate);
+
 BEGIN {
         my (%CFG, $c_src);
         # PublicInbox::Spawn will set PERL_INLINE_DIRECTORY
-        # to ~/.cache/public-inbox/inline-c if it exists
+        # to ~/.cache/public-inbox/inline-c if it exists and Inline::C works
         my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} //
                 die 'PERL_INLINE_DIRECTORY not defined';
-        my $f = "$inline_dir/.public-inbox.lock";
-        open my $fh, '+>', $f or die "open($f): $!";
 
         # CentOS 7.x ships Inline 0.53, 0.64+ has built-in locking
-        flock($fh, LOCK_EX) or die "LOCK_EX($f): $!\n";
+        my $lk = PublicInbox::Lock->new("$inline_dir/.public-inbox.lock");
+        my $fh = $lk->lock_acquire;
 
         my $pc = which($ENV{PKG_CONFIG} // 'pkg-config') //
                 die "pkg-config missing for libgit2";
         my ($dir) = (__FILE__ =~ m!\A(.+?)/[^/]+\z!);
-        my $ef = "$inline_dir/.public-inbox.pkg-config.err";
-        open my $err, '+>', $ef or die "open($ef): $!";
-        for my $x (qw(libgit2)) {
-                my $rdr = { 2 => $err };
-                my ($l, $pid) = popen_rd([$pc, '--libs', $x], undef, $rdr);
-                $l = do { local $/; <$l> };
-                waitpid($pid, 0);
-                next if $?;
-                (my $c, $pid) = popen_rd([$pc, '--cflags', $x], undef, $rdr);
-                $c = do { local $/; <$c> };
-                waitpid($pid, 0);
-                next if $?;
-
-                # note: we name C source files .h to prevent
-                # ExtUtils::MakeMaker from automatically trying to
-                # build them.
-                my $f = "$dir/gcf2_$x.h";
-                open(my $src, '<', $f) or die "E: open($f): $!";
-                chomp($l, $c);
-                local $/;
-                defined($c_src = <$src>) or die "read $f: $!";
-                $CFG{LIBS} = $l;
-                $CFG{CCFLAGSEX} = $c;
-                last;
-        }
-        unless ($c_src) {
-                seek($err, 0, SEEK_SET);
-                $err = do { local $/; <$err> };
-                die "E: libgit2 not installed: $err\n";
+        my $vals = {};
+        my $rdr = { 2 => \(my $err) };
+        my @switches = qw(modversion cflags libs);
+        for my $k (@switches) {
+                chomp(my $val = run_qx([$pc, "--$k", 'libgit2'], undef, $rdr));
+                die "E: libgit2 not installed: $err\n" if $?;
+                $vals->{$k} = $val;
         }
-        open my $oldout, '>&', \*STDOUT or die "dup(1): $!";
-        open my $olderr, '>&', \*STDERR or die "dup(2): $!";
-        open STDOUT, '>&', $fh or die "1>$f: $!";
-        open STDERR, '>&', $fh or die "2>$f: $!";
+        my $f = "$dir/gcf2_libgit2.h";
+        $c_src = PublicInbox::IO::try_cat $f or die "cat $f: $!";
+        # append pkg-config results to the source to ensure Inline::C
+        # can rebuild if there's changes (it doesn't seem to detect
+        # $CFG{CCFLAGSEX} nor $CFG{CPPFLAGS} changes)
+        $c_src .= "/* $pc --$_ libgit2 => $vals->{$_} */\n" for @switches;
+        open my $oldout, '>&', \*STDOUT;
+        open my $olderr, '>&', \*STDERR;
+        open STDOUT, '>&', $fh;
+        open STDERR, '>&', $fh;
         STDERR->autoflush(1);
         STDOUT->autoflush(1);
+        $CFG{CCFLAGSEX} = $vals->{cflags};
+        $CFG{LIBS} = $vals->{libs};
 
         # we use Capitalized and ALLCAPS for compatibility with old Inline::C
-        eval <<'EOM';
+        CORE::eval <<'EOM';
 use Inline C => Config => %CFG, BOOT => q[git_libgit2_init();];
 use Inline C => $c_src, BUILD_NOISY => 1;
 EOM
         $err = $@;
-        open(STDERR, '>&', $olderr) or warn "restore stderr: $!";
-        open(STDOUT, '>&', $oldout) or warn "restore stdout: $!";
+        open(STDERR, '>&', $olderr);
+        open(STDOUT, '>&', $oldout);
         if ($err) {
                 seek($fh, 0, SEEK_SET);
                 my @msg = <$fh>;
+                truncate($fh, 0);
                 die "Inline::C Gcf2 build failed:\n", $err, "\n", @msg;
         }
 }
 
 sub add_alt ($$) {
-        my ($gcf2, $objdir) = @_;
+        my ($gcf2, $git_dir) = @_;
+        my $objdir = PublicInbox::Git->new($git_dir)->git_path('objects');
 
         # libgit2 (tested 0.27.7+dfsg.1-0.2 and 0.28.3+dfsg.1-1~bpo10+1
         # in Debian) doesn't handle relative epochs properly when nested
@@ -89,23 +79,13 @@ sub add_alt ($$) {
         # to refer to $V2INBOX_DIR/git/$EPOCH.git/objects
         #
         # See https://bugs.debian.org/975607
-        if (open(my $fh, '<', "$objdir/info/alternates")) {
-                chomp(my @abs_alt = grep(m!^/!, <$fh>));
-                $gcf2->add_alternate($_) for @abs_alt;
+        if (my $s = PublicInbox::IO::try_cat("$objdir/info/alternates")) {
+                $gcf2->add_alternate($_) for ($s =~ m!^(/[^\n]+)\n!gms);
         }
         $gcf2->add_alternate($objdir);
         1;
 }
 
-sub have_unlinked_files () {
-        # FIXME: port gcf2-like over to git.git so we won't need to
-        # deal with libgit2
-        return 1 if $^O ne 'linux';
-        open my $fh, '<', "/proc/$$/maps" or return;
-        while (<$fh>) { return 1 if /\.(?:idx|pack) \(deleted\)$/ }
-        undef;
-}
-
 # Usage: $^X -MPublicInbox::Gcf2 -e PublicInbox::Gcf2::loop [EXPIRE-TIMEOUT]
 # (see lib/PublicInbox/Gcf2Client.pm)
 sub loop (;$) {
@@ -114,23 +94,24 @@ sub loop (;$) {
         my (%seen, $check_at);
         STDERR->autoflush(1);
         STDOUT->autoflush(1);
+        my $pid = $$;
 
         while (<STDIN>) {
                 chomp;
                 my ($oid, $git_dir) = split(/ /, $_, 2);
-                $seen{$git_dir} //= add_alt($gcf2, "$git_dir/objects");
+                $seen{$git_dir} //= add_alt($gcf2, $git_dir);
                 if (!$gcf2->cat_oid(1, $oid)) {
                         # retry once if missing.  We only get unabbreviated OIDs
                         # from SQLite or Xapian DBs, here, so malicious clients
                         # can't trigger excessive retries:
-                        warn "I: $$ $oid missing, retrying in $git_dir\n";
+                        warn "# $$ $oid missing, retrying in $git_dir\n";
 
                         $gcf2 = new();
-                        %seen = ($git_dir => add_alt($gcf2,"$git_dir/objects"));
+                        %seen = ($git_dir => add_alt($gcf2, $git_dir));
                         $check_at = clock_gettime(CLOCK_MONOTONIC) + $exp;
 
                         if ($gcf2->cat_oid(1, $oid)) {
-                                warn "I: $$ $oid found after retry\n";
+                                warn "# $$ $oid found after retry\n";
                         } else {
                                 warn "W: $$ $oid missing after retry\n";
                                 print "$oid missing\n"; # mimic git-cat-file
@@ -138,10 +119,13 @@ sub loop (;$) {
                 } else { # check expiry to deal with deleted pack files
                         my $now = clock_gettime(CLOCK_MONOTONIC);
                         $check_at //= $now + $exp;
-                        if ($now > $check_at && have_unlinked_files()) {
+                        if ($now > $check_at) {
                                 undef $check_at;
-                                $gcf2 = new();
-                                %seen = ();
+                                if (!$ck_unlinked_packs ||
+                                                $ck_unlinked_packs->($pid)) {
+                                        $gcf2 = new();
+                                        %seen = ();
+                                }
                         }
                 }
         }
diff --git a/lib/PublicInbox/Gcf2Client.pm b/lib/PublicInbox/Gcf2Client.pm
index 09c3aa06..07ff7dcb 100644
--- a/lib/PublicInbox/Gcf2Client.pm
+++ b/lib/PublicInbox/Gcf2Client.pm
@@ -1,15 +1,18 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # connects public-inbox processes to PublicInbox::Gcf2::loop()
 package PublicInbox::Gcf2Client;
-use strict;
+use v5.12;
 use parent qw(PublicInbox::DS);
 use PublicInbox::Git;
 use PublicInbox::Gcf2; # fails if Inline::C or libgit2-dev isn't available
 use PublicInbox::Spawn qw(spawn);
 use Socket qw(AF_UNIX SOCK_STREAM);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+use PublicInbox::Syscall qw(EPOLLIN);
+use PublicInbox::IO;
+use autodie qw(socketpair);
+
 # fields:
 #        sock => socket to Gcf2::loop
 # The rest of these fields are compatible with what PublicInbox::Git
@@ -18,68 +21,39 @@ use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
 #        pid.owner => process which spawned {pid}
 #        in => same as {sock}, for compatibility with PublicInbox::Git
 #        inflight => array (see PublicInbox::Git)
-#        rbuf => scalarref, may be non-existent or empty
 sub new  {
-        my ($rdr) = @_;
+        my ($opt) = @_;
         my $self = bless {}, __PACKAGE__;
         # ensure the child process has the same @INC we do:
         my $env = { PERL5LIB => join(':', @INC) };
-        my ($s1, $s2);
-        socketpair($s1, $s2, AF_UNIX, SOCK_STREAM, 0) or die "socketpair $!";
-        $rdr //= {};
-        $rdr->{0} = $rdr->{1} = $s2;
-        my $cmd = [$^X, qw[-MPublicInbox::Gcf2 -e PublicInbox::Gcf2::loop]];
-        $self->{'pid.owner'} = $$;
-        $self->{pid} = spawn($cmd, $env, $rdr);
+        socketpair(my $s1, my $s2, AF_UNIX, SOCK_STREAM, 0);
         $s1->blocking(0);
+        $opt->{0} = $opt->{1} = $s2;
+        my $cmd = [$^X, $^W ? ('-w') : (),
+                        qw[-MPublicInbox::Gcf2 -e PublicInbox::Gcf2::loop]];
         $self->{inflight} = [];
-        $self->{in} = $s1;
-        $self->SUPER::new($s1, EPOLLIN|EPOLLET);
-}
-
-sub fail {
-        my $self = shift;
-        $self->close; # PublicInbox::DS::close
-        PublicInbox::Git::fail($self, @_);
+        PublicInbox::IO::attach_pid($s1, spawn($cmd, $env, $opt),
+                        \&PublicInbox::Git::gcf_drain, $self->{inflight});
+        $self->{epwatch} = \undef; # for Git->cleanup
+        $self->SUPER::new($s1, EPOLLIN);
 }
 
 sub gcf2_async ($$$;$) {
         my ($self, $req, $cb, $arg) = @_;
-        my $inflight = $self->{inflight} or return $self->close;
-
-        # {wbuf} is rare, I hope:
-        cat_async_step($self, $inflight) if $self->{wbuf};
-
-        $self->fail("gcf2c write: $!") if !$self->write($req) && !$self->{sock};
-        push @$inflight, $req, $cb, $arg;
+        my $inflight = $self->gcf_inflight or return;
+        PublicInbox::Git::write_all($self, $req, \&cat_async_step, $inflight);
+        push @$inflight, \$req, $cb, $arg; # ref prevents Git.pm retries
 }
 
 # ensure PublicInbox::Git::cat_async_step never calls cat_async_retry
 sub alternates_changed {}
 
-# DS::event_loop will call this
-sub event_step {
-        my ($self) = @_;
-        $self->flush_write;
-        $self->close if !$self->{in} || !$self->{sock}; # process died
-        my $inflight = $self->{inflight};
-        if ($inflight && @$inflight) {
-                cat_async_step($self, $inflight);
-                return $self->close unless $self->{in}; # process died
-
-                # ok, more to do, requeue for fairness
-                $self->requeue if @$inflight || exists($self->{rbuf});
-        }
-}
-
-sub DESTROY {
-        my ($self) = @_;
-        delete $self->{sock}; # if outside event_loop
-        PublicInbox::Git::DESTROY($self);
-}
-
 no warnings 'once';
 
-*cat_async_step = \&PublicInbox::Git::cat_async_step;
+*gcf_inflight = \&PublicInbox::Git::gcf_inflight; # for event_step
+*cat_async_step = \&PublicInbox::Git::cat_async_step; # for event_step
+*event_step = \&PublicInbox::Git::event_step;
+*fail = \&PublicInbox::Git::fail;
+*DESTROY = \&PublicInbox::Git::DESTROY;
 
 1;
diff --git a/lib/PublicInbox/GetlineBody.pm b/lib/PublicInbox/GetlineBody.pm
deleted file mode 100644
index 0e781224..00000000
--- a/lib/PublicInbox/GetlineBody.pm
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-
-# Wrap a pipe or file for PSGI streaming response bodies and calls the
-# end callback when the object goes out-of-scope.
-# This depends on rpipe being _blocking_ on getline.
-#
-# This is only used by generic PSGI servers and not public-inbox-httpd
-package PublicInbox::GetlineBody;
-use strict;
-use warnings;
-
-sub new {
-        my ($class, $rpipe, $end, $end_arg, $buf, $filter) = @_;
-        bless {
-                rpipe => $rpipe,
-                end => $end,
-                end_arg => $end_arg,
-                initial_buf => $buf,
-                filter => $filter,
-        }, $class;
-}
-
-# close should always be called after getline returns undef,
-# but a client aborting a connection can ruin our day; so lets
-# hope our underlying PSGI server does not leak references, here.
-sub DESTROY { $_[0]->close }
-
-sub getline {
-        my ($self) = @_;
-        my $rpipe = $self->{rpipe} or return; # EOF was set on previous call
-        my $buf = delete($self->{initial_buf}) // $rpipe->getline;
-        delete($self->{rpipe}) unless defined $buf; # set EOF for next call
-        if (my $filter = $self->{filter}) {
-                $buf = $filter->translate($buf);
-        }
-        $buf;
-}
-
-sub close {
-        my ($self) = @_;
-        my ($end, $end_arg) = delete @$self{qw(end end_arg)};
-        $end->($end_arg) if $end;
-}
-
-1;
diff --git a/lib/PublicInbox/GetlineResponse.pm b/lib/PublicInbox/GetlineResponse.pm
new file mode 100644
index 00000000..290cce74
--- /dev/null
+++ b/lib/PublicInbox/GetlineResponse.pm
@@ -0,0 +1,40 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# For generic PSGI servers (not public-inbox-httpd/netd) which assumes their
+# getline response bodies can be backpressure-aware for slow clients
+# This depends on rpipe being _blocking_ on getline.
+package PublicInbox::GetlineResponse;
+use v5.12;
+
+sub response {
+        my ($qsp) = @_;
+        my ($res, $rbuf);
+        do { # read header synchronously
+                sysread($qsp->{rpipe}, $rbuf, 65536);
+                $res = $qsp->parse_hdr_done($rbuf); # fills $bref
+        } until defined($res);
+        my ($wcb, $filter) = $qsp->yield_pass(undef, $res) or return;
+        my $self = $res->[2] = bless {
+                qsp => $qsp,
+                filter => $filter,
+        }, __PACKAGE__;
+        my ($bref) = @{delete $qsp->{yield_parse_hdr}};
+        $self->{rbuf} = $$bref if $$bref ne '';
+        $wcb->($res);
+}
+
+sub getline {
+        my ($self) = @_;
+        my $rpipe = $self->{qsp}->{rpipe} // do {
+                delete($self->{qsp})->finish;
+                return; # EOF was set on previous call
+        };
+        my $buf = delete($self->{rbuf}) // $rpipe->getline;
+        $buf // delete($self->{qsp}->{rpipe}); # set EOF for next call
+        $self->{filter} ? $self->{filter}->translate($buf) : $buf;
+}
+
+sub close {}
+
+1;
diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index b2ae75c8..af12f141 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -9,27 +9,34 @@
 package PublicInbox::Git;
 use strict;
 use v5.10.1;
-use parent qw(Exporter);
+use parent qw(Exporter PublicInbox::DS);
+use autodie qw(socketpair read);
 use POSIX ();
-use IO::Handle; # ->autoflush
-use Errno qw(EINTR EAGAIN ENOENT);
+use Socket qw(AF_UNIX SOCK_STREAM);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+use Errno qw(EAGAIN);
 use File::Glob qw(bsd_glob GLOB_NOSORT);
 use File::Spec ();
-use Time::HiRes qw(stat);
-use PublicInbox::Spawn qw(popen_rd spawn);
+use PublicInbox::Spawn qw(spawn popen_rd run_qx which);
+use PublicInbox::IO qw(read_all try_cat);
 use PublicInbox::Tmpfile;
-use IO::Poll qw(POLLIN);
 use Carp qw(croak carp);
-use Digest::SHA ();
-use PublicInbox::DS qw(dwaitpid);
-our @EXPORT_OK = qw(git_unquote git_quote);
-our $PIPE_BUFSIZ = 65536; # Linux default
+use PublicInbox::SHA qw(sha_all);
+our %HEXLEN2SHA = (40 => 1, 64 => 256);
+our %OFMT2HEXLEN = (sha1 => 40, sha256 => 64);
+our @EXPORT_OK = qw(git_unquote git_quote %HEXLEN2SHA %OFMT2HEXLEN
+                        $ck_unlinked_packs);
 our $in_cleanup;
-our $RDTIMEO = 60_000; # milliseconds
 our $async_warn; # true in read-only daemons
 
-use constant MAX_INFLIGHT => (POSIX::PIPE_BUF * 3) /
-        65; # SHA-256 hex size + "\n" in preparation for git using non-SHA1
+# committerdate:unix is git 2.9.4+ (2017-05-05), so using raw instead
+my @MODIFIED_DATE = qw[for-each-ref --sort=-committerdate
+                        --format=%(committerdate:raw) --count=1];
+
+use constant {
+        MAX_INFLIGHT => 18, # arbitrary, formerly based on PIPE_BUF
+        BATCH_CMD_VER => v2.36.0, # git 2.36+
+};
 
 my %GIT_ESC = (
         a => "\a",
@@ -44,6 +51,28 @@ my %GIT_ESC = (
 );
 my %ESC_GIT = map { $GIT_ESC{$_} => $_ } keys %GIT_ESC;
 
+my $EXE_ST = ''; # pack('dd', st_dev, st_ino); # no `q' in some 32-bit builds
+my ($GIT_EXE, $GIT_VER);
+
+sub check_git_exe () {
+        $GIT_EXE = which('git') // die "git not found in $ENV{PATH}";
+        my @st = stat(_) or die "stat($GIT_EXE): $!"; # can't do HiRes w/ _
+        my $st = pack('dd', $st[0], $st[1]);
+        if ($st ne $EXE_ST) {
+                my $v = run_qx([ $GIT_EXE, '--version' ]);
+                die "$GIT_EXE --version: \$?=$?" if $?;
+                $v =~ /\b([0-9]+(?:\.[0-9]+){2})/ or die
+                        "$GIT_EXE --version output: $v # unparseable";
+                $GIT_VER = eval("v$1") // die "BUG: bad vstring: $1 ($v)";
+                $EXE_ST = $st;
+        }
+        $GIT_EXE;
+}
+
+sub git_version {
+        check_git_exe();
+        $GIT_VER;
+}
 
 # unquote pathnames used by git, see quote.c::unquote_c_style.c in git.git
 sub git_unquote ($) {
@@ -63,34 +92,42 @@ sub git_quote ($) {
 
 sub new {
         my ($class, $git_dir) = @_;
+        $git_dir .= '/';
         $git_dir =~ tr!/!/!s;
-        $git_dir =~ s!/*\z!!s;
+        chop $git_dir;
         # may contain {-tmp} field for File::Temp::Dir
-        bless { git_dir => $git_dir, alt_st => '', -git_path => {} }, $class
+        my %dedupe = ($git_dir => undef);
+        bless { git_dir => (keys %dedupe)[0] }, $class
 }
 
 sub git_path ($$) {
         my ($self, $path) = @_;
         $self->{-git_path}->{$path} //= do {
-                local $/ = "\n";
-                chomp(my $str = $self->qx(qw(rev-parse --git-path), $path));
-
-                # git prior to 2.5.0 did not understand --git-path
-                if ($str eq "--git-path\n$path") {
-                        $str = "$self->{git_dir}/$path";
+                my $d = "$self->{git_dir}/$path";
+                if (-e $d) {
+                        $d;
+                } else {
+                        local $/ = "\n";
+                        my $rdr = { 2 => \my $err };
+                        my $s = $self->qx([qw(rev-parse --git-path), $path],
+                                        undef, $rdr);
+                        chomp $s;
+
+                        # git prior to 2.5.0 did not understand --git-path
+                        $s eq "--git-path\n$path" ? $d : $s;
                 }
-                $str;
         };
 }
 
 sub alternates_changed {
         my ($self) = @_;
         my $alt = git_path($self, 'objects/info/alternates');
+        use Time::HiRes qw(stat);
         my @st = stat($alt) or return 0;
 
         # can't rely on 'q' on some 32-bit builds, but `d' works
         my $st = pack('dd', $st[10], $st[7]); # 10: ctime, 7: size
-        return 0 if $self->{alt_st} eq $st;
+        return 0 if ($self->{alt_st} // '') eq $st;
         $self->{alt_st} = $st; # always a true value
 }
 
@@ -103,124 +140,124 @@ sub object_format {
 
 sub last_check_err {
         my ($self) = @_;
-        my $fh = $self->{err_c} or return;
-        sysseek($fh, 0, 0) or $self->fail("sysseek failed: $!");
-        defined(sysread($fh, my $buf, -s $fh)) or
-                        $self->fail("sysread failed: $!");
+        my $fh = $self->{err_c} or return '';
+        sysseek($fh, 0, 0) or $self->fail("sysseek: $!");
+        my $size = -s $fh or return '';
+        sysread($fh, my $buf, $size) // $self->fail("sysread: $!");
+        truncate($fh, 0) or $self->fail("truncate: $!");
         $buf;
 }
 
-sub _bidi_pipe {
-        my ($self, $batch, $in, $out, $pid, $err) = @_;
-        if ($self->{$pid}) {
-                if (defined $err) { # "err_c"
-                        my $fh = $self->{$err};
-                        sysseek($fh, 0, 0) or $self->fail("sysseek failed: $!");
-                        truncate($fh, 0) or $self->fail("truncate failed: $!");
-                }
-                return;
+sub gcf_drain { # awaitpid cb
+        my ($pid, $inflight, $bc) = @_;
+        while (@$inflight) {
+                my ($req, $cb, $arg) = splice(@$inflight, 0, 3);
+                $req = $$req if ref($req);
+                $bc and $req =~ s/\A(?:contents|info) //;
+                $req =~ s/ .*//; # drop git_dir for Gcf2Client
+                eval { $cb->(undef, $req, undef, undef, $arg) };
+                warn "E: (in abort) $req: $@" if $@;
         }
-        pipe(my ($out_r, $out_w)) or $self->fail("pipe failed: $!");
-        my $rdr = { 0 => $out_r, pgid => 0 };
+}
+
+sub _sock_cmd {
+        my ($self, $batch, $err_c) = @_;
+        $self->{sock} and Carp::confess('BUG: {sock} exists');
+        socketpair(my $s1, my $s2, AF_UNIX, SOCK_STREAM, 0);
+        $s1->blocking(0);
+        my $opt = { pgid => 0, 0 => $s2, 1 => $s2 };
         my $gd = $self->{git_dir};
         if ($gd =~ s!/([^/]+/[^/]+)\z!/!) {
-                $rdr->{-C} = $gd;
+                $opt->{-C} = $gd;
                 $gd = $1;
         }
-        my @cmd = (qw(git), "--git-dir=$gd",
-                        qw(-c core.abbrev=40 cat-file), $batch);
-        if ($err) {
-                my $id = "git.$self->{git_dir}$batch.err";
-                my $fh = tmpfile($id) or $self->fail("tmpfile($id): $!");
-                $self->{$err} = $fh;
-                $rdr->{2} = $fh;
-        }
-        my ($in_r, $p) = popen_rd(\@cmd, undef, $rdr);
-        $self->{$pid} = $p;
-        $self->{"$pid.owner"} = $$;
-        $out_w->autoflush(1);
-        if ($^O eq 'linux') { # 1031: F_SETPIPE_SZ
-                fcntl($out_w, 1031, 4096);
-                fcntl($in_r, 1031, 4096) if $batch eq '--batch-check';
-        }
-        $self->{$out} = $out_w;
-        $self->{$in} = $in_r;
-}
-
-sub poll_in ($) { IO::Poll::_poll($RDTIMEO, fileno($_[0]), my $ev = POLLIN) }
-
-sub my_read ($$$) {
-        my ($fh, $rbuf, $len) = @_;
-        my $left = $len - length($$rbuf);
-        my $r;
-        while ($left > 0) {
-                $r = sysread($fh, $$rbuf, $PIPE_BUFSIZ, length($$rbuf));
-                if ($r) {
-                        $left -= $r;
-                } elsif (defined($r)) { # EOF
-                        return 0;
-                } else {
-                        next if ($! == EAGAIN and poll_in($fh));
-                        next if $! == EINTR; # may be set by sysread or poll_in
-                        return; # unrecoverable error
-                }
-        }
-        my $no_pad = substr($$rbuf, 0, $len, '');
-        \$no_pad;
-}
-
-sub my_readline ($$) {
-        my ($fh, $rbuf) = @_;
-        while (1) {
-                if ((my $n = index($$rbuf, "\n")) >= 0) {
-                        return substr($$rbuf, 0, $n + 1, '');
-                }
-                my $r = sysread($fh, $$rbuf, $PIPE_BUFSIZ, length($$rbuf))
-                                                                and next;
 
-                # return whatever's left on EOF
-                return substr($$rbuf, 0, length($$rbuf)+1, '') if defined($r);
-
-                next if ($! == EAGAIN and poll_in($fh));
-                next if $! == EINTR; # may be set by sysread or poll_in
-                return; # unrecoverable error
+        # git 2.31.0+ supports -c core.abbrev=no, don't bother with
+        # core.abbrev=64 since not many releases had SHA-256 prior to 2.31
+        my $abbr = $GIT_VER lt v2.31.0 ? 40 : 'no';
+        my @cmd = ($GIT_EXE, "--git-dir=$gd", '-c', "core.abbrev=$abbr",
+                        'cat-file', "--$batch");
+        if ($err_c) {
+                my $id = "git.$self->{git_dir}.$batch.err";
+                $self->{err_c} = $opt->{2} = tmpfile($id, undef, 1) or
+                                                $self->fail("tmpfile($id): $!");
         }
+        my $inflight = []; # TODO consider moving this into the IO object
+        my $pid = spawn(\@cmd, undef, $opt);
+        $self->{sock} = PublicInbox::IO::attach_pid($s1, $pid,
+                                \&gcf_drain, $inflight, $self->{-bc});
+        $self->{inflight} = $inflight;
 }
 
 sub cat_async_retry ($$) {
-        my ($self, $inflight) = @_;
+        my ($self, $old_inflight) = @_;
 
         # {inflight} may be non-existent, but if it isn't we delete it
         # here to prevent cleanup() from waiting:
-        delete $self->{inflight};
-        cleanup($self);
+        my ($sock, $epwatch) = delete @$self{qw(sock epwatch inflight)};
+        $self->SUPER::close if $epwatch;
+        my $new_inflight = batch_prepare($self);
+
+        while (my ($oid, $cb, $arg) = splice(@$old_inflight, 0, 3)) {
+                write_all($self, $oid."\n", \&cat_async_step, $new_inflight);
+                $oid = \$oid if !@$new_inflight; # to indicate oid retried
+                push @$new_inflight, $oid, $cb, $arg;
+        }
+        $sock->close if $sock; # only safe once old_inflight is empty
+        cat_async_step($self, $new_inflight); # take one step
+}
 
-        $self->{inflight} = $inflight;
-        batch_prepare($self);
-        my $buf = '';
-        for (my $i = 0; $i < @$inflight; $i += 3) {
-                $buf .= "$inflight->[$i]\n";
+sub gcf_inflight ($) {
+        my ($self) = @_;
+        # FIXME: the first {sock} check can succeed but Perl can complain
+        # about calling ->owner_pid on an undefined value.  Not sure why or
+        # how this happens but t/imapd.t can complain about it, sometimes.
+        if ($self->{sock}) {
+                if (eval { $self->{sock}->owner_pid == $$ }) {
+                        return $self->{inflight};
+                } elsif ($@) {
+                        no warnings 'uninitialized';
+                        warn "E: $self sock=$self->{sock}: owner_pid failed: ".
+                                "$@ (continuing...)";
+                }
+                delete @$self{qw(sock inflight)};
+        } else {
+                $self->close;
         }
-        print { $self->{out} } $buf or $self->fail("write error: $!");
-        my $req = shift @$inflight;
-        unshift(@$inflight, \$req); # \$ref to indicate retried
+        undef;
+}
 
-        cat_async_step($self, $inflight); # take one step
+# returns true if prefetch is successful
+sub async_prefetch {
+        my ($self, $oid, $cb, $arg) = @_;
+        my $inflight = gcf_inflight($self) or return;
+        return if @$inflight;
+        substr($oid, 0, 0) = 'contents ' if $self->{-bc};
+        write_all($self, "$oid\n", \&cat_async_step, $inflight);
+        push(@$inflight, $oid, $cb, $arg);
 }
 
 sub cat_async_step ($$) {
         my ($self, $inflight) = @_;
-        die 'BUG: inflight empty or odd' if scalar(@$inflight) < 3;
+        croak 'BUG: inflight empty or odd' if scalar(@$inflight) < 3;
         my ($req, $cb, $arg) = @$inflight[0, 1, 2];
-        my $rbuf = delete($self->{rbuf}) // \(my $new = '');
         my ($bref, $oid, $type, $size);
-        my $head = my_readline($self->{in}, $rbuf);
+        my $head = $self->{sock}->my_readline;
+        my $cmd = ref($req) ? $$req : $req;
         # ->fail may be called via Gcf2Client.pm
+        my $info = $self->{-bc} && substr($cmd, 0, 5) eq 'info ';
         if ($head =~ /^([0-9a-f]{40,}) (\S+) ([0-9]+)$/) {
                 ($oid, $type, $size) = ($1, $2, $3 + 0);
-                $bref = my_read($self->{in}, $rbuf, $size + 1) or
-                        $self->fail(defined($bref) ? 'read EOF' : "read: $!");
-                chop($$bref) eq "\n" or $self->fail('LF missing after blob');
+                unless ($info) { # --batch-command
+                        $bref = $self->{sock}->my_bufread($size + 1) or
+                                $self->fail(defined($bref) ?
+                                                'read EOF' : "read: $!");
+                        chop($$bref) eq "\n" or
+                                        $self->fail('LF missing after blob');
+                }
+        } elsif ($info && $head =~ / (missing|ambiguous)\n/) {
+                $type = $1;
+                $oid = substr($cmd, 5); # remove 'info '
         } elsif ($head =~ s/ missing\n//s) {
                 $oid = $head;
                 # ref($req) indicates it's already been retried
@@ -229,27 +266,34 @@ sub cat_async_step ($$) {
                         return cat_async_retry($self, $inflight);
                 }
                 $type = 'missing';
-                $oid = ref($req) ? $$req : $req if $oid eq '';
+                if ($oid eq '') {
+                        $oid = $cmd;
+                        $oid =~ s/\A(?:contents|info) // if $self->{-bc};
+                }
         } else {
                 my $err = $! ? " ($!)" : '';
                 $self->fail("bad result from async cat-file: $head$err");
         }
-        $self->{rbuf} = $rbuf if $$rbuf ne '';
         splice(@$inflight, 0, 3); # don't retry $cb on ->fail
         eval { $cb->($bref, $oid, $type, $size, $arg) };
-        async_err($self, $req, $oid, $@, 'cat') if $@;
+        async_err($self, $req, $oid, $@, $info ? 'check' : 'cat') if $@;
 }
 
 sub cat_async_wait ($) {
         my ($self) = @_;
-        my $inflight = $self->{inflight} or return;
-        while (scalar(@$inflight)) {
-                cat_async_step($self, $inflight);
-        }
+        my $inflight = gcf_inflight($self) or return;
+        cat_async_step($self, $inflight) while (scalar(@$inflight));
 }
 
 sub batch_prepare ($) {
-        _bidi_pipe($_[0], qw(--batch in out pid));
+        my ($self) = @_;
+        check_git_exe();
+        if ($GIT_VER ge BATCH_CMD_VER) {
+                $self->{-bc} = 1;
+                _sock_cmd($self, 'batch-command', 1);
+        } else {
+                _sock_cmd($self, 'batch');
+        }
 }
 
 sub _cat_file_cb {
@@ -266,55 +310,86 @@ sub cat_file {
 }
 
 sub check_async_step ($$) {
-        my ($self, $inflight_c) = @_;
-        die 'BUG: inflight empty or odd' if scalar(@$inflight_c) < 3;
-        my ($req, $cb, $arg) = @$inflight_c[0, 1, 2];
-        my $rbuf = delete($self->{rbuf_c}) // \(my $new = '');
-        chomp(my $line = my_readline($self->{in_c}, $rbuf));
+        my ($ck, $inflight) = @_;
+        croak 'BUG: inflight empty or odd' if scalar(@$inflight) < 3;
+        my ($req, $cb, $arg) = @$inflight[0, 1, 2];
+        chomp(my $line = $ck->{sock}->my_readline);
         my ($hex, $type, $size) = split(/ /, $line);
 
-        # Future versions of git.git may have type=ambiguous, but for now,
-        # we must handle 'dangling' below (and maybe some other oddball
-        # stuff):
+        # git <2.21 would show `dangling' (2.21+ shows `ambiguous')
         # https://public-inbox.org/git/20190118033845.s2vlrb3wd3m2jfzu@dcvr/T/
-        if ($hex eq 'dangling' || $hex eq 'notdir' || $hex eq 'loop') {
-                my $ret = my_read($self->{in_c}, $rbuf, $type + 1);
-                $self->fail(defined($ret) ? 'read EOF' : "read: $!") if !$ret;
+        if ($hex eq 'dangling') {
+                my $ret = $ck->{sock}->my_bufread($type + 1);
+                $ck->fail(defined($ret) ? 'read EOF' : "read: $!") if !$ret;
         }
-        $self->{rbuf_c} = $rbuf if $$rbuf ne '';
-        splice(@$inflight_c, 0, 3); # don't retry $cb on ->fail
-        eval { $cb->($hex, $type, $size, $arg, $self) };
-        async_err($self, $req, $hex, $@, 'check') if $@;
+        splice(@$inflight, 0, 3); # don't retry $cb on ->fail
+        eval { $cb->(undef, $hex, $type, $size, $arg) };
+        async_err($ck, $req, $hex, $@, 'check') if $@;
 }
 
 sub check_async_wait ($) {
         my ($self) = @_;
-        my $inflight_c = $self->{inflight_c} or return;
-        while (scalar(@$inflight_c)) {
-                check_async_step($self, $inflight_c);
-        }
+        return cat_async_wait($self) if $self->{-bc};
+        my $ck = $self->{ck} or return;
+        my $inflight = gcf_inflight($ck) or return;
+        check_async_step($ck, $inflight) while (scalar(@$inflight));
+}
+
+# git <2.36
+sub ck {
+        $_[0]->{ck} //= bless { git_dir => $_[0]->{git_dir} },
+                                'PublicInbox::GitCheck';
 }
 
 sub check_async_begin ($) {
         my ($self) = @_;
         cleanup($self) if alternates_changed($self);
-        _bidi_pipe($self, qw(--batch-check in_c out_c pid_c err_c));
-        die 'BUG: already in async check' if $self->{inflight_c};
-        $self->{inflight_c} = [];
+        check_git_exe();
+        if ($GIT_VER ge BATCH_CMD_VER) {
+                $self->{-bc} = 1;
+                _sock_cmd($self, 'batch-command', 1);
+        } else {
+                _sock_cmd($self = ck($self), 'batch-check', 1);
+        }
+}
+
+sub write_all {
+        my ($self, $buf, $read_step, $inflight) = @_;
+        $self->{sock} // Carp::confess 'BUG: no {sock}';
+        Carp::confess('BUG: not an array') if ref($inflight) ne 'ARRAY';
+        $read_step->($self, $inflight) while @$inflight >= MAX_INFLIGHT;
+        do {
+                my $w = syswrite($self->{sock}, $buf);
+                if (defined $w) {
+                        return if $w == length($buf);
+                        substr($buf, 0, $w, ''); # sv_chop
+                } elsif ($! != EAGAIN) {
+                        $self->fail("write: $!");
+                }
+                $read_step->($self, $inflight);
+        } while (1);
 }
 
 sub check_async ($$$$) {
         my ($self, $oid, $cb, $arg) = @_;
-        my $inflight_c = $self->{inflight_c} // check_async_begin($self);
-        while (scalar(@$inflight_c) >= MAX_INFLIGHT) {
-                check_async_step($self, $inflight_c);
+        my $inflight;
+        if ($self->{-bc}) { # likely as time goes on
+batch_command:
+                $inflight = gcf_inflight($self) // cat_async_begin($self);
+                substr($oid, 0, 0) = 'info ';
+                write_all($self, "$oid\n", \&cat_async_step, $inflight);
+        } else { # accounts for git upgrades while we're running:
+                my $ck = $self->{ck}; # undef OK, maybe set in check_async_begin
+                $inflight = ($ck ? gcf_inflight($ck) : undef)
+                                 // check_async_begin($self);
+                goto batch_command if $self->{-bc};
+                write_all($self->{ck}, "$oid\n", \&check_async_step, $inflight);
         }
-        print { $self->{out_c} } $oid, "\n" or $self->fail("write error: $!");
-        push(@$inflight_c, $oid, $cb, $arg);
+        push(@$inflight, $oid, $cb, $arg);
 }
 
 sub _check_cb { # check_async callback
-        my ($hex, $type, $size, $result) = @_;
+        my (undef, $hex, $type, $size, $result) = @_;
         @$result = ($hex, $type, $size);
 }
 
@@ -325,48 +400,15 @@ sub check {
         check_async_wait($self);
         my ($hex, $type, $size) = @$result;
 
-        # Future versions of git.git may show 'ambiguous', but for now,
-        # we must handle 'dangling' below (and maybe some other oddball
-        # stuff):
+        # git <2.21 would show `dangling' (2.21+ shows `ambiguous')
         # https://public-inbox.org/git/20190118033845.s2vlrb3wd3m2jfzu@dcvr/T/
-        return if $type eq 'missing' || $type eq 'ambiguous';
-        return if $hex eq 'dangling' || $hex eq 'notdir' || $hex eq 'loop';
+        return if $type =~ /\A(?:missing|ambiguous)\z/ || $hex eq 'dangling';
         ($hex, $type, $size);
 }
 
-sub _destroy {
-        my ($self, $rbuf, $in, $out, $pid, $err) = @_;
-        delete @$self{($rbuf, $in, $out)};
-        delete $self->{$err} if $err; # `err_c'
-
-        # GitAsyncCat::event_step may delete {pid}
-        my $p = delete $self->{$pid} or return;
-        dwaitpid($p) if $$ == $self->{"$pid.owner"};
-}
-
-sub async_abort ($) {
-        my ($self) = @_;
-        while (scalar(@{$self->{inflight_c} // []}) ||
-                        scalar(@{$self->{inflight} // []})) {
-                for my $c ('', '_c') {
-                        my $q = $self->{"inflight$c"} or next;
-                        while (@$q) {
-                                my ($req, $cb, $arg) = splice(@$q, 0, 3);
-                                $req = $$req if ref($req);
-                                $req =~ s/ .*//; # drop git_dir for Gcf2Client
-                                eval { $cb->(undef, $req, undef, undef, $arg) };
-                                warn "E: (in abort) $req: $@" if $@;
-                        }
-                        delete $self->{"inflight$c"};
-                        delete $self->{"rbuf$c"};
-                }
-        }
-        cleanup($self);
-}
-
-sub fail { # may be augmented in subclasses
+sub fail {
         my ($self, $msg) = @_;
-        async_abort($self);
+        $self->close;
         croak(ref($self) . ' ' . ($self->{git_dir} // '') . ": $msg");
 }
 
@@ -377,6 +419,11 @@ sub async_err ($$$$$) {
         $async_warn ? carp($msg) : $self->fail($msg);
 }
 
+sub cmd {
+        my $self = shift;
+        [ $GIT_EXE // check_git_exe(), "--git-dir=$self->{git_dir}", @_ ]
+}
+
 # $git->popen(qw(show f00)); # or
 # $git->popen(qw(show f00), { GIT_CONFIG => ... }, { 2 => ... });
 sub popen {
@@ -391,12 +438,12 @@ sub qx {
         my $fh = popen(@_);
         if (wantarray) {
                 my @ret = <$fh>;
-                close $fh; # caller should check $?
+                $fh->close; # caller should check $?
                 @ret;
         } else {
                 local $/;
                 my $ret = <$fh>;
-                close $fh; # caller should check $?
+                $fh->close; # caller should check $?
                 $ret;
         }
 }
@@ -408,12 +455,16 @@ sub date_parse {
         } $self->qx('rev-parse', map { "--since=$_" } @_);
 }
 
+sub _active ($) {
+        scalar(@{gcf_inflight($_[0]) // []}) ||
+                ($_[0]->{ck} && scalar(@{gcf_inflight($_[0]->{ck}) // []}))
+}
+
 # check_async and cat_async may trigger the other, so ensure they're
 # both completely done by using this:
 sub async_wait_all ($) {
         my ($self) = @_;
-        while (scalar(@{$self->{inflight_c} // []}) ||
-                        scalar(@{$self->{inflight} // []})) {
+        while (_active($self)) {
                 check_async_wait($self);
                 cat_async_wait($self);
         }
@@ -422,15 +473,11 @@ sub async_wait_all ($) {
 # returns true if there are pending "git cat-file" processes
 sub cleanup {
         my ($self, $lazy) = @_;
-        return 1 if $lazy && (scalar(@{$self->{inflight_c} // []}) ||
-                                scalar(@{$self->{inflight} // []}));
+        ($lazy && _active($self)) and
+                return $self->{epwatch} ? watch_async($self) : 1;
         local $in_cleanup = 1;
-        delete $self->{async_cat};
         async_wait_all($self);
-        delete $self->{inflight};
-        delete $self->{inflight_c};
-        _destroy($self, qw(rbuf in out pid));
-        _destroy($self, qw(rbuf_c in_c out_c pid_c err_c));
+        $_->close for ($self, (delete($self->{ck}) // ()));
         undef;
 }
 
@@ -441,135 +488,194 @@ sub packed_bytes {
         my ($self) = @_;
         my $n = 0;
         my $pack_dir = git_path($self, 'objects/pack');
-        foreach my $p (bsd_glob("$pack_dir/*.pack", GLOB_NOSORT)) {
-                $n += -s $p;
-        }
+        $n += (-s $_ // 0) for (bsd_glob("$pack_dir/*.pack", GLOB_NOSORT));
         $n
 }
 
-sub DESTROY { cleanup(@_) }
+sub DESTROY { cleanup($_[0]) }
 
 sub local_nick ($) {
         # don't show full FS path, basename should be OK:
-        $_[0]->{git_dir} =~ m!/([^/]+?)(?:/*\.git/*)?\z! ? "$1.git" : '???';
+        $_[0]->{nick} // ($_[0]->{git_dir} =~ m!/([^/]+?)(?:/*\.git/*)?\z! ?
+                        "$1.git" : undef);
 }
 
 sub host_prefix_url ($$) {
         my ($env, $url) = @_;
         return $url if index($url, '//') >= 0;
-        my $scheme = $env->{'psgi.url_scheme'};
         my $host_port = $env->{HTTP_HOST} //
                 "$env->{SERVER_NAME}:$env->{SERVER_PORT}";
-        "$scheme://$host_port". ($env->{SCRIPT_NAME} || '/') . $url;
+        my $sn = $env->{SCRIPT_NAME} // '';
+        "$env->{'psgi.url_scheme'}://\L$host_port\E$sn/$url";
+}
+
+sub base_url { # for coderepos, PSGI-only
+        my ($self, $env) = @_; # env - PSGI env
+        my $nick = $self->{nick} // return undef;
+        my $url = host_prefix_url($env, '');
+        # for mount in Plack::Builder
+        $url .= '/' if substr($url, -1, 1) ne '/';
+        $url . $nick . '/';
 }
 
+sub isrch {} # TODO
+
 sub pub_urls {
         my ($self, $env) = @_;
         if (my $urls = $self->{cgit_url}) {
-                return map { host_prefix_url($env, $_) } @$urls;
+                map { host_prefix_url($env, $_) } @$urls;
+        } else {
+                (base_url($self, $env) // '???');
         }
-        (local_nick($self));
 }
 
 sub cat_async_begin {
         my ($self) = @_;
         cleanup($self) if $self->alternates_changed;
-        $self->batch_prepare;
-        die 'BUG: already in async' if $self->{inflight};
-        $self->{inflight} = [];
+        die 'BUG: already in async' if gcf_inflight($self);
+        batch_prepare($self);
 }
 
 sub cat_async ($$$;$) {
         my ($self, $oid, $cb, $arg) = @_;
-        my $inflight = $self->{inflight} // cat_async_begin($self);
-        while (scalar(@$inflight) >= MAX_INFLIGHT) {
-                cat_async_step($self, $inflight);
-        }
-        print { $self->{out} } $oid, "\n" or $self->fail("write error: $!");
+        my $inflight = gcf_inflight($self) // cat_async_begin($self);
+        substr($oid, 0, 0) = 'contents ' if $self->{-bc};
+        write_all($self, $oid."\n", \&cat_async_step, $inflight);
         push(@$inflight, $oid, $cb, $arg);
 }
 
 # returns the modified time of a git repo, same as the "modified" field
 # of a grokmirror manifest
-sub modified ($) {
-        # committerdate:unix is git 2.9.4+ (2017-05-05), so using raw instead
-        my $fh = popen($_[0], qw[for-each-ref --sort=-committerdate
-                                --format=%(committerdate:raw) --count=1]);
+sub modified ($;$) {
+        my $fh = $_[1] // popen($_[0], @MODIFIED_DATE);
         (split(/ /, <$fh> // time))[0] + 0; # integerize for JSON
 }
 
+sub cat_desc ($) {
+        my $desc = try_cat($_[0]);
+        chomp $desc;
+        utf8::decode($desc);
+        $desc =~ s/\s+/ /smg;
+        $desc eq '' ? undef : $desc;
+}
+
+sub description {
+        cat_desc("$_[0]->{git_dir}/description") // 'Unnamed repository';
+}
+
+sub cloneurl {
+        my ($self, $env) = @_;
+        $self->{cloneurl} // do {
+                my @urls = split(/\s+/s, try_cat("$self->{git_dir}/cloneurl"));
+                scalar(@urls) ? ($self->{cloneurl} = \@urls) : undef;
+        } // [ substr(base_url($self, $env), 0, -1) ];
+}
+
 # for grokmirror, which doesn't read gitweb.description
 # templates/hooks--update.sample and git-multimail in git.git
 # only match "Unnamed repository", not the full contents of
 # templates/this--description in git.git
 sub manifest_entry {
         my ($self, $epoch, $default_desc) = @_;
-        my $fh = $self->popen('show-ref');
-        my $dig = Digest::SHA->new(1);
-        while (read($fh, my $buf, 65536)) {
-                $dig->add($buf);
-        }
-        close $fh or return; # empty, uninitialized git repo
-        undef $fh; # for open, below
-        my $git_dir = $self->{git_dir};
-        my $ent = {
-                fingerprint => $dig->hexdigest,
-                reference => undef,
-                modified => modified($self),
-        };
-        chomp(my $owner = $self->qx('config', 'gitweb.owner'));
-        utf8::decode($owner);
-        $ent->{owner} = $owner eq '' ? undef : $owner;
-        my $desc = '';
-        if (open($fh, '<', "$git_dir/description")) {
-                local $/ = "\n";
-                chomp($desc = <$fh>);
-                utf8::decode($desc);
-        }
-        $desc = 'Unnamed repository' if $desc eq '';
-        if (defined $epoch && $desc =~ /\AUnnamed repository/) {
-                $desc = "$default_desc [epoch $epoch]";
+        check_git_exe();
+        my $gd = $self->{git_dir};
+        my @git = ($GIT_EXE, "--git-dir=$gd");
+        my $sr = popen_rd([@git, 'show-ref']);
+        my $own = popen_rd([@git, qw(config gitweb.owner)]);
+        my $mod = popen_rd([@git, @MODIFIED_DATE]);
+        my $buf = description($self);
+        if (defined $epoch && index($buf, 'Unnamed repository') == 0) {
+                $buf = "$default_desc [epoch $epoch]";
         }
-        $ent->{description} = $desc;
-        if (open($fh, '<', "$git_dir/objects/info/alternates")) {
+        my $ent = { description => $buf, reference => undef };
+        if (open(my $alt, '<', "$gd/objects/info/alternates")) {
                 # n.b.: GitPython doesn't seem to handle comments or C-quoted
                 # strings like native git does; and we don't for now, either.
                 local $/ = "\n";
-                chomp(my @alt = <$fh>);
+                chomp(my @alt = <$alt>);
 
                 # grokmirror only supports 1 alternate for "reference",
                 if (scalar(@alt) == 1) {
-                        my $objdir = "$git_dir/objects";
-                        my $ref = File::Spec->rel2abs($alt[0], $objdir);
-                        $ref =~ s!/[^/]+/?\z!!; # basename
-                        $ent->{reference} = $ref;
+                        $buf = File::Spec->rel2abs($alt[0], "$gd/objects");
+                        $buf =~ s!/[^/]+/?\z!!; # basename
+                        $ent->{reference} = $buf;
                 }
         }
+        $ent->{fingerprint} = sha_all(1, $sr)->hexdigest;
+        $sr->close or return; # empty, uninitialized git repo
+        $ent->{modified} = modified(undef, $mod);
+        chomp($buf = <$own> // '');
+        utf8::decode($buf);
+        $ent->{owner} = $buf eq '' ? undef : $buf;
         $ent;
 }
 
+our $ck_unlinked_packs = $^O eq 'linux' ? sub {
+        # FIXME: port gcf2-like over to git.git so we won't need to
+        # deal with libgit2
+        my $s = try_cat "/proc/$_[0]/maps";
+        $s =~ /\.(?:idx|pack) \(deleted\)/s ? 1 : undef;
+} : undef;
+
 # returns true if there are pending cat-file processes
 sub cleanup_if_unlinked {
         my ($self) = @_;
-        return cleanup($self, 1) if $^O ne 'linux';
+        $ck_unlinked_packs or return cleanup($self, 1);
         # Linux-specific /proc/$PID/maps access
         # TODO: support this inside git.git
-        my $ret = 0;
-        for my $fld (qw(pid pid_c)) {
-                my $pid = $self->{$fld} // next;
-                open my $fh, '<', "/proc/$pid/maps" or return cleanup($self, 1);
-                while (<$fh>) {
-                        # n.b. we do not restart for unlinked multi-pack-index
-                        # since it's not too huge, and the startup cost may
-                        # be higher.
-                        /\.(?:idx|pack) \(deleted\)$/ and
-                                return cleanup($self, 1);
-                }
-                ++$ret;
+        my $nr_live = 0;
+        for my $obj ($self, ($self->{ck} // ())) {
+                my $sock = $obj->{sock} // next;
+                my $pid = $sock->attached_pid // next;
+                $ck_unlinked_packs->($pid) and return cleanup($self, 1);
+                ++$nr_live;
+        }
+        $nr_live;
+}
+
+sub event_step {
+        my ($self) = @_;
+        my $inflight = gcf_inflight($self);
+        if ($inflight && @$inflight) {
+                $self->cat_async_step($inflight);
+                return $self->close unless $self->{sock};
+                # don't loop here to keep things fair, but we must requeue
+                # if there's already-read data in pi_io_rbuf
+                $self->requeue if $self->{sock}->has_rbuf;
         }
-        $ret;
 }
 
+sub schedule_cleanup {
+        my ($self) = @_;
+        PublicInbox::DS::add_uniq_timer($self+0, 30, \&cleanup, $self, 1);
+}
+
+# idempotently registers with DS epoll/kqueue/select/poll
+sub watch_async ($) {
+        my ($self) = @_;
+        schedule_cleanup($self);
+        $self->{epwatch} //= do {
+                $self->SUPER::new($self->{sock}, EPOLLIN);
+                \undef;
+        }
+}
+
+sub close {
+        my ($self) = @_;
+        my $sock = $self->{sock};
+        delete @$self{qw(-bc err_c inflight)};
+        delete($self->{epwatch}) ? $self->SUPER::close : delete($self->{sock});
+        $sock->close if $sock; # calls gcf_drain via awaitpid
+}
+
+package PublicInbox::GitCheck; # only for git <2.36
+use v5.12;
+our @ISA = qw(PublicInbox::Git);
+no warnings 'once';
+
+# for event_step
+*cat_async_step = \&PublicInbox::Git::check_async_step;
+
 1;
 __END__
 =pod
diff --git a/lib/PublicInbox/GitAsyncCat.pm b/lib/PublicInbox/GitAsyncCat.pm
index cea3f539..f57e0336 100644
--- a/lib/PublicInbox/GitAsyncCat.pm
+++ b/lib/PublicInbox/GitAsyncCat.pm
@@ -1,91 +1,51 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-#
-# internal class used by PublicInbox::Git + PublicInbox::DS
-# This parses the output pipe of "git cat-file --batch"
 package PublicInbox::GitAsyncCat;
-use strict;
-use parent qw(PublicInbox::DS Exporter);
-use POSIX qw(WNOHANG);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
-our @EXPORT = qw(ibx_async_cat ibx_async_prefetch);
-use PublicInbox::Git ();
+use v5.12;
+use parent qw(Exporter);
+our @EXPORT = qw(ibx_async_cat ibx_async_prefetch async_check);
 
 our $GCF2C; # singleton PublicInbox::Gcf2Client
 
-sub close {
-        my ($self) = @_;
-        if (my $git = delete $self->{git}) {
-                $git->async_abort;
-        }
-        $self->SUPER::close; # PublicInbox::DS::close
-}
-
-sub event_step {
-        my ($self) = @_;
-        my $git = $self->{git} or return;
-        return $self->close if ($git->{in} // 0) != ($self->{sock} // 1);
-        my $inflight = $git->{inflight};
-        if ($inflight && @$inflight) {
-                $git->cat_async_step($inflight);
-
-                # child death?
-                if (($git->{in} // 0) != ($self->{sock} // 1)) {
-                        $self->close;
-                } elsif (@$inflight || exists $git->{rbuf}) {
-                        # ok, more to do, requeue for fairness
-                        $self->requeue;
-                }
-        } elsif ((my $pid = waitpid($git->{pid}, WNOHANG)) > 0) {
-                # May happen if the child process is killed by a BOFH
-                # (or segfaults)
-                delete $git->{pid};
-                warn "E: git $pid exited with \$?=$?\n";
-                $self->close;
-        }
-}
-
 sub ibx_async_cat ($$$$) {
         my ($ibx, $oid, $cb, $arg) = @_;
-        my $git = $ibx->git;
+        my $isrch = $ibx->{isrch};
+        my $git = $isrch ? $isrch->{es}->git : ($ibx->{git} // $ibx->git);
         # {topdir} means ExtSearch (likely [extindex "all"]) with potentially
-        # 100K alternates.  git(1) has a proposed patch for 100K alternates:
-        # <https://lore.kernel.org/git/20210624005806.12079-1-e@80x24.org/>
-        if (!defined($ibx->{topdir}) && ($GCF2C //= eval {
+        # 100K alternates.  git v2.33+ can handle 100k alternates fairly well.
+        if (!$isrch && !defined($ibx->{topdir}) && !defined($git->{-tmp}) &&
+                ($GCF2C //= eval {
                 require PublicInbox::Gcf2Client;
                 PublicInbox::Gcf2Client::new();
         } // 0)) { # 0: do not retry if libgit2 or Inline::C are missing
-                $GCF2C->gcf2_async(\"$oid $git->{git_dir}\n", $cb, $arg);
+                $GCF2C->gcf2_async("$oid $git->{git_dir}\n", $cb, $arg);
                 \undef;
         } else { # read-only end of git-cat-file pipe
                 $git->cat_async($oid, $cb, $arg);
-                $git->{async_cat} //= do {
-                        my $self = bless { git => $git }, __PACKAGE__;
-                        $git->{in}->blocking(0);
-                        $self->SUPER::new($git->{in}, EPOLLIN|EPOLLET);
-                        \undef; # this is a true ref()
-                };
+                $git->watch_async;
         }
 }
 
+sub async_check ($$$$) {
+        my ($ibx, $oidish, $cb, $arg) = @_; # $ibx may be $ctx
+        my $git = $ibx->{git} // $ibx->git;
+        $git->check_async($oidish, $cb, $arg);
+        ($git->{ck} // $git)->watch_async;
+}
+
 # this is safe to call inside $cb, but not guaranteed to enqueue
-# returns true if successful, undef if not.
+# returns true if successful, undef if not.  For fairness, we only
+# prefetch if there's no in-flight requests.
 sub ibx_async_prefetch {
         my ($ibx, $oid, $cb, $arg) = @_;
         my $git = $ibx->git;
         if (!defined($ibx->{topdir}) && $GCF2C) {
-                if (!$GCF2C->{wbuf}) {
+                if (!@{$GCF2C->gcf_inflight // []}) {
                         $oid .= " $git->{git_dir}\n";
-                        return $GCF2C->gcf2_async(\$oid, $cb, $arg); # true
-                }
-        } elsif ($git->{async_cat} && (my $inflight = $git->{inflight})) {
-                # we could use MAX_INFLIGHT here w/o the halving,
-                # but lets not allow one client to monopolize a git process
-                if (@$inflight < int(PublicInbox::Git::MAX_INFLIGHT/2)) {
-                        print { $git->{out} } $oid, "\n" or
-                                                $git->fail("write error: $!");
-                        return push(@$inflight, $oid, $cb, $arg);
+                        return $GCF2C->gcf2_async($oid, $cb, $arg); # true
                 }
+        } elsif ($git->{epwatch}) {
+                return $git->async_prefetch($oid, $cb, $arg);
         }
         undef;
 }
diff --git a/lib/PublicInbox/GitCredential.pm b/lib/PublicInbox/GitCredential.pm
index b18bba1e..bb225ff3 100644
--- a/lib/PublicInbox/GitCredential.pm
+++ b/lib/PublicInbox/GitCredential.pm
@@ -1,34 +1,36 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# git-credential wrapper with built-in .netrc fallback
 package PublicInbox::GitCredential;
-use strict;
+use v5.12;
 use PublicInbox::Spawn qw(popen_rd);
+use autodie qw(close pipe);
 
 sub run ($$;$) {
         my ($self, $op, $lei) = @_;
         my ($in_r, $in_w, $out_r);
         my $cmd = [ qw(git credential), $op ];
-        pipe($in_r, $in_w) or die "pipe: $!";
+        pipe($in_r, $in_w);
         if ($lei) { # we'll die if disconnected:
-                pipe($out_r, my $out_w) or die "pipe: $!";
+                pipe($out_r, my $out_w);
                 $lei->send_exec_cmd([ $in_r, $out_w ], $cmd, {});
         } else {
                 $out_r = popen_rd($cmd, undef, { 0 => $in_r });
         }
-        close $in_r or die "close in_r: $!";
+        close $in_r;
 
         my $out = '';
         for my $k (qw(url protocol host username password)) {
-                defined(my $v = $self->{$k}) or next;
+                my $v = $self->{$k} // next;
                 die "`$k' contains `\\n' or `\\0'\n" if $v =~ /[\n\0]/;
                 $out .= "$k=$v\n";
         }
-        $out .= "\n";
-        print $in_w $out or die "print (git credential $op): $!";
-        close $in_w or die "close (git credential $op): $!";
+        say $in_w $out;
+        close $in_w;
         return $out_r if $op eq 'fill';
         <$out_r> and die "unexpected output from `git credential $op'\n";
-        close $out_r or die "`git credential $op' failed: \$!=$! \$?=$?\n";
+        $out_r->close or die "`git credential $op' failed: \$!=$! \$?=$?\n";
 }
 
 sub check_netrc {
@@ -59,7 +61,7 @@ sub fill {
                 /\A([^=]+)=(.*)\z/ or die "bad line: $_\n";
                 $self->{$1} = $2;
         }
-        close $out_r or die "git credential fill failed: \$!=$! \$?=$?\n";
+        $out_r->close or die "git credential fill failed: \$!=$! \$?=$?\n";
         $self->{filled} = 1;
 }
 
diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm
index ba3a8f20..396aa783 100644
--- a/lib/PublicInbox/GitHTTPBackend.pm
+++ b/lib/PublicInbox/GitHTTPBackend.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # when no endpoints match, fallback to this and serve a static file
@@ -9,13 +9,14 @@ use v5.10.1;
 use Fcntl qw(:seek);
 use IO::Handle; # ->flush
 use HTTP::Date qw(time2str);
+use PublicInbox::Limiter;
 use PublicInbox::Qspawn;
 use PublicInbox::Tmpfile;
 use PublicInbox::WwwStatic qw(r @NO_CACHE);
 use Carp ();
 
 # 32 is same as the git-daemon connection limit
-my $default_limiter = PublicInbox::Qspawn::Limiter->new(32);
+my $default_limiter = PublicInbox::Limiter->new(32);
 
 # n.b. serving "description" and "cloneurl" should be innocuous enough to
 # not cause problems.  serving "config" might...
@@ -23,13 +24,10 @@ my @text = qw[HEAD info/refs info/attributes
         objects/info/(?:http-alternates|alternates|packs)
         cloneurl description];
 
-my @binary = qw!
-        objects/[a-f0-9]{2}/[a-f0-9]{38}
-        objects/pack/pack-[a-f0-9]{40}\.(?:pack|idx)
-        !;
+my @binary = ('objects/[a-f0-9]{2}/[a-f0-9]{38,62}',
+        'objects/pack/pack-[a-f0-9]{40,64}\.(?:pack|idx)');
 
 our $ANY = join('|', @binary, @text, 'git-upload-pack');
-my $BIN = join('|', @binary);
 my $TEXT = join('|', @text);
 
 sub serve {
@@ -62,13 +60,13 @@ sub serve_dumb {
 
         my $h = [];
         my $type;
-        if ($path =~ m!\Aobjects/[a-f0-9]{2}/[a-f0-9]{38}\z!) {
+        if ($path =~ m!\Aobjects/[a-f0-9]{2}/[a-f0-9]{38,62}\z!) {
                 $type = 'application/x-git-loose-object';
                 cache_one_year($h);
-        } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40}\.pack\z!) {
+        } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40,64}\.pack\z!) {
                 $type = 'application/x-git-packed-objects';
                 cache_one_year($h);
-        } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40}\.idx\z!) {
+        } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40,64}\.idx\z!) {
                 $type = 'application/x-git-packed-objects-toc';
                 cache_one_year($h);
         } elsif ($path =~ /\A(?:$TEXT)\z/o) {
@@ -81,10 +79,10 @@ sub serve_dumb {
         PublicInbox::WwwStatic::response($env, $h, $path, $type);
 }
 
-sub git_parse_hdr { # {parse_hdr} for Qspawn
-        my ($r, $bref, $dumb_args) = @_;
+sub ghb_parse_hdr { # header parser for Qspawn
+        my ($r, $bref, @dumb_args) = @_;
         my $res = parse_cgi_headers($r, $bref) or return; # incomplete
-        $res->[0] == 403 ? serve_dumb(@$dumb_args) : $res;
+        $res->[0] == 403 ? serve_dumb(@dumb_args) : $res;
 }
 
 # returns undef if 403 so it falls back to dumb HTTP
@@ -107,8 +105,9 @@ sub serve_smart {
         $env{GIT_HTTP_EXPORT_ALL} = '1';
         $env{PATH_TRANSLATED} = "$git->{git_dir}/$path";
         my $rdr = input_prepare($env) or return r(500);
+        $rdr->{quiet} = 1;
         my $qsp = PublicInbox::Qspawn->new([qw(git http-backend)], \%env, $rdr);
-        $qsp->psgi_return($env, $limiter, \&git_parse_hdr, [$env, $git, $path]);
+        $qsp->psgi_yield($env, $limiter, \&ghb_parse_hdr, $env, $git, $path);
 }
 
 sub input_prepare {
@@ -131,8 +130,8 @@ sub input_prepare {
         { 0 => $in };
 }
 
-sub parse_cgi_headers {
-        my ($r, $bref) = @_;
+sub parse_cgi_headers { # {parse_hdr} for Qspawn
+        my ($r, $bref, $ctx) = @_;
         return r(500) unless defined $r && $r >= 0;
         $$bref =~ s/\A(.*?)\r?\n\r?\n//s or return $r == 0 ? r(500) : undef;
         my $h = $1;
@@ -146,7 +145,18 @@ sub parse_cgi_headers {
                         push @h, $k, $v;
                 }
         }
-        [ $code, \@h ]
+
+        # fallback to WwwCoderepo if cgit 404s
+        if ($code == 404 && $ctx->{www} && !$ctx->{_coderepo_tried}++) {
+                my $wcb = delete $ctx->{env}->{'qspawn.wcb'};
+                $ctx->{env}->{'plack.skip-deflater'} = 1; # prevent 2x gzip
+                $ctx->{env}->{'qspawn.fallback'} = $code;
+                my $res = $ctx->{www}->coderepo->srv($ctx);
+                $ctx->{env}->{'qspawn.wcb'} = $wcb;
+                $res; # CODE or ARRAY ref
+        } else {
+                [ $code, \@h ]
+        }
 }
 
 1;
diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm
index e37f1f76..8b630f25 100644
--- a/lib/PublicInbox/GzipFilter.pm
+++ b/lib/PublicInbox/GzipFilter.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # In public-inbox <=1.5.0, public-inbox-httpd favored "getline"
@@ -18,6 +18,7 @@ use Compress::Raw::Zlib qw(Z_OK);
 use PublicInbox::CompressNoop;
 use PublicInbox::Eml;
 use PublicInbox::GitAsyncCat;
+use Carp qw(carp);
 
 our @EXPORT_OK = qw(gzf_maybe);
 my %OPT = (-WindowBits => 15 + 16, -AppendOutput => 1);
@@ -92,30 +93,24 @@ sub gone { # what: search/over/mm
         undef;
 }
 
-# for GetlineBody (via Qspawn) when NOT using $env->{'pi-httpd.async'}
+# for GetlineResponse (via Qspawn) when NOT using $env->{'pi-httpd.async'}
 # Also used for ->getline callbacks
-sub translate ($$) {
-        my $self = $_[0]; # $_[1] => input
+sub translate {
+        my $self = shift; # $_[1] => input
 
         # allocate the zlib context lazily here, instead of in ->new.
         # Deflate contexts are memory-intensive and this object may
         # be sitting in the Qspawn limiter queue for a while.
-        my $gz = $self->{gz} //= gzip_or_die();
-        my $zbuf = delete($self->{zbuf});
-        if (defined $_[1]) { # my $buf = $_[1];
-                my $err = $gz->deflate($_[1], $zbuf);
-                die "gzip->deflate: $err" if $err != Z_OK;
-                return $zbuf if length($zbuf) >= 8192;
-
-                $self->{zbuf} = $zbuf;
-                '';
+        $self->{gz} //= gzip_or_die();
+        if (defined $_[0]) { # my $buf = $_[1];
+                zmore($self, @_);
+                length($self->{zbuf}) >= 8192 ? delete($self->{zbuf}) : '';
         } else { # undef == EOF
-                my $err = $gz->flush($zbuf);
-                die "gzip->flush: $err" if $err != Z_OK;
-                $zbuf;
+                $self->zflush;
         }
 }
 
+# returns PublicInbox::HTTP::{Chunked,Identity}
 sub http_out ($) {
         my ($self) = @_;
         $self->{http_out} // do {
@@ -128,73 +123,86 @@ sub http_out ($) {
         };
 }
 
+# returns undef if HTTP client disconnected, may return 0
+# because ->translate can return ''
 sub write {
-        # my $ret = bytes::length($_[1]); # XXX does anybody care?
-        http_out($_[0])->write(translate($_[0], $_[1]));
+        my $self = shift;
+        http_out($self)->write($self->translate(@_));
+}
+
+sub zfh {
+        $_[0]->{zfh} // do {
+                open($_[0]->{zfh}, '>>', \($_[0]->{pbuf} //= '')) or
+                        die "open: $!";
+                $_[0]->{zfh}
+        };
 }
 
 # similar to ->translate; use this when we're sure we know we have
 # more data to buffer after this
 sub zmore {
-        my $self = $_[0]; # $_[1] => input
+        my $self = shift;
+        my $zfh = delete $self->{zfh};
+        if (@_ > 1 || $zfh) {
+                print { $zfh // zfh($self) } @_;
+                @_ = (delete $self->{pbuf});
+                delete $self->{zfh};
+        };
         http_out($self);
-        my $err = $self->{gz}->deflate($_[1], $self->{zbuf});
-        die "gzip->deflate: $err" if $err != Z_OK;
-        undef;
+        my $err;
+        ($err = $self->{gz}->deflate($_[0], $self->{zbuf})) == Z_OK or
+                die "gzip->deflate: $err";
 }
 
 # flushes and returns the final bit of gzipped data
-sub zflush ($;$) {
-        my $self = $_[0]; # $_[1] => final input (optional)
-        my $zbuf = delete $self->{zbuf};
-        my $gz = delete $self->{gz};
+sub zflush ($;@) {
+        my $self = shift; # $_[1..Inf] => final input (optional)
+        zmore($self, @_) if scalar(@_) || $self->{zfh};
+        # not a bug, recursing on DS->write failure
+        my $gz = delete $self->{gz} // return '';
         my $err;
-        if (defined $_[1]) {
-                $err = $gz->deflate($_[1], $zbuf);
-                die "gzip->deflate: $err" if $err != Z_OK;
-        }
-        $err = $gz->flush($zbuf);
-        die "gzip->flush: $err" if $err != Z_OK;
+        my $zbuf = delete $self->{zbuf};
+        ($err = $gz->flush($zbuf)) == Z_OK or die "gzip->flush: $err";
         $zbuf;
 }
 
 sub close {
         my ($self) = @_;
         my $http_out = http_out($self) // return;
-        $http_out->write(zflush($self));
-        delete($self->{http_out})->close;
+        $http_out->write($self->zflush);
+        (delete($self->{http_out}) // return)->close;
 }
 
-sub bail  {
+sub bail {
         my $self = shift;
-        if (my $env = $self->{env}) {
-                warn @_, "\n";
-                my $http = $env->{'psgix.io'} or return; # client abort
-                eval { $http->close }; # should hit our close
-                warn "E: error in http->close: $@" if $@;
-                eval { $self->close }; # just in case...
-                warn "E: error in self->close: $@" if $@;
-        } else {
-                warn @_, "\n";
-        }
+        carp @_;
+        my $env = $self->{env} or return;
+        my $http = $env->{'psgix.io'} or return; # client abort
+        eval { $http->close }; # should hit our close
+        carp "E: error in http->close: $@" if $@;
+        eval { $self->close }; # just in case...
+        carp "E: error in self->close: $@" if $@;
 }
 
 # this is public-inbox-httpd-specific
 sub async_blob_cb { # git->cat_async callback
         my ($bref, $oid, $type, $size, $self) = @_;
-        my $http = $self->{env}->{'psgix.io'};
+        my $http = $self->{env}->{'psgix.io'}; # PublicInbox::HTTP
         $http->{forward} or return; # client aborted
-        my $smsg = $self->{smsg} or bail($self, 'BUG: no smsg');
-        if (!defined($oid)) {
+        my $smsg = $self->{smsg} or return bail($self, 'BUG: no smsg');
+        $type // return
+                bail($self, "abort: $smsg->{blob} $self->{ibx}->{inboxdir}");
+        if ($type ne 'blob') {
                 # it's possible to have TOCTOU if an admin runs
                 # public-inbox-(edit|purge), just move onto the next message
-                warn "E: $smsg->{blob} missing in $self->{ibx}->{inboxdir}\n";
+                warn "E: $smsg->{blob} $type in $self->{ibx}->{inboxdir}\n";
                 return $http->next_step($self->can('async_next'));
         }
-        $smsg->{blob} eq $oid or bail($self, "BUG: $smsg->{blob} != $oid");
+        $smsg->{blob} eq $oid or return
+                bail($self, "BUG: $smsg->{blob} != $oid");
         eval { $self->async_eml(PublicInbox::Eml->new($bref)) };
-        bail($self, "E: async_eml: $@") if $@;
-        if ($self->{-low_prio}) {
+        return bail($self, "E: async_eml: $@") if $@;
+        if ($self->{-low_prio}) { # run via PublicInbox::WWW::event_step
                 push(@{$self->{www}->{-low_prio_q}}, $self) == 1 and
                                 PublicInbox::DS::requeue($self->{www});
         } else {
diff --git a/lib/PublicInbox/HTTP.pm b/lib/PublicInbox/HTTP.pm
index 76e978a2..7162732e 100644
--- a/lib/PublicInbox/HTTP.pm
+++ b/lib/PublicInbox/HTTP.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Generic PSGI server for convenience.  It aims to provide
@@ -43,7 +43,13 @@ use Errno qw(EAGAIN);
 our $MAX_REQUEST_BUFFER = $ENV{GIT_HTTP_MAX_REQUEST_BUFFER} ||
                         (10 * 1024 * 1024);
 
-open(my $null_io, '<', '/dev/null') or die "failed to open /dev/null: $!";
+open(my $null_io, '<', '/dev/null') or die "open /dev/null: $!";
+{
+        my @n = stat($null_io) or die "stat(/dev/null): $!";
+        my @i = stat(STDIN) or die "stat(STDIN): $!";
+        $null_io = *STDIN{IO} if "@n[0, 1]" eq "@i[0, 1]";
+}
+
 my $http_date;
 my $prev = 0;
 sub http_date () {
@@ -52,13 +58,13 @@ sub http_date () {
 }
 
 sub new ($$$) {
-        my ($class, $sock, $addr, $httpd) = @_;
-        my $self = bless { httpd => $httpd }, $class;
+        my ($class, $sock, $addr, $srv_env) = @_;
+        my $self = bless { srv_env => $srv_env }, $class;
         my $ev = EPOLLIN;
         my $wbuf;
         if ($sock->can('accept_SSL') && !$sock->accept_SSL) {
-                return CORE::close($sock) if $! != EAGAIN;
-                $ev = PublicInbox::TLS::epollbit() or return CORE::close($sock);
+                return $sock->close if $! != EAGAIN;
+                $ev = PublicInbox::TLS::epollbit() or return $sock->close;
                 $wbuf = [ \&PublicInbox::DS::accept_tls_step ];
         }
         $self->{wbuf} = $wbuf if $wbuf;
@@ -69,8 +75,8 @@ sub new ($$$) {
 
 sub event_step { # called by PublicInbox::DS
         my ($self) = @_;
-
-        return unless $self->flush_write && $self->{sock};
+        local $SIG{__WARN__} = $self->{srv_env}->{'pi-httpd.warn_cb'};
+        return unless $self->flush_write && $self->{sock} && !$self->{forward};
 
         # only read more requests if we've drained the write buffer,
         # otherwise we can be buffering infinitely w/o backpressure
@@ -78,7 +84,7 @@ sub event_step { # called by PublicInbox::DS
         return read_input($self) if ref($self->{env});
 
         my $rbuf = $self->{rbuf} // (\(my $x = ''));
-        my %env = %{$self->{httpd}->{env}}; # full hash copy
+        my %env = %{$self->{srv_env}}; # full hash copy
         my $r;
         while (($r = parse_http_request($$rbuf, \%env)) < 0) {
                 # We do not support Trailers in chunked requests, for
@@ -135,7 +141,7 @@ sub app_dispatch {
         $env->{REMOTE_ADDR} = $self->{remote_addr};
         $env->{REMOTE_PORT} = $self->{remote_port};
         if (defined(my $host = $env->{HTTP_HOST})) {
-                $host =~ s/:([0-9]+)\z// and $env->{SERVER_PORT} = $1;
+                $host =~ s/:([0-9]+)\z// and $env->{SERVER_PORT} = $1 + 0;
                 $env->{SERVER_NAME} = $host;
         }
         if (defined $input) {
@@ -145,7 +151,7 @@ sub app_dispatch {
         # note: NOT $self->{sock}, we want our close (+ PublicInbox::DS::close),
         # to do proper cleanup:
         $env->{'psgix.io'} = $self; # for ->close or async_pass
-        my $res = Plack::Util::run_app($self->{httpd}->{app}, $env);
+        my $res = Plack::Util::run_app($env->{'pi-httpd.app'}, $env);
         eval {
                 if (ref($res) eq 'CODE') {
                         $res->(sub { response_write($self, $env, $_[0]) });
@@ -224,6 +230,13 @@ sub identity_write ($$) {
 
 sub response_done {
         my ($self, $alive) = @_;
+        if (my $forward = delete $self->{forward}) { # avoid recursion
+                eval { $forward->close };
+                if ($@) {
+                        warn "response forward->close error: $@";
+                        return $self->close; # idempotent
+                }
+        }
         delete $self->{env}; # we're no longer busy
         # HEAD requests set $alive = 3 so we don't send "0\r\n\r\n";
         $self->write(\"0\r\n\r\n") if $alive == 2;
@@ -262,14 +275,6 @@ sub getline_pull {
                 warn "response ->getline error: $@";
                 $self->close;
         }
-        # avoid recursion
-        if (delete $self->{forward}) {
-                eval { $forward->close };
-                if ($@) {
-                        warn "response ->close error: $@";
-                        $self->close; # idempotent
-                }
-        }
         response_done($self, delete $self->{alive});
 }
 
@@ -449,11 +454,12 @@ sub next_step {
 # They may be exposed to the PSGI application when the PSGI app
 # returns a CODE ref for "push"-based responses
 package PublicInbox::HTTP::Chunked;
-use strict;
+use v5.12;
 
 sub write {
         # ([$http], $buf) = @_;
-        PublicInbox::HTTP::chunked_write($_[0]->[0], $_[1])
+        PublicInbox::HTTP::chunked_write($_[0]->[0], $_[1]);
+        $_[0]->[0]->{sock} ? length($_[1]) : undef;
 }
 
 sub close {
@@ -462,12 +468,13 @@ sub close {
 }
 
 package PublicInbox::HTTP::Identity;
-use strict;
+use v5.12;
 our @ISA = qw(PublicInbox::HTTP::Chunked);
 
 sub write {
         # ([$http], $buf) = @_;
         PublicInbox::HTTP::identity_write($_[0]->[0], $_[1]);
+        $_[0]->[0]->{sock} ? length($_[1]) : undef;
 }
 
 1;
diff --git a/lib/PublicInbox/HTTPD.pm b/lib/PublicInbox/HTTPD.pm
index 715e4538..6a6347d8 100644
--- a/lib/PublicInbox/HTTPD.pm
+++ b/lib/PublicInbox/HTTPD.pm
@@ -9,21 +9,23 @@ use strict;
 use Plack::Util ();
 use Plack::Builder;
 use PublicInbox::HTTP;
-use PublicInbox::HTTPD::Async;
 
-sub pi_httpd_async { PublicInbox::HTTPD::Async->new(@_) }
+# we have a different env for ever listener socket for
+# SERVER_NAME, SERVER_PORT and psgi.url_scheme
+# envs: listener FD => PSGI env
+sub new { bless { envs => {}, err => \*STDERR }, __PACKAGE__ }
 
-sub new {
-        my ($class, $sock, $app, $client) = @_;
-        my $n = getsockname($sock) or die "not a socket: $sock $!\n";
+# this becomes {srv_env} in PublicInbox::HTTP
+sub env_for ($$$) {
+        my ($self, $srv, $client) = @_;
+        my $n = getsockname($srv) or die "not a socket: $srv $!\n";
         my ($host, $port) = PublicInbox::Daemon::host_with_port($n);
-
-        my %env = (
+        {
                 SERVER_NAME => $host,
                 SERVER_PORT => $port,
                 SCRIPT_NAME => '',
                 'psgi.version' => [ 1, 1 ],
-                'psgi.errors' => \*STDERR,
+                'psgi.errors' => $self->{err},
                 'psgi.url_scheme' => $client->can('accept_SSL') ?
                                         'https' : 'http',
                 'psgi.nonblocking' => Plack::Util::TRUE,
@@ -40,26 +42,26 @@ sub new {
                 # this to limit git-http-backend(1) parallelism.
                 # We also check for the truthiness of this to
                 # detect when to use async paths for slow blobs
-                'pi-httpd.async' => \&pi_httpd_async
-        );
-        bless { app => $app, env => \%env }, $class;
+                'pi-httpd.async' => 1,
+                'pi-httpd.app' => $self->{app},
+                'pi-httpd.warn_cb' => $self->{warn_cb},
+        }
 }
 
-my %httpds; # per-listen-FD mapping for HTTPD->{env}->{SERVER_<NAME|PORT>}
-my $default_app; # ugh...
-
-sub refresh {
-        if (@main::ARGV) {
-                eval { $default_app = Plack::Util::load_psgi(@ARGV) };
-                if ($@) {
-                        die $@,
-"$0 runs in /, command-line paths must be absolute\n";
-                }
+sub refresh_groups {
+        my ($self) = @_;
+        my $app;
+        $self->{psgi} //= $main::ARGV[0] if @main::ARGV;
+        if ($self->{psgi}) {
+                eval { $app = Plack::Util::load_psgi($self->{psgi}) };
+                die $@, <<EOM if $@;
+$0 runs in /, command-line paths must be absolute
+EOM
         } else {
                 require PublicInbox::WWW;
                 my $www = PublicInbox::WWW->new;
                 $www->preload;
-                $default_app = builder {
+                $app = builder {
                         eval { enable 'ReverseProxy' };
                         $@ and warn <<EOM;
 Plack::Middleware::ReverseProxy missing,
@@ -69,14 +71,18 @@ EOM
                         sub { $www->call(@_) };
                 };
         }
-        %httpds = (); # invalidate cache
+        $_->{'pi-httpd.app'} = $app for values %{$self->{envs}};
+        $self->{app} = $app;
 }
 
-sub post_accept { # Listener->{post_accept}
-        my ($client, $addr, $srv) = @_; # $_[3] - tls_wrap (unused)
-        my $httpd = $httpds{fileno($srv)} //=
-                                __PACKAGE__->new($srv, $default_app, $client);
-        PublicInbox::HTTP->new($client, $addr, $httpd),
+sub post_accept_cb { # for Listener->{post_accept}
+        my ($self) = @_;
+        sub {
+                my ($client, $addr, $srv) = @_; # $_[4] - tls_wrap (unused)
+                PublicInbox::HTTP->new($client, $addr,
+                                $self->{envs}->{fileno($srv)} //=
+                                        env_for($self, $srv, $client));
+        }
 }
 
 1;
diff --git a/lib/PublicInbox/HTTPD/Async.pm b/lib/PublicInbox/HTTPD/Async.pm
deleted file mode 100644
index 1651da88..00000000
--- a/lib/PublicInbox/HTTPD/Async.pm
+++ /dev/null
@@ -1,105 +0,0 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-#
-# XXX This is a totally unstable API for public-inbox internal use only
-# This is exposed via the 'pi-httpd.async' key in the PSGI env hash.
-# The name of this key is not even stable!
-# Currently intended for use with read-only pipes with expensive
-# processes such as git-http-backend(1), cgit(1)
-#
-# fields:
-# http: PublicInbox::HTTP ref
-# fh: PublicInbox::HTTP::{Identity,Chunked} ref (can ->write + ->close)
-# cb: initial read callback
-# arg: arg for {cb}
-# end_obj: CODE or object which responds to ->event_step when ->close is called
-package PublicInbox::HTTPD::Async;
-use strict;
-use parent qw(PublicInbox::DS);
-use Errno qw(EAGAIN);
-use PublicInbox::Syscall qw(EPOLLIN);
-
-# This is called via: $env->{'pi-httpd.async'}->()
-# $io is a read-only pipe ($rpipe) for now, but may be a
-# bidirectional socket in the future.
-sub new {
-        my ($class, $io, $cb, $arg, $end_obj) = @_;
-
-        # no $io? call $cb at the top of the next event loop to
-        # avoid recursion:
-        unless (defined($io)) {
-                PublicInbox::DS::requeue($cb ? $cb : $arg);
-                die '$end_obj unsupported w/o $io' if $end_obj;
-                return;
-        }
-        my $self = bless {
-                cb => $cb, # initial read callback
-                arg => $arg, # arg for $cb
-                end_obj => $end_obj, # like END{}, can ->event_step
-        }, $class;
-        my $pp = tied *$io;
-        $pp->{fh}->blocking(0) // die "$io->blocking(0): $!";
-        $self->SUPER::new($io, EPOLLIN);
-}
-
-sub event_step {
-        my ($self) = @_;
-        if (my $cb = delete $self->{cb}) {
-                # this may call async_pass when headers are done
-                $cb->(my $refcnt_guard = delete $self->{arg});
-        } elsif (my $sock = $self->{sock}) {
-                my $http = $self->{http};
-                # $self->{sock} is a read pipe for git-http-backend or cgit
-                # and 65536 is the default Linux pipe size
-                my $r = sysread($sock, my $buf, 65536);
-                if ($r) {
-                        $self->{fh}->write($buf); # may call $http->close
-                        # let other clients get some work done, too
-                        return if $http->{sock}; # !closed
-
-                        # else: fall through to close below...
-                } elsif (!defined $r && $! == EAGAIN) {
-                        return; # EPOLLIN means we'll be notified
-                }
-
-                # Done! Error handling will happen in $self->{fh}->close
-                # called by end_obj->event_step handler
-                delete $http->{forward};
-                $self->close; # queues end_obj->event_step to be called
-        } # else { # we may've been requeued but closed by $http
-}
-
-# once this is called, all data we read is passed to the
-# to the PublicInbox::HTTP instance ($http) via $fh->write
-sub async_pass {
-        my ($self, $http, $fh, $bref) = @_;
-        # In case the client HTTP connection ($http) dies, it
-        # will automatically close this ($self) object.
-        $http->{forward} = $self;
-
-        # write anything we overread when we were reading headers
-        $fh->write($$bref); # PublicInbox:HTTP::{chunked,identity}_wcb
-
-        # we're done with this, free this memory up ASAP since the
-        # calls after this may use much memory:
-        $$bref = undef;
-
-        $self->{http} = $http;
-        $self->{fh} = $fh;
-}
-
-# may be called as $forward->close in PublicInbox::HTTP or EOF (event_step)
-sub close {
-        my $self = $_[0];
-        $self->SUPER::close; # DS::close
-
-        # we defer this to the next timer loop since close is deferred
-        if (my $end_obj = delete $self->{end_obj}) {
-                # this calls $end_obj->event_step
-                # (likely PublicInbox::Qspawn::event_step,
-                #  NOT PublicInbox::HTTPD::Async::event_step)
-                PublicInbox::DS::requeue($end_obj);
-        }
-}
-
-1;
diff --git a/lib/PublicInbox/Hval.pm b/lib/PublicInbox/Hval.pm
index 00b3c8b4..963dbb71 100644
--- a/lib/PublicInbox/Hval.pm
+++ b/lib/PublicInbox/Hval.pm
@@ -4,15 +4,16 @@
 # represents a header value in various forms.  Used for HTML generation
 # in our web interface(s)
 package PublicInbox::Hval;
+use v5.10.1; # be careful about unicode_strings in v5.12;
 use strict;
-use warnings;
 use Encode qw(find_encoding);
 use PublicInbox::MID qw/mid_clean mid_escape/;
 use base qw/Exporter/;
 our @EXPORT_OK = qw/ascii_html obfuscate_addrs to_filename src_escape
-                to_attr prurl mid_href fmt_ts ts2str/;
+                to_attr prurl mid_href fmt_ts ts2str utf8_maybe/;
 use POSIX qw(strftime);
 my $enc_ascii = find_encoding('us-ascii');
+use File::Spec;
 
 # safe-ish acceptable filename pattern for portability
 our $FN = '[a-zA-Z0-9][a-zA-Z0-9_\-\.]+[a-zA-Z0-9]'; # needs \z anchor
@@ -69,7 +70,16 @@ sub prurl ($$) {
                 $u = $host_match[0] // $u->[0];
                 # fall through to below:
         }
-        index($u, '//') == 0 ? "$env->{'psgi.url_scheme'}:$u" : $u;
+        my $dslash = index($u, '//');
+        if ($dslash == 0) {
+                "$env->{'psgi.url_scheme'}:$u"
+        } elsif ($dslash < 0 && substr($u, 0, 1) ne '/' &&
+                        substr(my $path = $env->{PATH_INFO}, 0, 1) eq '/') {
+                # this won't touch the FS at all:
+                File::Spec->abs2rel("/$u", $path);
+        } else {
+                $u;
+        }
 }
 
 # for misguided people who believe in this stuff, give them a
@@ -118,7 +128,7 @@ $ESCAPES{'/'} = ':'; # common
 sub to_attr ($) {
         my ($str) = @_;
 
-        # git would never do this to us:
+        # git would never do this to us, mail diff uses // to prevent anchors:
         return if index($str, '//') >= 0;
 
         my $first = '';
@@ -135,6 +145,15 @@ sub to_attr ($) {
 sub ts2str ($) { strftime('%Y%m%d%H%M%S', gmtime($_[0])) };
 
 # human-friendly format
-sub fmt_ts ($) { strftime('%Y-%m-%d %k:%M', gmtime($_[0])) }
+sub fmt_ts ($) {
+        # strftime %k is not portable and leading zeros in %H slow me down
+        my (undef, $M, $H, $d, $m, $Y) = gmtime $_[0];
+        sprintf '%u-%02u-%02u % 2u:%02u', $Y + 1900, $m + 1, $d, $H, $M;
+}
+
+sub utf8_maybe ($) {
+        utf8::decode($_[0]);
+        utf8::valid($_[0]) or utf8::encode($_[0]); # non-UTF-8 data exists
+}
 
 1;
diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm
index d47e4c2f..b12533cb 100644
--- a/lib/PublicInbox/IMAP.pm
+++ b/lib/PublicInbox/IMAP.pm
@@ -36,18 +36,10 @@ use parent qw(PublicInbox::DS);
 use PublicInbox::Eml;
 use PublicInbox::EmlContentFoo qw(parse_content_disposition);
 use PublicInbox::DS qw(now);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
 use PublicInbox::GitAsyncCat;
 use Text::ParseWords qw(parse_line);
 use Errno qw(EAGAIN);
-use PublicInbox::IMAPsearchqp;
-
-my $Address;
-for my $mod (qw(Email::Address::XS Mail::Address)) {
-        eval "require $mod" or next;
-        $Address = $mod and last;
-}
-die "neither Email::Address::XS nor Mail::Address loaded: $@" if !$Address;
+use PublicInbox::Address;
 
 sub LINE_MAX () { 8000 } # RFC 2683 3.2.1.5
 
@@ -99,29 +91,15 @@ undef %FETCH_NEED;
 my $valid_range = '[0-9]+|[0-9]+:[0-9]+|[0-9]+:\*';
 $valid_range = qr/\A(?:$valid_range)(?:,(?:$valid_range))*\z/;
 
-sub greet ($) {
+sub do_greet {
         my ($self) = @_;
         my $capa = capa($self);
         $self->write(\"* OK [$capa] public-inbox-imapd ready\r\n");
 }
 
-sub new ($$$) {
-        my ($class, $sock, $imapd) = @_;
-        my $self = bless { imapd => $imapd }, 'PublicInbox::IMAP_preauth';
-        my $ev = EPOLLIN;
-        my $wbuf;
-        if ($sock->can('accept_SSL') && !$sock->accept_SSL) {
-                return CORE::close($sock) if $! != EAGAIN;
-                $ev = PublicInbox::TLS::epollbit() or return CORE::close($sock);
-                $wbuf = [ \&PublicInbox::DS::accept_tls_step, \&greet ];
-        }
-        $self->SUPER::new($sock, $ev | EPOLLONESHOT);
-        if ($wbuf) {
-                $self->{wbuf} = $wbuf;
-        } else {
-                greet($self);
-        }
-        $self;
+sub new {
+        my (undef, $sock, $imapd) = @_;
+        (bless { imapd => $imapd }, 'PublicInbox::IMAP_preauth')->greet($sock)
 }
 
 sub logged_in { 1 }
@@ -136,7 +114,7 @@ sub capa ($) {
                 $capa .= ' COMPRESS=DEFLATE';
         } else {
                 if (!($self->{sock} // $self)->can('accept_SSL') &&
-                        $self->{imapd}->{accept_tls}) {
+                        $self->{imapd}->{ssl_ctx_opt}) {
                         $capa .= ' STARTTLS';
                 }
                 $capa .= ' AUTH=ANONYMOUS';
@@ -153,6 +131,7 @@ sub login_success ($$) {
 sub auth_challenge_ok ($) {
         my ($self) = @_;
         my $tag = delete($self->{-login_tag}) or return;
+        $self->{anon} = 1;
         login_success($self, $tag);
 }
 
@@ -365,21 +344,18 @@ sub idle_done ($$) {
         "$idle_tag OK Idle done\r\n";
 }
 
-sub ensure_slices_exist ($$$) {
-        my ($imapd, $ibx, $max) = @_;
-        defined(my $mb_top = $ibx->{newsgroup}) or return;
+sub ensure_slices_exist ($$) {
+        my ($imapd, $ibx) = @_;
+        my $mb_top = $ibx->{newsgroup} // return;
         my $mailboxes = $imapd->{mailboxes};
-        my @created;
-        for (my $i = int($max/UID_SLICE); $i >= 0; --$i) {
+        my $list = $imapd->{mailboxlist}; # may be undef, just autoviv + noop
+        for (my $i = int($ibx->art_max/UID_SLICE); $i >= 0; --$i) {
                 my $sub_mailbox = "$mb_top.$i";
                 last if exists $mailboxes->{$sub_mailbox};
                 $mailboxes->{$sub_mailbox} = $ibx;
                 $sub_mailbox =~ s/\Ainbox\./INBOX./i; # more familiar to users
-                push @created, $sub_mailbox;
+                push @$list, qq[* LIST (\\HasNoChildren) "." $sub_mailbox\r\n]
         }
-        return unless @created;
-        my $l = $imapd->{mailboxlist} or return;
-        push @$l, map { qq[* LIST (\\HasNoChildren) "." $_\r\n] } @created;
 }
 
 sub inbox_lookup ($$;$) {
@@ -399,9 +375,11 @@ sub inbox_lookup ($$;$) {
                         $self->{ibx} = $ibx;
                         $self->{uo2m} = uo2m_ary_new($self, \$exists);
                 } else {
-                        $exists = $over->imap_exists;
+                        my $uid_end = $uid_base + UID_SLICE;
+                        $exists = $over->imap_exists($uid_base, $uid_end);
                 }
-                ensure_slices_exist($self->{imapd}, $ibx, $over->max);
+                delete $ibx->{-art_max};
+                ensure_slices_exist($self->{imapd}, $ibx);
         } else {
                 if ($examine) {
                         $self->{uid_base} = $uid_base;
@@ -410,9 +388,9 @@ sub inbox_lookup ($$;$) {
                 }
                 # if "INBOX.foo.bar" is selected and "INBOX.foo.bar.0",
                 # check for new UID ranges (e.g. "INBOX.foo.bar.1")
-                if (my $z = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) {
-                        ensure_slices_exist($self->{imapd}, $z,
-                                                $z->over(1)->max);
+                if (my $ibx = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) {
+                        delete $ibx->{-art_max};
+                        ensure_slices_exist($self->{imapd}, $ibx);
                 }
         }
         ($ibx, $exists, $uidmax + 1, $uid_base);
@@ -441,8 +419,10 @@ sub _esc ($) {
         if (!defined($v)) {
                 'NIL';
         } elsif ($v =~ /[{"\r\n%*\\\[]/) { # literal string
+                utf8::encode($v);
                 '{' . length($v) . "}\r\n" . $v;
         } else { # quoted string
+                utf8::encode($v);
                 qq{"$v"}
         }
 }
@@ -452,7 +432,7 @@ sub addr_envelope ($$;$) {
         my $v = $eml->header_raw($x) //
                 ($y ? $eml->header_raw($y) : undef) // return 'NIL';
 
-        my @x = $Address->parse($v) or return 'NIL';
+        my @x = PublicInbox::Address::objects($v) or return 'NIL';
         '(' . join('',
                 map { '(' . join(' ',
                                 _esc($_->name), 'NIL',
@@ -577,22 +557,6 @@ sub fetch_body ($;$) {
         join('', @hold);
 }
 
-sub requeue_once ($) {
-        my ($self) = @_;
-        # COMPRESS users all share the same DEFLATE context.
-        # Flush it here to ensure clients don't see
-        # each other's data
-        $self->zflush;
-
-        # no recursion, schedule another call ASAP,
-        # but only after all pending writes are done.
-        # autovivify wbuf:
-        my $new_size = push(@{$self->{wbuf}}, \&long_step);
-
-        # wbuf may be populated by $cb, no need to rearm if so:
-        $self->requeue if $new_size == 1;
-}
-
 sub fetch_run_ops {
         my ($self, $smsg, $bref, $ops, $partial) = @_;
         my $uid = $smsg->{num};
@@ -606,26 +570,37 @@ sub fetch_run_ops {
         $self->msg_more(")\r\n");
 }
 
+sub requeue { # overrides PublicInbox::DS::requeue
+        my ($self) = @_;
+        if ($self->{anon}) { # AUTH=ANONYMOUS gets high priority
+                $self->SUPER::requeue;
+        } else { # low priority
+                push(@{$self->{imapd}->{-authed_q}}, $self) == 1 and
+                        PublicInbox::DS::requeue($self->{imapd});
+        }
+}
+
 sub fetch_blob_cb { # called by git->cat_async via ibx_async_cat
         my ($bref, $oid, $type, $size, $fetch_arg) = @_;
         my ($self, undef, $msgs, $range_info, $ops, $partial) = @$fetch_arg;
         my $ibx = $self->{ibx} or return $self->close; # client disconnected
         my $smsg = shift @$msgs or die 'BUG: no smsg';
-        if (!defined($oid)) {
+        if (!defined($type)) {
+                warn "E: git aborted on $oid / $smsg->{blob} $ibx->{inboxdir}";
+                return $self->close;
+        } elsif ($type ne 'blob') {
                 # it's possible to have TOCTOU if an admin runs
                 # public-inbox-(edit|purge), just move onto the next message
-                warn "E: $smsg->{blob} missing in $ibx->{inboxdir}\n";
-                return requeue_once($self);
-        } else {
-                $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid";
+                warn "E: $smsg->{blob} $type in $ibx->{inboxdir}\n";
+                return $self->requeue_once;
         }
+        $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid";
         my $pre;
-        if (!$self->{wbuf} && (my $nxt = $msgs->[0])) {
-                $pre = ibx_async_prefetch($ibx, $nxt->{blob},
+        ($self->{anon} && !$self->{wbuf} && $msgs->[0]) and
+                $pre = ibx_async_prefetch($ibx, $msgs->[0]->{blob},
                                         \&fetch_blob_cb, $fetch_arg);
-        }
         fetch_run_ops($self, $smsg, $bref, $ops, $partial);
-        $pre ? $self->zflush : requeue_once($self);
+        $pre ? $self->dflush : $self->requeue_once;
 }
 
 sub emit_rfc822 {
@@ -683,7 +658,7 @@ sub op_eml_new { $_[4] = PublicInbox::Eml->new($_[3]) }
 # s/From / fixes old bug from import (pre-a0c07cba0e5d8b6a)
 sub to_crlf_full {
         ${$_[0]} =~ s/(?<!\r)\n/\r\n/sg;
-        ${$_[0]} =~ s/\A[\r\n]*From [^\r\n]*\r\n//s;
+        PublicInbox::Eml::strip_from(${$_[0]});
 }
 
 sub op_crlf_bref { to_crlf_full($_[3]) }
@@ -1027,7 +1002,7 @@ sub fetch_compile ($) {
         # stabilize partial order for consistency and ease-of-debugging:
         if (scalar keys %partial) {
                 $need |= NEED_BLOB;
-                $r[2] = [ map { [ $_, @{$partial{$_}} ] } sort keys %partial ];
+                @{$r[2]} = map { [ $_, @{$partial{$_}} ] } sort keys %partial;
         }
 
         push @op, $OP_EML_NEW if ($need & (EML_HDR|EML_BDY));
@@ -1050,7 +1025,7 @@ sub fetch_compile ($) {
 
         # r[1] = [ $key1, $cb1, $key2, $cb2, ... ]
         use sort 'stable'; # makes output more consistent
-        $r[1] = [ map { ($_->[2], $_->[1]) } sort { $a->[0] <=> $b->[0] } @op ];
+        @{$r[1]} = map { ($_->[2], $_->[1]) } sort { $a->[0] <=> $b->[0] } @op;
         @r;
 }
 
@@ -1065,7 +1040,7 @@ sub cmd_uid_fetch ($$$$;@) {
         my $range_info = range_step($self, \$range_csv);
         return "$tag $range_info\r\n" if !ref($range_info);
         uo2m_hibernate($self) if $cb == \&fetch_blob; # slow, save RAM
-        long_response($self, $cb, $tag, [], $range_info, $ops, $partial);
+        $self->long_response($cb, $tag, [], $range_info, $ops, $partial);
 }
 
 sub cmd_fetch ($$$$;@) {
@@ -1080,7 +1055,7 @@ sub cmd_fetch ($$$$;@) {
         my $range_info = range_step($self, \$range_csv);
         return "$tag $range_info\r\n" if !ref($range_info);
         uo2m_hibernate($self) if $cb == \&fetch_blob; # slow, save RAM
-        long_response($self, $cb, $tag, [], $range_info, $ops, $partial);
+        $self->long_response($cb, $tag, [], $range_info, $ops, $partial);
 }
 
 sub msn_convert ($$) {
@@ -1106,6 +1081,7 @@ sub search_uid_range { # long_response
 
 sub parse_imap_query ($$) {
         my ($self, $query) = @_;
+        # IMAPsearchqp gets loaded in IMAPD->refresh_groups
         my $q = PublicInbox::IMAPsearchqp::parse($self, $query);
         if (ref($q)) {
                 my $max = $self->{ibx}->over(1)->max;
@@ -1124,7 +1100,7 @@ sub search_common {
         my ($sql, $range_info) = delete @$q{qw(sql range_info)};
         if (!scalar(keys %$q)) { # overview.sqlite3
                 $self->msg_more('* SEARCH');
-                long_response($self, \&search_uid_range,
+                $self->long_response(\&search_uid_range,
                                 $tag, $sql, $range_info, $want_msn);
         } elsif ($q = $q->{xap}) {
                 my $srch = $self->{ibx}->isrch or
@@ -1187,46 +1163,11 @@ sub process_line ($$) {
         my $err = $@;
         if ($err && $self->{sock}) {
                 $l =~ s/\r?\n//s;
-                err($self, 'error from: %s (%s)', $l, $err);
+                warn("error from: $l ($err)\n");
                 $tag //= '*';
-                $res = "$tag BAD program fault - command not performed\r\n";
+                $res = \"$tag BAD program fault - command not performed\r\n";
         }
-        return 0 unless defined $res;
-        $self->write($res);
-}
-
-sub long_step {
-        my ($self) = @_;
-        # wbuf is unset or empty, here; {long} may add to it
-        my ($fd, $cb, $t0, @args) = @{$self->{long_cb}};
-        my $more = eval { $cb->($self, @args) };
-        if ($@ || !$self->{sock}) { # something bad happened...
-                delete $self->{long_cb};
-                my $elapsed = now() - $t0;
-                if ($@) {
-                        err($self,
-                            "%s during long response[$fd] - %0.6f",
-                            $@, $elapsed);
-                }
-                out($self, " deferred[$fd] aborted - %0.6f", $elapsed);
-                $self->close;
-        } elsif ($more) { # $self->{wbuf}:
-                # control passed to ibx_async_cat if $more == \undef
-                requeue_once($self) if !ref($more);
-        } else { # all done!
-                delete $self->{long_cb};
-                my $elapsed = now() - $t0;
-                my $fd = fileno($self->{sock});
-                out($self, " deferred[$fd] done - %0.6f", $elapsed);
-                my $wbuf = $self->{wbuf}; # do NOT autovivify
-
-                $self->requeue unless $wbuf && @$wbuf;
-        }
-}
-
-sub err ($$;@) {
-        my ($self, $fmt, @args) = @_;
-        printf { $self->{imapd}->{err} } $fmt."\n", @args;
+        defined($res) ? $self->write($res) : 0;
 }
 
 sub out ($$;@) {
@@ -1234,22 +1175,10 @@ sub out ($$;@) {
         printf { $self->{imapd}->{out} } $fmt."\n", @args;
 }
 
-sub long_response ($$;@) {
-        my ($self, $cb, @args) = @_; # cb returns true if more, false if done
-
-        my $sock = $self->{sock} or return;
-        # make sure we disable reading during a long response,
-        # clients should not be sending us stuff and making us do more
-        # work while we are stream a response to them
-        $self->{long_cb} = [ fileno($sock), $cb, now(), @args ];
-        long_step($self); # kick off!
-        undef;
-}
-
 # callback used by PublicInbox::DS for any (e)poll (in/out/hup/err)
 sub event_step {
         my ($self) = @_;
-
+        local $SIG{__WARN__} = $self->{imapd}->{warn_cb};
         return unless $self->flush_write && $self->{sock} && !$self->{long_cb};
 
         # only read more requests if we've drained the write buffer,
@@ -1283,10 +1212,6 @@ sub event_step {
         $self->requeue unless $pending;
 }
 
-sub compressed { undef }
-
-sub zflush {} # overridden by IMAPdeflate
-
 # RFC 4978
 sub cmd_compress ($$$) {
         my ($self, $tag, $alg) = @_;
@@ -1296,21 +1221,21 @@ sub cmd_compress ($$$) {
         # CRIME made TLS compression obsolete
         # return "$tag NO [COMPRESSIONACTIVE]\r\n" if $self->tls_compressed;
 
-        PublicInbox::IMAPdeflate->enable($self, $tag);
+        PublicInbox::IMAPdeflate->enable($self) or return
+                                \"$tag BAD failed to activate compression\r\n";
+        PublicInbox::DS::write($self, \"$tag OK DEFLATE active\r\n");
         $self->requeue;
         undef
 }
 
 sub cmd_starttls ($$) {
         my ($self, $tag) = @_;
-        my $sock = $self->{sock} or return;
-        if ($sock->can('stop_SSL') || $self->compressed) {
+        (($self->{sock} // return)->can('stop_SSL') || $self->compressed) and
                 return "$tag BAD TLS or compression already enabled\r\n";
-        }
-        my $opt = $self->{imapd}->{accept_tls} or
+        $self->{imapd}->{ssl_ctx_opt} or
                 return "$tag BAD can not initiate TLS negotiation\r\n";
         $self->write(\"$tag OK begin TLS negotiation now\r\n");
-        $self->{sock} = IO::Socket::SSL->start_SSL($sock, %$opt);
+        PublicInbox::TLS::start($self->{sock}, $self->{imapd});
         $self->requeue if PublicInbox::DS::accept_tls_step($self);
         undef;
 }
@@ -1342,4 +1267,8 @@ our @ISA = qw(PublicInbox::IMAP);
 
 sub logged_in { 0 }
 
+package PublicInbox::IMAPdeflate;
+use PublicInbox::DSdeflate;
+our @ISA = qw(PublicInbox::DSdeflate PublicInbox::IMAP);
+
 1;
diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm
index d8814324..42dc2a9f 100644
--- a/lib/PublicInbox/IMAPD.pm
+++ b/lib/PublicInbox/IMAPD.pm
@@ -1,122 +1,85 @@
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# represents an IMAPD (currently a singleton),
-# see script/public-inbox-imapd for how it is used
+# represents an IMAPD, see script/public-inbox-imapd for how it is used
 package PublicInbox::IMAPD;
 use strict;
 use v5.10.1;
 use PublicInbox::Config;
-use PublicInbox::ConfigIter;
 use PublicInbox::InboxIdle;
-use PublicInbox::IMAPdeflate; # loads PublicInbox::IMAP
+use PublicInbox::IMAP;
 use PublicInbox::DummyInbox;
 my $dummy = bless { uidvalidity => 0 }, 'PublicInbox::DummyInbox';
 
 sub new {
         my ($class) = @_;
         bless {
-                mailboxes => {},
+                # mailboxes => {},
                 err => \*STDERR,
                 out => \*STDOUT,
-                # accept_tls => { SSL_server => 1, ..., SSL_reuse_ctx => ... }
+                # ssl_ctx_opt => { SSL_cert_file => ..., SSL_key_file => ... }
                 # pi_cfg => PublicInbox::Config
                 # idler => PublicInbox::InboxIdle
         }, $class;
 }
 
-sub imapd_refresh_ibx { # pi_cfg->each_inbox cb
-        my ($ibx, $imapd) = @_;
-        my $ngname = $ibx->{newsgroup} or return;
+sub _refresh_ibx { # pi_cfg->each_inbox cb
+        my ($ibx, $imapd, $cache, $dummies) = @_;
+        my $ngname = $ibx->{newsgroup} // return;
 
-        # We require lower-case since IMAP mailbox names are
-        # case-insensitive (but -nntpd matches INN in being
-        # case-sensitive
-        if ($ngname =~ m![^a-z0-9/_\.\-\~\@\+\=:]! ||
-                        # don't confuse with 50K slices
-                        $ngname =~ /\.[0-9]+\z/) {
-                warn "mailbox name invalid: newsgroup=`$ngname'\n";
+        if ($ngname =~ /\.[0-9]+\z/) { # don't confuse with 50K slices
+                warn "E: mailbox name invalid: newsgroup=`$ngname' (ignored)\n";
                 return;
         }
-        $ibx->over or return;
-        $ibx->{over} = undef;
-
-        # RFC 3501 2.3.1.1 -  "A good UIDVALIDITY value to use in
-        # this case is a 32-bit representation of the creation
-        # date/time of the mailbox"
-        eval { $ibx->uidvalidity };
-        my $mm = delete($ibx->{mm}) or return;
-        defined($ibx->{uidvalidity}) or return;
-        PublicInbox::IMAP::ensure_slices_exist($imapd, $ibx, $mm->max);
-
-        # preload to avoid fragmentation:
-        $ibx->description;
-        $ibx->base_url;
-
-        # ensure dummies are selectable
-        my $dummies = $imapd->{dummies};
-        do {
-                $dummies->{$ngname} = $dummy;
-        } while ($ngname =~ s/\.[^\.]+\z//);
-}
-
-sub imapd_refresh_finalize {
-        my ($imapd, $pi_cfg) = @_;
-        my $mailboxes;
-        if (my $next = delete $imapd->{imapd_next}) {
-                $imapd->{mailboxes} = delete $next->{mailboxes};
-                $mailboxes = delete $next->{dummies};
-        } else {
-                $mailboxes = delete $imapd->{dummies};
-        }
-        %$mailboxes = (%$mailboxes, %{$imapd->{mailboxes}});
-        $imapd->{mailboxes} = $mailboxes;
-        $imapd->{mailboxlist} = [
-                map { $_->[2] }
-                sort { $a->[0] cmp $b->[0] || $a->[1] <=> $b->[1] }
-                map {
-                        my $u = $_; # capitalize "INBOX" for user-familiarity
-                        $u =~ s/\Ainbox(\.|\z)/INBOX$1/i;
-                        if ($mailboxes->{$_} == $dummy) {
-                                [ $u, -1,
-                                  qq[* LIST (\\HasChildren) "." $u\r\n]]
-                        } else {
-                                $u =~ /\A(.+)\.([0-9]+)\z/ or
-                                        die "BUG: `$u' has no slice digit(s)";
-                                [ $1, $2 + 0,
-                                  qq[* LIST (\\HasNoChildren) "." $u\r\n] ]
-                        }
-                } keys %$mailboxes
-        ];
-        $imapd->{pi_cfg} = $pi_cfg;
-        if (my $idler = $imapd->{idler}) {
-                $idler->refresh($pi_cfg);
-        }
-}
-
-sub imapd_refresh_step { # PublicInbox::ConfigIter cb
-        my ($pi_cfg, $section, $imapd) = @_;
-        if (defined($section)) {
-                return if $section !~ m!\Apublicinbox\.([^/]+)\z!;
-                my $ibx = $pi_cfg->lookup_name($1) or return;
-                imapd_refresh_ibx($ibx, $imapd->{imapd_next});
-        } else { # undef == "EOF"
-                imapd_refresh_finalize($imapd, $pi_cfg);
+        my $ce = $cache->{$ngname};
+        %$ibx = (%$ibx, %$ce) if $ce;
+        # only valid if msgmap and over works:
+        if (defined($ibx->uidvalidity)) {
+                # fill ->{mailboxes}:
+                PublicInbox::IMAP::ensure_slices_exist($imapd, $ibx);
+                # preload to avoid fragmentation:
+                $ibx->description;
+                # ensure dummies are selectable:
+                do {
+                        $dummies->{$ngname} = $dummy;
+                } while ($ngname =~ s/\.[^\.]+\z//);
         }
+        delete @$ibx{qw(mm over)};
 }
 
 sub refresh_groups {
         my ($self, $sig) = @_;
         my $pi_cfg = PublicInbox::Config->new;
-        if ($sig) { # SIGHUP is handled through the event loop
-                $self->{imapd_next} = { dummies => {}, mailboxes => {} };
-                my $iter = PublicInbox::ConfigIter->new($pi_cfg,
-                                                \&imapd_refresh_step, $self);
-                $iter->event_step;
-        } else { # initial start is synchronous
-                $self->{dummies} = {};
-                $pi_cfg->each_inbox(\&imapd_refresh_ibx, $self);
-                imapd_refresh_finalize($self, $pi_cfg);
+        require PublicInbox::IMAPsearchqp;
+        $self->{mailboxes} = $pi_cfg->{-imap_mailboxes} // do {
+                my $mailboxes = $self->{mailboxes} = {};
+                my $cache = eval { $pi_cfg->ALL->misc->nntpd_cache_load } // {};
+                my $dummies = {};
+                $pi_cfg->each_inbox(\&_refresh_ibx, $self, $cache, $dummies);
+                %$mailboxes = (%$dummies, %$mailboxes);
+                @{$pi_cfg->{-imap_mailboxlist}} = map { $_->[2] }
+                        sort { $a->[0] cmp $b->[0] || $a->[1] <=> $b->[1] }
+                        map {
+                                # capitalize "INBOX" for user-familiarity
+                                my $u = $_;
+                                $u =~ s/\Ainbox(\.|\z)/INBOX$1/i;
+                                if ($mailboxes->{$_} == $dummy) {
+                                        [ $u, -1,
+                                          qq[* LIST (\\HasChildren) "." $u\r\n]]
+                                } else {
+                                        $u =~ /\A(.+)\.([0-9]+)\z/ or die
+"BUG: `$u' has no slice digit(s)";
+                                        [ $1, $2 + 0, '* LIST '.
+                                          qq[(\\HasNoChildren) "." $u\r\n] ]
+                                }
+                        } keys %$mailboxes;
+                $pi_cfg->{-imap_mailboxes} = $mailboxes;
+        };
+        $self->{mailboxlist} = $pi_cfg->{-imap_mailboxlist} //
+                        die 'BUG: no mailboxlist';
+        $self->{pi_cfg} = $pi_cfg;
+        if (my $idler = $self->{idler}) {
+                $idler->refresh($pi_cfg);
         }
 }
 
@@ -124,4 +87,11 @@ sub idler_start {
         $_[0]->{idler} //= PublicInbox::InboxIdle->new($_[0]->{pi_cfg});
 }
 
+sub event_step { # called vai requeue for low-priority IMAP clients
+        my ($self) = @_;
+        my $imap = shift(@{$self->{-authed_q}}) // return;
+        PublicInbox::DS::requeue($self) if scalar(@{$self->{-authed_q}});
+        $imap->event_step; # PublicInbox::IMAP::event_step
+}
+
 1;
diff --git a/lib/PublicInbox/IMAPdeflate.pm b/lib/PublicInbox/IMAPdeflate.pm
deleted file mode 100644
index d5929ef2..00000000
--- a/lib/PublicInbox/IMAPdeflate.pm
+++ /dev/null
@@ -1,126 +0,0 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# TODO: reduce duplication from PublicInbox::NNTPdeflate
-
-# RFC 4978
-package PublicInbox::IMAPdeflate;
-use strict;
-use warnings;
-use 5.010_001;
-use base qw(PublicInbox::IMAP);
-use Compress::Raw::Zlib;
-
-my %IN_OPT = (
-        -Bufsize => 1024,
-        -WindowBits => -15, # RFC 1951
-        -AppendOutput => 1,
-);
-
-# global deflate context and buffer
-my $zbuf = \(my $buf = '');
-my $zout;
-{
-        my $err;
-        ($zout, $err) = Compress::Raw::Zlib::Deflate->new(
-                # nnrpd (INN) and Compress::Raw::Zlib favor MemLevel=9,
-                # the zlib C library and git use MemLevel=8 as the default
-                # -MemLevel => 9,
-                -Bufsize => 65536, # same as nnrpd
-                -WindowBits => -15, # RFC 1951
-                -AppendOutput => 1,
-        );
-        $err == Z_OK or die "Failed to initialize zlib deflate stream: $err";
-}
-
-sub enable {
-        my ($class, $self, $tag) = @_;
-        my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT);
-        if ($err != Z_OK) {
-                $self->err("Inflate->new failed: $err");
-                $self->write(\"$tag BAD failed to activate compression\r\n");
-                return;
-        }
-        $self->write(\"$tag OK DEFLATE active\r\n");
-        bless $self, $class;
-        $self->{zin} = $in;
-}
-
-# overrides PublicInbox::NNTP::compressed
-sub compressed { 1 }
-
-sub do_read ($$$$) {
-        my ($self, $rbuf, $len, $off) = @_;
-
-        my $zin = $self->{zin} or return; # closed
-        my $doff;
-        my $dbuf = delete($self->{dbuf}) // '';
-        $doff = length($dbuf);
-        my $r = PublicInbox::DS::do_read($self, \$dbuf, $len, $doff) or return;
-
-        # Workaround inflate bug appending to OOK scalars:
-        # <https://rt.cpan.org/Ticket/Display.html?id=132734>
-        # We only have $off if the client is pipelining, and pipelining
-        # is where our substr() OOK optimization in event_step makes sense.
-        if ($off) {
-                my $copy = $$rbuf;
-                undef $$rbuf;
-                $$rbuf = $copy;
-        }
-
-        # assert(length($$rbuf) == $off) as far as NNTP.pm is concerned
-        # -ConsumeInput is true, so $dbuf is automatically emptied
-        my $err = $zin->inflate($dbuf, $rbuf);
-        if ($err == Z_OK) {
-                $self->{dbuf} = $dbuf if $dbuf ne '';
-                $r = length($$rbuf) and return $r;
-                # nothing ready, yet, get more, later
-                $self->requeue;
-        } else {
-                delete $self->{zin};
-                $self->close;
-        }
-        0;
-}
-
-# override PublicInbox::DS::msg_more
-sub msg_more ($$) {
-        my $self = $_[0];
-
-        # $_[1] may be a reference or not for ->deflate
-        my $err = $zout->deflate($_[1], $zbuf);
-        $err == Z_OK or die "->deflate failed $err";
-        1;
-}
-
-sub zflush ($) {
-        my ($self) = @_;
-
-        my $deflated = $zbuf;
-        $zbuf = \(my $next = '');
-
-        my $err = $zout->flush($deflated, Z_FULL_FLUSH);
-        $err == Z_OK or die "->flush failed $err";
-
-        # We can still let the lower socket layer do buffering:
-        PublicInbox::DS::msg_more($self, $$deflated);
-}
-
-# compatible with PublicInbox::DS::write, so $_[1] may be a reference or not
-sub write ($$) {
-        my $self = $_[0];
-        return PublicInbox::DS::write($self, $_[1]) if ref($_[1]) eq 'CODE';
-
-        my $deflated = $zbuf;
-        $zbuf = \(my $next = '');
-
-        # $_[1] may be a reference or not for ->deflate
-        my $err = $zout->deflate($_[1], $deflated);
-        $err == Z_OK or die "->deflate failed $err";
-        $err = $zout->flush($deflated, Z_FULL_FLUSH);
-        $err == Z_OK or die "->flush failed $err";
-
-        # We can still let the socket layer do buffering:
-        PublicInbox::DS::write($self, $deflated);
-}
-
-1;
diff --git a/lib/PublicInbox/IMAPsearchqp.pm b/lib/PublicInbox/IMAPsearchqp.pm
index 9f0c1205..0c37220c 100644
--- a/lib/PublicInbox/IMAPsearchqp.pm
+++ b/lib/PublicInbox/IMAPsearchqp.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # IMAP search query parser.  cf RFC 3501
 
@@ -279,6 +279,8 @@ sub parse {
         my $sql = '';
         %$q = (sql => \$sql, imap => $imap); # imap = PublicInbox::IMAP obj
         # $::RD_TRACE = 1;
+        local $::RD_ERRORS = undef;
+        local $::RD_WARN = undef;
         my $res = eval { $prd->search_key(uc($query)) };
         return $@ if $@ && $@ =~ /\A(?:BAD|NO) /;
         return 'BAD unexpected result' if !$res || $res != $q;
diff --git a/lib/PublicInbox/IO.pm b/lib/PublicInbox/IO.pm
new file mode 100644
index 00000000..5654f3b0
--- /dev/null
+++ b/lib/PublicInbox/IO.pm
@@ -0,0 +1,150 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# supports reaping of children tied to a pipe or socket
+package PublicInbox::IO;
+use v5.12;
+use parent qw(IO::Handle Exporter);
+use PublicInbox::DS qw(awaitpid);
+our @EXPORT_OK = qw(poll_in read_all try_cat write_file);
+use Carp qw(croak);
+use IO::Poll qw(POLLIN);
+use Errno qw(EINTR EAGAIN);
+# don't autodie in top-level for Perl 5.16.3 (and maybe newer versions)
+# we have our own ->close, so we scope autodie into each sub
+
+sub waitcb { # awaitpid callback
+        my ($pid, $errref, $cb, @args) = @_;
+        $$errref = $? if $errref; # sets .cerr for _close
+        $cb->($pid, @args) if $cb; # may clobber $?
+}
+
+sub attach_pid {
+        my ($io, $pid, @cb_arg) = @_;
+        bless $io, __PACKAGE__;
+        # we share $err (and not $self) with awaitpid to avoid a ref cycle
+        ${*$io}{pi_io_reap} = [ $$, $pid, \(my $err) ];
+        awaitpid($pid, \&waitcb, \$err, @cb_arg);
+        $io;
+}
+
+sub attached_pid {
+        my ($io) = @_;
+        ${${*$io}{pi_io_reap} // []}[1];
+}
+
+sub owner_pid {
+        my ($io) = @_;
+        ${${*$io}{pi_io_reap} // [-1]}[0];
+}
+
+# caller cares about error result if they call close explicitly
+# reap->[2] may be set before this is called via waitcb
+sub close {
+        my ($io) = @_;
+        my $ret = $io->SUPER::close;
+        my $reap = delete ${*$io}{pi_io_reap};
+        return $ret unless $reap && $reap->[0] == $$;
+        if (defined ${$reap->[2]}) { # reap_pids already reaped asynchronously
+                $? = ${$reap->[2]};
+        } else { # wait synchronously
+                my $w = awaitpid($reap->[1]);
+        }
+        $? ? '' : $ret;
+}
+
+sub DESTROY {
+        my ($io) = @_;
+        my $reap = delete ${*$io}{pi_io_reap};
+        if ($reap && $reap->[0] == $$) {
+                $io->SUPER::close;
+                awaitpid($reap->[1]);
+        }
+        $io->SUPER::DESTROY;
+}
+
+sub write_file ($$@) { # mode, filename, LIST (for print)
+        use autodie qw(open close);
+        open(my $fh, shift, shift);
+        print $fh @_;
+        defined(wantarray) && !wantarray ? $fh : close $fh;
+}
+
+sub poll_in ($;$) {
+        IO::Poll::_poll($_[1] // -1, fileno($_[0]), my $ev = POLLIN);
+}
+
+sub read_all ($;$$$) { # pass $len=0 to read until EOF for :utf8 handles
+        use autodie qw(read);
+        my ($io, $len, $bref, $off) = @_;
+        $bref //= \(my $buf);
+        $off //= 0;
+        my $r = 0;
+        if (my $left = $len //= -s $io) { # known size (binmode :raw/:unix)
+                do { # retry for binmode :unix
+                        $r = read($io, $$bref, $left, $off += $r) or croak(
+                                "read($io) premature EOF ($left/$len remain)");
+                } while ($left -= $r);
+        } else { # read until EOF
+                while (($r = read($io, $$bref, 65536, $off += $r))) {}
+        }
+        wantarray ? split(/^/sm, $$bref) : $$bref
+}
+
+sub try_cat ($) {
+        my ($path) = @_;
+        open(my $fh, '<', $path) or return '';
+        read_all $fh;
+}
+
+# TODO: move existing HTTP/IMAP/NNTP/POP3 uses of rbuf here
+sub my_bufread {
+        my ($io, $len) = @_;
+        my $rbuf = ${*$io}{pi_io_rbuf} //= \(my $new = '');
+        my $left = $len - length($$rbuf);
+        my $r;
+        while ($left > 0) {
+                $r = sysread($io, $$rbuf, $left, length($$rbuf));
+                if ($r) {
+                        $left -= $r;
+                } elsif (defined($r)) { # EOF
+                        return 0;
+                } else {
+                        next if ($! == EAGAIN and poll_in($io));
+                        next if $! == EINTR; # may be set by sysread or poll_in
+                        return; # unrecoverable error
+                }
+        }
+        my $no_pad = substr($$rbuf, 0, $len, '');
+        delete(${*$io}{pi_io_rbuf}) if $$rbuf eq '';
+        \$no_pad;
+}
+
+# always uses "\n"
+sub my_readline {
+        my ($io) = @_;
+        my $rbuf = ${*$io}{pi_io_rbuf} //= \(my $new = '');
+        while (1) {
+                if ((my $n = index($$rbuf, "\n")) >= 0) {
+                        my $ret = substr($$rbuf, 0, $n + 1, '');
+                        delete(${*$io}{pi_io_rbuf}) if $$rbuf eq '';
+                        return $ret;
+                }
+                my $r = sysread($io, $$rbuf, 65536, length($$rbuf));
+                if (!defined($r)) {
+                        next if ($! == EAGAIN and poll_in($io));
+                        next if $! == EINTR; # may be set by sysread or poll_in
+                        return; # unrecoverable error
+                } elsif ($r == 0) { # return whatever's left on EOF
+                        delete(${*$io}{pi_io_rbuf});
+                        return $$rbuf;
+                } # else { continue
+        }
+}
+
+sub has_rbuf {
+        my ($io) = @_;
+        defined(${*$io}{pi_io_rbuf});
+}
+
+1;
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index 67e86a43..a5cae6f2 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # base class for remote IPC calls and workqueues, requires Storable or Sereal
@@ -8,18 +8,17 @@
 # use ipc_do when you need work done on a certain process
 # use wq_io_do when your work can be done on any idle worker
 package PublicInbox::IPC;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(Exporter);
+use autodie qw(close fork pipe read socketpair sysread);
 use Carp qw(croak);
-use PublicInbox::DS qw(dwaitpid);
+use PublicInbox::DS qw(awaitpid);
 use PublicInbox::Spawn;
 use PublicInbox::OnDestroy;
 use PublicInbox::WQWorker;
-use Socket qw(AF_UNIX MSG_EOR SOCK_STREAM);
+use Socket qw(AF_UNIX SOCK_STREAM SOCK_SEQPACKET);
 my $MY_MAX_ARG_STRLEN = 4096 * 33; # extra 4K for serialization
-my $SEQPACKET = eval { Socket::SOCK_SEQPACKET() }; # portable enough?
-our @EXPORT_OK = qw(ipc_freeze ipc_thaw);
+our @EXPORT_OK = qw(ipc_freeze ipc_thaw nproc_shards);
 my ($enc, $dec);
 # ->imports at BEGIN turns sereal_*_with_object into custom ops on 5.14+
 # and eliminate method call overhead
@@ -42,8 +41,8 @@ if ($enc && $dec) { # should be custom ops
         *ipc_thaw = \&Storable::thaw;
 }
 
-my $recv_cmd = PublicInbox::Spawn->can('recv_cmd4');
-my $send_cmd = PublicInbox::Spawn->can('send_cmd4') // do {
+our $recv_cmd = PublicInbox::Spawn->can('recv_cmd4');
+our $send_cmd = PublicInbox::Spawn->can('send_cmd4') // do {
         require PublicInbox::CmdIPC4;
         $recv_cmd //= PublicInbox::CmdIPC4->can('recv_cmd4');
         PublicInbox::CmdIPC4->can('send_cmd4');
@@ -55,9 +54,9 @@ my $send_cmd = PublicInbox::Spawn->can('send_cmd4') // do {
 
 sub _get_rec ($) {
         my ($r) = @_;
-        defined(my $len = <$r>) or return;
+        my $len = <$r> // return;
         chop($len) eq "\n" or croak "no LF byte in $len";
-        defined(my $n = read($r, my $buf, $len)) or croak "read error: $!";
+        my $n = read($r, my $buf, $len);
         $n == $len or croak "short read: $n != $len";
         ipc_thaw($buf);
 }
@@ -96,19 +95,15 @@ sub ipc_worker_loop ($$$) {
 
 # starts a worker if Sereal or Storable is installed
 sub ipc_worker_spawn {
-        my ($self, $ident, $oldset, $fields) = @_;
+        my ($self, $ident, $oldset, $fields, @cb_args) = @_;
         return if ($self->{-ipc_ppid} // -1) == $$; # idempotent
         delete(@$self{qw(-ipc_req -ipc_res -ipc_ppid -ipc_pid)});
-        pipe(my ($r_req, $w_req)) or die "pipe: $!";
-        pipe(my ($r_res, $w_res)) or die "pipe: $!";
+        pipe(my $r_req, my $w_req);
+        pipe(my $r_res, my $w_res);
         my $sigset = $oldset // PublicInbox::DS::block_signals();
         $self->ipc_atfork_prepare;
-        my $seed = rand(0xffffffff);
-        my $pid = fork // die "fork: $!";
+        my $pid = PublicInbox::DS::do_fork;
         if ($pid == 0) {
-                srand($seed);
-                eval { Net::SSLeay::randomize() };
-                eval { PublicInbox::DS->Reset };
                 delete @$self{qw(-wq_s1 -wq_s2 -wq_workers -wq_ppid)};
                 $w_req = $r_res = undef;
                 $w_res->autoflush(1);
@@ -133,29 +128,20 @@ sub ipc_worker_spawn {
         $self->{-ipc_req} = $w_req;
         $self->{-ipc_res} = $r_res;
         $self->{-ipc_ppid} = $$;
+        awaitpid($pid, \&ipc_worker_reap, $self, @cb_args);
         $self->{-ipc_pid} = $pid;
 }
 
-sub ipc_worker_reap { # dwaitpid callback
-        my ($args, $pid) = @_;
-        my ($self, @uargs) = @$args;
+sub ipc_worker_reap { # awaitpid callback
+        my ($pid, $self, $cb, @args) = @_;
         delete $self->{-wq_workers}->{$pid};
-        return $self->{-reap_do}->($args, $pid) if $self->{-reap_do};
+        return $cb->($pid, $self, @args) if $cb;
         return if !$?;
         my $s = $? & 127;
         # TERM(15) is our default exit signal, PIPE(13) is likely w/ pager
         warn "$self->{-wq_ident} PID:$pid died \$?=$?\n" if $s != 15 && $s != 13
 }
 
-sub wq_wait_async {
-        my ($self, $cb, @uargs) = @_;
-        local $PublicInbox::DS::in_loop = 1;
-        $self->{-reap_async} = 1;
-        $self->{-reap_do} = $cb;
-        my @pids = keys %{$self->{-wq_workers}};
-        dwaitpid($_, \&ipc_worker_reap, [ $self, @uargs ]) for @pids;
-}
-
 # for base class, override in sub classes
 sub ipc_atfork_prepare {}
 
@@ -170,7 +156,7 @@ sub ipc_atfork_child {
 
 # idempotent, can be called regardless of whether worker is active or not
 sub ipc_worker_stop {
-        my ($self, $args) = @_;
+        my ($self) = @_;
         my ($pid, $ppid) = delete(@$self{qw(-ipc_pid -ipc_ppid)});
         my ($w_req, $r_res) = delete(@$self{qw(-ipc_req -ipc_res)});
         if (!$w_req && !$r_res) {
@@ -179,18 +165,7 @@ sub ipc_worker_stop {
         }
         die 'no PID with IPC pipes' unless $pid;
         $w_req = $r_res = undef;
-
-        return if $$ != $ppid;
-        dwaitpid($pid, \&ipc_worker_reap, [$self, $args]);
-}
-
-# use this if we have multiple readers reading curl or "pigz -dc"
-# and writing to the same store
-sub ipc_lock_init {
-        my ($self, $f) = @_;
-        $f // die 'BUG: no filename given';
-        require PublicInbox::Lock;
-        $self->{-ipc_lock} //= bless { lock_path => $f }, 'PublicInbox::Lock'
+        awaitpid($pid) if $$ == $ppid; # for non-event loop
 }
 
 sub _wait_return ($$) {
@@ -204,8 +179,6 @@ sub _wait_return ($$) {
 sub ipc_do {
         my ($self, $sub, @args) = @_;
         if (my $w_req = $self->{-ipc_req}) { # run in worker
-                my $ipc_lock = $self->{-ipc_lock};
-                my $lock = $ipc_lock ? $ipc_lock->lock_for_scope : undef;
                 if (defined(wantarray)) {
                         my $r_res = $self->{-ipc_res} or die 'no ipc_res';
                         _send_rec($w_req, [ wantarray, $sub, @args ]);
@@ -234,15 +207,12 @@ sub recv_and_run {
         my $n = length($buf) or return 0;
         my $nfd = 0;
         for my $fd (@fds) {
-                if (open(my $cmdfh, '+<&=', $fd)) {
-                        $self->{$nfd++} = $cmdfh;
-                        $cmdfh->autoflush(1);
-                } else {
-                        die "$$ open(+<&=$fd) (FD:$nfd): $!";
-                }
+                open(my $cmdfh, '+<&=', $fd);
+                $self->{$nfd++} = $cmdfh;
+                $cmdfh->autoflush(1);
         }
         while ($full_stream && $n < $len) {
-                my $r = sysread($s2, $buf, $len - $n, $n) // croak "read: $!";
+                my $r = sysread($s2, $buf, $len - $n, $n);
                 croak "read EOF after $n/$len bytes" if $r == 0;
                 $n = length($buf);
         }
@@ -256,12 +226,19 @@ sub recv_and_run {
         $n;
 }
 
-sub wq_worker_loop ($$) {
-        my ($self, $bcast2) = @_;
+sub sock_defined { # PublicInbox::DS::post_loop_do CB
+        my ($wqw) = @_;
+        defined($wqw->{sock});
+}
+
+sub wq_worker_loop ($$$) {
+        my ($self, $bcast2, $oldset) = @_;
         my $wqw = PublicInbox::WQWorker->new($self, $self->{-wq_s2});
         PublicInbox::WQWorker->new($self, $bcast2) if $bcast2;
-        PublicInbox::DS->SetPostLoopCallback(sub { $wqw->{sock} });
-        PublicInbox::DS::event_loop();
+        local @PublicInbox::DS::post_loop_do = (\&sock_defined, $wqw);
+        my $sig = delete($self->{wq_sig});
+        $sig->{CHLD} //= \&PublicInbox::DS::enqueue_reap;
+        PublicInbox::DS::event_loop($sig, $oldset);
         PublicInbox::DS->Reset;
 }
 
@@ -272,56 +249,40 @@ sub do_sock_stream { # via wq_io_do, for big requests
 
 sub wq_broadcast {
         my ($self, $sub, @args) = @_;
-        if (my $wkr = $self->{-wq_workers}) {
-                my $buf = ipc_freeze([$sub, @args]);
-                for my $bcast1 (values %$wkr) {
-                        my $sock = $bcast1 // $self->{-wq_s1} // next;
-                        send($sock, $buf, MSG_EOR) // croak "send: $!";
-                        # XXX shouldn't have to deal with EMSGSIZE here...
-                }
-        } else {
-                eval { $self->$sub(@args) };
-                warn "wq_broadcast: $@" if $@;
+        my $wkr = $self->{-wq_workers} or Carp::confess('no -wq_workers');
+        my $buf = ipc_freeze([$sub, @args]);
+        for my $bcast1 (values %$wkr) {
+                my $sock = $bcast1 // $self->{-wq_s1} // next;
+                send($sock, $buf, 0) // croak "send: $!";
+                # XXX shouldn't have to deal with EMSGSIZE here...
         }
 }
 
 sub stream_in_full ($$$) {
         my ($s1, $fds, $buf) = @_;
-        socketpair(my $r, my $w, AF_UNIX, SOCK_STREAM, 0) or
-                croak "socketpair: $!";
+        socketpair(my $r, my $w, AF_UNIX, SOCK_STREAM, 0);
         my $n = $send_cmd->($s1, [ fileno($r) ],
                         ipc_freeze(['do_sock_stream', length($buf)]),
-                        MSG_EOR) // croak "sendmsg: $!";
+                        0) // croak "sendmsg: $!";
         undef $r;
         $n = $send_cmd->($w, $fds, $buf, 0) // croak "sendmsg: $!";
-        while ($n < length($buf)) {
-                my $x = syswrite($w, $buf, length($buf) - $n, $n) //
-                                croak "syswrite: $!";
-                croak "syswrite wrote 0 bytes" if $x == 0;
-                $n += $x;
-        }
+        print $w substr($buf, $n) if $n < length($buf); # need > 2G on Linux
+        close $w; # autodies
 }
 
 sub wq_io_do { # always async
         my ($self, $sub, $ios, @args) = @_;
-        if (my $s1 = $self->{-wq_s1}) { # run in worker
-                my $fds = [ map { fileno($_) } @$ios ];
-                my $buf = ipc_freeze([$sub, @args]);
-                if (length($buf) > $MY_MAX_ARG_STRLEN) {
-                        stream_in_full($s1, $fds, $buf);
-                } else {
-                        my $n = $send_cmd->($s1, $fds, $buf, MSG_EOR);
-                        return if defined($n); # likely
-                        $!{ETOOMANYREFS} and
-                                croak "sendmsg: $! (check RLIMIT_NOFILE)";
-                        $!{EMSGSIZE} ? stream_in_full($s1, $fds, $buf) :
-                                croak("sendmsg: $!");
-                }
+        my $s1 = $self->{-wq_s1} or Carp::confess('no -wq_s1');
+        my $fds = [ map { fileno($_) } @$ios ];
+        my $buf = ipc_freeze([$sub, @args]);
+        if (length($buf) > $MY_MAX_ARG_STRLEN) {
+                stream_in_full($s1, $fds, $buf);
         } else {
-                @$self{0..$#$ios} = @$ios;
-                eval { $self->$sub(@args) };
-                warn "wq_io_do: $@" if $@;
-                delete @$self{0..$#$ios}; # don't close
+                my $n = $send_cmd->($s1, $fds, $buf, 0);
+                return if defined($n); # likely
+                $!{ETOOMANYREFS} and croak "sendmsg: $! (check RLIMIT_NOFILE)";
+                $!{EMSGSIZE} ? stream_in_full($s1, $fds, $buf) :
+                        croak("sendmsg: $!");
         }
 }
 
@@ -339,7 +300,7 @@ sub wq_sync_run {
 sub wq_do {
         my ($self, $sub, @args) = @_;
         if (defined(wantarray)) {
-                pipe(my ($r, $w)) or die "pipe: $!";
+                pipe(my $r, my $w);
                 wq_io_do($self, 'wq_sync_run', [ $w ], wantarray, $sub, @args);
                 undef $w;
                 _wait_return($r, $sub);
@@ -348,18 +309,30 @@ sub wq_do {
         }
 }
 
-sub _wq_worker_start ($$$$) {
-        my ($self, $oldset, $fields, $one) = @_;
+sub prepare_nonblock {
+        ($_[0]->{-wq_s1} // die 'BUG: no {-wq_s1}')->blocking(0);
+        require PublicInbox::WQBlocked;
+}
+
+sub wq_nonblock_do { # always async
+        my ($self, $sub, @args) = @_;
+        my $buf = ipc_freeze([$sub, @args]);
+        if ($self->{wqb}) { # saturated once, assume saturated forever
+                $self->{wqb}->flush_send($buf);
+        } else {
+                $send_cmd->($self->{-wq_s1}, [], $buf, 0) //
+                        ($!{EAGAIN} ? PublicInbox::WQBlocked->new($self, $buf)
+                                        : croak("sendmsg: $!"));
+        }
+}
+
+sub _wq_worker_start {
+        my ($self, $oldset, $fields, $one, @cb_args) = @_;
         my ($bcast1, $bcast2);
-        $one or socketpair($bcast1, $bcast2, AF_UNIX, $SEQPACKET, 0) or
-                                                        die "socketpair: $!";
-        my $seed = rand(0xffffffff);
-        my $pid = fork // die "fork: $!";
+        $one or socketpair($bcast1, $bcast2, AF_UNIX, SOCK_SEQPACKET, 0);
+        my $pid = PublicInbox::DS::do_fork;
         if ($pid == 0) {
-                srand($seed);
-                eval { Net::SSLeay::randomize() };
                 undef $bcast1;
-                eval { PublicInbox::DS->Reset };
                 delete @$self{qw(-wq_s1 -wq_ppid)};
                 $self->{-wq_worker_nr} =
                                 keys %{delete($self->{-wq_workers}) // {}};
@@ -373,24 +346,22 @@ sub _wq_worker_start ($$$$) {
                         local @$self{keys %$fields} = values(%$fields);
                         my $on_destroy = $self->ipc_atfork_child;
                         local @SIG{keys %SIG} = values %SIG;
-                        PublicInbox::DS::sig_setmask($oldset);
-                        wq_worker_loop($self, $bcast2);
+                        wq_worker_loop($self, $bcast2, $oldset);
                 };
                 warn "worker $self->{-wq_ident} PID:$$ died: $@" if $@;
                 undef $end; # trigger exit
         } else {
                 $self->{-wq_workers}->{$pid} = $bcast1;
+                awaitpid($pid, \&ipc_worker_reap, $self, @cb_args);
         }
 }
 
 # starts workqueue workers if Sereal or Storable is installed
 sub wq_workers_start {
-        my ($self, $ident, $nr_workers, $oldset, $fields) = @_;
-        ($send_cmd && $recv_cmd && defined($SEQPACKET)) or return;
+        my ($self, $ident, $nr_workers, $oldset, $fields, @cb_args) = @_;
+        ($send_cmd && $recv_cmd) or return;
         return if $self->{-wq_s1}; # idempotent
-        $self->{-wq_s1} = $self->{-wq_s2} = undef;
-        socketpair($self->{-wq_s1}, $self->{-wq_s2}, AF_UNIX, $SEQPACKET, 0) or
-                die "socketpair: $!";
+        socketpair($self->{-wq_s1}, $self->{-wq_s2},AF_UNIX, SOCK_SEQPACKET, 0);
         $self->ipc_atfork_prepare;
         $nr_workers //= $self->{-wq_nr_workers}; # was set earlier
         my $sigset = $oldset // PublicInbox::DS::block_signals();
@@ -398,17 +369,21 @@ sub wq_workers_start {
         $self->{-wq_ident} = $ident;
         my $one = $nr_workers == 1;
         $self->{-wq_nr_workers} = $nr_workers;
-        _wq_worker_start($self, $sigset, $fields, $one) for (1..$nr_workers);
+        for (1..$nr_workers) {
+                _wq_worker_start($self, $sigset, $fields, $one, @cb_args);
+        }
         PublicInbox::DS::sig_setmask($sigset) unless $oldset;
         $self->{-wq_ppid} = $$;
 }
 
 sub wq_close {
         my ($self) = @_;
+        if (my $wqb = delete $self->{wqb}) {
+                $wqb->enq_close;
+        }
         delete @$self{qw(-wq_s1 -wq_s2)} or return;
-        return if $self->{-reap_async};
-        my @pids = keys %{$self->{-wq_workers}};
-        dwaitpid($_, \&ipc_worker_reap, [ $self ]) for @pids;
+        return if ($self->{-wq_ppid} // -1) != $$;
+        awaitpid($_) for keys %{$self->{-wq_workers}};
 }
 
 sub wq_kill {
@@ -424,23 +399,52 @@ sub DESTROY {
         ipc_worker_stop($self);
 }
 
+# _SC_NPROCESSORS_ONLN = 84 on both Linux glibc and musl,
+# emitted using: $^X devel/sysdefs-list
+my %NPROCESSORS_ONLN = (
+        linux => 84,
+        freebsd => 58,
+        dragonfly => 58,
+        openbsd => 503,
+        netbsd => 1002
+);
+
 sub detect_nproc () {
-        # _SC_NPROCESSORS_ONLN = 84 on both Linux glibc and musl
-        return POSIX::sysconf(84) if $^O eq 'linux';
-        return POSIX::sysconf(58) if $^O eq 'freebsd';
-        # TODO: more OSes
+        my $n = $NPROCESSORS_ONLN{$^O};
+        return POSIX::sysconf($n) if defined $n;
 
-        # getconf(1) is POSIX, but *NPROCESSORS* vars are not
+        # getconf(1) is POSIX, but *NPROCESSORS* vars are not even if
+        # glibc, {Free,Net,Open}BSD all support them.
         for (qw(_NPROCESSORS_ONLN NPROCESSORS_ONLN)) {
                 `getconf $_ 2>/dev/null` =~ /^(\d+)$/ and return $1;
         }
-        for my $nproc (qw(nproc gnproc)) { # GNU coreutils nproc
-                `$nproc 2>/dev/null` =~ /^(\d+)$/ and return $1;
+        # note: GNU nproc(1) checks CPU affinity, which is nice but
+        # isn't remotely portable
+        undef
+}
+
+# SATA storage lags behind what CPUs are capable of, so relying on
+# nproc(1) can be misleading and having extra Xapian shards is a
+# waste of FDs and space.  It can also lead to excessive IO latency
+# and slow things down.  Users on NVME or other fast storage can
+# use the NPROC env or switches in our script/public-inbox-* programs
+# to increase Xapian shards
+our $NPROC_MAX_DEFAULT = 4;
+
+sub nproc_shards ($) {
+        my ($creat_opt) = @_;
+        my $n = $creat_opt->{nproc} if ref($creat_opt) eq 'HASH';
+        $n //= $ENV{NPROC};
+        if (!$n) {
+                # assume 2 cores if not detectable or zero
+                state $NPROC_DETECTED = PublicInbox::IPC::detect_nproc() || 2;
+                $n = $NPROC_DETECTED;
+                $n = $NPROC_MAX_DEFAULT if $n > $NPROC_MAX_DEFAULT;
         }
 
-        # should we bother with `sysctl hw.ncpu`?  Those only give
-        # us total processor count, not online processor count.
-        undef
+        # subtract for the main process and git-fast-import
+        $n -= 1;
+        $n < 1 ? 1 : $n;
 }
 
 1;
diff --git a/lib/PublicInbox/IdxStack.pm b/lib/PublicInbox/IdxStack.pm
index 54d480bd..7681ee6f 100644
--- a/lib/PublicInbox/IdxStack.pm
+++ b/lib/PublicInbox/IdxStack.pm
@@ -1,16 +1,18 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # temporary stack for public-inbox-index
+# FIXME: needs to support multi-hash in the same repo once git itself can
 package PublicInbox::IdxStack;
-use v5.10.1;
-use strict;
+use v5.12;
 use Fcntl qw(:seek);
 use constant PACK_FMT => eval { pack('Q', 1) } ? 'A1QQH*H*' : 'A1IIH*H*';
+use autodie qw(open seek);
+use PublicInbox::IO qw(read_all);
 
 # start off in write-only mode
 sub new {
-        open(my $io, '+>', undef) or die "open: $!";
+        open(my $io, '+>', undef);
         # latest_cmt is still useful when the newest revision is a `d'(elete),
         # otherwise we favor $sync->{latest_cmt} for checkpoints and {quit}
         bless { wr => $io, latest_cmt => $_[1] }, __PACKAGE__
@@ -27,7 +29,7 @@ sub push_rec {
                 $self->{rec_size} = length($rec);
                 $self->{unpack_fmt} = $fmt;
         };
-        print { $self->{wr} } $rec or die "print: $!";
+        print { $self->{wr} } $rec;
         $self->{tot_size} += length($rec);
 }
 
@@ -49,12 +51,8 @@ sub pop_rec {
         my $sz = $self->{rec_size} or return;
         my $rec_pos = $self->{tot_size} -= $sz;
         return if $rec_pos < 0;
-        my $io = $self->{rd};
-        seek($io, $rec_pos, SEEK_SET) or die "seek: $!";
-        my $r = read($io, my $buf, $sz);
-        defined($r) or die "read: $!";
-        $r == $sz or die "read($r != $sz)";
-        unpack($self->{unpack_fmt}, $buf);
+        seek($self->{rd}, $rec_pos, SEEK_SET);
+        unpack($self->{unpack_fmt}, read_all($self->{rd}, $sz));
 }
 
 1;
diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 60ce7b66..ed34d548 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # git fast-import-based ssoma-mda MDA replacement
@@ -6,10 +6,9 @@
 # and public-inbox-watch. Not the WWW or NNTP code which only
 # requires read-only access.
 package PublicInbox::Import;
-use strict;
+use v5.12;
 use parent qw(PublicInbox::Lock);
-use v5.10.1;
-use PublicInbox::Spawn qw(run_die popen_rd);
+use PublicInbox::Spawn qw(run_die run_qx spawn);
 use PublicInbox::MID qw(mids mid2path);
 use PublicInbox::Address;
 use PublicInbox::Smsg;
@@ -18,13 +17,16 @@ use PublicInbox::ContentHash qw(content_digest);
 use PublicInbox::MDA;
 use PublicInbox::Eml;
 use POSIX qw(strftime);
+use autodie qw(socketpair);
+use Carp qw(croak);
+use Socket qw(AF_UNIX SOCK_STREAM);
+use PublicInbox::IO qw(read_all);
 
 sub default_branch () {
         state $default_branch = do {
-                my $r = popen_rd([qw(git config --global init.defaultBranch)],
+                my $h = run_qx([qw(git config --global init.defaultBranch)],
                                  { GIT_CONFIG => undef });
-                chomp(my $h = <$r> // '');
-                close $r;
+                chomp $h;
                 $h eq '' ? 'refs/heads/master' : "refs/heads/$h";
         }
 }
@@ -55,11 +57,10 @@ sub new {
 # idempotent start function
 sub gfi_start {
         my ($self) = @_;
-
-        return ($self->{in}, $self->{out}) if $self->{in};
-
-        my ($in_r, $out_r, $out_w);
-        pipe($out_r, $out_w) or die "pipe failed: $!";
+        my $io = $self->{io};
+        return $io if $io;
+        socketpair($io, my $s2, AF_UNIX, SOCK_STREAM, 0);
+        $io->autoflush(1);
 
         $self->lock_acquire;
         eval {
@@ -72,21 +73,20 @@ sub gfi_start {
                         die "fatal: ls-tree -r -z --name-only $ref: \$?=$?" if $?;
                         $self->{-tree} = { map { $_ => 1 } split(/\0/, $t) };
                 }
-                $in_r = $self->{in} = $git->popen(qw(fast-import
-                                        --quiet --done --date-format=raw),
-                                        undef, { 0 => $out_r });
-                $out_w->autoflush(1);
-                $self->{out} = $out_w;
+                my $gfi = [ 'git', "--git-dir=$git->{git_dir}", qw(fast-import
+                                --quiet --done --date-format=raw) ];
+                my $pid = spawn($gfi, undef, { 0 => $s2, 1 => $s2 });
                 $self->{nchg} = 0;
+                $self->{io} = PublicInbox::IO::attach_pid($io, $pid);
         };
         if ($@) {
                 $self->lock_release;
                 die $@;
         }
-        ($in_r, $out_w);
+        $self->{io};
 }
 
-sub wfail () { die "write to fast-import failed: $!" }
+sub wfail () { croak "write to fast-import failed: $!" }
 
 sub now_raw () { time . ' +0000' }
 
@@ -98,60 +98,43 @@ sub norm_body ($) {
 }
 
 # only used for v1 (ssoma) inboxes
-sub _check_path ($$$$) {
-        my ($r, $w, $tip, $path) = @_;
+sub _check_path ($$$) {
+        my ($io, $tip, $path) = @_;
         return if $tip eq '';
-        print $w "ls $tip $path\n" or wfail;
+        print $io "ls $tip $path\n" or wfail;
         local $/ = "\n";
-        defined(my $info = <$r>) or die "EOF from fast-import: $!";
+        my $info = <$io> // die "EOF from fast-import: $!";
         $info =~ /\Amissing / ? undef : $info;
 }
 
-sub _cat_blob ($$$) {
-        my ($r, $w, $oid) = @_;
-        print $w "cat-blob $oid\n" or wfail;
+sub _cat_blob ($$) {
+        my ($io, $oid) = @_;
+        print $io "cat-blob $oid\n" or wfail;
         local $/ = "\n";
-        my $info = <$r>;
-        defined $info or die "EOF from fast-import / cat-blob: $!";
+        my $info = <$io> // die "EOF from fast-import / cat-blob: $!";
         $info =~ /\A[a-f0-9]{40,} blob ([0-9]+)\n\z/ or return;
-        my $left = $1;
-        my $offset = 0;
-        my $buf = '';
-        my $n;
-        while ($left > 0) {
-                $n = read($r, $buf, $left, $offset);
-                defined($n) or die "read cat-blob failed: $!";
-                $n == 0 and die 'fast-export (cat-blob) died';
-                $left -= $n;
-                $offset += $n;
-        }
-        $n = read($r, my $lf, 1);
-        defined($n) or die "read final byte of cat-blob failed: $!";
-        die "bad read on final byte: <$lf>" if $lf ne "\n";
-
-        # fixup some bugginess in old versions:
-        $buf =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+        my $buf = read_all($io, my $len = $1 + 1);
+        my $lf = chop $buf;
+        croak "bad read on final byte: <$lf>" if $lf ne "\n";
         \$buf;
 }
 
 sub cat_blob {
         my ($self, $oid) = @_;
-        my ($r, $w) = $self->gfi_start;
-        _cat_blob($r, $w, $oid);
+        _cat_blob($self->{io} // return, $oid);
 }
 
 sub check_remove_v1 {
-        my ($r, $w, $tip, $path, $mime) = @_;
+        my ($io, $tip, $path, $mime) = @_;
 
-        my $info = _check_path($r, $w, $tip, $path) or return ('MISSING',undef);
+        my $info = _check_path($io, $tip, $path) or return ('MISSING',undef);
         $info =~ m!\A100644 blob ([a-f0-9]{40,})\t!s or die "not blob: $info";
         my $oid = $1;
-        my $msg = _cat_blob($r, $w, $oid) or die "BUG: cat-blob $1 failed";
-        my $cur = PublicInbox::Eml->new($msg);
-        my $cur_s = $cur->header('Subject');
-        $cur_s = '' unless defined $cur_s;
-        my $cur_m = $mime->header('Subject');
-        $cur_m = '' unless defined $cur_m;
+        my $bref = _cat_blob($io, $oid) or die "BUG: cat-blob $1 failed";
+        PublicInbox::Eml::strip_from($$bref);
+        my $cur = PublicInbox::Eml->new($bref);
+        my $cur_s = $cur->header('Subject') // '';
+        my $cur_m = $mime->header('Subject') // '';
         if ($cur_s ne $cur_m || norm_body($cur) ne norm_body($mime)) {
                 return ('MISMATCH', $cur);
         }
@@ -160,16 +143,15 @@ sub check_remove_v1 {
 
 sub checkpoint {
         my ($self) = @_;
-        return unless $self->{in};
-        print { $self->{out} } "checkpoint\n" or wfail;
+        print { $self->{io} // return } "checkpoint\n" or wfail;
         undef;
 }
 
 sub progress {
         my ($self, $msg) = @_;
-        return unless $self->{in};
-        print { $self->{out} } "progress $msg\n" or wfail;
-        readline($self->{in}) eq "progress $msg\n" or die
+        my $io = $self->{io} or return;
+        print $io "progress $msg\n" or wfail;
+        readline($io) eq "progress $msg\n" or die
                 "progress $msg not received\n";
         undef;
 }
@@ -185,8 +167,8 @@ sub _update_git_info ($$) {
                 my $env = { GIT_INDEX_FILE => $index };
                 run_die([@cmd, qw(read-tree -m -v -i), $self->{ref}], $env);
         }
-        eval { run_die([@cmd, 'update-server-info']) };
         my $ibx = $self->{ibx};
+        eval { run_die([@cmd, 'update-server-info']) } if $ibx;
         if ($ibx && $ibx->version == 1 && -d "$ibx->{inboxdir}/public-inbox" &&
                                 eval { require PublicInbox::SearchIdx }) {
                 eval {
@@ -195,7 +177,10 @@ sub _update_git_info ($$) {
                 };
                 warn "$ibx->{inboxdir} index failed: $@\n" if $@;
         }
-        eval { run_die([@cmd, qw(gc --auto)]) } if $do_gc;
+        if ($do_gc) {
+                my @quiet = (-t STDERR ? () : '-q');
+                eval { run_die([@cmd, qw(gc --auto), @quiet]) }
+        }
 }
 
 sub barrier {
@@ -216,10 +201,9 @@ sub barrier {
 # used for v2
 sub get_mark {
         my ($self, $mark) = @_;
-        die "not active\n" unless $self->{in};
-        my ($r, $w) = $self->gfi_start;
-        print $w "get-mark $mark\n" or wfail;
-        defined(my $oid = <$r>) or die "get-mark failed, need git 2.6.0+\n";
+        my $io = $self->{io} or croak "not active\n";
+        print $io "get-mark $mark\n" or wfail;
+        my $oid = <$io> // die "get-mark failed, need git 2.6.0+\n";
         chomp($oid);
         $oid;
 }
@@ -236,11 +220,11 @@ sub remove {
         my $path_type = $self->{path_type};
         my ($path, $err, $cur, $blob);
 
-        my ($r, $w) = $self->gfi_start;
+        my $io = gfi_start($self);
         my $tip = $self->{tip};
         if ($path_type eq '2/38') {
                 $path = mid2path(v1_mid0($mime));
-                ($err, $cur) = check_remove_v1($r, $w, $tip, $path, $mime);
+                ($err, $cur) = check_remove_v1($io, $tip, $path, $mime);
                 return ($err, $cur) if $err;
         } else {
                 my $sref;
@@ -252,7 +236,7 @@ sub remove {
                 }
                 my $len = length($$sref);
                 $blob = $self->{mark}++;
-                print $w "blob\nmark :$blob\ndata $len\n",
+                print $io "blob\nmark :$blob\ndata $len\n",
                         $$sref, "\n" or wfail;
         }
 
@@ -260,22 +244,22 @@ sub remove {
         my $commit = $self->{mark}++;
         my $parent = $tip =~ /\A:/ ? $tip : undef;
         unless ($parent) {
-                print $w "reset $ref\n" or wfail;
+                print $io "reset $ref\n" or wfail;
         }
         my $ident = $self->{ident};
         my $now = now_raw();
         $msg //= 'rm';
         my $len = length($msg) + 1;
-        print $w "commit $ref\nmark :$commit\n",
+        print $io "commit $ref\nmark :$commit\n",
                 "author $ident $now\n",
                 "committer $ident $now\n",
                 "data $len\n$msg\n\n",
                 'from ', ($parent ? $parent : $tip), "\n" or wfail;
         if (defined $path) {
-                print $w "D $path\n\n" or wfail;
+                print $io "D $path\n\n" or wfail;
         } else {
-                clean_tree_v2($self, $w, 'd');
-                print $w "M 100644 :$blob d\n\n" or wfail;
+                clean_tree_v2($self, $io, 'd');
+                print $io "M 100644 :$blob d\n\n" or wfail;
         }
         $self->{nchg}++;
         (($self->{tip} = ":$commit"), $cur);
@@ -337,11 +321,38 @@ sub extract_cmt_info ($;$) {
 # kill potentially confusing/misleading headers
 our @UNWANTED_HEADERS = (qw(Bytes Lines Content-Length),
                         qw(Status X-Status));
+our $DROP_UNIQUE_UNSUB;
 sub drop_unwanted_headers ($) {
         my ($eml) = @_;
         for (@UNWANTED_HEADERS, @PublicInbox::MDA::BAD_HEADERS) {
                 $eml->header_set($_);
         }
+
+        # We don't want public-inbox readers to be able to unsubcribe the
+        # address which does archiving.  WARNING: this breaks DKIM if the
+        # mailing list sender follows RFC 8058, section 4; but breaking DKIM
+        # (or have senders ignore RFC 8058 sec. 4) is preferable to having
+        # saboteurs unsubscribing independent archivists:
+        if ($DROP_UNIQUE_UNSUB && grep(/\AList-Unsubscribe=One-Click\z/,
+                                $eml->header_raw('List-Unsubscribe-Post'))) {
+                for (qw(List-Unsubscribe-Post List-Unsubscribe)) {
+                        $eml->header_set($_)
+                }
+        }
+}
+
+sub load_config ($;$) {
+        my ($cfg, $do_exit) = @_;
+        my $v = $cfg->{lc 'publicinboxImport.dropUniqueUnsubscribe'};
+        if (defined $v) {
+                $DROP_UNIQUE_UNSUB = $cfg->git_bool($v) // do {
+                        warn <<EOM;
+E: publicinboxImport.dropUniqueUnsubscribe=$v in $cfg->{-f} is not boolean
+EOM
+                        $do_exit //= \&CORE::exit;
+                        $do_exit->(78); # EX_CONFIG
+                };
+        }
 }
 
 # used by V2Writable, too
@@ -365,11 +376,11 @@ sub v1_mid0 ($) {
         $mids->[0];
 }
 sub clean_tree_v2 ($$$) {
-        my ($self, $w, $keep) = @_;
+        my ($self, $io, $keep) = @_;
         my $tree = $self->{-tree} or return; #v2 only
         delete $tree->{$keep};
         foreach (keys %$tree) {
-                print $w "D $_\n" or wfail;
+                print $io "D $_\n" or wfail;
         }
         %$tree = ($keep => 1);
 }
@@ -388,10 +399,10 @@ sub add {
                 $path = 'm';
         }
 
-        my ($r, $w) = $self->gfi_start;
+        my $io = gfi_start($self);
         my $tip = $self->{tip};
         if ($path_type eq '2/38') {
-                _check_path($r, $w, $tip, $path) and return;
+                _check_path($io, $tip, $path) and return;
         }
 
         drop_unwanted_headers($mime);
@@ -405,8 +416,7 @@ sub add {
         my $raw_email = $mime->{-public_inbox_raw} // $mime->as_string;
         my $n = length($raw_email);
         $self->{bytes_added} += $n;
-        print $w "blob\nmark :$blob\ndata ", $n, "\n" or wfail;
-        print $w $raw_email, "\n" or wfail;
+        print $io "blob\nmark :$blob\ndata $n\n", $raw_email, "\n" or wfail;
 
         # v2: we need this for Xapian
         if ($smsg) {
@@ -433,19 +443,19 @@ sub add {
         my $parent = $tip =~ /\A:/ ? $tip : undef;
 
         unless ($parent) {
-                print $w "reset $ref\n" or wfail;
+                print $io "reset $ref\n" or wfail;
         }
 
-        print $w "commit $ref\nmark :$commit\n",
+        print $io "commit $ref\nmark :$commit\n",
                 "author $author $at\n",
-                "committer $self->{ident} $ct\n" or wfail;
-        print $w "data ", (length($subject) + 1), "\n",
+                "committer $self->{ident} $ct\n",
+                "data ", (length($subject) + 1), "\n",
                 $subject, "\n\n" or wfail;
         if ($tip ne '') {
-                print $w 'from ', ($parent ? $parent : $tip), "\n" or wfail;
+                print $io 'from ', ($parent ? $parent : $tip), "\n" or wfail;
         }
-        clean_tree_v2($self, $w, $path);
-        print $w "M 100644 :$blob $path\n\n" or wfail;
+        clean_tree_v2($self, $io, $path);
+        print $io "M 100644 :$blob $path\n\n" or wfail;
         $self->{nchg}++;
         $self->{tip} = ":$commit";
 }
@@ -461,32 +471,37 @@ my @INIT_FILES = ('HEAD' => undef, # filled in at runtime
 EOC
 
 sub init_bare {
-        my ($dir, $head) = @_; # or self
+        my ($dir, $head, $fmt) = @_; # or self
         $dir = $dir->{git}->{git_dir} if ref($dir);
         require File::Path;
-        File::Path::mkpath([ map { "$dir/$_" } qw(objects/info refs/heads) ]);
+        File::Path::make_path(map { $dir.$_ } qw(/objects/info /refs/heads));
         $INIT_FILES[1] //= 'ref: '.default_branch."\n";
         my @fn_contents = @INIT_FILES;
         $fn_contents[1] = "ref: refs/heads/$head\n" if defined $head;
+        $fn_contents[3] = <<EOM if defined($fmt) && $fmt ne 'sha1';
+[core]
+        repositoryFormatVersion = 1
+        filemode = true
+        bare = true
+[extensions]
+        objectFormat = $fmt
+EOM
         while (my ($fn, $contents) = splice(@fn_contents, 0, 2)) {
                 my $f = $dir.'/'.$fn;
                 next if -f $f;
-                open my $fh, '>', $f or die "open $f: $!";
-                print $fh $contents or die "print $f: $!";
-                close $fh or die "close $f: $!";
+                PublicInbox::IO::write_file '>', $f, $contents;
         }
 }
 
 # true if locked and active
-sub active { !!$_[0]->{out} }
+sub active { !!$_[0]->{io} }
 
 sub done {
         my ($self) = @_;
-        my $w = delete $self->{out} or return;
+        my $io = delete $self->{io} or return;
         eval {
-                my $r = delete $self->{in} or die 'BUG: missing {in} when done';
-                print $w "done\n" or wfail;
-                close $r or die "fast-import failed: $?"; # ProcessPipe::CLOSE
+                print $io "done\n" or wfail;
+                $io->close or croak "close fast-import \$?=$?"; # reaps
         };
         my $wait_err = $@;
         my $nchg = delete $self->{nchg};
@@ -499,13 +514,7 @@ sub done {
         die $wait_err if $wait_err;
 }
 
-sub atfork_child {
-        my ($self) = @_;
-        foreach my $f (qw(in out)) {
-                next unless defined($self->{$f});
-                close $self->{$f} or die "failed to close import[$f]: $!\n";
-        }
-}
+sub atfork_child { (delete($_[0]->{io}) // return)->close }
 
 sub digest2mid ($$;$) {
         my ($dig, $hdr, $fallback_time) = @_;
@@ -558,7 +567,7 @@ sub replace_oids {
         my $git = $self->{git};
         my @export = (qw(fast-export --no-data --use-done-feature), $old);
         my $rd = $git->popen(@export);
-        my ($r, $w) = $self->gfi_start;
+        my $io = gfi_start($self);
         my @buf;
         my $nreplace = 0;
         my @oids;
@@ -569,17 +578,14 @@ sub replace_oids {
                         push @buf, "reset $tmp\n";
                 } elsif (/^commit (?:.+)/) {
                         if (@buf) {
-                                print $w @buf or wfail;
+                                print $io @buf or wfail;
                                 @buf = ();
                         }
                         push @buf, "commit $tmp\n";
                 } elsif (/^data ([0-9]+)/) {
                         # only commit message, so $len is small:
-                        my $len = $1; # + 1 for trailing "\n"
                         push @buf, $_;
-                        my $n = read($rd, my $buf, $len) or die "read: $!";
-                        $len == $n or die "short read ($n < $len)";
-                        push @buf, $buf;
+                        push @buf, read_all($rd, my $len = $1);
                 } elsif (/^M 100644 ([a-f0-9]+) (\w+)/) {
                         my ($oid, $path) = ($1, $2);
                         $tree->{$path} = 1;
@@ -606,7 +612,7 @@ sub replace_oids {
                                 rewrite_commit($self, \@oids, \@buf, $mime);
                                 $nreplace++;
                         }
-                        print $w @buf, "\n" or wfail;
+                        print $io @buf, "\n" or wfail;
                         @buf = ();
                 } elsif ($_ eq "done\n") {
                         $done = 1;
@@ -617,9 +623,9 @@ sub replace_oids {
                         push @buf, $_;
                 }
         }
-        close $rd or die "close fast-export failed: $?";
+        $rd->close or die "E: git @export (\$?=$?)";
         if (@buf) {
-                print $w @buf or wfail;
+                print $io @buf or wfail;
         }
         die 'done\n not seen from fast-export' unless $done;
         chomp(my $cmt = $self->get_mark(":$mark")) if $nreplace;
diff --git a/lib/PublicInbox/In2Tie.pm b/lib/PublicInbox/In2Tie.pm
index ffe26a44..3689432b 100644
--- a/lib/PublicInbox/In2Tie.pm
+++ b/lib/PublicInbox/In2Tie.pm
@@ -1,10 +1,10 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # used to ensure PublicInbox::DS can call fileno() as a function
 # on Linux::Inotify2 objects
 package PublicInbox::In2Tie;
-use strict;
+use v5.12;
 use Symbol qw(gensym);
 
 sub io {
diff --git a/lib/PublicInbox/In3Event.pm b/lib/PublicInbox/In3Event.pm
new file mode 100644
index 00000000..f93dc0da
--- /dev/null
+++ b/lib/PublicInbox/In3Event.pm
@@ -0,0 +1,24 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# duck-type compatible with Linux::Inotify2::Event for pure Perl
+# PublicInbox::Inotify3 w/o callback support
+package PublicInbox::In3Event;
+use v5.12;
+
+sub w { $_[0]->[2] } # PublicInbox::In3Watch
+sub mask { $_[0]->[0] }
+sub name { $_[0]->[1] }
+
+sub fullname {
+        my ($name, $wname) = ($_[0]->[1], $_[0]->[2]->name);
+        length($name) ? "$wname/$name" : $wname;
+}
+
+my $buf = '';
+while (my ($sym, $mask) = each %PublicInbox::Inotify3::events) {
+        $buf .= "sub $sym { \$_[0]->[0] & $mask }\n";
+}
+eval $buf;
+
+1;
diff --git a/lib/PublicInbox/In3Watch.pm b/lib/PublicInbox/In3Watch.pm
new file mode 100644
index 00000000..bdb91869
--- /dev/null
+++ b/lib/PublicInbox/In3Watch.pm
@@ -0,0 +1,20 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# duck-type compatible with Linux::Inotify2::Watch for pure Perl
+# PublicInbox::Inotify3 for our needs, only
+package PublicInbox::In3Watch;
+use v5.12;
+
+sub mask { $_[0]->[1] }
+sub name { $_[0]->[2] }
+
+sub cancel {
+        my ($self) = @_;
+        my ($wd, $in3) = @$self[0, 3];
+        $in3 or return 1; # already canceled
+        pop @$self;
+        $in3->rm_watch($wd);
+}
+
+1;
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 1579d500..dd689221 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Represents a public-inbox (which may have multiple mailing addresses)
@@ -10,32 +10,22 @@ use PublicInbox::MID qw(mid2path);
 use PublicInbox::Eml;
 use List::Util qw(max);
 use Carp qw(croak);
+use PublicInbox::Compat qw(uniqstr);
 
-# returns true if further checking is required
+# in case DBs get replaced (Xapcmd does it for v1)
 sub check_inodes ($) {
         for (qw(over mm)) { $_[0]->{$_}->check_inodes if $_[0]->{$_} }
 }
 
+# search/over/mm hold onto FDs and description+cloneurl may get updated.
+# creating long-lived allocations in the same phase as short-lived
+# allocations also leads to fragmentation, so we don't want some stuff
+# living too long.
 sub do_cleanup {
         my ($ibx) = @_;
-        my $live;
-        if (defined $ibx->{git}) {
-                $live = $ibx->isa(__PACKAGE__) ? $ibx->{git}->cleanup(1)
-                                        : $ibx->{git}->cleanup_if_unlinked;
-                delete($ibx->{git}) unless $live;
-        }
-        if ($live) {
-                check_inodes($ibx);
-        } else {
-                delete(@$ibx{qw(over mm description cloneurl
-                                -imap_url -nntp_url)});
-        }
-        my $srch = $ibx->{search} // $ibx;
+        my ($srch) = delete @$ibx{qw(search over mm description cloneurl)};
+        $srch //= $ibx; # extsearch
         delete @$srch{qw(xdb qp)};
-        for my $git (@{$ibx->{-repo_objs} // []}) {
-                $live = 1 if $git->cleanup(1);
-        }
-        PublicInbox::DS::add_uniq_timer($ibx+0, 5, \&do_cleanup, $ibx) if $live;
 }
 
 sub _cleanup_later ($) {
@@ -54,8 +44,8 @@ sub _set_limiter ($$$) {
                 my $val = $self->{$mkey} or return;
                 my $lim;
                 if ($val =~ /\A[0-9]+\z/) {
-                        require PublicInbox::Qspawn;
-                        $lim = PublicInbox::Qspawn::Limiter->new($val);
+                        require PublicInbox::Limiter;
+                        $lim = PublicInbox::Limiter->new($val);
                 } elsif ($val =~ /\A[a-z][a-z0-9]*\z/) {
                         $lim = $pi_cfg->limiter($val);
                         warn "$mkey limiter=$val not found\n" if !$lim;
@@ -80,12 +70,8 @@ sub new {
                 delete $opts->{feedmax};
         }
         # allow any combination of multi-line or comma-delimited hide entries
-        my $hide = {};
-        if (defined(my $h = $opts->{hide})) {
-                foreach my $v (@$h) {
-                        $hide->{$_} = 1 foreach (split(/\s*,\s*/, $v));
-                }
-                $opts->{-hide} = $hide;
+        for $v (@{delete($opts->{hide}) // []}) {
+                $opts->{-'hide_'.$_} = 1 for split(/\s*,\s*/, $v);
         }
         bless $opts, $class;
 }
@@ -115,7 +101,6 @@ sub git {
                 my $g = PublicInbox::Git->new($git_dir);
                 my $lim = $self->{-httpbackend_limiter};
                 $g->{-httpbackend_limiter} = $lim if $lim;
-                _cleanup_later($self);
                 $g;
         };
 }
@@ -181,33 +166,18 @@ sub over {
         } // ($req ? croak("E: $@") : undef);
 }
 
-sub try_cat {
-        my ($path) = @_;
-        open(my $fh, '<', $path) or return '';
-        local $/;
-        <$fh> // '';
-}
-
-sub cat_desc ($) {
-        my $desc = try_cat($_[0]);
-        local $/ = "\n";
-        chomp $desc;
-        utf8::decode($desc);
-        $desc =~ s/\s+/ /smg;
-        $desc eq '' ? undef : $desc;
-}
-
 sub description {
         my ($self) = @_;
-        ($self->{description} //= cat_desc("$self->{inboxdir}/description")) //
+        ($self->{description} //=
+                PublicInbox::Git::cat_desc("$self->{inboxdir}/description")) //
                 '($INBOX_DIR/description missing)';
 }
 
 sub cloneurl {
         my ($self) = @_;
         $self->{cloneurl} // do {
-                my $s = try_cat("$self->{inboxdir}/cloneurl");
-                my @urls = split(/\s+/s, $s);
+                my @urls = split(/\s+/s,
+                        PublicInbox::IO::try_cat "$self->{inboxdir}/cloneurl");
                 scalar(@urls) ? ($self->{cloneurl} = \@urls) : undef;
         } // [];
 }
@@ -220,7 +190,8 @@ sub base_url {
                 $url .= '/' if $url !~ m!/\z!;
                 return $url .= $self->{name} . '/';
         }
-        # called from a non-PSGI environment (e.g. NNTP/POP3):
+        # called from a non-PSGI environment or cross-inbox environment
+        # where multiple inboxes can have different domains
         my $url = $self->{url} // return undef;
         $url = $url->[0] // return undef;
         # expand protocol-relative URLs to HTTPS if we're
@@ -230,8 +201,9 @@ sub base_url {
         $url;
 }
 
+# imapserver, nntpserver configs are used here:
 sub _x_url ($$$) {
-        my ($self, $x, $ctx) = @_; # $x is "nntp" or "imap"
+        my ($self, $x, $ctx) = @_; # $x is "imap" or "nntp"
         # no checking for nntp_usable here, we can point entirely
         # to non-local servers or users run by a different user
         my $ns = $self->{"${x}server"} //
@@ -253,7 +225,7 @@ sub _x_url ($$$) {
                                 if ($group) {
                                         $u .= '/' if $u !~ m!/\z!;
                                         $u .= $group;
-                                } else { # n.b. IMAP uses "newsgroup"
+                                } else { # n.b. IMAP and POP3 use "newsgroup"
                                         warn <<EOM;
 publicinbox.$self->{name}.${x}mirror=$_ missing newsgroup name
 EOM
@@ -263,18 +235,36 @@ EOM
                         # nntp://news.example.com/alt.example
                         push @m, $u;
                 }
-
-                # List::Util::uniq requires Perl 5.26+, maybe we
-                # can use it by 2030 or so
-                my %seen;
-                @urls = grep { !$seen{$_}++ } (@urls, @m);
+                @urls = uniqstr @urls, @m;
         }
         \@urls;
 }
 
 # my ($self, $ctx) = @_;
-sub nntp_url { $_[0]->{-nntp_url} //= _x_url($_[0], 'nntp', $_[1]) }
 sub imap_url { $_[0]->{-imap_url} //= _x_url($_[0], 'imap', $_[1]) }
+sub nntp_url { $_[0]->{-nntp_url} //= _x_url($_[0], 'nntp', $_[1]) }
+
+sub pop3_url {
+        my ($self, $ctx) = @_;
+        $self->{-pop3_url} //= do {
+                my $ps = $self->{'pop3server'} //
+                       $ctx->{www}->{pi_cfg}->get_all('publicinbox.pop3server');
+                my $group = $self->{newsgroup};
+                my @urls;
+                ($ps && $group) and
+                        @urls = map { m!\Apop3?s?://! ? $_ : "pop3://$_" } @$ps;
+                if (my $mi = $self->{'pop3mirror'}) {
+                        my @m = map { m!\Apop3?s?://! ? $_ : "pop3://$_" } @$mi;
+                        @urls = uniqstr @urls, @m;
+                }
+                my $n = 0;
+                for (@urls) { $n += s!/+\z!! }
+                warn <<EOM if $n;
+W: pop3server and/or pop3mirror URLs should not end with trailing slash `/'
+EOM
+                \@urls;
+        }
+}
 
 sub nntp_usable {
         my ($self) = @_;
@@ -327,11 +317,6 @@ sub msg_by_mid ($$) {
         $smsg ? msg_by_smsg($self, $smsg) : msg_by_path($self, mid2path($mid));
 }
 
-sub recent {
-        my ($self, $opts, $after, $before) = @_;
-        $self->over->recent($opts, $after, $before);
-}
-
 sub modified {
         my ($self) = @_;
         if (my $over = $self->over) {
@@ -373,7 +358,7 @@ sub unsubscribe_unlock {
 # called by inotify
 sub on_unlock {
         my ($self) = @_;
-        check_inodes($self);
+        check_inodes($self); # DB files may be replaced while holding lock
         my $subs = $self->{unlock_subs} or return;
         for my $obj (values %$subs) {
                 eval { $obj->on_inbox_unlock($self) };
@@ -385,6 +370,16 @@ sub uidvalidity { $_[0]->{uidvalidity} //= eval { $_[0]->mm->created_at } }
 
 sub eidx_key { $_[0]->{newsgroup} // $_[0]->{inboxdir} }
 
+# only used by NNTP, so we need ->mm anyways
+sub art_min { $_[0]->{-art_min} //= eval { $_[0]->mm(1)->min } }
+
+# used by IMAP, too, which tries to avoid ->mm (but ->{mm} is likely
+# faster since it's smaller iff available)
+sub art_max {
+        $_[0]->{-art_max} //= eval { $_[0]->{mm}->max } //
+                                eval { $_[0]->over(1)->max };
+}
+
 sub mailboxid { # rfc 8474, 8620, 8621
         my ($self, $imap_slice) = @_;
         my $pfx = defined($imap_slice) ? $self->{newsgroup} : $self->{name};
@@ -397,4 +392,6 @@ sub mailboxid { # rfc 8474, 8620, 8621
                 sprintf('-%x', uidvalidity($self) // 0)
 }
 
+sub thing_type { 'public inbox' }
+
 1;
diff --git a/lib/PublicInbox/InboxIdle.pm b/lib/PublicInbox/InboxIdle.pm
index 2781b3e1..3c4d4a68 100644
--- a/lib/PublicInbox/InboxIdle.pm
+++ b/lib/PublicInbox/InboxIdle.pm
@@ -1,18 +1,18 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # fields:
 # inot: Linux::Inotify2-like object
 # pathmap => { inboxdir => [ ibx, watch1, watch2, watch3... ] } mapping
 package PublicInbox::InboxIdle;
-use strict;
+use v5.12;
 use parent qw(PublicInbox::DS);
 use PublicInbox::Syscall qw(EPOLLIN);
 my $IN_MODIFY = 0x02; # match Linux inotify
 my $ino_cls;
-if ($^O eq 'linux' && eval { require Linux::Inotify2; 1 }) {
-        $IN_MODIFY = Linux::Inotify2::IN_MODIFY();
-        $ino_cls = 'Linux::Inotify2';
+if ($^O eq 'linux' && eval { require PublicInbox::Inotify }) {
+        $IN_MODIFY = PublicInbox::Inotify::IN_MODIFY();
+        $ino_cls = 'PublicInbox::Inotify';
 } elsif (eval { require PublicInbox::KQNotify }) {
         $IN_MODIFY = PublicInbox::KQNotify::NOTE_WRITE();
         $ino_cls = 'PublicInbox::KQNotify';
@@ -30,11 +30,11 @@ sub in2_arm ($$) { # PublicInbox::Config::each_inbox callback
         my $old_ibx = $cur->[0];
         $cur->[0] = $ibx;
         if ($old_ibx) {
-                $ibx->{unlock_subs} and
-                        die "BUG: $dir->{unlock_subs} should not exist";
+                my $u = $ibx->{unlock_subs};
                 $ibx->{unlock_subs} = $old_ibx->{unlock_subs};
+                %{$ibx->{unlock_subs}} = (%$u, %{$ibx->{unlock_subs}}) if $u;
 
-                # Linux::Inotify2::Watch::name matches if watches are the
+                # *::Inotify*::Watch::name matches if watches are the
                 # same, no point in replacing a watch of the same name
                 if ($cur->[1]->name eq $lock) {
                         $self->{on_unlock}->{$lock} = $ibx;
@@ -48,11 +48,9 @@ sub in2_arm ($$) { # PublicInbox::Config::each_inbox callback
                 $self->{on_unlock}->{$w->name} = $ibx;
         } else {
                 warn "E: ".ref($inot)."->watch($lock, IN_MODIFY) failed: $!\n";
-                if ($!{ENOSPC} && $^O eq 'linux') {
-                        warn <<"";
-I: consider increasing /proc/sys/fs/inotify/max_user_watches
+                warn <<"" if $!{ENOSPC} && $^O eq 'linux';
+# consider increasing /proc/sys/fs/inotify/max_user_watches
 
-                }
         }
 
         # TODO: detect deleted packs (and possibly other files)
@@ -89,7 +87,7 @@ sub new {
 sub event_step {
         my ($self) = @_;
         eval {
-                my @events = $self->{inot}->read; # Linux::Inotify2::read
+                my @events = $self->{inot}->read; # PublicInbox::Inotify3::read
                 my $on_unlock = $self->{on_unlock};
                 for my $ev (@events) {
                         my $fn = $ev->fullname // next; # cancelled
diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm
index 17dfbe18..8e95cb28 100644
--- a/lib/PublicInbox/InboxWritable.pm
+++ b/lib/PublicInbox/InboxWritable.pm
@@ -1,25 +1,18 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Extends read-only Inbox for writing
 package PublicInbox::InboxWritable;
 use strict;
 use v5.10.1;
-use parent qw(PublicInbox::Inbox Exporter);
+use parent qw(PublicInbox::Inbox PublicInbox::Umask Exporter);
 use PublicInbox::Import;
+use PublicInbox::IO qw(read_all);
 use PublicInbox::Filter::Base qw(REJECT);
 use Errno qw(ENOENT);
 our @EXPORT_OK = qw(eml_from_path);
 use Fcntl qw(O_RDONLY O_NONBLOCK);
 
-use constant {
-        PERM_UMASK => 0,
-        OLD_PERM_GROUP => 1,
-        OLD_PERM_EVERYBODY => 2,
-        PERM_GROUP => 0660,
-        PERM_EVERYBODY => 0664,
-};
-
 sub new {
         my ($class, $ibx, $creat_opt) = @_;
         return $ibx if ref($ibx) eq $class;
@@ -122,9 +115,8 @@ sub filter {
 sub eml_from_path ($) {
         my ($path) = @_;
         if (sysopen(my $fh, $path, O_RDONLY|O_NONBLOCK)) {
-                return unless -f $fh; # no FIFOs or directories
-                my $str = do { local $/; <$fh> } or return;
-                PublicInbox::Eml->new(\$str);
+                return unless -f $fh && -s _; # no FIFOs or directories
+                PublicInbox::Eml->new(\(my $str = read_all($fh, -s _)));
         } else { # ENOENT is common with Maildir
                 warn "failed to open $path: $!\n" if $! != ENOENT;
                 undef;
@@ -176,64 +168,6 @@ sub import_mbox {
         $im->done;
 }
 
-sub _read_git_config_perm {
-        my ($self) = @_;
-        chomp(my $perm = $self->git->qx('config', 'core.sharedRepository'));
-        $perm;
-}
-
-sub _git_config_perm {
-        my $self = shift;
-        my $perm = scalar @_ ? $_[0] : _read_git_config_perm($self);
-        return PERM_UMASK if (!defined($perm) || $perm eq '');
-        return PERM_UMASK if ($perm eq 'umask');
-        return PERM_GROUP if ($perm eq 'group');
-        if ($perm =~ /\A(?:all|world|everybody)\z/) {
-                return PERM_EVERYBODY;
-        }
-        return PERM_GROUP if ($perm =~ /\A(?:true|yes|on|1)\z/);
-        return PERM_UMASK if ($perm =~ /\A(?:false|no|off|0)\z/);
-
-        my $i = oct($perm);
-        return PERM_UMASK if ($i == PERM_UMASK);
-        return PERM_GROUP if ($i == OLD_PERM_GROUP);
-        return PERM_EVERYBODY if ($i == OLD_PERM_EVERYBODY);
-
-        if (($i & 0600) != 0600) {
-                die "core.sharedRepository mode invalid: ".
-                    sprintf('%.3o', $i) . "\nOwner must have permissions\n";
-        }
-        ($i & 0666);
-}
-
-sub _umask_for {
-        my ($perm) = @_; # _git_config_perm return value
-        my $rv = $perm;
-        return umask if $rv == 0;
-
-        # set +x bit if +r or +w were set
-        $rv |= 0100 if ($rv & 0600);
-        $rv |= 0010 if ($rv & 0060);
-        $rv |= 0001 if ($rv & 0006);
-        (~$rv & 0777);
-}
-
-sub with_umask {
-        my ($self, $cb, @arg) = @_;
-        my $old = umask($self->{umask} //= umask_prepare($self));
-        my $rv = eval { $cb->(@arg) };
-        my $err = $@;
-        umask $old;
-        die $err if $err;
-        $rv;
-}
-
-sub umask_prepare {
-        my ($self) = @_;
-        my $perm = _git_config_perm($self);
-        _umask_for($perm);
-}
-
 sub cleanup ($) {
         delete @{$_[0]}{qw(over mm git search)};
 }
@@ -245,4 +179,31 @@ sub git_dir_latest {
                 "$self->{inboxdir}/git/$$max.git" : undef;
 }
 
+# for unconfigured inboxes
+sub detect_indexlevel ($) {
+        my ($ibx) = @_;
+
+        my $over = $ibx->over;
+        my $srch = $ibx->search;
+        delete @$ibx{qw(over search)}; # don't leave open FDs lying around
+
+        # brand new or never before indexed inboxes default to full
+        return 'full' unless $over;
+        my $l = 'basic';
+        return $l unless $srch;
+        if (my $xdb = $srch->xdb) {
+                $l = 'full';
+                my $m = $xdb->get_metadata('indexlevel');
+                if ($m eq 'medium') {
+                        $l = $m;
+                } elsif ($m ne '') {
+                        warn <<"";
+$ibx->{inboxdir} has unexpected indexlevel in Xapian: $m
+
+                }
+                $ibx->{-skip_docdata} = 1 if $xdb->get_metadata('skip_docdata');
+        }
+        $l;
+}
+
 1;
diff --git a/lib/PublicInbox/Inotify.pm b/lib/PublicInbox/Inotify.pm
new file mode 100644
index 00000000..c4f1ae84
--- /dev/null
+++ b/lib/PublicInbox/Inotify.pm
@@ -0,0 +1,47 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# wrap Linux::Inotify2 XS module, support pure Perl via `syscall' someday
+package PublicInbox::Inotify;
+use v5.12;
+our @ISA;
+BEGIN { # prefer pure Perl since it works out-of-the-box
+        my $isa;
+        for my $m (qw(PublicInbox::Inotify3 Linux::Inotify2)) {
+                eval "require $m";
+                next if $@;
+                $isa = $m;
+        }
+        if ($isa) {
+                push @ISA, $isa;
+                my $buf = '';
+                for (qw(IN_MOVED_TO IN_CREATE IN_DELETE IN_DELETE_SELF
+                                IN_MOVE_SELF IN_MOVED_FROM IN_MODIFY)) {
+                        $buf .= "*$_ = \\&PublicInbox::Inotify3::$_;\n";
+                }
+                eval $buf;
+                die $@ if $@;
+        } else {
+                die <<EOM;
+W: inotify syscall numbers unknown on your platform and
+W: Linux::Inotify2 missing: $@
+W: public-inbox hackers welcome the plain-text output of ./devel/sysdefs-list
+W: at meta\@public-inbox.org
+EOM
+        }
+};
+
+sub new {
+        $_[0]->SUPER::new // do {
+                my $msg = $!{EMFILE} ? <<EOM : "$_[0]->new: $!\n";
+inotify_init/inotify_init1: $!
+You may need to raise the `fs.inotify.max_user_instances' sysctl limit.
+Consult your OS documentation and/or sysctl(8) + sysctl.conf(5) manpages.
+EOM
+                $msg =~ s/^/E: /smg;
+                require Carp;
+                Carp::croak($msg);
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/Inotify3.pm b/lib/PublicInbox/Inotify3.pm
new file mode 100644
index 00000000..4f337a7a
--- /dev/null
+++ b/lib/PublicInbox/Inotify3.pm
@@ -0,0 +1,115 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Implements most Linux::Inotify2 functionality we need in pure Perl
+# Anonymous sub support isn't supported since it's expensive in the
+# best case and likely leaky in older Perls (e.g. 5.16.3)
+package PublicInbox::Inotify3;
+use v5.12;
+use autodie qw(open);
+use PublicInbox::Syscall ();
+use Carp;
+use Scalar::Util ();
+
+# this fails if undefined no unsupported platforms
+use constant $PublicInbox::Syscall::INOTIFY;
+our %events;
+
+# extracted from devel/sysdefs-list output, these should be arch-independent
+BEGIN {
+%events = (
+        IN_ACCESS => 0x1,
+        IN_ALL_EVENTS => 0xfff,
+        IN_ATTRIB => 0x4,
+        IN_CLOSE => 0x18,
+        IN_CLOSE_NOWRITE => 0x10,
+        IN_CLOSE_WRITE => 0x8,
+        IN_CREATE => 0x100,
+        IN_DELETE => 0x200,
+        IN_DELETE_SELF => 0x400,
+        IN_DONT_FOLLOW => 0x2000000,
+        IN_EXCL_UNLINK => 0x4000000,
+        IN_IGNORED => 0x8000,
+        IN_ISDIR => 0x40000000,
+        IN_MASK_ADD => 0x20000000,
+        IN_MODIFY => 0x2,
+        IN_MOVE => 0xc0,
+        IN_MOVED_FROM => 0x40,
+        IN_MOVED_TO => 0x80,
+        IN_MOVE_SELF => 0x800,
+        IN_ONESHOT => 0x80000000,
+        IN_ONLYDIR => 0x1000000,
+        IN_OPEN => 0x20,
+        IN_Q_OVERFLOW => 0x4000,
+        IN_UNMOUNT => 0x2000,
+);
+} # /BEGIN
+use constant \%events;
+require PublicInbox::In3Event; # uses %events
+require PublicInbox::In3Watch; # uses SYS_inotify_rm_watch
+
+use constant autocancel =>
+        (IN_IGNORED|IN_UNMOUNT|IN_ONESHOT|IN_DELETE_SELF);
+
+sub new {
+        open my $fh, '+<&=', syscall(SYS_inotify_init1, IN_CLOEXEC);
+        bless { fh => $fh }, __PACKAGE__;
+}
+
+sub read {
+        my ($self) = @_;
+        my (@ret, $wd, $mask, $len, $name, $size, $buf);
+        my $r = sysread($self->{fh}, my $rbuf, 8192);
+        if ($r) {
+                while ($r) {
+                        ($wd, $mask, undef, $len) = unpack('lLLL', $rbuf);
+                        $size = 16 + $len; # 16: sizeof(struct inotify_event)
+                        substr($rbuf, 0, 16, '');
+                        $name = $len ? unpack('Z*', substr($rbuf, 0, $len, ''))
+                                        : undef;
+                        $r -= $size;
+                        next if $self->{ignore}->{$wd};
+                        my $ev = bless [$mask, $name], 'PublicInbox::In3Event';
+                        push @ret, $ev;
+                        if (my $w = $self->{w}->{$wd}) {
+                                $ev->[2] = $w;
+                                $w->cancel if $ev->mask & autocancel;
+                        } elsif ($mask & IN_Q_OVERFLOW) {
+                                carp 'E: IN_Q_OVERFLOW, too busy? (non-fatal)'
+                        } else {
+                                carp "BUG? wd:$wd unknown (non-fatal)";
+                        }
+                }
+        } elsif (defined($r) || ($!{EAGAIN} || $!{EINTR})) {
+        } else {
+                croak "inotify read: $!";
+        }
+        delete $self->{ignore};
+        @ret;
+}
+
+sub fileno { CORE::fileno($_[0]->{fh}) }
+
+sub fh { $_[0]->{fh} }
+
+sub blocking { shift->{fh}->blocking(@_) }
+
+sub watch {
+        my ($self, $name, $mask, $cb) = @_;
+        croak "E: $cb not supported" if $cb; # too much memory
+        my $wd = syscall(SYS_inotify_add_watch, $self->fileno, $name, $mask);
+        return if $wd < 0;
+        my $w = bless [ $wd, $mask, $name, $self ], 'PublicInbox::In3Watch';
+        $self->{w}->{$wd} = $w;
+        Scalar::Util::weaken($w->[3]); # ugh
+        $w;
+}
+
+sub rm_watch {
+        my ($self, $wd) = @_;
+        delete $self->{w}->{$wd};
+        $self->{ignore}->{$wd} = 1; # is this needed?
+        syscall(SYS_inotify_rm_watch, $self->fileno, $wd) < 0 ? undef : 1;
+}
+
+1;
diff --git a/lib/PublicInbox/InputPipe.pm b/lib/PublicInbox/InputPipe.pm
index e1e26e20..ee5bda59 100644
--- a/lib/PublicInbox/InputPipe.pm
+++ b/lib/PublicInbox/InputPipe.pm
@@ -1,36 +1,52 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# for reading pipes and sockets off the DS event loop
+# for reading pipes, sockets, and TTYs off the DS event loop
 package PublicInbox::InputPipe;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+use PublicInbox::Syscall qw(EPOLLIN);
 
 sub consume {
         my ($in, $cb, @args) = @_;
         my $self = bless { cb => $cb, args => \@args }, __PACKAGE__;
-        eval { $self->SUPER::new($in, EPOLLIN|EPOLLET) };
-        return $self->requeue if $@; # regular file
-        $in->blocking(0); # pipe or socket
+        eval { $self->SUPER::new($in, EPOLLIN) };
+        if ($@) { # regular file (but not w/ select|IO::Poll backends)
+                $self->{-need_rq} = 1;
+                $self->requeue;
+        } elsif (-p _ || -S _) { # O_NONBLOCK for sockets and pipes
+                $in->blocking(0);
+        }
+        $self;
+}
+
+sub close { # idempotent
+        my ($self) = @_;
+        $self->{-need_rq} ? delete($self->{sock}) : $self->SUPER::close
 }
 
 sub event_step {
         my ($self) = @_;
         my $r = sysread($self->{sock} // return, my $rbuf, 65536);
-        if ($r) {
-                $self->{cb}->(@{$self->{args} // []}, $rbuf);
-                return $self->requeue; # may be regular file or pipe
-        }
-        if (defined($r)) { # EOF
-                $self->{cb}->(@{$self->{args} // []}, '');
-        } elsif ($!{EAGAIN}) {
-                return;
-        } else { # another error
-                $self->{cb}->(@{$self->{args} // []}, undef)
+        eval {
+                if ($r) {
+                        $self->{cb}->($self, @{$self->{args}}, $rbuf);
+                        $self->requeue if $self->{-need_rq};
+                } elsif (defined($r)) { # EOF
+                        $self->{cb}->($self, @{$self->{args}}, '');
+                        $self->close
+                } elsif ($!{EAGAIN}) { # rely on EPOLLIN
+                } elsif ($!{EINTR}) { # rely on EPOLLIN for sockets/pipes
+                        $self->requeue if $self->{-need_rq};
+                } else { # another error
+                        $self->{cb}->($self, @{$self->{args}}, undef);
+                        $self->close;
+                }
+        };
+        if ($@) {
+                warn "E: $@";
+                $self->close;
         }
-        $self->{sock}->blocking ? delete($self->{sock}) : $self->close
 }
 
 1;
diff --git a/lib/PublicInbox/Isearch.pm b/lib/PublicInbox/Isearch.pm
index df940e76..62112171 100644
--- a/lib/PublicInbox/Isearch.pm
+++ b/lib/PublicInbox/Isearch.pm
@@ -1,12 +1,11 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Provides everything the PublicInbox::Search object does;
 # but uses global ExtSearch (->ALL) with an eidx_key query to
 # emulate per-Inbox search using ->ALL.
 package PublicInbox::Isearch;
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::ExtSearch;
 use PublicInbox::Search;
 
@@ -49,10 +48,10 @@ SELECT MAX(docid) FROM xref3 WHERE ibx_id = ? AND xnum >= ? AND xnum <= ?
                 if (defined($r[1]) && defined($r[0])) {
                         $opt{limit} = $r[1] - $r[0] + 1;
                 } else {
-                        $r[1] //= 0xffffffff;
+                        $r[1] //= $self->{es}->xdb->get_lastdocid;
                         $r[0] //= 0;
                 }
-                $opt{uid_range} = \@r;
+                $opt{uid_range} = \@r; # these are fed to Xapian and SQLite
         }
         $self->{es}->mset($str, \%opt);
 }
@@ -69,12 +68,11 @@ sub mset_to_artnums {
                         $range = 'AND xnum >= ? AND xnum <= ?';
                         @r = @$r;
                 }
-                my $rows = $self->{es}->over->dbh->
-                        selectall_arrayref(<<"", undef, $ibx_id, @$docids, @r);
+                return $self->{es}->over->dbh->
+                        selectcol_arrayref(<<"", undef, $ibx_id, @$docids, @r);
 SELECT xnum FROM xref3 WHERE ibx_id = ? AND docid IN ($qmarks) $range
 ORDER BY xnum ASC
 
-                return [ map { $_->[0] } @$rows ];
         }
 
         my $rows = $self->{es}->over->dbh->
@@ -125,4 +123,9 @@ sub has_threadid { 1 }
 
 sub help { $_[0]->{es}->help }
 
+sub xh_args { # prep getopt args to feed to xap_helper.h socket
+        my ($self, $opt) = @_; # TODO uid_range
+        ($self->{es}->xh_args, '-O', $self->{eidx_key});
+}
+
 1;
diff --git a/lib/PublicInbox/KQNotify.pm b/lib/PublicInbox/KQNotify.pm
index 7efb8b60..2efa887d 100644
--- a/lib/PublicInbox/KQNotify.pm
+++ b/lib/PublicInbox/KQNotify.pm
@@ -1,52 +1,35 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # implements the small subset of Linux::Inotify2 functionality we use
 # using IO::KQueue on *BSD systems.
 package PublicInbox::KQNotify;
-use strict;
-use v5.10.1;
+use v5.12;
+use parent qw(PublicInbox::FakeInotify);
 use IO::KQueue;
 use PublicInbox::DSKQXS; # wraps IO::KQueue for fork-safe DESTROY
-use PublicInbox::FakeInotify qw(fill_dirlist on_dir_change);
-use Time::HiRes qw(stat);
+use Errno qw(ENOENT);
 
 # NOTE_EXTEND detects rename(2), NOTE_WRITE detects link(2)
 sub MOVED_TO_OR_CREATE () { NOTE_EXTEND|NOTE_WRITE }
 
 sub new {
         my ($class) = @_;
-        bless { dskq => PublicInbox::DSKQXS->new, watch => {} }, $class;
+        bless { dskq => PublicInbox::DSKQXS->new }, $class;
 }
 
 sub watch {
         my ($self, $path, $mask) = @_;
-        my ($fh, $watch);
-        if (-d $path) {
-                opendir($fh, $path) or return;
-                my @st = stat($fh);
-                $watch = bless [ $fh, $path, $st[10] ],
-                        'PublicInbox::KQNotify::Watchdir';
-        } else {
-                open($fh, '<', $path) or return;
-                $watch = bless [ $fh, $path ],
-                        'PublicInbox::KQNotify::Watch';
-        }
-        my $ident = fileno($fh);
+        my $dir_delete = $mask & NOTE_DELETE ? 1 : 0;
+        my $w = $self->watch_open($path, \$dir_delete) or return;
+        $w->[2] = pop @$w; # ctime is unused by this subclass
+        my $ident = fileno($w->[2]) // die "BUG: bad fileno $w->[2]: $!";
         $self->{dskq}->{kq}->EV_SET($ident, # ident (fd)
                 EVFILT_VNODE, # filter
                 EV_ADD | EV_CLEAR, # flags
                 $mask, # fflags
-                0, 0); # data, udata
-        if ($mask & (MOVED_TO_OR_CREATE|NOTE_DELETE|NOTE_LINK|NOTE_REVOKE)) {
-                $self->{watch}->{$ident} = $watch;
-                if ($mask & (NOTE_DELETE|NOTE_LINK|NOTE_REVOKE)) {
-                        fill_dirlist($self, $path, $fh)
-                }
-        } else {
-                die "TODO Not implemented: $mask";
-        }
-        $watch;
+                0, $dir_delete); # data, udata
+        $self->{watch}->{$ident} = $w;
 }
 
 # emulate Linux::Inotify::fileno
@@ -63,54 +46,31 @@ sub blocking {}
 # behave like Linux::Inotify2->read
 sub read {
         my ($self) = @_;
-        my @kevents = $self->{dskq}->{kq}->kevent(0);
         my $events = [];
-        my @gone;
-        my $watch = $self->{watch};
-        for my $kev (@kevents) {
+        for my $kev ($self->{dskq}->{kq}->kevent(0)) {
                 my $ident = $kev->[KQ_IDENT];
-                my $mask = $kev->[KQ_FFLAGS];
-                my ($dh, $path, $old_ctime) = @{$watch->{$ident}};
-                if (!defined($old_ctime)) {
-                        push @$events,
-                                bless(\$path, 'PublicInbox::FakeInotify::Event')
-                } elsif ($mask & (MOVED_TO_OR_CREATE|NOTE_DELETE|NOTE_LINK|
-                                NOTE_REVOKE|NOTE_RENAME)) {
-                        my @new_st = stat($path);
-                        if (!@new_st && $!{ENOENT}) {
-                                push @$events, bless(\$path,
-                                                'PublicInbox::FakeInotify::'.
-                                                'SelfGoneEvent');
-                                push @gone, $ident;
-                                delete $self->{dirlist}->{$path};
-                                next;
-                        }
-                        if (!@new_st) {
-                                warn "unhandled stat($path) error: $!\n";
-                                next;
-                        }
-                        $watch->{$ident}->[3] = $new_st[10]; # ctime
-                        rewinddir($dh);
-                        on_dir_change($events, $dh, $path, $old_ctime,
-                                        $self->{dirlist});
+                my $w = $self->{watch}->{$ident} or next;
+                if (!@$w) { # cancelled
+                        delete($self->{watch}->{$ident});
+                        next;
+                }
+                my $dir_delete = $kev->[KQ_UDATA];
+                my ($old_dev, $old_ino, $fh, $path) = @$w;
+                my @new_st = stat($path);
+                warn "W: stat($path): $!\n" if !@new_st && $! != ENOENT;
+                if (!@new_st || "$old_dev $old_ino" ne "@new_st[0,1]") {
+                        push(@$events, $self->gone($ident, $path));
+                        next;
+                }
+                if (-d _) {
+                        rewinddir($fh);
+                        $self->on_dir_change($events, $fh, $path, $dir_delete);
+                } else {
+                        push @$events, bless(\$path,
+                                        'PublicInbox::FakeInotify::Event');
                 }
         }
-        delete @$watch{@gone};
         @$events;
 }
 
-package PublicInbox::KQNotify::Watch;
-use strict;
-
-sub name { $_[0]->[1] }
-
-sub cancel { close $_[0]->[0] or die "close: $!" }
-
-package PublicInbox::KQNotify::Watchdir;
-use strict;
-
-sub name { $_[0]->[1] }
-
-sub cancel { closedir $_[0]->[0] or die "closedir: $!" }
-
 1;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d81ca296..81f940fe 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -9,8 +9,9 @@ package PublicInbox::LEI;
 use v5.12;
 use parent qw(PublicInbox::DS PublicInbox::LeiExternal
         PublicInbox::LeiQuery);
+use autodie qw(bind chdir fork open pipe socket socketpair syswrite unlink);
 use Getopt::Long ();
-use Socket qw(AF_UNIX SOCK_SEQPACKET MSG_EOR pack_sockaddr_un);
+use Socket qw(AF_UNIX SOCK_SEQPACKET pack_sockaddr_un);
 use Errno qw(EPIPE EAGAIN ECONNREFUSED ENOENT ECONNRESET);
 use Cwd qw(getcwd);
 use POSIX qw(strftime);
@@ -18,33 +19,35 @@ use IO::Handle ();
 use Fcntl qw(SEEK_SET);
 use PublicInbox::Config;
 use PublicInbox::Syscall qw(EPOLLIN);
-use PublicInbox::DS qw(dwaitpid);
-use PublicInbox::Spawn qw(spawn popen_rd);
+use PublicInbox::Spawn qw(run_wait popen_rd run_qx);
 use PublicInbox::Lock;
 use PublicInbox::Eml;
 use PublicInbox::Import;
 use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::IPC;
 use Time::HiRes qw(stat); # ctime comparisons for config cache
-use File::Path qw(mkpath);
+use File::Path ();
 use File::Spec;
+use Carp qw(carp);
 use Sys::Syslog qw(openlog syslog closelog);
 our $quit = \&CORE::exit;
-our ($current_lei, $errors_log, $listener, $oldset, $dir_idle,
-        $recv_cmd, $send_cmd);
+our ($current_lei, $errors_log, $listener, $oldset, $dir_idle);
 my $GLP = Getopt::Long::Parser->new;
 $GLP->configure(qw(gnu_getopt no_ignore_case auto_abbrev));
 my $GLP_PASS = Getopt::Long::Parser->new;
 $GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through));
 
-our %PATH2CFG; # persistent for socket daemon
-our $MDIR2CFGPATH; # /path/to/maildir => { /path/to/config => [ ino watches ] }
+our (%PATH2CFG, # persistent for socket daemon
+$MDIR2CFGPATH, # location => { /path/to/config => [ ino watches ] }
+$OPT, # shared between optparse and opt_dash callback (for Getopt::Long)
+$daemon_pid
+);
 
 # TBD: this is a documentation mechanism to show a subcommand
 # (may) pass options through to another command:
 sub pass_through { $GLP_PASS }
 
-my $OPT;
-sub opt_dash ($$) {
+sub opt_dash ($$) { # callback runs inside optparse
         my ($spec, $re_str) = @_; # 'limit|n=i', '([0-9]+)'
         my ($key) = ($spec =~ m/\A([a-z]+)/g);
         my $cb = sub { # Getopt::Long "<>" catch-all handler
@@ -158,7 +161,7 @@ our @diff_opt = qw(unified|U=i output-indicator-new=s output-indicator-old=s
         rename-empty! check ws-error-highlight=s full-index binary
         abbrev:i break-rewrites|B:s find-renames|M:s find-copies:s
         find-copies-harder irreversible-delete|D l=i diff-filter=s
-        S=s G=s find-object=s pickaxe-all pickaxe-regex O=s R
+        S=s G=s find-object=s pickaxe-all pickaxe-regex R
         relative:s text|a ignore-cr-at-eol ignore-space-at-eol
         ignore-space-change|b ignore-all-space|w ignore-blank-lines
         inter-hunk-context=i function-context|W exit-code ext-diff
@@ -197,8 +200,8 @@ our %CMD = ( # sorted in order of importance/use:
 'rediff' => [ '--stdin|LOCATION...',
                 'regenerate a diff with different options',
         'stdin|', # /|\z/ must be first for lone dash
-        qw(git-dir=s@ cwd! verbose|v+ color:s no-color drq:1 dequote-only:1),
-        @diff_opt, @lxs_opt, @net_opt, @c_opt ],
+        qw(git-dir=s@ cwd! verbose|v+ color:s no-color drq:1 dequote-only:1
+        order-file=s), @diff_opt, @lxs_opt, @net_opt, @c_opt ],
 
 'mail-diff' => [ '--stdin|LOCATION...', 'diff the contents of emails',
         'stdin|', # /|\z/ must be first for lone dash
@@ -229,20 +232,21 @@ our %CMD = ( # sorted in order of importance/use:
 'rm' => [ '--stdin|LOCATION...',
         'remove a message from the index and prevent reindexing',
         'stdin|', # /|\z/ must be first for lone dash
-        qw(in-format|F=s lock=s@), @net_opt, @c_opt ],
+        qw(in-format|F=s lock=s@ commit-delay=i), @net_opt, @c_opt ],
 'plonk' => [ '--threads|--from=IDENT',
         'exclude mail matching From: or threads from non-Message-ID searches',
         qw(stdin| threads|t from|f=s mid=s oid=s), @c_opt ],
-'tag' => [ 'KEYWORDS...',
+tag => [ 'KEYWORDS... LOCATION...|--stdin',
         'set/unset keywords and/or labels on message(s)',
-        qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@),
+        qw(stdin| in-format|F=s input|i=s@ oid=s@ mid=s@ commit-delay=i),
         @net_opt, @c_opt, pass_through('-kw:foo for delete') ],
 
 'purge-mailsource' => [ 'LOCATION|--all',
         'remove imported messages from IMAP, Maildirs, and MH',
         qw(exact! all jobs:i indexed), @c_opt ],
 
-'add-watch' => [ 'LOCATION...', 'watch for new messages and flag changes',
+'add-watch' => [ 'LOCATION... [LABELS...]',
+        'watch for new messages and flag changes',
         qw(poll-interval=s state=s recursive|r), @c_opt ],
 'rm-watch' => [ 'LOCATION...', 'remove specified watch(es)',
         qw(recursive|r), @c_opt ],
@@ -253,14 +257,17 @@ our %CMD = ( # sorted in order of importance/use:
 'forget-watch' => [ '{WATCH_NUMBER|--prune}', 'stop and forget a watch',
         qw(prune), @c_opt ],
 
-'index' => [ 'LOCATION...', 'one-time index from URL or filesystem',
+'reindex' => [ '', 'reindex all locally-indexed messages', @c_opt ],
+
+'index' => [ 'LOCATION... [LABELS...]', 'one-time index from URL or filesystem',
         qw(in-format|F=s kw! offset=i recursive|r exclude=s include|I=s
         verbose|v+ incremental!), @net_opt, # mainly for --proxy=
          @c_opt ],
-'import' => [ 'LOCATION...|--stdin',
+import => [ 'LOCATION...|--stdin [LABELS...]',
         'one-time import/update from URL or filesystem',
         qw(stdin| offset=i recursive|r exclude=s include|I=s new-only
-        lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
+        lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!
+        commit-delay=i sort|s:s@),
         @net_opt, @c_opt ],
 'forget-mail-sync' => [ 'LOCATION...',
         'forget sync information for a mail folder', @c_opt ],
@@ -272,13 +279,15 @@ our %CMD = ( # sorted in order of importance/use:
         qw(all:s mode=s), @net_opt, @c_opt ],
 'convert' => [ 'LOCATION...|--stdin',
         'one-time conversion from URL or filesystem to another format',
-        qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!),
+        qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s lock=s@ kw!
+                rsyncable sort|s:s@),
         @net_opt, @c_opt ],
 'p2q' => [ 'LOCATION_OR_COMMIT...|--stdin',
         "use a patch to generate a query for `lei q --stdin'",
         qw(stdin| in-format|F=s want|w=s@ uri debug), @net_opt, @c_opt ],
 'config' => [ '[...]', sub {
-                'git-config(1) wrapper for '._config_path($_[0]);
+                'git-config(1) wrapper for '._config_path($_[0]). "\n" .
+        '-l/--list and other common git-config uses are supported'
         }, qw(config-file|system|global|file|f=s), # for conflict detection
          qw(edit|e c=s@ C=s@), pass_through('git config') ],
 'inspect' => [ 'ITEMS...|--stdin', 'inspect lei/store and/or local external',
@@ -312,6 +321,9 @@ our %CMD = ( # sorted in order of importance/use:
 my $stdin_formats = [ 'MAIL_FORMAT|eml|mboxrd|mboxcl2|mboxcl|mboxo',
                         'specify message input format' ];
 my $ls_format = [ 'OUT|plain|json|null', 'listing output format' ];
+my $sort_out = [ 'VAL|received|relevance|docid',
+                "order of results is `--output'-dependent"];
+my $sort_in = [ 'sequence|mtime|size', 'sort input (format-dependent)' ];
 
 # we use \x{a0} (non-breaking SP) to avoid wrapping in PublicInbox::LeiHelp
 my %OPTDESC = (
@@ -344,6 +356,7 @@ my %OPTDESC = (
 'no-torsocks' => 'alias for --torsocks=no',
 'save!' =>  "do not save a search for `lei up'",
 'import-remote!' => 'do not memoize remote messages into local store',
+'import-before!' => 'do not import before writing to output (DANGEROUS)',
 
 'type=s' => [ 'any|mid|git', 'disambiguate type' ],
 
@@ -397,8 +410,10 @@ my %OPTDESC = (
                 'include specified external(s) in search' ],
 'only|O=s@        q' => [ 'LOCATION',
                 'only use specified external(s) for search' ],
-'jobs=s        q' => [ '[SEARCH_JOBS][,WRITER_JOBS]',
-                'control number of search and writer jobs' ],
+'jobs|j=s' => [ 'JOBSPEC',
+                'control number of query and writer jobs' .
+                "integers delimited by `,', either of which may be omitted"
+                ],
 'jobs|j=i        add-external' => 'set parallelism when indexing after --mirror',
 
 'in-format|F=s' => $stdin_formats,
@@ -416,8 +431,10 @@ my %OPTDESC = (
 'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
 'offset=i' => ['OFF', 'search result offset (default: 0)'],
 
-'sort|s=s' => [ 'VAL|received|relevance|docid',
-                "order of results is `--output'-dependent"],
+'sort|s=s        q' => $sort_out,
+'sort|s=s        lcat' => $sort_out,
+'sort|s:s@        convert' => $sort_in,
+'sort|s:s@        import' => $sort_in,
 'reverse|r' => 'reverse search results', # like sort(1)
 
 'boost=i' => 'increase/decrease priority of results (default: 0)',
@@ -451,6 +468,7 @@ my %OPTDESC = (
 'z|0' => 'use NUL \\0 instead of newline (CR) to delimit lines',
 
 'signal|s=s' => [ 'SIG', 'signal to send lei-daemon (default: TERM)' ],
+'edit|e        config' => 'open an editor to modify the lei config file',
 ); # %OPTDESC
 
 my %CONFIG_KEYS = (
@@ -462,7 +480,7 @@ my @WQ_KEYS = qw(lxs l2m ikw pmd wq1 lne v2w); # internal workers
 sub _drop_wq {
         my ($self) = @_;
         for my $wq (grep(defined, delete(@$self{@WQ_KEYS}))) {
-                $wq->wq_kill('-TERM');
+                $wq->wq_kill(-POSIX::SIGTERM());
                 $wq->DESTROY;
         }
 }
@@ -470,17 +488,18 @@ sub _drop_wq {
 # pronounced "exit": x_it(1 << 8) => exit(1); x_it(13) => SIGPIPE
 sub x_it ($$) {
         my ($self, $code) = @_;
-        local $current_lei = $self;
         # make sure client sees stdout before exit
         $self->{1}->autoflush(1) if $self->{1};
         stop_pager($self);
         if ($self->{pkt_op_p}) { # worker => lei-daemon
                 $self->{pkt_op_p}->pkt_do('x_it', $code);
+                exit($code >> 8) if $$ != $daemon_pid;
         } elsif ($self->{sock}) { # lei->daemon => lei(1) client
-                send($self->{sock}, "x_it $code", MSG_EOR);
+                send($self->{sock}, "x_it $code", 0);
         } elsif ($quit == \&CORE::exit) { # an admin (one-shot) command
                 exit($code >> 8);
         } # else ignore if client disconnected
+        $self->dclose if $$ == $daemon_pid;
 }
 
 sub err ($;@) {
@@ -489,7 +508,7 @@ sub err ($;@) {
         my @eor = (substr($_[-1]//'', -1, 1) eq "\n" ? () : ("\n"));
         print $err @_, @eor and return;
         my $old_err = delete $self->{2};
-        close($old_err) if $! == EPIPE && $old_err;
+        $old_err->close if $! == EPIPE && $old_err;
         $err = $self->{2} = ($self->{pgr} // [])->[2] // *STDERR{GLOB};
         print $err @_, @eor or print STDERR @_, @eor;
 }
@@ -504,8 +523,7 @@ sub qfin { # show message on finalization (LeiFinmsg)
 
 sub fail_handler ($;$$) {
         my ($lei, $code, $io) = @_;
-        local $current_lei = $lei;
-        close($io) if $io; # needed to avoid warnings on SIGPIPE
+        $io->close if $io; # needed to avoid warnings on SIGPIPE
         _drop_wq($lei);
         x_it($lei, $code // (1 << 8));
 }
@@ -514,13 +532,17 @@ sub sigpipe_handler { # handles SIGPIPE from @WQ_KEYS workers
         fail_handler($_[0], 13, delete $_[0]->{1});
 }
 
-sub fail ($$;$) {
-        my ($self, $msg, $exit_code) = @_;
-        local $current_lei = $self;
-        $self->{failed}++;
-        warn(substr($msg, -1, 1) eq "\n" ? $msg : "$msg\n") if defined $msg;
-        $self->{pkt_op_p}->pkt_do('fail_handler') if $self->{pkt_op_p};
-        x_it($self, ($exit_code // 1) << 8);
+sub fail ($;@) {
+        my ($lei, @msg) = @_;
+        my $exit_code = ($msg[0]//'') =~ /\A-?[0-9]+\z/ ? shift(@msg) : undef;
+        local $current_lei = $lei;
+        $lei->{failed}++;
+        if (@msg) {
+                push @msg, "\n" if substr($msg[-1], -1, 1);
+                warn @msg;
+        }
+        $lei->{pkt_op_p}->pkt_do('fail_handler') if $lei->{pkt_op_p};
+        x_it($lei, $exit_code // (1 << 8));
         undef;
 }
 
@@ -540,18 +562,17 @@ sub child_error { # passes non-fatal curl exit codes to user
         local $current_lei = $self;
         $child_error ||= 1 << 8;
         warn(substr($msg, -1, 1) eq "\n" ? $msg : "$msg\n") if defined $msg;
+        $self->{child_error} ||= $child_error;
         if ($self->{pkt_op_p}) { # to top lei-daemon
                 $self->{pkt_op_p}->pkt_do('child_error', $child_error);
         } elsif ($self->{sock}) { # to lei(1) client
-                send($self->{sock}, "child_error $child_error", MSG_EOR);
-        } else { # non-lei admin command
-                $self->{child_error} ||= $child_error;
+                send($self->{sock}, "child_error $child_error", 0);
         } # else noop if client disconnected
 }
 
 sub note_sigpipe { # triggers sigpipe_handler
         my ($self, $fd) = @_;
-        close(delete($self->{$fd})); # explicit close silences Perl warning
+        delete($self->{$fd})->close; # explicit close silences Perl warning
         $self->{pkt_op_p}->pkt_do('sigpipe_handler') if $self->{pkt_op_p};
         x_it($self, 13);
 }
@@ -559,20 +580,21 @@ sub note_sigpipe { # triggers sigpipe_handler
 sub _lei_atfork_child {
         my ($self, $persist) = @_;
         # we need to explicitly close things which are on stack
+        my $cfg = $self->{cfg};
+        delete @$cfg{qw(-watches -lei_note_event)};
         if ($persist) {
-                open $self->{3}, '<', '/' or die "open(/) $!";
+                open $self->{3}, '<', '/';
                 fchdir($self);
                 close($_) for (grep(defined, delete @$self{qw(0 1 2 sock)}));
-                if (my $cfg = $self->{cfg}) {
-                        delete @$cfg{qw(-lei_store -watches -lei_note_event)};
-                }
+                delete $cfg->{-lei_store};
         } else { # worker, Net::NNTP (Net::Cmd) uses STDERR directly
-                open STDERR, '+>&='.fileno($self->{2}) or warn "open $!";
+                open STDERR, '+>&='.fileno($self->{2}); # idempotent w/ fileno
                 STDERR->autoflush(1);
+                $self->{2} = \*STDERR;
                 POSIX::setpgid(0, $$) // die "setpgid(0, $$): $!";
         }
         close($_) for (grep(defined, delete @$self{qw(old_1 au_done)}));
-        delete $self->{-socks};
+        close($_) for (@{delete($self->{-socks}) // []});
         if (my $op_c = delete $self->{pkt_op_c}) {
                 close(delete $op_c->{sock});
         }
@@ -584,7 +606,7 @@ sub _lei_atfork_child {
         $dir_idle->force_close if $dir_idle;
         undef $dir_idle;
         %PATH2CFG = ();
-        $MDIR2CFGPATH = {};
+        $MDIR2CFGPATH = undef;
         eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
         undef $errors_log;
         $quit = \&CORE::exit;
@@ -617,8 +639,9 @@ sub pkt_op_pair {
 }
 
 sub incr {
-        my ($self, $field, $nr) = @_;
-        $self->{counters}->{$field} += $nr;
+        my $lei = shift;
+        $lei->{incr_pid} = $$ if @_;
+        while (my ($f, $n) = splice(@_, 0, 2)) { $lei->{$f} += $n }
 }
 
 sub pkt_ops {
@@ -641,12 +664,12 @@ sub workers_start {
         my $end = $lei->pkt_op_pair;
         my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
         $flds->{lei} = $lei;
-        $wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
+        $wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds,
+                $wq->can('_wq_done_wait') // \&wq_done_wait, $lei);
         delete $lei->{pkt_op_p};
         my $op_c = delete $lei->{pkt_op_c};
         @$end = ();
         $lei->event_step_init;
-        $wq->wq_wait_async($wq->can('_wq_done_wait') // \&wq_done_wait, $lei);
         ($op_c, $ops);
 }
 
@@ -680,7 +703,7 @@ sub optparse ($$$) {
         # allow _complete --help to complete, not show help
         return 1 if substr($cmd, 0, 1) eq '_';
         $self->{cmd} = $cmd;
-        $OPT = $self->{opt} //= {};
+        local $OPT = $self->{opt} //= {};
         my $info = $CMD{$cmd} // [ '[...]' ];
         my ($proto, undef, @spec) = @$info;
         my $glp = ref($spec[-1]) eq ref($GLP) ? pop(@spec) : $GLP;
@@ -700,6 +723,14 @@ sub optparse ($$$) {
         # "-" aliases "stdin" or "clear"
         $OPT->{$lone_dash} = ${$OPT->{$lone_dash}} if defined $lone_dash;
 
+        if ($proto =~ s/\s*\[?(?:KEYWORDS|LABELS)\.\.\.\]?\s*//g) {
+                require PublicInbox::LeiInput;
+                my @err = PublicInbox::LeiInput::vmd_mod_extract($self, $argv);
+                return $self->fail(join("\n", @err)) if @err;
+        } else {
+                warn "proto $proto\n" if $cmd =~ /(add-watch|tag|index)/;
+        }
+
         my $i = 0;
         my $POS_ARG = '[A-Z][A-Z0-9_]+';
         my ($err, $inf);
@@ -727,11 +758,10 @@ sub optparse ($$$) {
                                         # w/o args means stdin
                                         if ($sw eq 'stdin' && !@$argv &&
                                                         (-p $self->{0} ||
-                                                         -f _) && -r _) {
+                                                         -f _)) {
                                                 $OPT->{stdin} //= 1;
                                         }
-                                        $ok = defined($OPT->{$sw});
-                                        last if $ok;
+                                        $ok = defined($OPT->{$sw}) and last;
                                 } elsif (defined($argv->[$i])) {
                                         $ok = 1;
                                         $i++;
@@ -752,38 +782,7 @@ sub optparse ($$$) {
         $err ? fail($self, "usage: lei $cmd $proto\nE: $err") : 1;
 }
 
-sub _tmp_cfg { # for lei -c <name>=<value> ...
-        my ($self) = @_;
-        my $cfg = _lei_cfg($self, 1);
-        require File::Temp;
-        my $ft = File::Temp->new(TEMPLATE => 'lei_cfg-XXXX', TMPDIR => 1);
-        my $tmp = { '-f' => $ft->filename, -tmp => $ft };
-        $ft->autoflush(1);
-        print $ft <<EOM or return fail($self, "$tmp->{-f}: $!");
-[include]
-        path = $cfg->{-f}
-EOM
-        $tmp = $self->{cfg} = bless { %$cfg, %$tmp }, ref($cfg);
-        for (@{$self->{opt}->{c}}) {
-                /\A([^=\.]+\.[^=]+)(?:=(.*))?\z/ or return fail($self, <<EOM);
-`-c $_' is not of the form -c <name>=<value>'
-EOM
-                my $name = $1;
-                my $value = $2 // 1;
-                _config($self, '--add', $name, $value);
-                if (defined(my $v = $tmp->{$name})) {
-                        if (ref($v) eq 'ARRAY') {
-                                push @$v, $value;
-                        } else {
-                                $tmp->{$name} = [ $v, $value ];
-                        }
-                } else {
-                        $tmp->{$name} = $value;
-                }
-        }
-}
-
-sub lazy_cb ($$$) {
+sub lazy_cb ($$$) { # $pfx is _complete_ or lei_
         my ($self, $cmd, $pfx) = @_;
         my $ucmd = $cmd;
         $ucmd =~ tr/-/_/;
@@ -796,11 +795,19 @@ sub lazy_cb ($$$) {
                 $pkg->can($pfx.$ucmd) : undef;
 }
 
+sub do_env {
+        my $lei = shift;
+        fchdir($lei);
+        my $cb = shift // return ($lei, %{$lei->{env}}) ;
+        local ($current_lei, %ENV) = ($lei, %{$lei->{env}});
+        $cb = $lei->can($cb) if !ref($cb); # $cb may be a scalar sub name
+        eval { $cb->($lei, @_) };
+        $lei->fail($@) if $@;
+}
+
 sub dispatch {
         my ($self, $cmd, @argv) = @_;
-        fchdir($self);
-        local %ENV = %{$self->{env}};
-        local $current_lei = $self; # for __WARN__
+        local ($current_lei, %ENV) = do_env($self);
         $self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
         return _help($self, 'no command given') unless defined($cmd);
         # do not support Getopt bundling for this
@@ -812,14 +819,12 @@ sub dispatch {
         }
         if (my $cb = lazy_cb(__PACKAGE__, $cmd, 'lei_')) {
                 optparse($self, $cmd, \@argv) or return;
-                $self->{opt}->{c} and (_tmp_cfg($self) // return);
                 if (my $chdir = $self->{opt}->{C}) {
                         for my $d (@$chdir) {
                                 next if $d eq ''; # same as git(1)
-                                chdir $d or return fail($self, "cd $d: $!");
+                                chdir $d;
                         }
-                        open $self->{3}, '<', '.' or
-                                return fail($self, "open . $!");
+                        open($self->{3}, '<', '.');
                 }
                 $cb->($self, @argv);
         } elsif (grep(/\A-/, $cmd, @argv)) { # --help or -h only
@@ -837,28 +842,30 @@ sub _lei_cfg ($;$) {
         my $f = _config_path($self);
         my @st = stat($f);
         my $cur_st = @st ? pack('dd', $st[10], $st[7]) : ''; # 10:ctime, 7:size
-        my ($sto, $sto_dir, $watches, $lne);
-        if (my $cfg = $PATH2CFG{$f}) { # reuse existing object in common case
-                return ($self->{cfg} = $cfg) if $cur_st eq $cfg->{-st};
+        my ($sto, $sto_dir, $watches, $lne, $cfg);
+        if ($cfg = $PATH2CFG{$f}) { # reuse existing object in common case
+                ($cur_st eq $cfg->{-st} && !$self->{opt}->{c}) and
+                        return ($self->{cfg} = $cfg);
+                # reuse some fields below if they match:
                 ($sto, $sto_dir, $watches, $lne) =
                                 @$cfg{qw(-lei_store leistore.dir -watches
                                         -lei_note_event)};
         }
         if (!@st) {
-                unless ($creat) {
-                        delete $self->{cfg};
-                        return bless {}, 'PublicInbox::Config';
+                unless ($creat) { # any commands which write to cfg must creat
+                        $cfg = PublicInbox::Config->git_config_dump(
+                                                        '/dev/null', $self);
+                        return ($self->{cfg} = $cfg);
                 }
                 my ($cfg_dir) = ($f =~ m!(.*?/)[^/]+\z!);
-                -d $cfg_dir or mkpath($cfg_dir) or die "mkpath($cfg_dir): $!\n";
-                open my $fh, '>>', $f or die "open($f): $!\n";
+                File::Path::mkpath($cfg_dir);
+                open my $fh, '>>', $f;
                 @st = stat($fh) or die "fstat($f): $!\n";
                 $cur_st = pack('dd', $st[10], $st[7]);
                 qerr($self, "# $f created") if $self->{cmd} ne 'config';
         }
-        my $cfg = PublicInbox::Config->git_config_dump($f, $self->{2});
+        $cfg = PublicInbox::Config->git_config_dump($f, $self);
         $cfg->{-st} = $cur_st;
-        $cfg->{'-f'} = $f;
         if ($sto && canonpath_harder($sto_dir // store_path($self))
                         eq canonpath_harder($cfg->{'leistore.dir'} //
                                                 store_path($self))) {
@@ -870,7 +877,7 @@ sub _lei_cfg ($;$) {
                 # FIXME: use inotify/EVFILT_VNODE to detect unlinked configs
                 delete(@PATH2CFG{grep(!-f, keys %PATH2CFG)});
         }
-        $self->{cfg} = $PATH2CFG{$f} = $cfg;
+        $self->{cfg} = $self->{opt}->{c} ? $cfg : ($PATH2CFG{$f} = $cfg);
         refresh_watches($self);
         $cfg;
 }
@@ -886,16 +893,49 @@ sub _lei_store ($;$) {
         };
 }
 
+# returns true on success, undef
+# argv[0] eq `+e' means errors do not ->fail # (like `sh +e')
 sub _config {
         my ($self, @argv) = @_;
-        my %env = (%{$self->{env}}, GIT_CONFIG => undef);
+        my $err_ok = ($argv[0] // '') eq '+e' ? shift(@argv) : undef;
+        my %env;
+        my %opt = map { $_ => $self->{$_} } (0..2);
         my $cfg = _lei_cfg($self, 1);
-        my $cmd = [ qw(git config -f), $cfg->{'-f'}, @argv ];
-        my %rdr = map { $_ => $self->{$_} } (0..2);
-        waitpid(spawn($cmd, \%env, \%rdr), 0);
+        my $opt_c = delete local $cfg->{-opt_c};
+        my @file_arg;
+        if ($opt_c) {
+                my ($set, $get, $nondash);
+                for (@argv) { # order matters for git-config
+                        if (!$nondash) {
+                                if (/\A--(?:add|rename-section|remove-section|
+                                                replace-all|
+                                                unset-all|unset)\z/x) {
+                                        ++$set;
+                                } elsif ($_ eq '-l' || $_ eq '--list' ||
+                                                /\A--get/) {
+                                        ++$get;
+                                } elsif (/\A-/) { # -z and such
+                                } else {
+                                        ++$nondash;
+                                }
+                        } else {
+                                ++$nondash;
+                        }
+                }
+                if ($set || ($nondash//0) > 1 && !$get) {
+                        @file_arg = ('-f', $cfg->{-f});
+                        $env{GIT_CONFIG} = $file_arg[1];
+                } else { # OK, we can use `-c n=v' for read-only
+                        $cfg->{-opt_c} = $opt_c;
+                        $env{GIT_CONFIG} = undef;
+                }
+        }
+        my $cmd = $cfg->config_cmd(\%env, \%opt);
+        push @$cmd, @file_arg, @argv;
+        run_wait($cmd, \%env, \%opt) ? ($err_ok ? undef : fail($self, $?)) : 1;
 }
 
-sub lei_daemon_pid { puts shift, $$ }
+sub lei_daemon_pid { puts shift, $daemon_pid }
 
 sub lei_daemon_kill {
         my ($self) = @_;
@@ -1000,9 +1040,10 @@ sub start_mua {
                 $io->[0] = $self->{1} if $self->{opt}->{stdin} && -t $self->{1};
                 send_exec_cmd($self, $io, \@cmd, {});
         }
-        if ($self->{lxs} && $self->{au_done}) { # kick wait_startq
-                syswrite($self->{au_done}, 'q' x ($self->{lxs}->{jobs} // 0));
-        }
+
+        # kick wait_startq:
+        syswrite($self->{au_done}, 'q') if $self->{lxs} && $self->{au_done};
+
         return unless -t $self->{2}; # XXX how to determine non-TUI MUAs?
         $self->{opt}->{quiet} = 1;
         delete $self->{-progress};
@@ -1011,9 +1052,11 @@ sub start_mua {
 
 sub send_exec_cmd { # tell script/lei to execute a command
         my ($self, $io, $cmd, $env) = @_;
-        my $sock = $self->{sock} // die 'lei client gone';
-        my $fds = [ map { fileno($_) } @$io ];
-        $send_cmd->($sock, $fds, exec_buf($cmd, $env), MSG_EOR);
+        $PublicInbox::IPC::send_cmd->(
+                        $self->{sock} // die('lei client gone'),
+                        [ map { fileno($_) } @$io ],
+                        exec_buf($cmd, $env), 0) //
+                Carp::croak("sendmsg: $!");
 }
 
 sub poke_mua { # forces terminal MUAs to wake up and hopefully notice new mail
@@ -1023,7 +1066,7 @@ sub poke_mua { # forces terminal MUAs to wake up and hopefully notice new mail
         while (my $op = shift(@$alerts)) {
                 if ($op eq ':WINCH') {
                         # hit the process group that started the MUA
-                        send($sock, '-WINCH', MSG_EOR) if $sock;
+                        send($sock, '-WINCH', 0) if $sock;
                 } elsif ($op eq ':bell') {
                         out($self, "\a");
                 } elsif ($op =~ /(?<!\\),/) { # bare ',' (not ',,')
@@ -1032,7 +1075,7 @@ sub poke_mua { # forces terminal MUAs to wake up and hopefully notice new mail
                         my $cmd = $1; # run an arbitrary command
                         require Text::ParseWords;
                         $cmd = [ Text::ParseWords::shellwords($cmd) ];
-                        send($sock, exec_buf($cmd, {}), MSG_EOR) if $sock;
+                        send($sock, exec_buf($cmd, {}), 0) if $sock;
                 } else {
                         warn("W: unsupported --alert=$op\n"); # non-fatal
                 }
@@ -1056,16 +1099,15 @@ sub path_to_fd {
 # caller needs to "-t $self->{1}" to check if tty
 sub start_pager {
         my ($self, $new_env) = @_;
-        my $fh = popen_rd([qw(git var GIT_PAGER)]);
-        chomp(my $pager = <$fh> // '');
-        close($fh) or warn "`git var PAGER' error: \$?=$?";
+        chomp(my $pager = run_qx([qw(git var GIT_PAGER)]));
+        warn "`git var PAGER' error: \$?=$?" if $?;
         return if $pager eq 'cat' || $pager eq '';
         $new_env //= {};
         $new_env->{LESS} //= 'FRX';
         $new_env->{LV} //= '-c';
         $new_env->{MORE} = $new_env->{LESS} if $^O eq 'freebsd';
-        pipe(my ($r, $wpager)) or return warn "pipe: $!";
-        my $rdr = { 0 => $r, 1 => $self->{1}, 2 => $self->{2} };
+        my $rdr = { 1 => $self->{1}, 2 => $self->{2} };
+        CORE::pipe($rdr->{0}, my $wpager) or return warn "pipe: $!";
         my $pgr = [ undef, @$rdr{1, 2} ];
         my $env = $self->{env};
         if ($self->{sock}) { # lei(1) process runs it
@@ -1085,17 +1127,17 @@ sub pgr_err {
         my ($self, @msg) = @_;
         return warn(@msg) unless $self->{sock} && -t $self->{2};
         start_pager($self, { LESS => 'RX' }); # no 'F' so we prompt
-        print { $self->{2} } @msg;
+        say { $self->{2} } @msg, '# -quit pager to continue-';
         $self->{2}->autoflush(1);
         stop_pager($self);
-        send($self->{sock}, 'wait', MSG_EOR); # wait for user to quit pager
+        send($self->{sock}, 'wait', 0); # wait for user to quit pager
 }
 
 sub stop_pager {
         my ($self) = @_;
         my $pgr = delete($self->{pgr}) or return;
         $self->{2} = $pgr->[2];
-        close(delete($self->{1})) if $self->{1};
+        delete($self->{1})->close if $self->{1};
         $self->{1} = $pgr->[1];
 }
 
@@ -1105,24 +1147,22 @@ sub accept_dispatch { # Listener {post_accept} callback
         my $self = bless { sock => $sock }, __PACKAGE__;
         vec(my $rvec = '', fileno($sock), 1) = 1;
         select($rvec, undef, undef, 60) or
-                return send($sock, 'timed out waiting to recv FDs', MSG_EOR);
+                return send($sock, 'timed out waiting to recv FDs', 0);
         # (4096 * 33) >MAX_ARG_STRLEN
-        my @fds = $recv_cmd->($sock, my $buf, 4096 * 33) or return; # EOF
+        my @fds = $PublicInbox::IPC::recv_cmd->($sock, my $buf, 4096 * 33) or
+                return; # EOF
         if (!defined($fds[0])) {
                 warn(my $msg = "recv_cmd failed: $!");
-                return send($sock, $msg, MSG_EOR);
+                return send($sock, $msg, 0);
         } else {
                 my $i = 0;
-                for my $fd (@fds) {
-                        open($self->{$i++}, '+<&=', $fd) and next;
-                        send($sock, "open(+<&=$fd) (FD=$i): $!", MSG_EOR);
-                }
-                $i == 4 or return send($sock, 'not enough FDs='.($i-1), MSG_EOR)
+                open($self->{$i++}, '+<&=', $_) for @fds;
+                $i == 4 or return send($sock, 'not enough FDs='.($i-1), 0)
         }
         # $ENV_STR = join('', map { "\0$_=$ENV{$_}" } keys %ENV);
         # $buf = "$argc\0".join("\0", @ARGV).$ENV_STR."\0\0";
         substr($buf, -2, 2, '') eq "\0\0" or  # s/\0\0\z//
-                return send($sock, 'request command truncated', MSG_EOR);
+                return send($sock, 'request command truncated', 0);
         my ($argc, @argv) = split(/\0/, $buf, -1);
         undef $buf;
         my %env = map { split(/=/, $_, 2) } splice(@argv, $argc);
@@ -1145,11 +1185,11 @@ sub event_step {
         local %ENV = %{$self->{env}};
         local $current_lei = $self;
         eval {
-                my @fds = $recv_cmd->($self->{sock} // return, my $buf, 4096);
+                my @fds = $PublicInbox::IPC::recv_cmd->(
+                        $self->{sock} // return, my $buf, 4096);
                 if (scalar(@fds) == 1 && !defined($fds[0])) {
                         return if $! == EAGAIN;
                         die "recvmsg: $!" if $! != ECONNRESET;
-                        $buf = '';
                         @fds = (); # for open loop below:
                 }
                 for (@fds) { open my $rfh, '+<&=', $_ }
@@ -1167,7 +1207,7 @@ sub event_step {
                         die "unrecognized client signal: $buf";
                 }
                 my $s = $self->{-socks} // []; # lei up --all
-                @$s = grep { send($_, $buf, MSG_EOR) } @$s;
+                @$s = grep { send($_, $buf, 0) } @$s;
         };
         if (my $err = $@) {
                 eval { $self->fail($err) };
@@ -1185,8 +1225,6 @@ sub event_step_init {
         };
 }
 
-sub noop {}
-
 sub oldset { $oldset }
 
 sub dump_and_clear_log {
@@ -1203,48 +1241,87 @@ sub dump_and_clear_log {
 sub cfg2lei ($) {
         my ($cfg) = @_;
         my $lei = bless { env => { %{$cfg->{-env}} } }, __PACKAGE__;
-        open($lei->{0}, '<&', \*STDIN) or die "dup 0: $!";
-        open($lei->{1}, '>>&', \*STDOUT) or die "dup 1: $!";
-        open($lei->{2}, '>>&', \*STDERR) or die "dup 2: $!";
-        open($lei->{3}, '<', '/') or die "open /: $!";
-        my ($x, $y);
-        socketpair($x, $y, AF_UNIX, SOCK_SEQPACKET, 0) or die "socketpair: $!";
+        open($lei->{0}, '<&', \*STDIN);
+        open($lei->{1}, '>>&', \*STDOUT);
+        open($lei->{2}, '>>&', \*STDERR);
+        open($lei->{3}, '<', '/');
+        socketpair(my $x, my $y, AF_UNIX, SOCK_SEQPACKET, 0);
         $lei->{sock} = $x;
         require PublicInbox::LeiSelfSocket;
         PublicInbox::LeiSelfSocket->new($y); # adds to event loop
         $lei;
 }
 
+sub note_event ($@) { # runs lei_note_event for a given config file
+        my ($cfg_f, @args) = @_;
+        my $cfg = $PATH2CFG{$cfg_f} // return;
+        eval { cfg2lei($cfg)->dispatch('note-event', @args) };
+        carp "E: note-event $cfg_f: $@\n" if $@;
+}
+
 sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
         my ($ev) = @_; # Linux::Inotify2::Event or duck type
         my $fn = $ev->fullname;
         if ($fn =~ m!\A(.+)/(new|cur)/([^/]+)\z!) { # Maildir file
-                my ($mdir, $nc, $bn) = ($1, $2, $3);
-                $nc = '' if $ev->IN_DELETE || $ev->IN_MOVED_FROM;
-                for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) {
-                        my $cfg = $PATH2CFG{$f} // next;
-                        eval {
-                                my $lei = cfg2lei($cfg);
-                                $lei->dispatch('note-event',
-                                                "maildir:$mdir", $nc, $bn, $fn);
-                        };
-                        warn "E: note-event $f: $@\n" if $@;
+                my ($loc, $new_cur, $bn) = ("maildir:$1", $2, $3);
+                $new_cur = '' if $ev->IN_DELETE || $ev->IN_MOVED_FROM;
+                for my $cfg_f (keys %{$MDIR2CFGPATH->{$loc} // {}}) {
+                        note_event($cfg_f, $loc, $new_cur, $bn, $fn);
                 }
-        }
+        } elsif ($fn =~ m!\A(.+)/([0-9]+)\z!) { # MH mail message file
+                my ($loc, $n, $new_cur) = ("mh:$1", $2, '+');
+                $new_cur = '' if $ev->IN_DELETE || $ev->IN_MOVED_FROM;
+                for my $cfg_f (keys %{$MDIR2CFGPATH->{$loc} // {}}) {
+                        note_event($cfg_f, $loc, $new_cur, $n, $fn);
+                }
+        } elsif ($fn =~ m!\A(.+)/\.mh_sequences\z!) { # reread flags
+                my $loc = "mh:$1";
+                for my $cfg_f (keys %{$MDIR2CFGPATH->{$loc} // {}}) {
+                        note_event($cfg_f, $loc, '.mh_sequences')
+                }
+        } # else we don't care
         if ($ev->can('cancel') && ($ev->IN_IGNORE || $ev->IN_UNMOUNT)) {
                 $ev->cancel;
         }
         if ($fn =~ m!\A(.+)/(?:new|cur)\z! && !-e $fn) {
-                delete $MDIR2CFGPATH->{$1};
+                delete $MDIR2CFGPATH->{"maildir:$1"};
         }
-        if (!-e $fn) { # config file or Maildir gone
-                for my $cfgpaths (values %$MDIR2CFGPATH) {
-                        delete $cfgpaths->{$fn};
-                }
+        if (!-e $fn) { # config file, Maildir, or MH dir gone
+                delete $_->{$fn} for values %$MDIR2CFGPATH; # config file
+                delete @$MDIR2CFGPATH{"maildir:$fn", "mh:$fn"};
                 delete $PATH2CFG{$fn};
         }
 }
 
+sub can_stay_alive { # PublicInbox::DS::post_loop_do cb
+        my ($path, $dev_ino_expect) = @_;
+        if (my @st = defined($$path) ? stat($$path) : ()) {
+                if ($dev_ino_expect ne pack('dd', $st[0], $st[1])) {
+                        warn "$$path dev/ino changed, quitting\n";
+                        $$path = undef;
+                }
+        } elsif (defined($$path)) { # ENOENT is common
+                warn "stat($$path): $!, quitting ...\n" if $! != ENOENT;
+                undef $$path;
+                $quit->();
+        }
+        return 1 if defined($$path);
+        my $n = PublicInbox::DS::close_non_busy() or do {
+                eval 'PublicInbox::LeiNoteEvent::flush_task()';
+                # drop stores only if no clients
+                for my $cfg (values %PATH2CFG) {
+                        my $lne = delete($cfg->{-lei_note_event});
+                        $lne->wq_close if $lne;
+                        my $sto = delete($cfg->{-lei_store}) // next;
+                        eval { $sto->wq_do('done') if $sto->{-wq_s1} };
+                        warn "E: $@ (dropping store for $cfg->{-f})" if $@;
+                        $sto->wq_close;
+                }
+        };
+        # returns true: continue, false: stop
+        $n + scalar(keys(%PublicInbox::DS::AWAIT_PIDS));
+}
+
 # lei(1) calls this when it can't connect
 sub lazy_start {
         my ($path, $errno, $narg) = @_;
@@ -1252,106 +1329,66 @@ sub lazy_start {
         my ($sock_dir) = ($path =~ m!\A(.+?)/[^/]+\z!);
         $errors_log = "$sock_dir/errors.log";
         my $addr = pack_sockaddr_un($path);
-        my $lk = bless { lock_path => $errors_log }, 'PublicInbox::Lock';
+        my $lk = PublicInbox::Lock->new($errors_log);
         umask(077) // die("umask(077): $!");
         $lk->lock_acquire;
-        socket($listener, AF_UNIX, SOCK_SEQPACKET, 0) or die "socket: $!";
+        socket($listener, AF_UNIX, SOCK_SEQPACKET, 0);
         if ($errno == ECONNREFUSED || $errno == ENOENT) {
                 return if connect($listener, $addr); # another process won
-                if ($errno == ECONNREFUSED && -S $path) {
-                        unlink($path) or die "unlink($path): $!";
-                }
+                unlink($path) if $errno == ECONNREFUSED && -S $path;
         } else {
                 $! = $errno; # allow interpolation to stringify in die
                 die "connect($path): $!";
         }
-        bind($listener, $addr) or die "bind($path): $!";
+        bind($listener, $addr);
         $lk->lock_release;
         undef $lk;
         my @st = stat($path) or die "stat($path): $!";
         my $dev_ino_expect = pack('dd', $st[0], $st[1]); # dev+ino
-        local $oldset = PublicInbox::DS::block_signals();
-        if ($narg == 5) {
-                $send_cmd = PublicInbox::Spawn->can('send_cmd4');
-                $recv_cmd = PublicInbox::Spawn->can('recv_cmd4') // do {
-                        require PublicInbox::CmdIPC4;
-                        $send_cmd = PublicInbox::CmdIPC4->can('send_cmd4');
-                        PublicInbox::CmdIPC4->can('recv_cmd4');
-                } // do {
-                        $send_cmd = PublicInbox::Syscall->can('send_cmd4');
-                        PublicInbox::Syscall->can('recv_cmd4');
-                };
-        }
-        $recv_cmd or die <<"";
+        local $oldset = PublicInbox::DS::block_signals(POSIX::SIGALRM);
+        die "incompatible narg=$narg" if $narg != 5;
+        $PublicInbox::IPC::send_cmd or die <<"";
 (Socket::MsgHdr || Inline::C) missing/unconfigured (narg=$narg);
 
         require PublicInbox::Listener;
         require PublicInbox::PktOp;
         (-p STDOUT) or die "E: stdout must be a pipe\n";
-        open(STDIN, '+>>', $errors_log) or die "open($errors_log): $!";
+        open(STDIN, '+>>', $errors_log);
         STDIN->autoflush(1);
         dump_and_clear_log();
         POSIX::setsid() > 0 or die "setsid: $!";
-        my $pid = fork // die "fork: $!";
+        my $pid = fork;
         return if $pid;
         $0 = "lei-daemon $path";
-        local %PATH2CFG;
-        local $MDIR2CFGPATH;
+        local (%PATH2CFG, $MDIR2CFGPATH);
+        local $daemon_pid = $$;
         $listener->blocking(0);
         my $exit_code;
         my $pil = PublicInbox::Listener->new($listener, \&accept_dispatch);
         local $quit = do {
                 my (undef, $eof_p) = PublicInbox::PktOp->pair;
                 sub {
-                        $exit_code //= shift;
+                        $exit_code //= eval("POSIX::SIG$_[0] + 128") if @_;
+                        $dir_idle->close if $dir_idle; # EPOLL_CTL_DEL
+                        $dir_idle = undef; # let RC take care of it
                         eval 'PublicInbox::LeiNoteEvent::flush_task()';
-                        my $lis = $pil or exit($exit_code);
+                        my $lis = $pil or exit($exit_code // 0);
                         # closing eof_p triggers \&noop wakeup
                         $listener = $eof_p = $pil = $path = undef;
                         $lis->close; # DS::close
-                        PublicInbox::DS->SetLoopTimeout(1000);
                 };
         };
-        my $sig = {
-                CHLD => \&PublicInbox::DS::enqueue_reap,
-                QUIT => $quit,
-                INT => $quit,
-                TERM => $quit,
-                HUP => \&noop,
-                USR1 => \&noop,
-                USR2 => \&noop,
-        };
+        my $sig = { CHLD => \&PublicInbox::DS::enqueue_reap };
+        $sig->{$_} = $quit for qw(QUIT INT TERM);
+        $sig->{$_} = \&PublicInbox::Config::noop for qw(HUP USR1 USR2);
         require PublicInbox::DirIdle;
         local $dir_idle = PublicInbox::DirIdle->new(sub {
-                # just rely on wakeup to hit PostLoopCallback set below
+                # just rely on wakeup to hit post_loop_do
                 dir_idle_handler($_[0]) if $_[0]->fullname ne $path;
         });
         $dir_idle->add_watches([$sock_dir]);
-        PublicInbox::DS->SetPostLoopCallback(sub {
-                my ($dmap, undef) = @_;
-                if (@st = defined($path) ? stat($path) : ()) {
-                        if ($dev_ino_expect ne pack('dd', $st[0], $st[1])) {
-                                warn "$path dev/ino changed, quitting\n";
-                                $path = undef;
-                        }
-                } elsif (defined($path)) { # ENOENT is common
-                        warn "stat($path): $!, quitting ...\n" if $! != ENOENT;
-                        undef $path;
-                        $quit->();
-                }
-                return 1 if defined($path);
-                my $n = 0;
-                for my $s (values %$dmap) {
-                        $s->can('busy') or next;
-                        if ($s->busy) {
-                                ++$n;
-                        } else {
-                                $s->close;
-                        }
-                }
-                $n; # true: continue, false: stop
-        });
-
+        local @PublicInbox::DS::post_loop_do = (\&can_stay_alive,
+                                                \$path, $dev_ino_expect);
         # STDIN was redirected to /dev/null above, closing STDERR and
         # STDOUT will cause the calling `lei' client process to finish
         # reading the <$daemon> pipe.
@@ -1359,13 +1396,13 @@ sub lazy_start {
                 $current_lei ? err($current_lei, @_) : warn(
                   strftime('%Y-%m-%dT%H:%M:%SZ', gmtime(time))," $$ ", @_);
         };
-        open STDERR, '>&STDIN' or die "redirect stderr failed: $!";
-        open STDOUT, '>&STDIN' or die "redirect stdout failed: $!";
+        local $SIG{PIPE} = 'IGNORE';
+        local $SIG{ALRM} = 'IGNORE';
+        open STDERR, '>&STDIN';
+        open STDOUT, '>&STDIN';
         # $daemon pipe to `lei' closed, main loop begins:
         eval { PublicInbox::DS::event_loop($sig, $oldset) };
         warn "event loop error: $@\n" if $@;
-        # exit() may trigger waitpid via various DESTROY, ensure interruptible
-        PublicInbox::DS::sig_setmask($oldset);
         dump_and_clear_log();
         exit($exit_code // 0);
 }
@@ -1376,9 +1413,10 @@ sub busy { 1 } # prevent daemon-shutdown if client is connected
 # can immediately reread it
 sub DESTROY {
         my ($self) = @_;
-        if (my $counters = delete $self->{counters}) {
-                for my $k (sort keys %$counters) {
-                        my $nr = $counters->{$k};
+        if (defined($self->{incr_pid}) && $self->{incr_pid} == $$) {
+                for my $k (sort(grep(/\A-nr_/, keys %$self))) {
+                        my $nr = $self->{$k};
+                        substr($k, 0, length('-nr_'), '');
                         $self->child_error(0, "$nr $k messages");
                 }
         }
@@ -1388,9 +1426,8 @@ sub DESTROY {
         # preserve $? for ->fail or ->x_it code
 }
 
-sub wq_done_wait { # dwaitpid callback
-        my ($arg, $pid) = @_;
-        my ($wq, $lei) = @$arg;
+sub wq_done_wait { # awaitpid cb (via wq_eof)
+        my ($pid, $wq, $lei) = @_;
         local $current_lei = $lei;
         my $err_type = $lei->{-err_type};
         $? and $lei->child_error($?,
@@ -1400,8 +1437,7 @@ sub wq_done_wait { # dwaitpid callback
 
 sub fchdir {
         my ($lei) = @_;
-        my $dh = $lei->{3} // die 'BUG: lei->{3} (CWD) gone';
-        chdir($dh) || die "fchdir: $!";
+        chdir($lei->{3} // die 'BUG: lei->{3} (CWD) gone');
 }
 
 sub wq_eof { # EOF callback for main daemon
@@ -1417,19 +1453,22 @@ sub watch_state_ok ($) {
         $state =~ /\Apause|(?:import|index|tag)-(?:ro|rw)\z/;
 }
 
-sub cancel_maildir_watch ($$) {
-        my ($d, $cfg_f) = @_;
-        my $w = delete $MDIR2CFGPATH->{$d}->{$cfg_f};
-        scalar(keys %{$MDIR2CFGPATH->{$d}}) or
-                delete $MDIR2CFGPATH->{$d};
-        for my $x (@{$w // []}) { $x->cancel }
+sub cancel_dir_watch ($$$) {
+        my ($type, $d, $cfg_f) = @_;
+        my $loc = "$type:".canonpath_harder($d);
+        my $w = delete $MDIR2CFGPATH->{$loc}->{$cfg_f};
+        delete $MDIR2CFGPATH->{$loc} if !(keys %{$MDIR2CFGPATH->{$loc}});
+        $_->cancel for @$w;
 }
 
-sub add_maildir_watch ($$) {
-        my ($d, $cfg_f) = @_;
-        if (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) {
-                my @w = $dir_idle->add_watches(["$d/cur", "$d/new"], 1);
-                push @{$MDIR2CFGPATH->{$d}->{$cfg_f}}, @w if @w;
+sub add_dir_watch ($$$) {
+        my ($type, $d, $cfg_f) = @_;
+        $d = canonpath_harder($d);
+        my $loc = "$type:$d";
+        my @dirs = $type eq 'mh' ? ($d) : ("$d/cur", "$d/new");
+        if (!exists($MDIR2CFGPATH->{$loc}->{$cfg_f})) {
+                my @w = $dir_idle->add_watches(\@dirs, 1);
+                push @{$MDIR2CFGPATH->{$loc}->{$cfg_f}}, @w if @w;
         }
 }
 
@@ -1442,24 +1481,20 @@ sub refresh_watches {
         my %seen;
         my $cfg_f = $cfg->{'-f'};
         for my $w (grep(/\Awatch\..+\.state\z/, keys %$cfg)) {
-                my $url = substr($w, length('watch.'), -length('.state'));
+                my $loc = substr($w, length('watch.'), -length('.state'));
                 require PublicInbox::LeiWatch;
-                $watches->{$url} //= PublicInbox::LeiWatch->new($url);
-                $seen{$url} = undef;
-                my $state = $cfg->get_1("watch.$url.state");
+                $watches->{$loc} //= PublicInbox::LeiWatch->new($loc);
+                $seen{$loc} = undef;
+                my $state = $cfg->get_1("watch.$loc.state");
                 if (!watch_state_ok($state)) {
-                        warn("watch.$url.state=$state not supported\n");
-                        next;
-                }
-                if ($url =~ /\Amaildir:(.+)/i) {
-                        my $d = canonpath_harder($1);
-                        if ($state eq 'pause') {
-                                cancel_maildir_watch($d, $cfg_f);
-                        } else {
-                                add_maildir_watch($d, $cfg_f);
-                        }
+                        warn("watch.$loc.state=$state not supported\n");
+                } elsif ($loc =~ /\A(maildir|mh):(.+)\z/i) {
+                        my ($type, $d) = ($1, $2);
+                        $state eq 'pause' ?
+                                cancel_dir_watch($type, $d, $cfg_f) :
+                                add_dir_watch($type, $d, $cfg_f);
                 } else { # TODO: imap/nntp/jmap
-                        $lei->child_error(0, "E: watch $url not supported, yet")
+                        $lei->child_error(0, "E: watch $loc not supported, yet")
                 }
         }
 
@@ -1467,29 +1502,28 @@ sub refresh_watches {
         my $lms = $lei->lms;
         if ($lms) {
                 $lms->lms_write_prepare;
-                for my $d ($lms->folders('maildir:')) {
-                        substr($d, 0, length('maildir:')) = '';
-
+                for my $loc ($lms->folders(qr/\A(?:maildir|mh):/)) {
+                        my $old = $loc;
+                        my ($type, $d) = split /:/, $loc, 2;
                         # fixup old bugs while we're iterating:
-                        my $cd = canonpath_harder($d);
-                        my $f = "maildir:$cd";
-                        $lms->rename_folder("maildir:$d", $f) if $d ne $cd;
-                        next if $watches->{$f}; # may be set to pause
+                        $d = canonpath_harder($d);
+                        $loc = "$type:$d";
+                        $lms->rename_folder($old, $loc) if $old ne $loc;
+                        next if $watches->{$loc}; # may be set to pause
                         require PublicInbox::LeiWatch;
-                        $watches->{$f} = PublicInbox::LeiWatch->new($f);
-                        $seen{$f} = undef;
-                        add_maildir_watch($cd, $cfg_f);
+                        $watches->{$loc} = PublicInbox::LeiWatch->new($loc);
+                        $seen{$loc} = undef;
+                        add_dir_watch($type, $d, $cfg_f);
                 }
         }
         if ($old) { # cull old non-existent entries
-                for my $url (keys %$old) {
-                        next if exists $seen{$url};
-                        delete $old->{$url};
-                        if ($url =~ /\Amaildir:(.+)/i) {
-                                my $d = canonpath_harder($1);
-                                cancel_maildir_watch($d, $cfg_f);
+                for my $loc (keys %$old) {
+                        next if exists $seen{$loc};
+                        delete $old->{$loc};
+                        if ($loc =~ /\A(maildir|mh):(.+)\z/i) {
+                                cancel_dir_watch($1, $2, $cfg_f);
                         } else { # TODO: imap/nntp/jmap
-                                $lei->child_error(0, "E: watch $url TODO");
+                                $lei->child_error(0, "E: watch $loc TODO");
                         }
                 }
         }
@@ -1517,22 +1551,22 @@ sub lms {
 
 sub sto_done_request {
         my ($lei, $wq) = @_;
-        return unless $lei->{sto};
+        return unless $lei->{sto} && $lei->{sto}->{-wq_s1};
         local $current_lei = $lei;
-        my $sock = $wq ? $wq->{lei_sock} : undef;
-        eval {
-                if ($sock //= $lei->{sock}) { # issue, async wait
-                        $lei->{sto}->wq_io_do('done', [ $sock ]);
-                } else { # forcibly wait
-                        my $wait = $lei->{sto}->wq_do('done');
-                }
-        };
+        if (my $n = $lei->{opt}->{'commit-delay'}) {
+                eval { $lei->{sto}->wq_do('schedule_commit', $n) };
+        } else {
+                my $s = ($wq ? $wq->{lei_sock} : undef) // $lei->{sock};
+                my $errfh = $lei->{2} // *STDERR{GLOB};
+                my @io = $s ? ($errfh, $s) : ($errfh);
+                eval { $lei->{sto}->wq_io_do('done', \@io) };
+        }
         warn($@) if $@;
 }
 
 sub cfg_dump ($$) {
         my ($lei, $f) = @_;
-        my $ret = eval { PublicInbox::Config->git_config_dump($f, $lei->{2}) };
+        my $ret = eval { PublicInbox::Config->git_config_dump($f, $lei) };
         return $ret if !$@;
         warn($@);
         undef;
@@ -1541,7 +1575,7 @@ sub cfg_dump ($$) {
 sub request_umask {
         my ($lei) = @_;
         my $s = $lei->{sock} // return;
-        send($s, 'umask', MSG_EOR) // die "send: $!";
+        send($s, 'umask', 0) // die "send: $!";
         vec(my $rvec = '', fileno($s), 1) = 1;
         select($rvec, undef, undef, 2) or die 'timeout waiting for umask';
         recv($s, my $v, 5, 0) // die "recv: $!";
@@ -1549,4 +1583,24 @@ sub request_umask {
         $u eq 'u' or warn "E: recv $v has no umask";
 }
 
+sub _stdin_cb { # PublicInbox::InputPipe::consume callback for --stdin
+        my (undef, $lei, $cb) = @_; # $_[-1] = $rbuf
+        $_[1] // return $lei->fail("error reading stdin: $!");
+        $lei->{stdin_buf} .= $_[-1];
+        do_env($lei, $cb) if $_[-1] eq '';
+}
+
+sub slurp_stdin {
+        my ($lei, $cb) = @_;
+        require PublicInbox::InputPipe;
+        my $in = $lei->{0};
+        if (-t $in) { # run cat via script/lei and read from it
+                $in = undef;
+                pipe($in, my $wr);
+                say { $lei->{2} } '# enter query, Ctrl-D when done';
+                send_exec_cmd($lei, [ $lei->{0}, $wr ], ['cat'], {});
+        }
+        PublicInbox::InputPipe::consume($in, \&_stdin_cb, $lei, $cb);
+}
+
 1;
diff --git a/lib/PublicInbox/LI2Wrap.pm b/lib/PublicInbox/LI2Wrap.pm
index 204850a6..d4792b25 100644
--- a/lib/PublicInbox/LI2Wrap.pm
+++ b/lib/PublicInbox/LI2Wrap.pm
@@ -5,7 +5,7 @@
 # Remove this when supported LTS/enterprise distros are all
 # Linux::Inotify2 >= 2.3
 package PublicInbox::LI2Wrap;
-use v5.10.1;
+use v5.12;
 our @ISA = qw(Linux::Inotify2);
 
 sub wrapclose {
diff --git a/lib/PublicInbox/LeiALE.pm b/lib/PublicInbox/LeiALE.pm
index cc9a2095..528de22c 100644
--- a/lib/PublicInbox/LeiALE.pm
+++ b/lib/PublicInbox/LeiALE.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # All Locals Ever: track lei/store + externals ever used as
@@ -6,10 +6,10 @@
 # and --only targets that haven't been through "lei add-external".
 # Typically: ~/.cache/lei/all_locals_ever.git
 package PublicInbox::LeiALE;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::LeiSearch PublicInbox::Lock);
 use PublicInbox::Git;
+use autodie qw(close open rename seek truncate);
 use PublicInbox::Import;
 use PublicInbox::LeiXSearch;
 use Fcntl qw(SEEK_SET);
@@ -54,11 +54,7 @@ sub refresh_externals {
         $self->git->cleanup;
         my $lk = $self->lock_for_scope;
         my $cur_lxs = ref($lxs)->new;
-        my $orig = do {
-                local $/;
-                readline($self->{lockfh}) //
-                                die "readline($self->{lock_path}): $!";
-        };
+        my $orig = PublicInbox::IO::read_all $self->{lockfh};
         my $new = '';
         my $old = '';
         my $gone = 0;
@@ -81,19 +77,16 @@ sub refresh_externals {
         if ($new ne '' || $gone) {
                 $self->{lockfh}->autoflush(1);
                 if ($gone) {
-                        seek($self->{lockfh}, 0, SEEK_SET) or die "seek: $!";
-                        truncate($self->{lockfh}, 0) or die "truncate: $!";
+                        seek($self->{lockfh}, 0, SEEK_SET);
+                        truncate($self->{lockfh}, 0);
                 } else {
                         $old = '';
                 }
                 print { $self->{lockfh} } $old, $new or die "print: $!";
         }
-        $new = $old = '';
+        $new = '';
         my $f = $self->git->{git_dir}.'/objects/info/alternates';
-        if (open my $fh, '<', $f) {
-                local $/;
-                $old = <$fh> // die "readline($f): $!";
-        }
+        $old = PublicInbox::IO::try_cat $f;
         for my $x (@ibxish) {
                 $new .= $lei->canonpath_harder($x->git->{git_dir})."/objects\n";
         }
@@ -103,10 +96,10 @@ sub refresh_externals {
         # this needs to be atomic since child processes may start
         # git-cat-file at any time
         my $tmp = "$f.$$.tmp";
-        open my $fh, '>', $tmp or die "open($tmp): $!";
-        print $fh $new or die "print($tmp): $!";
-        close $fh or die "close($tmp): $!";
-        rename($tmp, $f) or die "rename($tmp, $f): $!";
+        open my $fh, '>', $tmp;
+        print $fh $new;
+        close $fh;
+        rename($tmp, $f)
 }
 
 1;
diff --git a/lib/PublicInbox/LeiAddWatch.pm b/lib/PublicInbox/LeiAddWatch.pm
index 97e7a342..e2be5cee 100644
--- a/lib/PublicInbox/LeiAddWatch.pm
+++ b/lib/PublicInbox/LeiAddWatch.pm
@@ -15,24 +15,23 @@ sub lei_add_watch {
         my $state = $lei->{opt}->{'state'} // 'import-rw';
         $lei->watch_state_ok($state) or
                 return $lei->fail("invalid state: $state");
-        my $vmd_mod = $self->vmd_mod_extract(\@argv);
-        return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
         $self->prepare_inputs($lei, \@argv) or return;
         my @vmd;
-        while (my ($type, $vals) = each %$vmd_mod) {
+        while (my ($type, $vals) = each %{$lei->{vmd_mod}}) {
                 push @vmd, "$type:$_" for @$vals;
         }
         my $vmd0 = shift @vmd;
         for my $w (@{$self->{inputs}}) {
                 # clobber existing, allow multiple
                 if (defined($vmd0)) {
-                        $lei->_config("watch.$w.vmd", '--replace-all', $vmd0);
+                        $lei->_config("watch.$w.vmd", '--replace-all', $vmd0)
+                                or return;
                         for my $v (@vmd) {
-                                $lei->_config("watch.$w.vmd", $v);
+                                $lei->_config("watch.$w.vmd", $v) or return;
                         }
                 }
                 next if defined $cfg->{"watch.$w.state"};
-                $lei->_config("watch.$w.state", $state);
+                $lei->_config("watch.$w.state", $state) or return;
         }
         $lei->_lei_store(1); # create
         $lei->lms(1)->lms_write_prepare->add_folders(@{$self->{inputs}});
diff --git a/lib/PublicInbox/LeiAuth.pm b/lib/PublicInbox/LeiAuth.pm
index 9b09cecf..020dd125 100644
--- a/lib/PublicInbox/LeiAuth.pm
+++ b/lib/PublicInbox/LeiAuth.pm
@@ -1,8 +1,8 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Authentication worker for anything that needs auth for read/write IMAP
-# (eventually for read-only NNTP access)
+# and read-only NNTP access
 #
 # timelines
 # lei-daemon              |  LeiAuth worker #0      | other WQ workers
@@ -22,8 +22,7 @@
 #                         |
 # call net_merge_all_done ->-> do per-WQ-class defined actions
 package PublicInbox::LeiAuth;
-use strict;
-use v5.10.1;
+use v5.12;
 
 sub do_auth_atfork { # used by IPC WQ workers
         my ($self, $wq) = @_;
@@ -57,7 +56,7 @@ sub net_merge_all { # called in wq worker via wq_broadcast
 # called by top-level lei-daemon when first worker is done with auth
 # passes updated net auth info to current workers
 sub net_merge_continue {
-        my ($wq, $lei, $net_new) = @_;
+        my ($lei, $wq, $net_new) = @_;
         $wq->{-net_new} = $net_new; # for "lei up"
         $wq->wq_broadcast('PublicInbox::LeiAuth::net_merge_all', $net_new);
         $wq->net_merge_all_done($lei); # defined per-WQ
@@ -65,7 +64,7 @@ sub net_merge_continue {
 
 sub op_merge { # prepares PktOp->pair ops
         my ($self, $ops, $wq, $lei) = @_;
-        $ops->{net_merge_continue} = [ \&net_merge_continue, $wq, $lei ];
+        $ops->{net_merge_continue} = [ \&net_merge_continue, $lei, $wq ];
 }
 
 sub new { bless \(my $x), __PACKAGE__ }
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 004b156c..127cc81e 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # "lei blob $OID" command
@@ -7,8 +7,11 @@ package PublicInbox::LeiBlob;
 use strict;
 use v5.10.1;
 use parent qw(PublicInbox::IPC);
-use PublicInbox::Spawn qw(spawn popen_rd which);
+use PublicInbox::Spawn qw(run_wait run_qx which);
 use PublicInbox::DS;
+use PublicInbox::Eml;
+use PublicInbox::Git;
+use PublicInbox::IO qw(read_all);
 
 sub get_git_dir ($$) {
         my ($lei, $d) = @_;
@@ -21,10 +24,8 @@ sub get_git_dir ($$) {
         } else { # implicit --cwd, quiet errors
                 open $opt->{2}, '>', '/dev/null' or die "open /dev/null: $!";
         }
-        my ($r, $pid) = popen_rd($cmd, {GIT_DIR => undef}, $opt);
-        chomp(my $gd = do { local $/; <$r> });
-        waitpid($pid, 0) == $pid or die "BUG: waitpid @$cmd ($!)";
-        $? == 0 ? $gd : undef;
+        chomp(my $git_dir = run_qx($cmd, {GIT_DIR => undef}, $opt));
+        $? ? undef : $git_dir;
 }
 
 sub solver_user_cb { # called by solver when done
@@ -44,8 +45,7 @@ sub solver_user_cb { # called by solver when done
 
         my $cmd = [ 'git', "--git-dir=$gd", 'show', $oid ];
         my $rdr = { 1 => $lei->{1}, 2 => $lei->{2} };
-        waitpid(spawn($cmd, $lei->{env}, $rdr), 0);
-        $lei->child_error($?) if $?;
+        run_wait($cmd, $lei->{env}, $rdr) and $lei->child_error($?);
 }
 
 sub do_solve_blob { # via wq_do
@@ -70,7 +70,7 @@ sub do_solve_blob { # via wq_do
                         } @$git_dirs ],
                 user_cb => \&solver_user_cb,
                 uarg => $self,
-                # -cur_di, -qsp, -msg => temporary fields for Qspawn callbacks
+                # -cur_di, -msg => temporary fields for Qspawn callbacks
                 inboxes => [ $self->{lxs}->locals, @rmt ],
         }, 'PublicInbox::SolverGit';
         local $PublicInbox::DS::in_loop = 0; # waitpid synchronously
@@ -122,25 +122,23 @@ sub lei_blob {
                 my $cmd = [ 'git', '--git-dir='.$lei->ale->git->{git_dir},
                                 'cat-file', 'blob', $blob ];
                 if (defined $lei->{-attach_idx}) {
-                        my $fh = popen_rd($cmd, $lei->{env}, $rdr);
-                        require PublicInbox::Eml;
-                        my $buf = do { local $/; <$fh> };
-                        return extract_attach($lei, $blob, \$buf) if close($fh);
-                } else {
-                        $rdr->{1} = $lei->{1};
-                        waitpid(spawn($cmd, $lei->{env}, $rdr), 0);
+                        my $buf = run_qx($cmd, $lei->{env}, $rdr);
+                        return extract_attach($lei, $blob, \$buf) unless $?;
                 }
-                my $ce = $?;
-                return if $ce == 0;
+                $rdr->{1} = $lei->{1};
+                my $cerr = run_wait($cmd, $lei->{env}, $rdr) or return;
                 my $lms = $lei->lms;
-                if (my $bref = $lms ? $lms->local_blob($blob, 1) : undef) {
-                        defined($lei->{-attach_idx}) and
-                                return extract_attach($lei, $blob, $bref);
-                        return $lei->out($$bref);
-                } elsif ($opt->{mail}) {
-                        my $eh = $rdr->{2};
-                        seek($eh, 0, 0);
-                        return $lei->child_error($ce, do { local $/; <$eh> });
+                my $bref = ($lms ? $lms->local_blob($blob, 1) : undef) // do {
+                        my $sto = $lei->{sto} // $lei->_lei_store;
+                        $sto && $sto->{-wq_s1} ? $sto->wq_do('cat_blob', $blob)
+                                                : undef;
+                };
+                $bref and return $lei->{-attach_idx} ?
+                                        extract_attach($lei, $blob, $bref) :
+                                        $lei->out($$bref);
+                if ($opt->{mail}) {
+                        seek($rdr->{2}, 0, 0);
+                        return $lei->child_error($cerr, read_all($rdr->{2}));
                 } # else: fall through to solver below
         }
 
@@ -158,7 +156,7 @@ sub lei_blob {
         if ($lxs->remotes) {
                 require PublicInbox::LeiRemote;
                 $lei->{curl} //= which('curl') or return
-                        $lei->fail('curl needed for', $lxs->remotes);
+                        $lei->fail('curl needed for '.join(', ',$lxs->remotes));
                 $lei->_lei_store(1)->write_prepare($lei);
         }
         require PublicInbox::SolverGit;
diff --git a/lib/PublicInbox/LeiConfig.pm b/lib/PublicInbox/LeiConfig.pm
index 23be9aaf..a50ff2b6 100644
--- a/lib/PublicInbox/LeiConfig.pm
+++ b/lib/PublicInbox/LeiConfig.pm
@@ -1,9 +1,11 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-package PublicInbox::LeiConfig;
-use strict;
-use v5.10.1;
+package PublicInbox::LeiConfig; # subclassed by LeiEditSearch
+use v5.12;
 use PublicInbox::PktOp;
+use Fcntl qw(SEEK_SET);
+use autodie qw(open seek);
+use PublicInbox::IO qw(read_all);
 
 sub cfg_do_edit ($;$) {
         my ($self, $reason) = @_;
@@ -15,28 +17,39 @@ sub cfg_do_edit ($;$) {
         # run in script/lei foreground
         my ($op_c, $op_p) = PublicInbox::PktOp->pair;
         # $op_p will EOF when $EDITOR is done
-        $op_c->{ops} = { '' => [\&cfg_edit_done, $self] };
+        $op_c->{ops} = { '' => [\&cfg_edit_done, $lei, $self] };
         $lei->send_exec_cmd([ @$lei{qw(0 1 2)}, $op_p->{op_p} ], $cmd, $env);
 }
 
-sub cfg_edit_done { # PktOp
-        my ($self) = @_;
-        eval {
-                my $cfg = $self->{lei}->cfg_dump($self->{-f}, $self->{lei}->{2})
-                        // return cfg_do_edit($self, "\n");
-                $self->cfg_verify($cfg) if $self->can('cfg_verify');
+sub cfg_edit_done { # PktOp lei->do_env cb
+        my ($lei, $self) = @_;
+        open my $fh, '+>', undef;
+        my $cfg = do {
+                local $lei->{2} = $fh;
+                $lei->cfg_dump($self->{-f});
+        } or do {
+                seek($fh, 0, SEEK_SET);
+                return cfg_do_edit($self, read_all($fh));
         };
-        $self->{lei}->fail($@) if $@;
+        $self->cfg_verify($cfg) if $self->can('cfg_verify');
 }
 
 sub lei_config {
         my ($lei, @argv) = @_;
         $lei->{opt}->{'config-file'} and return $lei->fail(
                 "config file switches not supported by `lei config'");
-        return $lei->_config(@argv) unless $lei->{opt}->{edit};
-        my $f = $lei->_lei_cfg(1)->{-f};
-        my $self = bless { lei => $lei, -f => $f }, __PACKAGE__;
-        cfg_do_edit($self);
+        if ($lei->{opt}->{edit}) {
+                @argv and return $lei->fail(
+'--edit must be used without other arguments');
+                $lei->{opt}->{c} and return $lei->fail(
+"`-c $lei->{opt}->{c}->[0]' not allowed with --edit");
+                my $f = $lei->_lei_cfg(1)->{-f};
+                cfg_do_edit(bless { lei => $lei, -f => $f }, __PACKAGE__);
+        } elsif (@argv) { # let git-config do error-checking
+                $lei->_config(@argv);
+        } else {
+                $lei->_help('no options given');
+        }
 }
 
 1;
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 906f3026..4d4fceb2 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # front-end for the "lei convert" sub-command
@@ -28,21 +28,31 @@ sub input_maildir_cb {
         $self->{wcb}->(undef, { kw => $kw }, $eml);
 }
 
+sub input_mh_cb {
+        my ($dn, $bn, $kw, $eml, $self) = @_;
+        $self->{wcb}->(undef, { kw => $kw }, $eml);
+}
+
 sub process_inputs { # via wq_do
         my ($self) = @_;
-        local $PublicInbox::DS::in_loop = 0; # force synchronous dwaitpid
+        local $PublicInbox::DS::in_loop = 0; # force synchronous awaitpid
         $self->SUPER::process_inputs;
         my $lei = $self->{lei};
-        delete $lei->{1};
-        delete $self->{wcb}; # commit
-        my $nr = delete($lei->{-nr_write}) // 0;
-        $lei->qerr("# converted $nr messages");
+        my $l2m = delete $lei->{l2m};
+        delete $self->{wcb}; # may close connections
+        $l2m->finish_output($lei) if $l2m;
+        if (my $v2w = delete $lei->{v2w}) { $v2w->done } # may die
+        my $nr_w = delete($l2m->{-nr_write}) // 0;
+        my $d = (delete($l2m->{-nr_seen}) // 0) - $nr_w;
+        $d = $d ? " ($d duplicates)" : '';
+        $lei->qerr("# converted $nr_w messages$d");
 }
 
 sub lei_convert { # the main "lei convert" method
         my ($lei, @inputs) = @_;
         $lei->{opt}->{kw} //= 1;
         $lei->{opt}->{dedupe} //= 'none';
+        $lei->{input_opt}->{sort} = 1; # for LeiToMail conflict check
         my $self = bless {}, __PACKAGE__;
         my $ovv = PublicInbox::LeiOverview->new($lei, 'out-format');
         $lei->{l2m} or return
@@ -62,7 +72,7 @@ sub ipc_atfork_child {
         my ($self) = @_;
         my $lei = $self->{lei};
         $lei->_lei_atfork_child;
-        my $l2m = delete $lei->{l2m};
+        my $l2m = $lei->{l2m};
         if (my $net = $lei->{net}) { # may prompt user once
                 $net->{mics_cached} = $net->imap_common_init($lei);
                 $net->{nn_cached} = $net->nntp_common_init($lei);
diff --git a/lib/PublicInbox/LeiCurl.pm b/lib/PublicInbox/LeiCurl.pm
index 5ffade99..48c66ee9 100644
--- a/lib/PublicInbox/LeiCurl.pm
+++ b/lib/PublicInbox/LeiCurl.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # common option and torsocks(1) wrapping for curl(1)
@@ -7,8 +7,7 @@
 # n.b. curl may support a daemon/client model like lei someday:
 #   https://github.com/curl/curl/wiki/curl-tool-master-client
 package PublicInbox::LeiCurl;
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::Spawn qw(which);
 use PublicInbox::Config;
 
@@ -27,7 +26,7 @@ sub new {
         my ($cls, $lei, $curl) = @_;
         $curl //= which('curl') // return $lei->fail('curl not found');
         my $opt = $lei->{opt};
-        my @cmd = ($curl, qw(-Sf));
+        my @cmd = ($curl, qw(-gSf));
         $cmd[-1] .= 's' if $opt->{quiet}; # already the default for "lei q"
         $cmd[-1] .= 'v' if $opt->{verbose}; # we use ourselves, too
         for my $o ($lei->curl_opt) {
@@ -77,8 +76,8 @@ sub for_uri {
         my $pfx = torsocks($self, $lei, $uri) or return; # error
         if ($uri->scheme =~ /\Ahttps?\z/i) {
                 my $cfg = $lei->_lei_cfg;
-                my $p = $cfg ? $cfg->urlmatch('http.Proxy', $$uri) : undef;
-                push(@opt, "--proxy=$p") if defined($p);
+                my $p = $cfg ? $cfg->urlmatch('http.Proxy', $$uri, 1) : undef;
+                push(@opt, '--proxy', $p) if defined($p);
         }
         bless [ @$pfx, @$self, @opt, $uri->as_string ], ref($self);
 }
diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index 32f99cd0..eda54d79 100644
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -1,10 +1,9 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 package PublicInbox::LeiDedupe;
-use strict;
-use v5.10.1;
-use PublicInbox::ContentHash qw(content_hash git_sha);
-use Digest::SHA ();
+use v5.12;
+use PublicInbox::ContentHash qw(content_hash content_digest git_sha);
+use PublicInbox::SHA qw(sha256);
 
 # n.b. mutt sets most of these headers not sure about Bytes
 our @OID_IGNORE = qw(Status X-Status Content-Length Lines Bytes);
@@ -30,11 +29,9 @@ sub _oidbin ($) { defined($_[0]) ? pack('H*', $_[0]) : undef }
 
 sub smsg_hash ($) {
         my ($smsg) = @_;
-        my $dig = Digest::SHA->new(256);
         my $x = join("\0", @$smsg{qw(from to cc ds subject references mid)});
         utf8::encode($x);
-        $dig->add($x);
-        $dig->digest;
+        sha256($x);
 }
 
 # the paranoid option
@@ -72,7 +69,12 @@ sub dedupe_content ($) {
         my ($skv) = @_;
         (sub { # may be called in a child process
                 my ($eml) = @_; # $oidhex = $_[1], ignored
-                $skv->set_maybe(content_hash($eml), '');
+
+                # we must account for Message-ID via hash_mids, since
+                # (unlike v2 dedupe) Message-ID is not accounted for elsewhere:
+                $skv->set_maybe(content_digest($eml, PublicInbox::SHA->new(256),
+                                1 # hash_mids
+                                )->digest, '');
         }, sub {
                 my ($smsg) = @_;
                 $skv->set_maybe(smsg_hash($smsg), '');
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index d2396fa7..16f069da 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -38,7 +38,7 @@ sub export_kw_md { # LeiMailSync->each_src callback
                 } elsif ($! == EEXIST) { # lost race with lei/store?
                         return;
                 } elsif ($! != ENOENT) {
-                        $lei->child_error(1,
+                        $lei->child_error(0,
                                 "E: rename_noreplace($src -> $dst): $!");
                 } # else loop @try
         }
@@ -46,7 +46,7 @@ sub export_kw_md { # LeiMailSync->each_src callback
         # both tries failed
         my $oidhex = unpack('H*', $oidbin);
         my $src = "$mdir/{".join(',', @try)."}/$$id";
-        $lei->child_error(1, "rename_noreplace($src -> $dst) ($oidhex): $e");
+        $lei->child_error(0, "rename_noreplace($src -> $dst) ($oidhex): $e");
         for (@try) { return if -e "$mdir/$_/$$id" }
         $self->{lms}->clear_src("maildir:$mdir", $id);
 }
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 30bb1a45..31b9bd1e 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -1,11 +1,11 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # *-external commands of lei
 package PublicInbox::LeiExternal;
 use strict;
 use v5.10.1;
-use PublicInbox::Config;
+use PublicInbox::Config qw(glob2re);
 
 sub externals_each {
         my ($self, $cb, @arg) = @_;
@@ -44,40 +44,6 @@ sub ext_canonicalize {
         }
 }
 
-# TODO: we will probably extract glob2re into a separate module for
-# PublicInbox::Filter::Base and maybe other places
-my %re_map = ( '*' => '[^/]*?', '?' => '[^/]',
-                '[' => '[', ']' => ']', ',' => ',' );
-
-sub glob2re {
-        my $re = $_[-1]; # $_[0] may be $lei
-        my $p = '';
-        my $in_bracket = 0;
-        my $qm = 0;
-        my $schema_host_port = '';
-
-        # don't glob URL-looking things that look like IPv6
-        if ($re =~ s!\A([a-z0-9\+]+://\[[a-f0-9\:]+\](?::[0-9]+)?/)!!i) {
-                $schema_host_port = quotemeta $1; # "http://[::1]:1234"
-        }
-        my $changes = ($re =~ s!(.)!
-                $re_map{$p eq '\\' ? '' : do {
-                        if ($1 eq '[') { ++$in_bracket }
-                        elsif ($1 eq ']') { --$in_bracket }
-                        elsif ($1 eq ',') { ++$qm } # no change
-                        $p = $1;
-                }} // do {
-                        $p = $1;
-                        ($p eq '-' && $in_bracket) ? $p : (++$qm, "\Q$p")
-                }!sge);
-        # bashism (also supported by curl): {a,b,c} => (a|b|c)
-        $changes += ($re =~ s/([^\\]*)\\\{([^,]*,[^\\]*)\\\}/
-                        (my $in_braces = $2) =~ tr!,!|!;
-                        $1."($in_braces)";
-                        /sge);
-        ($changes - $qm) ? $schema_host_port.$re : undef;
-}
-
 # get canonicalized externals list matching $loc
 # $is_exclude denotes it's for --exclude
 # otherwise it's for --only/--include is assumed
@@ -88,7 +54,7 @@ sub get_externals {
         my @cur = externals_each($self);
         my $do_glob = !$self->{opt}->{globoff}; # glob by default
         if ($do_glob && (my $re = glob2re($loc))) {
-                @m = grep(m!$re!, @cur);
+                @m = grep(m!$re/?\z!, @cur);
                 return @m if scalar(@m);
         } elsif (index($loc, '/') < 0) { # exact basename match:
                 @m = grep(m!/\Q$loc\E/?\z!, @cur);
@@ -120,39 +86,34 @@ sub canonicalize_excludes {
 # returns an anonymous sub which returns an array of potential results
 sub complete_url_prepare {
         my $argv = $_[-1]; # $_[0] may be $lei
-        # Workaround bash word-splitting URLs to ['https', ':', '//' ...]
-        # Maybe there's a better way to go about this in
-        # contrib/completion/lei-completion.bash
-        my $re = '';
-        my $cur = pop(@$argv) // '';
+        # Workaround bash default COMP_WORDBREAKS splitting URLs to
+        # ['https', ':', '//', ...].  COMP_WORDBREAKS is global for all
+        # completions loaded, not just ours, so we can't change it.
+        # cf. contrib/completion/lei-completion.bash
+        my ($pfx, $cur)  = ('', pop(@$argv) // '');
         if (@$argv) {
                 my @x = @$argv;
-                if ($cur eq ':' && @x) {
+                if ($cur =~ /\A[:;=]\z/) { # COMP_WORDBREAKS + URL union
                         push @x, $cur;
                         $cur = '';
                 }
-                while (@x > 2 && $x[0] !~ /\A(?:http|nntp|imap)s?\z/i &&
-                                $x[1] ne ':') {
-                        shift @x;
-                }
-                if (@x >= 2) { # qw(https : hostname : 443) or qw(http :)
-                        $re = join('', @x);
-                } else { # just filter out the flags and hope for the best
-                        $re = join('', grep(!/^-/, @$argv));
+                while (@x && $pfx !~ m!\A(?: (?:[\+\-]?(?:L|kw):) |
+                                (?:(?:imap|nntp|http)s?:) |
+                                (?:--\w?\z)|(?:-\w?\z) )!x) {
+                        $pfx = pop(@x).$pfx;
                 }
-                $re = quotemeta($re);
         }
+        my $re = qr!\A\Q$pfx\E(\Q$cur\E.*)!;
         my $match_cb = sub {
                 # the "//;" here (for AUTH=ANONYMOUS) interacts badly with
                 # bash tab completion, strip it out for now since our commands
                 # work w/o it.  Not sure if there's a better solution...
                 $_[0] =~ s!//;AUTH=ANONYMOUS\@!//!i;
-                $_[0] =~ s!;!\\;!g;
                 # only return the part specified on the CLI
                 # don't duplicate if already 100% completed
-                $_[0] =~ /\A$re(\Q$cur\E.*)/ ? ($cur eq $1 ? () : $1) : ()
+                $_[0] =~ $re ? ($cur eq $1 ? () : $1) : ()
         };
-        wantarray ? ($re, $cur, $match_cb) : $match_cb;
+        wantarray ? ($pfx, $cur, $match_cb) : $match_cb;
 }
 
 1;
diff --git a/lib/PublicInbox/LeiForgetExternal.pm b/lib/PublicInbox/LeiForgetExternal.pm
index 07f0ac80..c8d1df38 100644
--- a/lib/PublicInbox/LeiForgetExternal.pm
+++ b/lib/PublicInbox/LeiForgetExternal.pm
@@ -16,8 +16,7 @@ sub lei_forget_external {
                         next if $seen{$l}++;
                         my $key = "external.$l.boost";
                         delete($cfg->{$key});
-                        $lei->_config('--unset', $key);
-                        if ($? == 0) {
+                        if ($lei->_config('+e', '--unset', $key)) {
                                 $lei->qerr("# $l forgotten ");
                         } elsif (($? >> 8) == 5) {
                                 warn("# $l not found\n");
@@ -32,14 +31,10 @@ sub lei_forget_external {
 sub _complete_forget_external {
         my ($lei, @argv) = @_;
         my $cfg = $lei->_lei_cfg or return ();
-        my ($cur, $re, $match_cb) = $lei->complete_url_prepare(\@argv);
-        # FIXME: bash completion off "http:" or "https:" when the last
-        # character is a colon doesn't work properly even if we're
-        # returning "//$HTTP_HOST/$PATH_INFO/", not sure why, could
-        # be a bash issue.
+        my ($pfx, $cur, $match_cb) = $lei->complete_url_prepare(\@argv);
         map {
                 $match_cb->(substr($_, length('external.')));
-        } grep(/\Aexternal\.$re\Q$cur/, @{$cfg->{-section_order}});
+        } grep(/\Aexternal\.\Q$pfx$cur/, @{$cfg->{-section_order}});
 }
 
 1;
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 2d91e4c4..5521188c 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -7,6 +7,7 @@ use strict;
 use v5.10.1;
 use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
 use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::Compat qw(uniqstr);
 
 # /^input_/ subs are used by (or override) PublicInbox::LeiInput superclass
 
@@ -40,8 +41,7 @@ sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
         my @oidbin = $lms ? $lms->name_oidbin($folder, $bn) : ();
         @oidbin > 1 and warn("W: $folder/*/$$bn not unique:\n",
                                 map { "\t".unpack('H*', $_)."\n" } @oidbin);
-        my %seen;
-        my @docids = sort { $a <=> $b } grep { !$seen{$_}++ }
+        my @docids = sort { $a <=> $b } uniqstr
                         map { $lse->over->oidbin_exists($_) } @oidbin;
         my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
         if (scalar @docids) {
@@ -53,6 +53,29 @@ sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
         }
 }
 
+sub input_mh_cb {
+        my ($mhdir, $n, $kw, $eml, $self) = @_;
+        substr($mhdir, 0, 0) = 'mh:'; # add prefix
+        my $lse = $self->{lse} //= $self->{lei}->{sto}->search;
+        my $lms = $self->{-lms_rw} //= $self->{lei}->lms; # may be 0 or undef
+        my @oidbin = $lms ? $lms->num_oidbin($mhdir, $n) : ();
+        @oidbin > 1 and warn("W: $mhdir/$n not unique:\n",
+                                map { "\t".unpack('H*', $_)."\n" } @oidbin);
+        my @docids = sort { $a <=> $b } uniqstr
+                        map { $lse->over->oidbin_exists($_) } @oidbin;
+        if (scalar @docids) {
+                $lse->kw_changed(undef, $kw, \@docids) or return;
+        }
+        if (defined $eml) {
+                my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
+                $vmd->{sync_info} = [ $mhdir, $n + 0 ] if $self->{-mail_sync};
+                $self->input_eml_cb($eml, $vmd);
+        }
+        # TODO:
+        # elsif (my $ikw = $self->{lei}->{ikw}) { # old message, kw only
+        #        $ikw->wq_io_do('ck_update_kw', [], "mh:$dir", $uid, $kw);
+}
+
 sub input_net_cb { # imap_each / nntp_each
         my ($uri, $uid, $kw, $eml, $self) = @_;
         if (defined $eml) {
@@ -71,9 +94,7 @@ sub do_import_index ($$@) {
         my $sto = $lei->_lei_store(1);
         $sto->write_prepare($lei);
         $self->{-import_kw} = $lei->{opt}->{kw} // 1;
-        my $vmd_mod = $self->vmd_mod_extract(\@inputs);
-        return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
-        $self->{all_vmd} = $vmd_mod if scalar keys %$vmd_mod;
+        $self->{all_vmd} = $lei->{vmd_mod} if keys %{$lei->{vmd_mod}};
         $lei->ale; # initialize for workers to read (before LeiPmdir->new)
         $self->{-mail_sync} = $lei->{opt}->{'mail-sync'} // 1;
         $self->prepare_inputs($lei, \@inputs) or return;
@@ -115,18 +136,24 @@ sub lei_import { # the main "lei import" method
 
 sub _complete_import {
         my ($lei, @argv) = @_;
-        my ($re, $cur, $match_cb) = $lei->complete_url_prepare(\@argv);
-        my @k = $lei->url_folder_cache->keys($argv[-1] // undef, 1);
+        my $has_arg = @argv;
+        my ($pfx, $cur, $match_cb) = $lei->complete_url_prepare(\@argv);
+        my @try = $has_arg ? ($pfx.$cur, $argv[-1]) : ($argv[-1]);
+        push(@try, undef) if defined $try[-1];
+        my (@f, @k);
+        for (@try) {
+                @k = $lei->url_folder_cache->keys($_, 1) and last;
+        }
         my @L = eval { $lei->_lei_store->search->all_terms('L') };
         push(@k, map { "+L:$_" } @L);
-        my @m = map { $match_cb->($_) } @k;
-        my %f = map { $_ => 1 } (@m ? @m : @k);
         if (my $lms = $lei->lms) {
-                @k = $lms->folders($argv[-1] // undef, 1);
-                @m = map { $match_cb->($_) } @k;
-                if (@m) { @f{@m} = @m } else { @f{@k} = @k }
+                for (@try) {
+                        @f = $lms->folders($_, 1) and last;
+                }
+                push @k, @f;
         }
-        keys %f;
+        my @m = map { $match_cb->($_) } @k;
+        @m ? @m : @k;
 }
 
 no warnings 'once';
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index 4dd938f5..765e23cd 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -7,6 +7,7 @@ package PublicInbox::LeiImportKw;
 use strict;
 use v5.10.1;
 use parent qw(PublicInbox::IPC);
+use PublicInbox::Compat qw(uniqstr);
 
 sub new {
         my ($cls, $lei) = @_;
@@ -35,11 +36,10 @@ sub ipc_atfork_child {
 sub ck_update_kw { # via wq_io_do
         my ($self, $url, $uid, $kw) = @_;
         my @oidbin = $self->{-lms_rw}->num_oidbin($url, $uid);
-        my $uid_url = "$url/;UID=$uid";
+        my $uid_url = index($url, 'mh:') == 0 ? $url.$uid : "$url/;UID=$uid";
         @oidbin > 1 and warn("W: $uid_url not unique:\n",
                                 map { "\t".unpack('H*', $_)."\n" } @oidbin);
-        my %seen;
-        my @docids = sort { $a <=> $b } grep { !$seen{$_}++ }
+        my @docids = sort { $a <=> $b } uniqstr
                 map { $self->{over}->oidbin_exists($_) } @oidbin;
         $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
         $self->{verbose} and $self->{lei}->qerr("# $uid_url => @$kw\n");
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index b3f3e1a0..0e329e58 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -35,7 +35,7 @@ sub lei_index {
 
 no warnings 'once';
 no strict 'refs';
-for my $m (qw(pmdir_cb input_net_cb)) {
+for my $m (qw(pmdir_cb input_net_cb input_mh_cb)) {
         *$m = PublicInbox::LeiImport->can($m);
 }
 
diff --git a/lib/PublicInbox/LeiInit.pm b/lib/PublicInbox/LeiInit.pm
index 27ce8169..94897e61 100644
--- a/lib/PublicInbox/LeiInit.pm
+++ b/lib/PublicInbox/LeiInit.pm
@@ -23,7 +23,7 @@ sub lei_init {
 
                 # some folks like symlinks and bind mounts :P
                 if (@dir && "@cur[1,0]" eq "@dir[1,0]") {
-                        $self->_config('leistore.dir', $dir);
+                        $self->_config('leistore.dir', $dir) or return;
                         $self->_lei_store(1)->done;
                         return $self->qerr("$exists (as $cur)");
                 }
@@ -31,7 +31,7 @@ sub lei_init {
 E: leistore.dir=$cur already initialized and it is not $dir
 
         }
-        $self->_config('leistore.dir', $dir);
+        $self->_config('leistore.dir', $dir) or return;
         $self->_lei_store(1)->done;
         $exists //= "# leistore.dir=$dir newly initialized";
         $self->qerr($exists);
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index a1dcc907..d003d983 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -1,14 +1,12 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # parent class for LeiImport, LeiConvert, LeiIndex
 package PublicInbox::LeiInput;
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::DS;
 use PublicInbox::Spawn qw(which popen_rd);
 use PublicInbox::InboxWritable qw(eml_from_path);
-use PublicInbox::AutoReap;
 
 # JMAP RFC 8621 4.1.1
 # https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
@@ -30,6 +28,8 @@ my %ERR = (
                 my ($label) = @_;
                 length($label) >= $L_MAX and
                         return "`$label' too long (must be <= $L_MAX)";
+                $label =~ /[A-Z]/ and
+                        return "`$label' must be lowercase";
                 $label =~ m{\A[a-z0-9_](?:[a-z0-9_\-\./\@,]*[a-z0-9])?\z} ?
                         undef : "`$label' is invalid";
         },
@@ -69,6 +69,11 @@ sub input_maildir_cb {
         $self->input_eml_cb($eml);
 }
 
+sub input_mh_cb {
+        my ($dn, $n, $kw, $eml, $self) = @_;
+        $self->input_eml_cb($eml);
+}
+
 sub input_net_cb { # imap_each, nntp_each cb
         my ($url, $uid, $kw, $eml, $self) = @_;
         $self->input_eml_cb($eml);
@@ -79,14 +84,12 @@ sub input_net_cb { # imap_each, nntp_each cb
 sub input_fh {
         my ($self, $ifmt, $fh, $name, @args) = @_;
         if ($ifmt eq 'eml') {
-                my $buf = do { local $/; <$fh> } //
-                        return $self->{lei}->child_error(0, <<"");
-error reading $name: $!
+                my $buf = eval { PublicInbox::IO::read_all $fh, 0 };
+                my $e = $@;
+                return $self->{lei}->child_error($?, <<"") if !$fh->close || $e;
+error reading $name: $! (\$?=$?) (\$@=$e)
 
-                # mutt pipes single RFC822 messages with a "From " line,
-                # but no Content-Length or "From " escaping.
-                # "git format-patch" also generates such files by default.
-                $buf =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+                PublicInbox::Eml::strip_from($buf);
 
                 # a user may feed just a body: git diff | lei rediff -U9
                 if ($self->{-force_eml}) {
@@ -113,15 +116,58 @@ sub handle_http_input ($$@) {
         push @$curl, '-s', @$curl_opt;
         my $cmd = $curl->for_uri($lei, $uri);
         $lei->qerr("# $cmd");
-        my ($fh, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
-        my $ar = PublicInbox::AutoReap->new($pid);
+        my $fh = popen_rd($cmd, undef, { 2 => $lei->{2} });
         grep(/\A--compressed\z/, @$curl) or
-                $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
+                $fh = IO::Uncompress::Gunzip->new($fh,
+                                        MultiStream => 1, AutoClose => 1);
         eval { $self->input_fh('mboxrd', $fh, $url, @args) };
-        my @err = ($@ ? $@ : ());
-        $ar->join;
-        push(@err, "\$?=$?") if $?;
-        $lei->child_error($?, "@$cmd failed: @err") if @err;
+        my $err = $@ ? ": $@" : '';
+        $lei->child_error($?, "@$cmd failed$err") if $err || $?;
+}
+
+sub oid2eml { # git->cat_async cb
+        my ($bref, $oid, $type, $size, $self) = @_;
+        if ($type eq 'blob') {
+                $self->input_eml_cb(PublicInbox::Eml->new($bref));
+        } else {
+                warn "W: $oid is type=$type\n";
+        }
+}
+
+sub each_ibx_eml_unindexed {
+        my ($self, $ibx, @args) = @_;
+        $ibx->isa('PublicInbox::Inbox') or return $self->{lei}->fail(<<EOM);
+unindexed extindex $ibx->{topdir} not supported
+EOM
+        require PublicInbox::SearchIdx;
+        my $n = $ibx->max_git_epoch;
+        my @g = defined($n) ? map { $ibx->git_epoch($_) } (0..$n) : ($ibx->git);
+        my $sync = { D => {}, ibx => $ibx }; # D => {} filters out deletes
+        my ($f, $at, $ct, $oid, $cmt);
+        for my $git (grep defined, @g) {
+                my $s = PublicInbox::SearchIdx::log2stack($sync, $git, 'HEAD');
+                while (($f, $at, $ct, $oid, $cmt) = $s->pop_rec) {
+                        $git->cat_async($oid, \&oid2eml, $self) if $f eq 'm';
+                }
+                $git->cleanup; # wait all
+        }
+}
+
+sub each_ibx_eml {
+        my ($self, $ibx, @args) = @_; # TODO: is @args used at all?
+        my $over = $ibx->over or return each_ibx_eml_unindexed(@_);
+        my $git = $ibx->git;
+        my $prev = 0;
+        my $smsg;
+        my $ids = $over->ids_after(\$prev);
+        while (@$ids) {
+                for (@$ids) {
+                        $smsg = $over->get_art($_) // next;
+                        $git->cat_async($smsg->{blob}, \&oid2eml, $self);
+                }
+                $ids = $over->ids_after(\$prev);
+        }
+        $git->cat_async_wait;
 }
 
 sub input_path_url {
@@ -149,7 +195,7 @@ sub input_path_url {
                 $ifmt = lc($1);
         } elsif ($input =~ /\.(?:patch|eml)\z/i) {
                 $ifmt = 'eml';
-        } elsif (-f $input && $input =~ m{\A(?:.+)/(?:new|cur)/([^/]+)\z}) {
+        } elsif ($input =~ m{\A(?:.+)/(?:new|cur)/([^/]+)\z} && -f $input) {
                 my $bn = $1;
                 my $fl = PublicInbox::MdirReader::maildir_basename_flags($bn);
                 return if index($fl, 'T') >= 0;
@@ -163,6 +209,10 @@ sub input_path_url {
         my $devfd = $lei->path_to_fd($input) // return;
         if ($devfd >= 0) {
                 $self->input_fh($ifmt, $lei->{$devfd}, $input, @args);
+        } elsif ($devfd < 0 && $input =~ m{\A(.+/)([0-9]+)\z} && -f $input) {
+                my ($dn, $n) = ($1, $2);
+                my $mhr = PublicInbox::MHreader->new($dn, $lei->{3});
+                $mhr->mh_read_one($n, $self->can('input_mh_cb'), $self);
         } elsif (-f $input && $ifmt eq 'eml') {
                 open my $fh, '<', $input or
                                         return $lei->fail("open($input): $!");
@@ -177,12 +227,9 @@ sub input_path_url {
                         $mbl->{fh} =
                              PublicInbox::MboxReader::zsfxcat($in, $zsfx, $lei);
                 }
-                local $PublicInbox::DS::in_loop = 0 if $zsfx; # dwaitpid
+                local $PublicInbox::DS::in_loop = 0 if $zsfx; # awaitpid
                 $self->input_fh($ifmt, $mbl->{fh}, $input, @args);
-        } elsif (-d _ && (-d "$input/cur" || -d "$input/new")) {
-                return $lei->fail(<<EOM) if $ifmt && $ifmt ne 'maildir';
-$input appears to be a maildir, not $ifmt
-EOM
+        } elsif (-d _ && $ifmt eq 'maildir') {
                 my $mdr = PublicInbox::MdirReader->new;
                 if (my $pmd = $self->{pmd}) {
                         $mdr->maildir_each_file($input,
@@ -193,15 +240,24 @@ EOM
                                                 $self->can('input_maildir_cb'),
                                                 $self, @args);
                 }
+        } elsif (-d _ && $ifmt eq 'mh') {
+                my $mhr = PublicInbox::MHreader->new($input.'/', $lei->{3});
+                $mhr->{sort} = $lei->{opt}->{sort} // [ 'sequence'];
+                $mhr->mh_each_eml($self->can('input_mh_cb'), $self, @args);
+        } elsif (-d _ && $ifmt =~ /\A(?:v1|v2)\z/) {
+                my $ibx = PublicInbox::Inbox->new({inboxdir => $input});
+                each_ibx_eml($self, $ibx, @args);
+        } elsif (-d _ && $ifmt eq 'extindex') {
+                my $esrch = PublicInbox::ExtSearch->new($input);
+                each_ibx_eml($self, $esrch, @args);
         } elsif ($self->{missing_ok} && !-e $input) { # don't ->fail
                 if ($lei->{cmd} eq 'p2q') {
                         my $fp = [ qw(git format-patch --stdout -1), $input ];
                         my $rdr = { 2 => $lei->{2} };
                         my $fh = popen_rd($fp, undef, $rdr);
                         eval { $self->input_fh('eml', $fh, $input, @args) };
-                        my @err = ($@ ? $@ : ());
-                        close($fh) or push @err, "\$?=$?";
-                        $lei->child_error($?, "@$fp failed: @err") if @err;
+                        my $err = $@ ? ": $@" : '';
+                        $lei->child_error($?, "@$fp failed$err") if $err || $?;
                 } else {
                         $self->folder_missing("$ifmt:$input");
                 }
@@ -257,6 +313,17 @@ sub prepare_http_input ($$$) {
         $self->{"-curl-$url"} = [ @curl_opt, $uri ]; # for handle_http_input
 }
 
+sub add_dir ($$$$) {
+        my ($lei, $istate, $ifmt, $input) = @_;
+        if ($istate->{-may_sync}) {
+                $$input = "$ifmt:".$lei->abs_path($$input);
+                push @{$istate->{-sync}->{ok}}, $$input if $istate->{-sync};
+        } else {
+                substr($$input, 0, 0) = "$ifmt:"; # prefix
+        }
+        push @{$istate->{$ifmt}}, $$input;
+}
+
 sub prepare_inputs { # returns undef on error
         my ($self, $lei, $inputs) = @_;
         my $in_fmt = $lei->{opt}->{'in-format'};
@@ -270,7 +337,8 @@ sub prepare_inputs { # returns undef on error
                 push @{$sync->{no}}, '/dev/stdin' if $sync;
         }
         my $net = $lei->{net}; # NetWriter may be created by l2m
-        my (@f, @md);
+        my @f;
+        my $istate = { -sync => $sync, -may_sync => $may_sync };
         # e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX
         for my $input (@$inputs) {
                 my $input_path = $input;
@@ -290,26 +358,24 @@ sub prepare_inputs { # returns undef on error
 --in-format=$in_fmt and `$ifmt:' conflict
 
                         }
-                        if ($ifmt =~ /\A(?:maildir|mh)\z/i) {
-                                push @{$sync->{ok}}, $input if $sync;
-                        } else {
-                                push @{$sync->{no}}, $input if $sync;
-                        }
+                        ($sync && $ifmt !~ /\A(?:maildir|mh)\z/i) and
+                                push(@{$sync->{no}}, $input);
                         my $devfd = $lei->path_to_fd($input_path) // return;
                         if ($devfd >= 0 || (-f $input_path || -p _)) {
                                 require PublicInbox::MboxLock;
                                 require PublicInbox::MboxReader;
                                 PublicInbox::MboxReader->reads($ifmt) or return
                                         $lei->fail("$ifmt not supported");
-                        } elsif (-d $input_path) {
-                                $ifmt eq 'maildir' or return
-                                        $lei->fail("$ifmt not supported");
-                                $may_sync and $input = 'maildir:'.
-                                                $lei->abs_path($input_path);
-                                push @md, $input;
-                        } elsif ($self->{missing_ok} && !-e _) {
+                        } elsif (-d $input_path) { # TODO extindex
+                                $ifmt =~ /\A(?:maildir|mh|v1|v2|extindex)\z/ or
+                                        return$lei->fail("$ifmt not supported");
+                                $input = $input_path;
+                                add_dir $lei, $istate, $ifmt, \$input;
+                        } elsif ($self->{missing_ok} &&
+                                        $ifmt =~ /\A(?:maildir|mh)\z/ &&
+                                        !-e $input_path) {
                                 # for "lei rm-watch" on missing Maildir
-                                $may_sync and $input = 'maildir:'.
+                                $may_sync and $input = "$ifmt:".
                                                 $lei->abs_path($input_path);
                         } else {
                                 my $m = "Unable to handle $input";
@@ -322,7 +388,7 @@ sub prepare_inputs { # returns undef on error
 $input is `eml', not --in-format=$in_fmt
 
                         push @{$sync->{no}}, $input if $sync;
-                } elsif (-f $input && $input =~ m{\A(.+)/(new|cur)/([^/]+)\z}) {
+                } elsif ($input =~ m{\A(.+)/(new|cur)/([^/]+)\z} && -f $input) {
                         # single file in a Maildir
                         my ($mdir, $nc, $bn) = ($1, $2, $3);
                         my $other = $mdir . ($nc eq 'new' ? '/cur' : '/new');
@@ -334,25 +400,41 @@ $input is `eml', not --in-format=$in_fmt
 
                         if ($sync) {
                                 $input = $lei->abs_path($mdir) . "/$nc/$bn";
-                                push @{$sync->{ok}}, $input if $sync;
+                                push @{$sync->{ok}}, $input;
                         }
                         require PublicInbox::MdirReader;
                 } else {
                         my $devfd = $lei->path_to_fd($input) // return;
-                        if ($devfd >= 0 || -f $input || -p _) {
+                        if ($devfd < 0 && $input =~ m{\A(.+)/([0-9]+)\z} &&
+                                        -f $input) { # single file in MH dir
+                                my ($mh, $n) = ($1, $2);
+                                lc($in_fmt//'eml') eq 'eml' or
+                                                return $lei->fail(<<"");
+$input is `eml', not --in-format=$in_fmt
+
+                                if ($sync) {
+                                        $input = $lei->abs_path($mh)."/$n";
+                                        push @{$sync->{ok}}, $input;
+                                }
+                                require PublicInbox::MHreader;
+                        } elsif ($devfd >= 0 || -f $input || -p _) {
                                 push @{$sync->{no}}, $input if $sync;
                                 push @f, $input;
                         } elsif (-d "$input/new" && -d "$input/cur") {
-                                if ($may_sync) {
-                                        $input = 'maildir:'.
-                                                $lei->abs_path($input);
-                                        push @{$sync->{ok}}, $input if $sync;
-                                }
-                                push @md, $input;
+                                add_dir $lei, $istate, 'maildir', \$input;
+                        } elsif (-e "$input/inbox.lock") {
+                                add_dir $lei, $istate, 'v2', \$input;
+                        } elsif (-e "$input/ssoma.lock") {
+                                add_dir $lei, $istate, 'v1', \$input;
+                        } elsif (-e "$input/ei.lock") {
+                                add_dir $lei, $istate, 'extindex', \$input;
+                        } elsif (-f "$input/.mh_sequences") {
+                                add_dir $lei, $istate, 'mh', \$input;
                         } elsif ($self->{missing_ok} && !-e $input) {
                                 if ($lei->{cmd} eq 'p2q') {
                                         # will run "git format-patch"
                                 } elsif ($may_sync) { # for lei rm-watch
+                                        # FIXME: support MH, here
                                         $input = 'maildir:'.
                                                 $lei->abs_path($input);
                                 }
@@ -380,20 +462,29 @@ $input is `eml', not --in-format=$in_fmt
                 $lei->{auth} //= PublicInbox::LeiAuth->new;
                 $lei->{net} //= $net;
         }
-        if (scalar(@md)) {
+        if (my $md = $istate->{maildir}) {
                 require PublicInbox::MdirReader;
                 if ($self->can('pmdir_cb')) {
                         require PublicInbox::LeiPmdir;
                         $self->{pmd} = PublicInbox::LeiPmdir->new($lei, $self);
                 }
+                grep(!m!\Amaildir:/!i, @$md) and die "BUG: @$md (no pfx)";
 
                 # start watching Maildirs ASAP
                 if ($may_sync && $lei->{sto}) {
-                        grep(!m!\Amaildir:/!i, @md) and die "BUG: @md (no pfx)";
-                        $lei->lms(1)->lms_write_prepare->add_folders(@md);
+                        $lei->lms(1)->lms_write_prepare->add_folders(@$md);
                         $lei->refresh_watches;
                 }
         }
+        if (my $mh = $istate->{mh}) {
+                require PublicInbox::MHreader;
+                grep(!m!\Amh:!i, @$mh) and die "BUG: @$mh (no pfx)";
+                if ($may_sync && $lei->{sto}) {
+                        $lei->lms(1)->lms_write_prepare->add_folders(@$mh);
+                        # $lei->refresh_watches; TODO
+                }
+        }
+        require PublicInbox::ExtSearch if $istate->{extindex};
         $self->{inputs} = $inputs;
 }
 
@@ -408,7 +499,7 @@ sub process_inputs {
         }
         # always commit first, even on error partial work is acceptable for
         # lei <import|tag|convert>
-        my $wait = $self->{lei}->{sto}->wq_do('done') if $self->{lei}->{sto};
+        $self->{lei}->sto_done_request;
         $self->{lei}->fail($err) if $err;
 }
 
@@ -432,23 +523,22 @@ sub input_only_net_merge_all_done {
 # for update_xvmd -> update_vmd
 # returns something like { "+L" => [ @Labels ], ... }
 sub vmd_mod_extract {
-        my $argv = $_[-1];
-        my $vmd_mod = {};
-        my @new_argv;
+        my ($lei, $argv) = @_;
+        my (@new_argv, @err);
         for my $x (@$argv) {
                 if ($x =~ /\A(\+|\-)(kw|L):(.+)\z/) {
                         my ($op, $pfx, $val) = ($1, $2, $3);
                         if (my $err = $ERR{$pfx}->($val)) {
-                                push @{$vmd_mod->{err}}, $err;
+                                push @err, $err;
                         } else { # set "+kw", "+L", "-L", "-kw"
-                                push @{$vmd_mod->{$op.$pfx}}, $val;
+                                push @{$lei->{vmd_mod}->{$op.$pfx}}, $val;
                         }
                 } else {
                         push @new_argv, $x;
                 }
         }
         @$argv = @new_argv;
-        $vmd_mod;
+        @err;
 }
 
 1;
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index d7775d4b..576ab2c7 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # "lei inspect" general purpose inspector for stuff in SQLite and
@@ -12,7 +12,6 @@ use parent qw(PublicInbox::IPC);
 use PublicInbox::Config;
 use PublicInbox::MID qw(mids);
 use PublicInbox::NetReader qw(imap_uri nntp_uri);
-use POSIX qw(strftime);
 use PublicInbox::LeiOverview;
 *iso8601 = \&PublicInbox::LeiOverview::iso8601;
 
@@ -97,7 +96,6 @@ sub _inspect_doc ($$) {
                 my $term = ($1 // '');
                 push @{$ent->{terms}->{$term}}, $tn;
         }
-        @$_ = sort(@$_) for values %{$ent->{terms} // {}};
         $cur = $doc->values_begin;
         $end = $doc->values_end;
         for (; $cur != $end; $cur++) {
@@ -235,7 +233,8 @@ sub inspect_argv { # via wq_do
         $lei->{1}->autoflush(0);
         $lei->out('[') if $multi;
         while (defined(my $x = shift @$argv)) {
-                inspect1($lei, $x, scalar(@$argv)) or return;
+                eval { inspect1($lei, $x, scalar(@$argv)) or return };
+                warn "E: $@\n" if $@;
         }
         $lei->out(']') if $multi;
 }
@@ -250,21 +249,13 @@ sub inspect_start ($$) {
         $self->wq_close;
 }
 
-sub ins_add { # InputPipe->consume callback
-        my ($lei) = @_; # $_[1] = $rbuf
-        if (defined $_[1]) {
-                $_[1] eq '' and return eval {
-                        my $str = delete $lei->{istr};
-                        $str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
-                        my $eml = PublicInbox::Eml->new(\$str);
-                        inspect_start($lei, [
-                                'blob:'.$lei->git_oid($eml)->hexdigest,
-                                map { "mid:$_" } @{mids($eml)} ]);
-                };
-                $lei->{istr} .= $_[1];
-        } else {
-                $lei->fail("error reading stdin: $!");
-        }
+sub do_inspect { # lei->do_env cb
+        my ($lei) = @_;
+        my $str = delete $lei->{stdin_buf};
+        PublicInbox::Eml::strip_from($str);
+        my $eml = PublicInbox::Eml->new(\$str);
+        inspect_start($lei, [ 'blob:'.$lei->git_oid($eml)->hexdigest,
+                        map { "mid:$_" } @{mids($eml)} ]);
 }
 
 sub lei_inspect {
@@ -281,8 +272,7 @@ sub lei_inspect {
                 return $lei->fail(<<'') if @argv;
 no args allowed on command-line with --stdin
 
-                require PublicInbox::InputPipe;
-                PublicInbox::InputPipe::consume($lei->{0}, \&ins_add, $lei);
+                $lei->slurp_stdin(\&do_inspect);
         } else {
                 inspect_start($lei, \@argv);
         }
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 8d89cb73..274a9605 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -122,17 +122,11 @@ could not extract Message-ID from $x
         @q ? join(' OR ', @q) : $lei->fail("no Message-ID in: @argv");
 }
 
-sub _stdin { # PublicInbox::InputPipe::consume callback for --stdin
-        my ($lei) = @_; # $_[1] = $rbuf
-        $_[1] // return $lei->fail("error reading stdin: $!");
-        return $lei->{mset_opt}->{qstr} .= $_[1] if $_[1] ne '';
-        eval {
-                $lei->fchdir;
-                my @argv = split(/\s+/, $lei->{mset_opt}->{qstr});
-                $lei->{mset_opt}->{qstr} = extract_all($lei, @argv) or return;
-                $lei->_start_query;
-        };
-        $lei->fail($@) if $@;
+sub do_lcat { # lei->do_env cb
+        my ($lei) = @_;
+        my @argv = split(/\s+/, delete($lei->{stdin_buf}));
+        $lei->{mset_opt}->{qstr} = extract_all($lei, @argv) or return;
+        $lei->_start_query;
 }
 
 sub lei_lcat {
@@ -151,9 +145,7 @@ sub lei_lcat {
                 return $lei->fail(<<'') if @argv;
 no args allowed on command-line with --stdin
 
-                require PublicInbox::InputPipe;
-                PublicInbox::InputPipe::consume($lei->{0}, \&_stdin, $lei);
-                return;
+                return $lei->slurp_stdin(\&do_lcat);
         }
         $lei->{mset_opt}->{qstr} = extract_all($lei, @argv) or return;
         $lei->_start_query;
diff --git a/lib/PublicInbox/LeiLsExternal.pm b/lib/PublicInbox/LeiLsExternal.pm
index dd2eb2e7..2cdd0c4d 100644
--- a/lib/PublicInbox/LeiLsExternal.pm
+++ b/lib/PublicInbox/LeiLsExternal.pm
@@ -5,6 +5,7 @@
 package PublicInbox::LeiLsExternal;
 use strict;
 use v5.10.1;
+use PublicInbox::Config qw(glob2re);
 
 # TODO: does this need JSON output?
 sub lei_ls_external {
@@ -12,7 +13,8 @@ sub lei_ls_external {
         my $do_glob = !$lei->{opt}->{globoff}; # glob by default
         my ($OFS, $ORS) = $lei->{opt}->{z} ? ("\0", "\0\0") : (" ", "\n");
         $filter //= '*';
-        my $re = $do_glob ? $lei->glob2re($filter) : undef;
+        my $re = $do_glob ? glob2re($filter) : undef;
+        $re .= '/?\\z' if defined $re;
         $re //= index($filter, '/') < 0 ?
                         qr!/\Q$filter\E/?\z! : # exact basename match
                         qr/\Q$filter\E/; # grep -F semantics
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index 50799270..ab6c1e60 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -19,7 +19,8 @@ sub input_path_url { # overrides LeiInput version
         if ($url =~ m!\Aimaps?://!i) {
                 my $uri = PublicInbox::URIimap->new($url);
                 my $sec = $lei->{net}->can('uri_section')->($uri);
-                my $mic = $lei->{net}->mic_get($uri);
+                my $mic = $lei->{net}->mic_get($uri) or
+                        return $lei->err("E: $uri");
                 my $l = $mic->folders_hash($uri->path); # server-side filter
                 @$l = map { $_->[2] } # undo Schwartzian transform below:
                         sort { $a->[0] cmp $b->[0] || $a->[1] <=> $b->[1] }
@@ -39,8 +40,13 @@ sub input_path_url { # overrides LeiInput version
                 }
         } elsif ($url =~ m!\A(?:nntps?|s?news)://!i) {
                 my $uri = PublicInbox::URInntps->new($url);
-                my $nn = $lei->{net}->nn_get($uri);
-                my $l = $nn->newsgroups($uri->group); # name => description
+                my $nn = $lei->{net}->nn_get($uri) or
+                        return $lei->err("E: $uri");
+                # $l = name => description
+                my $l = $nn->newsgroups($uri->group) // return $lei->err(<<EOM);
+E: $uri LIST NEWSGROUPS: ${\($lei->{net}->ndump($nn->message))}
+E: login may be required, try adding `-c nntp.debug' to your command
+EOM
                 my $sec = $lei->{net}->can('uri_section')->($uri);
                 if ($json) {
                         my $all = $nn->list;
diff --git a/lib/PublicInbox/LeiLsMailSync.pm b/lib/PublicInbox/LeiLsMailSync.pm
index 2b167b1d..1400d488 100644
--- a/lib/PublicInbox/LeiLsMailSync.pm
+++ b/lib/PublicInbox/LeiLsMailSync.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # front-end for the "lei ls-mail-sync" sub-command
@@ -6,13 +6,17 @@ package PublicInbox::LeiLsMailSync;
 use strict;
 use v5.10.1;
 use PublicInbox::LeiMailSync;
+use PublicInbox::Config qw(glob2re);
 
 sub lei_ls_mail_sync {
         my ($lei, $filter) = @_;
         my $lms = $lei->lms or return;
         my $opt = $lei->{opt};
-        my $re = $opt->{globoff} ? undef : $lei->glob2re($filter // '*');
-        $re //= qr/\Q$filter\E/;
+        my $re = $opt->{globoff} ? undef : glob2re($filter // '*');
+        $re .= '/?\\z' if defined $re;
+        $re //= index($filter, '/') < 0 ?
+                        qr!/\Q$filter\E/?\z! : # exact basename match
+                        qr/\Q$filter\E/; # grep -F semantics
         my @f = $lms->folders;
         @f = $opt->{'invert-match'} ? grep(!/$re/, @f) : grep(/$re/, @f);
         if ($opt->{'local'} && !$opt->{remote}) {
diff --git a/lib/PublicInbox/LeiMailDiff.pm b/lib/PublicInbox/LeiMailDiff.pm
index 2b4cfd9e..af6ecf82 100644
--- a/lib/PublicInbox/LeiMailDiff.pm
+++ b/lib/PublicInbox/LeiMailDiff.pm
@@ -4,74 +4,27 @@
 # The "lei mail-diff" sub-command, diffs input contents against
 # the first message of input
 package PublicInbox::LeiMailDiff;
-use strict;
-use v5.10.1;
-use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
-use File::Temp 0.19 (); # 0.19 for ->newdir
-use PublicInbox::Spawn qw(spawn which);
-use PublicInbox::MsgIter qw(msg_part_text);
-use File::Path qw(remove_tree);
-use PublicInbox::ContentHash qw(content_digest);
+use v5.12;
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput PublicInbox::MailDiff);
+use PublicInbox::Spawn qw(run_wait);
 require PublicInbox::LeiRediff;
-use Data::Dumper ();
-
-sub write_part { # Eml->each_part callback
-        my ($ary, $self) = @_;
-        my ($part, $depth, $idx) = @$ary;
-        if ($idx ne '1' || $self->{lei}->{opt}->{'raw-header'}) {
-                open my $fh, '>', "$self->{curdir}/$idx.hdr" or die "open: $!";
-                print $fh ${$part->{hdr}} or die "print $!";
-                close $fh or die "close $!";
-        }
-        my $ct = $part->content_type || 'text/plain';
-        my ($s, $err) = msg_part_text($part, $ct);
-        my $sfx = defined($s) ? 'txt' : 'bin';
-        open my $fh, '>', "$self->{curdir}/$idx.$sfx" or die "open: $!";
-        print $fh ($s // $part->body) or die "print $!";
-        close $fh or die "close $!";
-}
-
-sub dump_eml ($$$) {
-        my ($self, $dir, $eml) = @_;
-        local $self->{curdir} = $dir;
-        mkdir $dir or die "mkdir($dir): $!";
-        $eml->each_part(\&write_part, $self);
-
-        open my $fh, '>', "$dir/content_digest" or die "open: $!";
-        my $dig = PublicInbox::ContentDigestDbg->new($fh);
-        local $Data::Dumper::Useqq = 1;
-        local $Data::Dumper::Terse = 1;
-        content_digest($eml, $dig);
-        print $fh "\n", $dig->hexdigest, "\n" or die "print $!";
-        close $fh or die "close: $!";
-}
-
-sub prep_a ($$) {
-        my ($self, $eml) = @_;
-        $self->{tmp} = File::Temp->newdir('lei-mail-diff-XXXX', TMPDIR => 1);
-        dump_eml($self, "$self->{tmp}/a", $eml);
-}
 
 sub diff_a ($$) {
         my ($self, $eml) = @_;
-        ++$self->{nr};
-        my $dir = "$self->{tmp}/N$self->{nr}";
-        dump_eml($self, $dir, $eml);
+        my $dir = "$self->{tmp}/N".(++$self->{nr});
+        $self->dump_eml($dir, $eml);
         my $cmd = [ qw(git diff --no-index) ];
         my $lei = $self->{lei};
         PublicInbox::LeiRediff::_lei_diff_prepare($lei, $cmd);
         push @$cmd, qw(-- a), "N$self->{nr}";
         my $rdr = { -C => "$self->{tmp}" };
         @$rdr{1, 2} = @$lei{1, 2};
-        my $pid = spawn($cmd, $lei->{env}, $rdr);
-        waitpid($pid, 0);
-        $lei->child_error($?) if $?; # for git diff --exit-code
-        File::Path::remove_tree($self->{curdir});
+        run_wait($cmd, $lei->{env}, $rdr) and $lei->child_error($?);
 }
 
 sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
         my ($self, $eml) = @_;
-        $self->{tmp} ? diff_a($self, $eml) : prep_a($self, $eml);
+        $self->{tmp} ? diff_a($self, $eml) : $self->prep_a($eml);
 }
 
 sub lei_mail_diff {
@@ -82,24 +35,10 @@ sub lei_mail_diff {
         $lei->{opt}->{color} //= $isatty;
         $lei->start_pager if $isatty;
         $lei->{-err_type} = 'non-fatal';
+        $self->{-raw_hdr} = $lei->{opt}->{'raw-header'};
         $lei->wq1_start($self);
 }
 
 no warnings 'once';
 *net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
-
-package PublicInbox::ContentDigestDbg; # cf. PublicInbox::ContentDigest
-use strict;
-use v5.10.1;
-use Data::Dumper;
-
-sub new { bless { dig => Digest::SHA->new(256), fh => $_[1] }, __PACKAGE__ }
-
-sub add {
-        $_[0]->{dig}->add($_[1]);
-        print { $_[0]->{fh} } Dumper([split(/^/sm, $_[1])]) or die "print $!";
-}
-
-sub hexdigest { $_[0]->{dig}->hexdigest; }
-
 1;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 665206a8..c498421c 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -6,9 +6,12 @@ package PublicInbox::LeiMailSync;
 use strict;
 use v5.10.1;
 use parent qw(PublicInbox::Lock);
+use PublicInbox::Compat qw(uniqstr);
 use DBI qw(:sql_types); # SQL_BLOB
 use PublicInbox::ContentHash qw(git_sha);
 use Carp ();
+use PublicInbox::Git qw(%HEXLEN2SHA);
+use PublicInbox::IO qw(read_all);
 
 sub dbh_new {
         my ($self) = @_;
@@ -339,6 +342,17 @@ SELECT $op(uid) FROM blob2num WHERE fid = ?
         $ret;
 }
 
+# must be called with lock
+sub _forget_fids ($;@) {
+        my $dbh = shift;
+        $dbh->begin_work;
+        for my $t (qw(blob2name blob2num folders)) {
+                my $sth = $dbh->prepare_cached("DELETE FROM $t WHERE fid = ?");
+                $sth->execute($_) for @_;
+        }
+        $dbh->commit;
+}
+
 # returns a { location => [ list-of-ids-or-names ] } mapping
 sub locations_for {
         my ($self, $oidbin) = @_;
@@ -379,18 +393,28 @@ sub locations_for {
 
         $sth = $dbh->prepare('SELECT loc FROM folders WHERE fid = ? LIMIT 1');
         my $ret = {};
+        my $drop_fids = $dbh->{ReadOnly} ? undef : {};
         while (my ($fid, $ids) = each %fid2id) {
                 $sth->execute($fid);
                 my ($loc) = $sth->fetchrow_array;
                 unless (defined $loc) {
+                        my $del = '';
+                        if ($drop_fids) {
+                                $del = ' (deleting)';
+                                $drop_fids->{$fid} = $fid;
+                        }
                         my $oidhex = unpack('H*', $oidbin);
-                        warn "E: fid=$fid for $oidhex unknown:\n", map {
-                                        'E: '.(ref() ? $$_ : "#$_")."\n";
+                        warn "E: fid=$fid for $oidhex stale/unknown:\n", map {
+                                        'E: '.(ref() ? $$_ : "#$_")."$del\n";
                                 } @$ids;
                         next;
                 }
                 $ret->{$loc} = $ids;
         }
+        if ($drop_fids && scalar(values %$drop_fids)) {
+                my $lk = $self->lock_for_scope;
+                _forget_fids($self->{dbh}, values %$drop_fids);
+        }
         scalar(keys %$ret) ? $ret : undef;
 }
 
@@ -401,9 +425,13 @@ sub folders {
         my $re;
         if (defined($pfx[0])) {
                 $sql .= ' WHERE loc REGEXP ?'; # DBD::SQLite uses perlre
-                $re = !!$pfx[1] ? '.*' : '';
-                $re .= quotemeta($pfx[0]);
-                $re .= '.*';
+                if (ref($pfx[0])) { # assume qr// "Regexp"
+                        $re = $pfx[0];
+                } else {
+                        $re = !!$pfx[1] ? '.*' : '';
+                        $re .= quotemeta($pfx[0]);
+                        $re .= '.*';
+                }
         }
         my $sth = ($self->{dbh} //= dbh_new($self))->prepare($sql);
         $sth->bind_param(1, $re) if defined($re);
@@ -411,15 +439,24 @@ sub folders {
         map { $_->[0] } @{$sth->fetchall_arrayref};
 }
 
+sub blob_mismatch ($$$) {
+        my ($f, $oidhex, $rawref) = @_;
+        my $sha = $HEXLEN2SHA{length($oidhex)};
+        my $got = git_sha($sha, $rawref)->hexdigest;
+        $got eq $oidhex ? undef : warn("$f changed $oidhex => $got\n");
+}
+
 sub local_blob {
         my ($self, $oidhex, $vrfy) = @_;
         my $dbh = $self->{dbh} //= dbh_new($self);
+        my $oidbin = pack('H*', $oidhex);
+
         my $b2n = $dbh->prepare(<<'');
 SELECT f.loc,b.name FROM blob2name b
 LEFT JOIN folders f ON b.fid = f.fid
 WHERE b.oidbin = ?
 
-        $b2n->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+        $b2n->bind_param(1, $oidbin, SQL_BLOB);
         $b2n->execute;
         while (my ($d, $n) = $b2n->fetchrow_array) {
                 substr($d, 0, length('maildir:')) = '';
@@ -432,19 +469,28 @@ WHERE b.oidbin = ?
                         my $f = "$d/$x/$n";
                         open my $fh, '<', $f or next;
                         # some (buggy) Maildir writers are non-atomic:
-                        next unless -s $fh;
-                        local $/;
-                        my $raw = <$fh>;
-                        if ($vrfy) {
-                                my $got = git_sha(1, \$raw)->hexdigest;
-                                if ($got ne $oidhex) {
-                                        warn "$f changed $oidhex => $got\n";
-                                        next;
-                                }
-                        }
+                        my $raw = read_all($fh, -s $fh // next);
+                        next if $vrfy && blob_mismatch $f, $oidhex, \$raw;
                         return \$raw;
                 }
         }
+
+        # MH, except `uid' is not always unique (can be packed)
+        $b2n = $dbh->prepare(<<'');
+SELECT f.loc,b.uid FROM blob2num b
+LEFT JOIN folders f ON b.fid = f.fid
+WHERE b.oidbin = ? AND f.loc REGEXP '^mh:/'
+
+        $b2n->bind_param(1, $oidbin, SQL_BLOB);
+        $b2n->execute;
+        while (my ($f, $n) = $b2n->fetchrow_array) {
+                $f =~ s/\Amh://s or die "BUG: not MH: $f";
+                $f .= "/$n";
+                open my $fh, '<', $f or next;
+                my $raw = read_all($fh, -s $fh // next);
+                next if blob_mismatch $f, $oidhex, \$raw;
+                return \$raw;
+        }
         undef;
 }
 
@@ -520,20 +566,19 @@ EOM
 --all=@no not accepted (must be `local' and/or `remote')
 EOM
         }
-        my (%seen, @inc);
         my @all = $self->folders;
         for my $ok (@ok) {
                 if ($ok eq 'local') {
-                        @inc = grep(!m!\A[a-z0-9\+]+://!i, @all);
+                        push @$folders, grep(!m!\A[a-z0-9\+]+://!i, @all);
                 } elsif ($ok eq 'remote') {
-                        @inc = grep(m!\A[a-z0-9\+]+://!i, @all);
+                        push @$folders, grep(m!\A[a-z0-9\+]+://!i, @all);
                 } elsif ($ok ne '') {
                         return $lei->fail("--all=$all not understood");
                 } else {
-                        @inc = @all;
+                        push @$folders, @all;
                 }
-                push(@$folders, (grep { !$seen{$_}++ } @inc));
         }
+        @$folders = uniqstr @$folders;
         scalar(@$folders) || $lei->fail(<<EOM);
 no --mail-sync folders known to lei
 EOM
@@ -596,14 +641,10 @@ EOF
 sub forget_folders {
         my ($self, @folders) = @_;
         my $lk = $self->lock_for_scope;
-        for my $folder (@folders) {
-                my $fid = delete($self->{fmap}->{$folder}) //
-                        fid_for($self, $folder) // next;
-                for my $t (qw(blob2name blob2num folders)) {
-                        $self->{dbh}->do("DELETE FROM $t WHERE fid = ?",
-                                        undef, $fid);
-                }
-        }
+        _forget_fids($self->{dbh}, map {
+                delete($self->{fmap}->{$_}) //
+                        fid_for($self, $_) // ();
+        } @folders);
 }
 
 # only used for changing canonicalization errors
@@ -637,8 +678,8 @@ sub num_oidbin ($$$) {
 SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ? ORDER BY _rowid_
 EOM
         $sth->execute($fid, $uid);
-        my %uniq; # for public-inbox <= 1.7.0
-        grep { !$uniq{$_}++ } map { $_->[0] } @{$sth->fetchall_arrayref};
+        # for public-inbox <= 1.7.0:
+        uniqstr(map { $_->[0] } @{$sth->fetchall_arrayref});
 }
 
 sub name_oidbin ($$$) {
@@ -655,8 +696,7 @@ EOM
         $sth->bind_param(2, $nm, SQL_VARCHAR);
         $sth->execute;
         my @old = map { $_->[0] } @{$sth->fetchall_arrayref};
-        my %uniq; # for public-inbox <= 1.7.0
-        grep { !$uniq{$_}++ } (@bin, @old);
+        uniqstr @bin, @old # for public-inbox <= 1.7.0
 }
 
 sub imap_oidhex {
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index e20d30b4..5353ae61 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -1,32 +1,49 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # "lei add-external --mirror" support (also "public-inbox-clone");
 package PublicInbox::LeiMirror;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::IPC);
-use PublicInbox::Config;
-use PublicInbox::AutoReap;
 use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
 use IO::Compress::Gzip qw(gzip $GzipError);
-use PublicInbox::Spawn qw(popen_rd spawn);
+use PublicInbox::Spawn qw(spawn run_wait run_die run_qx);
+use PublicInbox::IO qw(write_file);
+use File::Path ();
 use File::Temp ();
+use File::Spec ();
 use Fcntl qw(SEEK_SET O_CREAT O_EXCL O_WRONLY);
 use Carp qw(croak);
+use URI;
+use PublicInbox::Config qw(glob2re);
+use PublicInbox::Inbox;
+use PublicInbox::LeiCurl;
+use PublicInbox::OnDestroy;
+use PublicInbox::SHA qw(sha256_hex sha_all);
+use POSIX qw(strftime);
+use PublicInbox::Admin qw(fmt_localtime);
+use autodie qw(chdir chmod close open pipe readlink
+                seek symlink sysopen sysseek truncate unlink);
 
-sub _wq_done_wait { # dwaitpid callback (via wq_eof)
-        my ($arg, $pid) = @_;
-        my ($mrr, $lei) = @$arg;
-        my $f = "$mrr->{dst}/mirror.done";
+our $LIVE; # pid => callback
+our $FGRP_TODO; # objstore -> [[ to resume ], [ to clone ]]
+our $TODO; # reference => [ non-fgrp mirror objects ]
+our @PUH; # post-update hooks
+
+sub keep_going ($) {
+        $LIVE && (!$_[0]->{lei}->{child_error} ||
+                $_[0]->{lei}->{opt}->{'keep-going'});
+}
+
+sub _wq_done_wait { # awaitpid cb (via wq_eof)
+        my ($pid, $mrr, $lei) = @_;
         if ($?) {
                 $lei->child_error($?);
-        } elsif (!unlink($f)) {
-                warn("unlink($f): $!\n") unless $!{ENOENT};
-        } else {
-                if ($lei->{cmd} ne 'public-inbox-clone') {
-                        $lei->lazy_cb('add-external', '_finish_'
-                                        )->($lei, $mrr->{dst});
+        } elsif (!$lei->{child_error}) {
+                if (!$mrr->{dry_run} && $lei->{cmd} ne 'public-inbox-clone') {
+                        require PublicInbox::LeiAddExternal;
+                        PublicInbox::LeiAddExternal::_finish_add_external(
+                                                        $lei, $mrr->{dst});
                 }
                 $lei->qerr("# mirrored $mrr->{src} => $mrr->{dst}");
         }
@@ -35,20 +52,27 @@ sub _wq_done_wait { # dwaitpid callback (via wq_eof)
 
 # for old installations without manifest.js.gz
 sub try_scrape {
-        my ($self) = @_;
+        my ($self, $fallback_manifest) = @_;
         my $uri = URI->new($self->{src});
         my $lei = $self->{lei};
         my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
         my $cmd = $curl->for_uri($lei, $uri, '--compressed');
         my $opt = { 0 => $lei->{0}, 2 => $lei->{2} };
-        my $fh = popen_rd($cmd, undef, $opt);
-        my $html = do { local $/; <$fh> } // die "read(curl $uri): $!";
-        close($fh) or return $lei->child_error($?, "@$cmd failed");
+        my $html = run_qx($cmd, undef, $opt);
+        return $lei->child_error($?, "@$cmd failed") if $?;
 
         # we grep with URL below, we don't want Subject/From headers
-        # making us clone random URLs
+        # making us clone random URLs.  This assumes remote instances
+        # prior to public-inbox 1.7.0
+        # 5b96edcb1e0d8252 (www: move mirror instructions to /text/, 2021-08-28)
         my @html = split(/<hr>/, $html);
         my @urls = ($html[-1] =~ m!\bgit clone --mirror ([a-z\+]+://\S+)!g);
+        if (!@urls && $fallback_manifest) {
+                warn <<EOM;
+W: failed to extract URLs from $uri, trying manifest.js.gz...
+EOM
+                return start_clone_url($self);
+        }
         my $url = $uri->as_string;
         chop($url) eq '/' or die "BUG: $uri not canonicalized";
 
@@ -57,9 +81,12 @@ sub try_scrape {
         if (my @v2_urls = grep(m!\A\Q$url\E/[0-9]+\z!, @urls)) {
                 my %v2_epochs = map {
                         my ($n) = (m!/([0-9]+)\z!);
-                        $n => URI->new($_)
+                        $n => [ URI->new($_), '' ]
                 } @v2_urls; # uniq
-                return clone_v2($self, \%v2_epochs);
+                clone_v2_prep($self, \%v2_epochs);
+                delete local $lei->{opt}->{epoch};
+                clone_all($self);
+                return;
         }
 
         # filter out common URLs served by WWW (e.g /$MSGID/T/)
@@ -84,56 +111,92 @@ sub clone_cmd {
         # e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
         push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
         push @cmd, qw(clone --mirror);
-        push @cmd, '-q' if $lei->{opt}->{quiet};
+        push @cmd, '-q' if $lei->{opt}->{quiet} ||
+                        ($lei->{opt}->{jobs} // 1) > 1;
         push @cmd, '-v' if $lei->{opt}->{verbose};
         # XXX any other options to support?
-        # --reference is tricky with multiple epochs...
+        # --reference is tricky with multiple epochs, but handled
+        # automatically if using manifest.js.gz
         @cmd;
 }
 
-sub ft_rename ($$$) {
-        my ($ft, $dst, $open_mode) = @_;
-        my $fn = $ft->filename;
-        my @st = stat($dst);
+sub ft_rename ($$$;$) {
+        my ($ft, $dst, $open_mode, $fh) = @_;
+        my @st = stat($fh // $dst);
         my $mode = @st ? ($st[2] & 07777) : ($open_mode & ~umask);
-        chmod($mode, $ft) or croak "E: chmod $fn: $!";
-        rename($fn, $dst) or croak "E: rename($fn => $ft): $!";
+        chmod($mode, $ft);
+        require File::Copy;
+        File::Copy::mv($ft->filename, $dst) or croak "E: mv($ft => $dst): $!";
         $ft->unlink_on_destroy(0);
 }
 
-sub _get_txt { # non-fatal
-        my ($self, $endpoint, $file, $mode) = @_;
-        my $uri = URI->new($self->{src});
+sub do_reap ($;$) {
+        my ($self, $jobs) = @_;
+        $jobs //= $self->{-jobs} //= $self->{lei}->{opt}->{jobs} // 1;
+        $jobs = 1 if $jobs < 1;
+        while (keys(%$LIVE) >= $jobs) {
+                my $pid = waitpid(-1, 0) // die "waitpid(-1): $!";
+                if (my $x = delete $LIVE->{$pid}) {
+                        my $cb = shift @$x;
+                        $cb->(@$x) if $cb;
+                } else {
+                        warn "reaped unknown PID=$pid ($?)\n";
+                }
+        }
+}
+
+sub _get_txt_start { # non-fatal
+        my ($self, $endpoint, $fini) = @_;
+        my $uri = URI->new($self->{cur_src} // $self->{src});
         my $lei = $self->{lei};
         my $path = $uri->path;
         chop($path) eq '/' or die "BUG: $uri not canonicalized";
         $uri->path("$path/$endpoint");
-        my $ft = File::Temp->new(TEMPLATE => "$file-XXXX", DIR => $self->{dst});
+        my $f = (split(m!/!, $endpoint))[-1];
+        my $ft = File::Temp->new(TEMPLATE => "$f-XXXX", TMPDIR => 1);
         my $opt = { 0 => $lei->{0}, 1 => $lei->{1}, 2 => $lei->{2} };
-        my $cmd = $self->{curl}->for_uri($lei, $uri,
-                                        qw(--compressed -R -o), $ft->filename);
-        my $cerr = run_reap($lei, $cmd, $opt);
-        return "$uri missing" if ($cerr >> 8) == 22;
-        return "# @$cmd failed (non-fatal)" if $cerr;
-        ft_rename($ft, "$self->{dst}/$file", $mode);
+        my $cmd = $self->{curl}->for_uri($lei, $uri, qw(--compressed -R -o),
+                                        $ft->filename);
+        do_reap($self);
+        $lei->qerr("# @$cmd");
+        return if $self->{dry_run};
+        $self->{"-get_txt.$endpoint"} = [ $ft, $cmd, $uri ];
+        $LIVE->{spawn($cmd, undef, $opt)} =
+                        [ \&_get_txt_done, $self, $endpoint, $fini ];
+}
+
+sub _get_txt_done { # returns true on error (non-fatal), undef on success
+        my ($self, $endpoint) = @_;
+        my ($fh, $cmd, $uri) = @{delete $self->{"-get_txt.$endpoint"}};
+        my $cerr = $?;
+        $? = 0; # don't influence normal lei exit
+        return warn("$uri missing\n") if ($cerr >> 8) == 22;
+        return warn("# @$cmd failed (non-fatal)\n") if $cerr;
+        seek($fh, 0, SEEK_SET);
+        $self->{"mtime.$endpoint"} = (stat($fh))[9];
+        $self->{"txt.$endpoint"} = PublicInbox::IO::read_all $fh, -s _;
         undef; # success
 }
 
-# tries the relatively new /$INBOX/_/text/config/raw endpoint
-sub _try_config {
+sub _write_inbox_config {
         my ($self) = @_;
-        my $dst = $self->{dst};
-        if (!-d $dst || !mkdir($dst)) {
-                require File::Path;
-                File::Path::mkpath($dst);
-                -d $dst or die "mkpath($dst): $!\n";
-        }
-        my $err = _get_txt($self,
-                        qw(_/text/config/raw inbox.config.example), 0444);
-        return warn($err, "\n") if $err;
-        my $f = "$self->{dst}/inbox.config.example";
-        my $cfg = PublicInbox::Config->git_config_dump($f, $self->{lei}->{2});
-        my $ibx = $self->{ibx} = {};
+        my $buf = delete($self->{'txt._/text/config/raw'}) // return;
+        my $dst = $self->{cur_dst} // $self->{dst};
+        my $f = "$dst/inbox.config.example";
+        my $mtime = delete $self->{'mtime._/text/config/raw'};
+        if (CORE::sysopen(my $fh, $f, O_CREAT|O_EXCL|O_WRONLY)) {
+                print $fh $buf;
+                chmod(0444 & ~umask, $fh);
+                $fh->flush or die "flush($f): $!";
+                if (defined $mtime) {
+                        utime($mtime, $mtime, $fh) or die "utime($f): $!";
+                }
+        } elsif (!$!{EEXIST}) {
+                die "open($f): $!";
+        }
+        my $cfg = PublicInbox::Config->git_config_dump($f,
+                                                { 2 => $self->{lei}->{2} });
+        my $ibx = $self->{ibx} = {}; # for indexing
         for my $sec (grep(/\Apublicinbox\./, @{$cfg->{-section_order}})) {
                 for (qw(address newsgroup nntpmirror)) {
                         $ibx->{$_} = $cfg->{"$sec.$_"};
@@ -143,35 +206,31 @@ sub _try_config {
 
 sub set_description ($) {
         my ($self) = @_;
-        my $f = "$self->{dst}/description";
-        open my $fh, '+>>', $f or die "open($f): $!";
-        seek($fh, 0, SEEK_SET) or die "seek($f): $!";
-        chomp(my $d = do { local $/; <$fh> } // die "read($f): $!");
-        if ($d eq '($INBOX_DIR/description missing)' ||
-                        $d =~ /^Unnamed repository/ || $d !~ /\S/) {
-                seek($fh, 0, SEEK_SET) or die "seek($f): $!";
-                truncate($fh, 0) or die "truncate($f): $!";
-                print $fh "mirror of $self->{src}\n" or die "print($f): $!";
-                close $fh or die "close($f): $!";
+        my $dst = $self->{cur_dst} // $self->{dst};
+        chomp(my $orig = PublicInbox::IO::try_cat("$dst/description"));
+        my $d = $orig;
+        while (defined($d) && ($d =~ m!^\(\$INBOX_DIR/description missing\)! ||
+                        $d =~ /^Unnamed repository/ || $d !~ /\S/)) {
+                $d = delete($self->{'txt.description'});
         }
+        $d //= 'mirror of '.($self->{cur_src} // $self->{src});
+        atomic_write($dst, 'description', $d."\n") if $d ne $orig;
 }
 
 sub index_cloned_inbox {
         my ($self, $iv) = @_;
         my $lei = $self->{lei};
-        my $err = _get_txt($self, qw(description description), 0666);
-        warn($err, "\n") if $err; # non fatal
-        eval { set_description($self) };
-        warn $@ if $@;
 
         # n.b. public-inbox-clone works w/o (SQLite || Xapian)
         # lei is useless without Xapian + SQLite
         if ($lei->{cmd} ne 'public-inbox-clone') {
+                require PublicInbox::InboxWritable;
+                require PublicInbox::Admin;
                 my $ibx = delete($self->{ibx}) // {
                         address => [ 'lei@example.com' ],
                         version => $iv,
                 };
-                $ibx->{inboxdir} = $self->{dst};
+                $ibx->{inboxdir} = $self->{cur_dst} // $self->{dst};
                 PublicInbox::Inbox->new($ibx);
                 PublicInbox::InboxWritable->new($ibx);
                 my $opt = {};
@@ -179,46 +238,408 @@ sub index_cloned_inbox {
                         my ($k) = ($sw =~ /\A([\w-]+)/);
                         $opt->{$k} = $lei->{opt}->{$k};
                 }
-                # force synchronous dwaitpid for v2:
+                # force synchronous awaitpid for v2:
                 local $PublicInbox::DS::in_loop = 0;
-                my $cfg = PublicInbox::Config->new(undef, $lei->{2});
+                my $cfg = PublicInbox::Config->new(undef, { 2 => $lei->{2} });
                 my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
                 local %ENV = (%ENV, %$env) if $env;
                 PublicInbox::Admin::progress_prepare($opt, $lei->{2});
                 PublicInbox::Admin::index_inbox($ibx, undef, $opt);
         }
-        open my $x, '>', "$self->{dst}/mirror.done"; # for _wq_done_wait
+        return if defined $self->{cur_dst}; # one of many repos to clone
 }
 
 sub run_reap {
         my ($lei, $cmd, $opt) = @_;
         $lei->qerr("# @$cmd");
-        my $ar = PublicInbox::AutoReap->new(spawn($cmd, undef, $opt));
-        $ar->join;
-        my $ret = $?;
+        my $ret = run_wait($cmd, undef, $opt);
         $? = 0; # don't let it influence normal exit
         $ret;
 }
 
-sub clone_v1 {
+sub start_cmd {
+        my ($self, $cmd, $opt, $fini) = @_;
+        do_reap($self);
+        $self->{lei}->qerr("# @$cmd");
+        return if $self->{dry_run};
+        $LIVE->{spawn($cmd, undef, $opt)} = [ \&reap_cmd, $self, $cmd, $fini ]
+}
+
+sub fetch_args ($$) {
+        my ($lei, $opt) = @_;
+        my @cmd; # (git --git-dir=...) to be added by caller
+        $opt->{$_} = $lei->{$_} for (0..2);
+        # we support "-c $key=$val" for arbitrary git config options
+        # e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
+        push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
+        push @cmd, 'fetch';
+        push @cmd, '-q' if $lei->{opt}->{quiet} ||
+                        ($lei->{opt}->{jobs} // 1) > 1;
+        push @cmd, '-v' if $lei->{opt}->{verbose};
+        push(@cmd, '-p') if $lei->{opt}->{prune};
+        PublicInbox::Git::git_version() ge v2.29.0 and
+                push(@cmd, '--no-write-fetch-head');
+        @cmd;
+}
+
+sub upr { # feed `git update-ref --stdin -z' verbosely
+        my ($lei, $w, $op, @rest) = @_; # ($ref, $oid) = @rest
+        $lei->qerr("# $op @rest") if $lei->{opt}->{verbose};
+        print $w "$op ", join("\0", @rest, '') or die "print(w): $!";
+}
+
+sub start_update_ref {
+        my ($fgrp) = @_;
+        pipe(my $r, my $w);
+        my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
+                qw(update-ref --stdin -z) ];
+        my $pack = PublicInbox::OnDestroy->new($$, \&satellite_done, $fgrp);
+        start_cmd($fgrp, $cmd, { 0 => $r, 2 => $fgrp->{lei}->{2} }, $pack);
+        close $r;
+        $fgrp->{dry_run} ? undef : $w;
+}
+
+sub upref_warn { warn "E: close(update-ref --stdin): $! (need git 1.8.5+)\n" }
+
+sub fgrp_update {
+        my ($fgrp) = @_;
+        return if !keep_going($fgrp);
+        my $srcfh = delete $fgrp->{srcfh} or return;
+        my $dstfh = delete $fgrp->{dstfh} or return;
+        seek($srcfh, 0, SEEK_SET);
+        seek($dstfh, 0, SEEK_SET);
+        my %src = map { chomp; split /\0/ } PublicInbox::IO::read_all $srcfh;
+        my %dst = map { chomp; split /\0/ } PublicInbox::IO::read_all $dstfh;
+        $srcfh = $dstfh = undef;
+        my $w = start_update_ref($fgrp) or return;
+        my $lei = $fgrp->{lei};
+        my $ndel;
+        for my $ref (keys %dst) {
+                my $new = delete $src{$ref};
+                my $old = $dst{$ref};
+                if (defined $new) {
+                        $new eq $old or
+                                upr($lei, $w, 'update', $ref, $new, $old);
+                } else {
+                        upr($lei, $w, 'delete', $ref, $old);
+                        ++$ndel;
+                }
+        }
+        # git's ref files backend doesn't allow directory/file conflicts
+        # between `delete' and `create' ops:
+        if ($ndel && scalar(keys %src)) {
+                $fgrp->{-create_refs} = \%src;
+        } else {
+                while (my ($ref, $oid) = each %src) {
+                        upr($lei, $w, 'create', $ref, $oid);
+                }
+        }
+        $w->close or upref_warn();
+}
+
+sub satellite_done {
+        my ($fgrp) = @_;
+        if (my $create = delete $fgrp->{-create_refs}) {
+                my $w = start_update_ref($fgrp) or return;
+                while (my ($ref, $oid) = each %$create) {
+                        upr($fgrp->{lei}, $w, 'create', $ref, $oid);
+                }
+                $w->close or upref_warn();
+        } else {
+                pack_refs($fgrp, $fgrp->{cur_dst});
+                run_puh($fgrp);
+        }
+}
+
+sub pack_refs {
+        my ($self, $git_dir) = @_;
+        my $cmd = [ 'git', "--git-dir=$git_dir", qw(pack-refs --all --prune) ];
+        start_cmd($self, $cmd, { 2 => $self->{lei}->{2} });
+}
+
+sub unlink_fetch_head ($) {
+        my ($git_dir) = @_;
+        return if CORE::unlink("$git_dir/FETCH_HEAD") || $!{ENOENT};
+        warn "W: unlink($git_dir/FETCH_HEAD): $!";
+}
+
+sub fgrpv_done {
+        my ($fgrpv) = @_;
+        return if !$LIVE;
+        my $first = $fgrpv->[0] // die 'BUG: no fgrpv->[0]';
+        return if !keep_going($first);
+        unlink_fetch_head($first->{-osdir}) if !$first->{dry_run};
+        pack_refs($first, $first->{-osdir}); # objstore refs always packed
+        for my $fgrp (@$fgrpv) {
+                my $rn = $fgrp->{-remote};
+                my %opt = ( 2 => $fgrp->{lei}->{2} );
+
+                my $update_ref = PublicInbox::OnDestroy->new($$,
+                                                        \&fgrp_update, $fgrp);
+
+                my $src = [ 'git', "--git-dir=$fgrp->{-osdir}", 'for-each-ref',
+                        "--format=refs/%(refname:lstrip=3)%00%(objectname)",
+                        "refs/remotes/$rn/" ];
+                open(my $sfh, '+>', undef);
+                $fgrp->{srcfh} = $sfh;
+                start_cmd($fgrp, $src, { %opt, 1 => $sfh }, $update_ref);
+                my $dst = [ 'git', "--git-dir=$fgrp->{cur_dst}", 'for-each-ref',
+                        '--format=%(refname)%00%(objectname)' ];
+                open(my $dfh, '+>', undef);
+                $fgrp->{dstfh} = $dfh;
+                start_cmd($fgrp, $dst, { %opt, 1 => $dfh }, $update_ref);
+        }
+}
+
+sub fgrp_fetch_all {
         my ($self) = @_;
+        my $todo = $FGRP_TODO;
+        $FGRP_TODO = \'BUG on further use';
+        keys(%$todo) or return;
+
+        # Rely on the fgrptmp remote groups in the config file rather
+        # than listing all remotes since the remote name list may exceed
+        # system argv limits:
+        my $grp = 'fgrptmp';
+
+        my @git = (@{$self->{-torsocks}}, 'git');
+        my $j = $self->{lei}->{opt}->{jobs};
+        my $opt = {};
+        my @fetch = do {
+                local $self->{lei}->{opt}->{jobs} = 1;
+                (fetch_args($self->{lei}, $opt), qw(--no-tags --multiple));
+        };
+        push(@fetch, "-j$j") if $j;
+        while (my ($osdir, $fgrp_old_new) = each %$todo) {
+                my $f = "$osdir/config";
+                return if !keep_going($self);
+                my ($old, $new) = @$fgrp_old_new;
+                @$old = sort { $b->{-sort} <=> $a->{-sort} } @$old;
+                # $new is ordered by {references}
+                my $cmd = ['git', "--git-dir=$osdir", qw(config -f), $f ];
+
+                # clobber settings from previous run atomically
+                for ("remotes.$grp", 'fetch.hideRefs') {
+                        my $c = [ @$cmd, '--unset-all', $_ ];
+                        $self->{lei}->qerr("# @$c");
+                        next if $self->{dry_run};
+                        run_wait($c, undef, $opt);
+                        die "E: @$c \$?=$?" if ($? && ($? >> 8) != 5);
+                }
+
+                # permanent configs:
+                my $cfg = PublicInbox::Config->git_config_dump($f);
+                for my $fgrp (@$old, @$new) {
+                        my $u = $fgrp->{-uri} // die 'BUG: no {-uri}';
+                        my $rn = $fgrp->{-remote} // die 'BUG: no {-remote}';
+                        for ("url=$u", "fetch=+refs/*:refs/remotes/$rn/*",
+                                        'tagopt=--no-tags') {
+                                my ($k, $v) = split(/=/, $_, 2);
+                                $k = "remote.$rn.$k";
+                                next if ($cfg->{$k} // '') eq $v;
+                                my $c = [@$cmd, $k, $v];
+                                $fgrp->{lei}->qerr("# @$c");
+                                next if $fgrp->{dry_run};
+                                run_die($c, undef, $opt);
+                        }
+                }
+
+                if (!$self->{dry_run}) {
+                        # update the config atomically via O_APPEND while
+                        # respecting git-config locking
+                        sysopen(my $lk, "$f.lock", O_CREAT|O_EXCL|O_WRONLY);
+                        open my $fh, '>>', $f;
+                        $fh->autoflush(1);
+                        my $buf = '';
+                        if (@$old) {
+                                $buf = "[fetch]\n\thideRefs = refs\n";
+                                $buf .= join('', map {
+                                        "\thideRefs = !refs/remotes/" .
+                                                "$_->{-remote}/\n";
+                                } @$old);
+                        }
+                        $buf .= join('', "[remotes]\n",
+                                (map { "\t$grp = $_->{-remote}\n" } @$old),
+                                (map { "\t$grp = $_->{-remote}\n" } @$new));
+                        print $fh $buf or die "print($f): $!";
+                        close $fh;
+                        unlink("$f.lock");
+                }
+                $cmd  = [ @git, "--git-dir=$osdir", @fetch, $grp ];
+                push @$old, @$new;
+                my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $old);
+                start_cmd($self, $cmd, $opt, $end);
+        }
+}
+
+# keep this idempotent for future use by public-inbox-fetch
+sub forkgroup_prep {
+        my ($self, $uri) = @_;
+        $self->{-ent} // return;
+        my $os = $self->{-objstore} // return;
+        my $fg = $self->{-ent}->{forkgroup} // return;
+        my $dir = "$os/$fg.git";
+        if (!-d $dir && !$self->{dry_run}) {
+                PublicInbox::Import::init_bare($dir);
+                write_file '+>>', "$dir/config", <<EOM;
+[repack]
+        useDeltaIslands = true
+[pack]
+        island = refs/remotes/([^/]+)/
+EOM
+        }
+        my $key = $self->{-key} // die 'BUG: no -key';
+        my $rn = substr(sha256_hex($key), 0, 16);
+        if (!-d $self->{cur_dst} && !$self->{dry_run}) {
+                PublicInbox::Import::init_bare($self->{cur_dst});
+                write_file '+>>', "$self->{cur_dst}/config", <<EOM;
+; rely on the "$rn" remote in the
+; $fg fork group for fetches
+; only uncomment the following iff you detach from fork groups
+; [remote "origin"]
+;        url = $uri
+;        fetch = +refs/*:refs/*
+;        mirror = true
+EOM
+        }
+        if (!$self->{dry_run}) {
+                my $alt = File::Spec->rel2abs("$dir/objects");
+                my $o = "$self->{cur_dst}/objects";
+                my $l = File::Spec->abs2rel($alt, File::Spec->rel2abs($o));
+                open my $fh, '+>>', my $f = "$o/info/alternates";
+                seek($fh, 0, SEEK_SET); # Perl did SEEK_END when it saw '>>'
+                my $seen = grep /\A\Q$l\E\n/, PublicInbox::IO::read_all $fh;
+                print $fh "$l\n" if !$seen;
+                close $fh;
+        }
+        bless {
+                %$self, -osdir => $dir, -remote => $rn, -uri => $uri
+        }, __PACKAGE__;
+}
+
+sub fp_done {
+        my ($self, $cmd, $cb, @arg) = @_;
+        if ($?) {
+                $self->{lei}->err("@$cmd failed (\$?=$?) (non-fatal)");
+                $? = 0; # don't let it influence normal exit
+        }
+        return if !keep_going($self);
+        my $fh = delete $self->{-show_ref} // die 'BUG: no show-ref output';
+        sysseek($fh, 0, SEEK_SET);
+        $self->{-ent} // die 'BUG: no -ent';
+        my $A = $self->{-ent}->{fingerprint} // die 'BUG: no fingerprint';
+        my $B = sha_all(1, $fh)->hexdigest;
+        return $cb->($self, @arg) if $A ne $B;
+        $self->{lei}->qerr("# $self->{-key} up-to-date");
+}
+
+sub cmp_fp_do {
+        my ($self, $cb, @arg) = @_;
+        # $cb is either resume_fetch or fgrp_enqueue
+        $self->{-ent} // return $cb->($self, @arg);
+        my $new = $self->{-ent}->{fingerprint} // return $cb->($self, @arg);
+        my $key = $self->{-key} // die 'BUG: no -key';
+        if (my $cur_ent = $self->{-local_manifest}->{$key}) {
+                # runs go_fetch->DESTROY run if eq
+                return if $cur_ent->{fingerprint} eq $new;
+        }
+        my $dst = $self->{cur_dst} // $self->{dst};
+        my $cmd = ['git', "--git-dir=$dst", 'show-ref'];
+        my $opt = { 2 => $self->{lei}->{2} };
+        open($opt->{1}, '+>', undef);
+        $self->{-show_ref} = $opt->{1};
+        do_reap($self);
+        $self->{lei}->qerr("# @$cmd");
+        $LIVE->{spawn($cmd, undef, $opt)} = [ \&fp_done, $self, $cmd,
+                                                $cb, @arg ];
+}
+
+sub resume_fetch {
+        my ($self, $uri, $fini) = @_;
+        return if !keep_going($self);
+        my $dst = $self->{cur_dst} // $self->{dst};
+        my @git = ('git', "--git-dir=$dst");
+        my $opt = { 2 => $self->{lei}->{2} };
+        my $rn = 'random'.int(rand(1 << 30));
+        for ("url=$uri", "fetch=+refs/*:refs/*", 'mirror=true') {
+                push @git, '-c', "remote.$rn.$_";
+        }
+        my $cmd = [ @{$self->{-torsocks}}, @git,
+                        fetch_args($self->{lei}, $opt), $rn ];
+        push @$cmd, '-P' if $self->{lei}->{prune}; # --prune-tags implied
+        my $run_puh = PublicInbox::OnDestroy->new($$, \&run_puh, $self, $fini);
+        ++$self->{chg}->{nr_chg};
+        start_cmd($self, $cmd, $opt, $run_puh);
+}
+
+sub fgrp_enqueue {
+        my ($fgrp, $end) = @_; # $end calls fgrp_fetch_all
+        return if !keep_going($fgrp);
+        ++$fgrp->{chg}->{nr_chg};
+        my $dst = $FGRP_TODO->{$fgrp->{-osdir}} //= [ [], [] ]; # [ old, new ]
+        push @{$dst->[defined($fgrp->{-sort} ? 0 : 1)]}, $fgrp;
+}
+
+sub clone_v1 {
+        my ($self, $end) = @_;
         my $lei = $self->{lei};
         my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
-        my $uri = URI->new($self->{src});
+        my $uri = URI->new($self->{cur_src} // $self->{src});
+        my $path = $uri->path;
+        $path =~ s!/*\z!! and $uri->path($path);
         defined($lei->{opt}->{epoch}) and
                 die "$uri is a v1 inbox, --epoch is not supported\n";
-        my $pfx = $curl->torsocks($lei, $uri) or return;
-        my $cmd = [ @$pfx, clone_cmd($lei, my $opt = {}),
-                        $uri->as_string, $self->{dst} ];
-        my $cerr = run_reap($lei, $cmd, $opt);
-        return $lei->child_error($cerr, "@$cmd failed") if $cerr;
-        _try_config($self);
-        write_makefile($self->{dst}, 1);
-        index_cloned_inbox($self, 1);
+        $self->{-torsocks} //= $curl->torsocks($lei, $uri) or return;
+        my $dst = $self->{cur_dst} // $self->{dst};
+        my $resume = -d $dst;
+        if ($resume) { # respect read-only cloned w/ --epoch=
+                my @st = stat(_); # for root
+                if (!-w _ || !($st[2] & 0222)) {
+                        warn "# skipping $dst, not writable\n";
+                        return;
+                }
+        }
+        my $fini = PublicInbox::OnDestroy->new($$, \&v1_done, $self);
+        if (my $fgrp = forkgroup_prep($self, $uri)) {
+                $fgrp->{-fini} = $fini;
+                if ($resume) {
+                        $fgrp->{-sort} = $fgrp->{-ent}->{modified};
+                        cmp_fp_do($fgrp, \&fgrp_enqueue, $end);
+                } else { # new repo, save for last
+                        fgrp_enqueue($fgrp, $end);
+                }
+        } elsif ($resume) {
+                cmp_fp_do($self, \&resume_fetch, $uri, $fini);
+        } else { # normal clone
+                my $cmd = [ @{$self->{-torsocks}},
+                                clone_cmd($lei, my $opt = {}), "$uri", $dst ];
+                if (defined($self->{-ent})) {
+                        if (defined(my $ref = $self->{-ent}->{reference})) {
+                                -e "$self->{dst}$ref" and
+                                        push @$cmd, '--reference',
+                                                "$self->{dst}$ref";
+                        }
+                }
+                ++$self->{chg}->{nr_chg};
+                start_cmd($self, $cmd, $opt, PublicInbox::OnDestroy->new($$,
+                                                \&run_puh, $self, $fini));
+        }
+        if (!$self->{-is_epoch} && $lei->{opt}->{'inbox-config'} =~
+                                /\A(?:always|v1)\z/s &&
+                        !-f "$dst/inbox.config.example") {
+                _get_txt_start($self, '_/text/config/raw', $fini);
+        }
+
+        my $d = $self->{-ent} ? $self->{-ent}->{description} : undef;
+        utf8::encode($self->{'txt.description'} = $d) if defined $d;
+        (!defined($d) && !$end) and
+                _get_txt_start($self, 'description', $fini);
+
+        $end or do_reap($self, 1); # for non-manifest clone
 }
 
 sub parse_epochs ($$) {
-        my ($opt_epochs, $v2_epochs) = @_; # $epcohs "LOW..HIGH"
+        my ($opt_epochs, $v2_epochs) = @_; # $epochs "LOW..HIGH"
         $opt_epochs // return; # undef => all epochs
         my ($lo, $dotdot, $hi, @extra) = split(/(\.\.)/, $opt_epochs);
         undef($lo) if ($lo // '') eq '';
@@ -260,12 +681,14 @@ EOM
         $want
 }
 
-sub init_placeholder ($$) {
-        my ($src, $edst) = @_;
+sub init_placeholder ($$$) {
+        my ($src, $edst, $ent) = @_;
         PublicInbox::Import::init_bare($edst);
-        my $f = "$edst/config";
-        open my $fh, '>>', $f or die "open($f): $!";
-        print $fh <<EOM or die "print($f): $!";
+        my @owner = defined($ent->{owner}) ? (<<EOM) : ();
+[gitweb]
+        owner = $ent->{owner}
+EOM
+        write_file '>>', "$edst/config", <<EOM, @owner;
 [remote "origin"]
         url = $src
         fetch = +refs/*:refs/*
@@ -273,20 +696,196 @@ sub init_placeholder ($$) {
 
 ; This git epoch was created read-only and "public-inbox-fetch"
 ; will not fetch updates for it unless write permission is added.
+; Hint: chmod +w $edst
 EOM
-        close $fh or die "close:($f): $!";
+        my %map = (head => 'HEAD', description => undef);
+        while (my ($key, $fn) = each %map) {
+                my $val = $ent->{$key} // next;
+                $fn //= $key;
+                write_file '>', "$edst/$fn", $val;
+        }
+}
+
+sub reap_cmd { # async, called via SIGCHLD
+        my ($self, $cmd) = @_;
+        my $cerr = $?;
+        $? = 0; # don't let it influence normal exit
+        $self->{lei}->child_error($cerr, "@$cmd failed (\$?=$cerr)") if $cerr;
+}
+
+sub up_fp_done {
+        my ($self) = @_;
+        return if !keep_going($self);
+        my $fh = delete $self->{-show_ref_up} // die 'BUG: no show-ref output';
+        sysseek($fh, 0, SEEK_SET);
+        $self->{-ent} // die 'BUG: no -ent';
+        my $A = $self->{-ent}->{fingerprint} // die 'BUG: no fingerprint';
+        my $B = sha_all(1, $fh)->hexdigest;
+        return if $A eq $B;
+        $self->{-ent}->{fingerprint} = $B;
+        push @{$self->{chg}->{fp_mismatch}}, $self->{-key};
+}
+
+sub atomic_write ($$$) {
+        my ($dn, $bn, $raw) = @_;
+        my $ft = File::Temp->new(DIR => $dn, TEMPLATE => "$bn-XXXX");
+        print $ft $raw;
+        $ft->flush or die "flush($ft): $!";
+        ft_rename($ft, "$dn/$bn", 0666);
+}
+
+sub run_next_puh {
+        my ($self) = @_;
+        my $puh = shift @{$self->{-puh_todo}} // return delete($self->{-fini});
+        my $fini = PublicInbox::OnDestroy->new($$, \&run_next_puh, $self);
+        my $cmd = [ @$puh, ($self->{cur_dst} // $self->{dst}) ];
+        my $opt = +{ map { $_ => $self->{lei}->{$_} } (0..2) };
+        start_cmd($self, $cmd, undef, $opt, $fini);
 }
 
-sub clone_v2 ($$;$) {
+sub run_puh {
+        my ($self, $fini) = @_;
+        $self->{-fini} = $fini;
+        @{$self->{-puh_todo}} = @PUH;
+        run_next_puh($self);
+}
+
+# modifies the to-be-written manifest entry, and sets values from it, too
+sub update_ent {
+        my ($self) = @_;
+        my $key = $self->{-key} // die 'BUG: no -key';
+        my $new = $self->{-ent}->{fingerprint};
+        my $cur = $self->{-local_manifest}->{$key}->{fingerprint} // "\0";
+        my $dst = $self->{cur_dst} // $self->{dst};
+        if (defined($new) && $new ne $cur) {
+                my $cmd = ['git', "--git-dir=$dst", 'show-ref'];
+                my $opt = { 2 => $self->{lei}->{2} };
+                open($opt->{1}, '+>', undef);
+                $self->{-show_ref_up} = $opt->{1};
+                my $done = PublicInbox::OnDestroy->new($$, \&up_fp_done, $self);
+                start_cmd($self, $cmd, $opt, $done);
+        }
+        $new = $self->{-ent}->{head};
+        $cur = $self->{-local_manifest}->{$key}->{head} // "\0";
+        if (defined($new) && $new ne $cur) {
+                # n.b. grokmirror writes raw contents to $dst/HEAD w/o locking
+                my $cmd = [ 'git', "--git-dir=$dst" ];
+                if ($new =~ s/\Aref: //) {
+                        push @$cmd, qw(symbolic-ref HEAD), $new;
+                } elsif ($new =~ /\A[a-f0-9]{40,}\z/) {
+                        push @$cmd, qw(update-ref --no-deref HEAD), $new;
+                } else {
+                        undef $cmd;
+                        warn "W: $key: {head} => `$new' not understood\n";
+                }
+                start_cmd($self, $cmd, { 2 => $self->{lei}->{2} }) if $cmd;
+        }
+        if (my $symlinks = $self->{-ent}->{symlinks}) {
+                my $top = File::Spec->rel2abs($self->{dst});
+                push @{$self->{-new_symlinks}}, @$symlinks;
+                for my $p (@$symlinks) {
+                        my $ln = "$top/$p";
+                        $ln =~ tr!/!/!s;
+                        my (undef, $dn, $bn) = File::Spec->splitpath($ln);
+                        File::Path::mkpath($dn);
+                        my $tgt = "$top/$key";
+                        $tgt = File::Spec->abs2rel($tgt, $dn);
+                        if (lstat($ln)) {
+                                if (-l _) {
+                                        next if readlink($ln) eq $tgt;
+                                        unlink($ln);
+                                } else {
+                                        push @{$self->{chg}->{badlink}}, $p;
+                                }
+                        }
+                        symlink($tgt, $ln);
+                        ++$self->{chg}->{nr_chg};
+                }
+        }
+        if (defined(my $t = $self->{-ent}->{modified})) {
+                my ($dn, $bn) = ("$dst/info/web", 'last-modified');
+                my $orig = PublicInbox::IO::try_cat("$dn/$bn");
+                $t = strftime('%F %T', gmtime($t))." +0000\n";
+                File::Path::mkpath($dn);
+                atomic_write($dn, $bn, $t) if $orig ne $t;
+        }
+
+        $new = $self->{-ent}->{owner} // return;
+        $cur = $self->{-local_manifest}->{$key}->{owner} // "\0";
+        return if $cur eq $new;
+        utf8::encode($new); # to octets
+        my $cmd = [ qw(git config -f), "$dst/config", 'gitweb.owner', $new ];
+        start_cmd($self, $cmd, { 2 => $self->{lei}->{2} });
+}
+
+sub v1_done { # called via OnDestroy
+        my ($self) = @_;
+        return if $self->{dry_run} || !keep_going($self);
+        _write_inbox_config($self);
+        my $dst = $self->{cur_dst} // $self->{dst};
+        unlink_fetch_head($dst);
+        update_ent($self) if $self->{-ent};
+        my $o = "$dst/objects";
+        if (CORE::open(my $fh, '<', my $fn = "$o/info/alternates")) {;
+                my $base = File::Spec->rel2abs($o);
+                my @l = <$fh>;
+                my $ft;
+                for (@l) {
+                        next unless m!\A/!;
+                        $_ = File::Spec->abs2rel($_, $base);
+                        $ft //= File::Temp->new(TEMPLATE => '.XXXX',
+                                                DIR => "$o/info");
+                }
+                if ($ft) {
+                        print $ft @l;
+                        $ft->flush or die "flush($ft): $!";
+                        ft_rename($ft, $fn, 0666, $fh);
+                }
+        }
+        eval { set_description($self) };
+        warn $@ if $@;
+        return if ($self->{-is_epoch} ||
+                $self->{lei}->{opt}->{'inbox-config'} ne 'always');
+        write_makefile($dst, 1);
+        index_cloned_inbox($self, 1);
+}
+
+sub v2_done { # called via OnDestroy
+        my ($self) = @_;
+        return if $self->{dry_run} || !keep_going($self);
+        my $dst = $self->{cur_dst} // $self->{dst};
+        require PublicInbox::Lock;
+        my $lk = PublicInbox::Lock->new("$dst/inbox.lock");
+        my $lck = $lk->lock_for_scope($$);
+        _write_inbox_config($self);
+        require PublicInbox::MultiGit;
+        my $mg = PublicInbox::MultiGit->new($dst, 'all.git', 'git');
+        $mg->fill_alternates;
+        for my $i ($mg->git_epochs) { $mg->epoch_cfg_set($i) }
+        for my $edst (@{delete($self->{-read_only}) // []}) {
+                my @st = stat($edst) or die "stat($edst): $!";
+                chmod($st[2] & 0555, $edst);
+        }
+        write_makefile($dst, 2);
+        undef $lck; # unlock
+        eval { set_description($self) };
+        warn $@ if $@;
+        index_cloned_inbox($self, 2);
+}
+
+sub clone_v2_prep ($$;$) {
         my ($self, $v2_epochs, $m) = @_; # $m => manifest.js.gz hashref
         my $lei = $self->{lei};
         my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
-        my $pfx = $curl->torsocks($lei, (values %$v2_epochs)[0]) or return;
-        my $dst = $self->{dst};
+        my $first_uri = (map { $_->[0] } values %$v2_epochs)[0];
+        $self->{-torsocks} //= $curl->torsocks($lei, $first_uri) or return;
+        my $dst = $self->{cur_dst} // $self->{dst};
         my $want = parse_epochs($lei->{opt}->{epoch}, $v2_epochs);
-        my (@src_edst, @read_only, @skip_nr);
+        my $task = $m ? bless { %$self }, __PACKAGE__ : $self;
+        my (@skip, $desc);
+        my $fini = PublicInbox::OnDestroy->new($$, \&v2_done, $task);
         for my $nr (sort { $a <=> $b } keys %$v2_epochs) {
-                my $uri = $v2_epochs->{$nr};
+                my ($uri, $key) = @{$v2_epochs->{$nr}};
                 my $src = $uri->as_string;
                 my $edst = $dst;
                 $src =~ m!/([0-9]+)(?:\.git)?\z! or die <<"";
@@ -294,60 +893,48 @@ failed to extract epoch number from $src
 
                 $1 + 0 == $nr or die "BUG: <$uri> miskeyed $1 != $nr";
                 $edst .= "/git/$nr.git";
+                my $ent;
+                if ($m) {
+                        $ent = $m->{$key} //
+                                die("BUG: `$key' not in manifest.js.gz");
+                        if (defined(my $d = $ent->{description})) {
+                                $d =~ s/ \[epoch [0-9]+\]\z//s;
+                                $desc = $d;
+                        }
+                }
                 if (!$want || $want->{$nr}) {
-                        push @src_edst, $src, $edst;
+                        my $etask = bless { %$task, -key => $key }, __PACKAGE__;
+                        $etask->{-ent} = $ent; # may have {reference}
+                        $etask->{cur_src} = $src;
+                        $etask->{cur_dst} = $edst;
+                        $etask->{-is_epoch} = $fini;
+                        my $ref = $ent->{reference} // '';
+                        push @{$TODO->{$ref}}, $etask;
+                        $self->{any_want}->{$key} = 1;
                 } else { # create a placeholder so users only need to chmod +w
-                        init_placeholder($src, $edst);
-                        push @read_only, $edst;
-                        push @skip_nr, $nr;
+                        init_placeholder($src, $edst, $ent);
+                        push @{$task->{-read_only}}, $edst;
+                        push @skip, $key;
                 }
         }
-        if (@skip_nr) { # filter out the epochs we skipped
-                my $re = join('|', @skip_nr);
-                my @del = grep(m!/git/$re\.git\z!, keys %$m);
-                delete @$m{@del};
-                $self->{-culled_manifest} = 1;
-        }
-        my $lk = bless { lock_path => "$dst/inbox.lock" }, 'PublicInbox::Lock';
-        _try_config($self);
-        my $on_destroy = $lk->lock_for_scope($$);
-        my @cmd = clone_cmd($lei, my $opt = {});
-        while (my ($src, $edst) = splice(@src_edst, 0, 2)) {
-                my $cmd = [ @$pfx, @cmd, $src, $edst ];
-                my $cerr = run_reap($lei, $cmd, $opt);
-                return $lei->child_error($cerr, "@$cmd failed") if $cerr;
-        }
-        require PublicInbox::MultiGit;
-        my $mg = PublicInbox::MultiGit->new($dst, 'all.git', 'git');
-        $mg->fill_alternates;
-        for my $i ($mg->git_epochs) { $mg->epoch_cfg_set($i) }
-        for my $edst (@read_only) {
-                my @st = stat($edst) or die "stat($edst): $!";
-                chmod($st[2] & 0555, $edst) or die "chmod(a-w, $edst): $!";
+        # filter out the epochs we skipped
+        $self->{chg}->{manifest} = 1 if $m && delete(@$m{@skip});
+
+        $self->{dry_run} or File::Path::mkpath($dst);
+
+        if ($lei->{opt}->{'inbox-config'} =~ /\A(?:always|v2)\z/s &&
+                        !-f "$dst/inbox.config.example") {
+                _get_txt_start($task, '_/text/config/raw', $fini);
         }
-        write_makefile($self->{dst}, 2);
-        undef $on_destroy; # unlock
-        index_cloned_inbox($self, 2);
-}
 
-# PSGI mount prefixes and manifest.js.gz prefixes don't always align...
-sub deduce_epochs ($$) {
-        my ($m, $path) = @_;
-        my ($v1_ent, @v2_epochs);
-        my $path_pfx = '';
-        $path =~ s!/+\z!!;
-        do {
-                $v1_ent = $m->{$path};
-                @v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
-        } while (!defined($v1_ent) && !@v2_epochs &&
-                $path =~ s!\A(/[^/]+)/!/! and $path_pfx .= $1);
-        ($path_pfx, $v1_ent ? $path : undef, @v2_epochs);
+        defined($desc) ? ($task->{'txt.description'} = $desc) :
+                _get_txt_start($task, 'description', $fini);
 }
 
 sub decode_manifest ($$$) {
         my ($fh, $fn, $uri) = @_;
         my $js;
-        my $gz = do { local $/; <$fh> } // die "slurp($fn): $!";
+        my $gz = PublicInbox::IO::read_all $fh;
         gunzip(\$gz => \$js, MultiStream => 1) or
                 die "gunzip($uri): $GunzipError\n";
         my $m = eval { PublicInbox::Config->json->decode($js) };
@@ -356,61 +943,318 @@ sub decode_manifest ($$$) {
         $m;
 }
 
+sub load_current_manifest ($) {
+        my ($self) = @_;
+        my $fn = $self->{-manifest} // return;
+        if (CORE::open(my $fh, '<', $fn)) {
+                decode_manifest($fh, $fn, $fn);
+        } elsif ($!{ENOENT}) { # non-fatal, we can just do it slowly
+                $self->{-initial_clone} or
+                        warn "W: open($fn): $! (non-fatal)\n";
+                undef;
+        } else {
+                die "E: open($fn): $!\n";
+        }
+}
+
+sub multi_inbox ($$$) {
+        my ($self, $path, $m) = @_;
+        my $incl = $self->{lei}->{opt}->{include};
+        my $excl = $self->{lei}->{opt}->{exclude};
+
+        # assuming everything not v2 is v1, for now
+        my @v1 = sort grep(!m!.+/git/[0-9]+\.git\z!, keys %$m);
+        my @v2_epochs = sort grep(m!.+/git/[0-9]+\.git\z!, keys %$m);
+        my $v2 = {};
+
+        for (@v2_epochs) {
+                m!\A(/.+)/git/[0-9]+\.git\z! or die "BUG: $_";
+                push @{$v2->{$1}}, $_;
+        }
+        my $n = scalar(keys %$v2) + scalar(@v1);
+        my @orig = defined($incl // $excl) ? (keys %$v2, @v1) : ();
+        if (defined $incl) {
+                my $re = '(?:'.join('\\z|', map {
+                                glob2re($_) // qr/\A\Q$_\E/
+                        } @$incl).'\\z)';
+                my @gone = delete @$v2{grep(!/$re/, keys %$v2)};
+                delete @$m{map { @$_ } @gone} and $self->{chg}->{manifest} = 1;
+                delete @$m{grep(!/$re/, @v1)} and $self->{chg}->{manifest} = 1;
+                @v1 = grep(/$re/, @v1);
+        }
+        if (defined $excl) {
+                my $re = '(?:'.join('\\z|', map {
+                                glob2re($_) // qr/\A\Q$_\E/
+                        } @$excl).'\\z)';
+                my @gone = delete @$v2{grep(/$re/, keys %$v2)};
+                delete @$m{map { @$_ } @gone} and $self->{chg}->{manifest} = 1;
+                delete @$m{grep(/$re/, @v1)} and $self->{chg}->{manifest} = 1;
+                @v1 = grep(!/$re/, @v1);
+        }
+        my $ret; # { v1 => [ ... ], v2 => { "/$inbox_name" => [ epochs ] }}
+        $ret->{v1} = \@v1 if @v1;
+        $ret->{v2} = $v2 if keys %$v2;
+        $ret //= @orig ? "Nothing to clone, available repositories:\n\t".
+                                join("\n\t", sort @orig)
+                        : "Nothing available to clone\n";
+        my $path_pfx = '';
+
+        # PSGI mount prefixes and manifest.js.gz prefixes don't always align...
+        if (@v2_epochs) {
+                until (grep(m!\A\Q$$path\E/git/[0-9]+\.git\z!,
+                                @v2_epochs) == @v2_epochs) {
+                        $$path =~ s!\A(/[^/]+)/!/! or last;
+                        $path_pfx .= $1;
+                }
+        } elsif (@v1) {
+                while (!defined($m->{$$path}) && $$path =~ s!\A(/[^/]+)/!/!) {
+                        $path_pfx .= $1;
+                }
+        }
+        ($path_pfx, $n, $ret);
+}
+
+sub clone_all {
+        my ($self, $m) = @_;
+        my $todo = $TODO;
+        $TODO = \'BUG on further use';
+        my $end = PublicInbox::OnDestroy->new($$, \&fgrp_fetch_all, $self);
+        {
+                my $nodep = delete $todo->{''};
+
+                # do not download unwanted deps
+                my $any_want = delete $self->{any_want};
+                my @unwanted = grep { !$any_want->{$_} } keys %$todo;
+                my @nodep = delete(@$todo{@unwanted});
+                push(@$nodep, @$_) for @nodep;
+
+                # handle no-dependency repos, first
+                for (@$nodep) {
+                        clone_v1($_, $end);
+                        return if !keep_going($self);
+                }
+        }
+        # resolve references, deepest, first:
+        while (scalar keys %$todo) {
+                for my $x (keys %$todo) {
+                        my ($nr, $nxt);
+                        # resolve multi-level references
+                        while ($m && defined($nxt = $m->{$x}->{reference})) {
+                                exists($todo->{$nxt}) or last;
+                                if (++$nr > 1000) {
+                                        $m->{$x}->{reference} = undef;
+                                        $m->{$nxt}->{reference} = undef;
+                                        warn <<EOM
+E: dependency loop detected (`$x' => `$nxt'), breaking
+EOM
+                                }
+                                $x = $nxt;
+                        }
+                        my $y = delete $todo->{$x} // next; # already done
+                        for (@$y) {
+                                clone_v1($_, $end);
+                                return if !keep_going($self);
+                        }
+                        last; # restart %$todo iteration
+                }
+        }
+
+        # $end->DESTROY will call fgrp_fetch_all once all references
+        # in $LIVE are gone, and do_reap will eventually drain $LIVE
+        $end = undef;
+        do_reap($self, 1);
+}
+
+sub dump_manifest ($$) {
+        my ($m, $ft) = @_;
+        # write the smaller manifest if epochs were skipped so
+        # users won't have to delete manifest if they +w an
+        # epoch they no longer want to skip
+        my $json = PublicInbox::Config->json->encode($m);
+        my $mtime = (stat($ft))[9];
+        seek($ft, 0, SEEK_SET);
+        truncate($ft, 0);
+        gzip(\$json => $ft) or die "gzip($ft): $GzipError";
+        $ft->flush or die "flush($ft): $!";
+        utime($mtime, $mtime, "$ft") or die "utime(..., $ft): $!";
+}
+
+sub dump_project_list ($$) {
+        my ($self, $m) = @_;
+        my $f = $self->{'-project-list'};
+        my $old = defined($f) ? PublicInbox::IO::try_cat($f) : '';
+        my %new;
+
+        open my $dh, '<', '.';
+        if (!$self->{dry_run} || -d $self->{dst}) {
+                chdir($self->{dst});
+        }
+        my @local = grep { -e $_ ? ($new{$_} = undef) : 1 } split(/\n/s, $old);
+        chdir($dh);
+
+        $new{substr($_, 1)} = 1 for keys %$m; # drop leading '/'
+        my @list = sort keys %new;
+        my @remote = grep { !defined($new{$_}) } @list;
+        my %lnk = map { substr($_, 1) => undef } @{$self->{-new_symlinks}};
+        @remote = grep { !exists($lnk{$_}) } @remote;
+
+        if (@remote) {
+                warn <<EOM;
+The following local repositories are ignored/gone from $self->{src}:
+EOM
+                warn "\t", $_, "\n" for @remote;
+
+                if ($self->{lei}->{opt}->{purge} && !$self->{dry_run}) {
+                        my $o = {};
+                        $o->{verbose} = 1 if $self->{lei}->{opt}->{verbose};
+                        my $dst = $self->{dst};
+                        File::Path::remove_tree(map { "$dst/$_" } @remote, $o);
+                        my %rm = map { $_ => undef } @remote;
+                        @list = grep { !exists($rm{$_}) } @list;
+                        $self->{lei}->qerr('# purged ');
+                }
+        }
+        if (defined($f) && @local) {
+                warn <<EOM;
+The following repos in $f no longer exist on the filesystem:
+EOM
+                warn "\t", $_, "\n" for @local;
+        }
+        $self->{chg}->{nr_chg} += scalar(@remote) + scalar(@local);
+        return if !defined($f) || $self->{dry_run};
+        my (undef, $dn, $bn) = File::Spec->splitpath($f);
+        my $new = join("\n", @list, '');
+        atomic_write($dn, $bn, $new) if $new ne $old;
+}
+
+# FIXME: this gets confused by single inbox instance w/ global manifest.js.gz
 sub try_manifest {
         my ($self) = @_;
         my $uri = URI->new($self->{src});
         my $lei = $self->{lei};
         my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
+        $self->{-torsocks} //= $curl->torsocks($lei, $uri) or return;
         my $path = $uri->path;
         chop($path) eq '/' or die "BUG: $uri not canonicalized";
-        $uri->path($path . '/manifest.js.gz');
-        my $pdir = $lei->rel2abs($self->{dst});
-        $pdir =~ s!/[^/]+/?\z!!;
-        my $ft = File::Temp->new(TEMPLATE => 'm-XXXX',
-                                UNLINK => 1, DIR => $pdir, SUFFIX => '.tmp');
-        my $fn = $ft->filename;
-        my ($bn) = ($fn =~ m!/([^/]+)\z!);
-        my $cmd = $curl->for_uri($lei, $uri, '-R', '-o', $bn);
-        my $opt = { -C => $pdir };
-        $opt->{$_} = $lei->{$_} for (0..2);
-        my $cerr = run_reap($lei, $cmd, $opt);
+        my $rmf = $lei->{opt}->{'remote-manifest'} // '/manifest.js.gz';
+        if ($rmf =~ m!\A[^/:]+://!) {
+                $uri = URI->new($rmf);
+        } else {
+                $rmf = "/$rmf" if index($rmf, '/') != 0;
+                $uri->path($path.$rmf);
+        }
+        my $manifest = $self->{-manifest} // "$self->{dst}/manifest.js.gz";
+        my %opt = (UNLINK => 1, SUFFIX => '.tmp', TMPDIR => 1);
+        if (!$self->{dry_run} && $manifest =~ m!\A(.+?)/[^/]+\z! and -d $1) {
+                $opt{DIR} = $1; # allows fast rename(2) w/o EXDEV
+                delete $opt{TMPDIR};
+        }
+        my $ft = File::Temp->new(TEMPLATE => '.manifest-XXXX', %opt);
+        my $cmd = $curl->for_uri($lei, $uri, qw(-R -o), $ft->filename);
+        push(@$cmd, '-z', $manifest) if -f $manifest;
+        my $mf_url = "$uri";
+        %opt = map { $_ => $lei->{$_} } (0..2);
+        my $cerr = run_reap($lei, $cmd, \%opt);
         if ($cerr) {
                 return try_scrape($self) if ($cerr >> 8) == 22; # 404 missing
                 return $lei->child_error($cerr, "@$cmd failed");
         }
-        my $m = eval { decode_manifest($ft, $fn, $uri) };
+
+        # bail out if curl -z/--timecond hit 304 Not Modified, $ft will be empty
+        if (-f $manifest && !-s $ft) {
+                $lei->child_error(127 << 8) if $lei->{opt}->{'exit-code'};
+                return $lei->qerr("# $manifest unchanged");
+        }
+
+        my $m = eval { decode_manifest($ft, $ft, $uri) };
         if ($@) {
                 warn $@;
                 return try_scrape($self);
         }
-        my ($path_pfx, $v1_path, @v2_epochs) = deduce_epochs($m, $path);
-        if (@v2_epochs) {
-                # It may be possible to have v1 + v2 in parallel someday:
-                warn(<<EOM) if defined $v1_path;
-# `$v1_path' appears to be a v1 inbox while v2 epochs exist:
-# @v2_epochs
-# ignoring $v1_path (use --inbox-version=1 to force v1 instead)
+        local $self->{chg} = {};
+        local $self->{-local_manifest} = load_current_manifest($self);
+        local $self->{-new_symlinks} = [];
+        my ($path_pfx, $n, $multi) = multi_inbox($self, \$path, $m);
+        return $lei->child_error(0, $multi) if !ref($multi);
+        my $v2 = delete $multi->{v2};
+        if ($v2) {
+                for my $name (sort keys %$v2) {
+                        my $epochs = delete $v2->{$name};
+                        my %v2_epochs = map {
+                                $uri->path($n > 1 ? $path_pfx.$path.$_
+                                                : $path_pfx.$_);
+                                my ($e) = ("$uri" =~ m!/([0-9]+)\.git\z!);
+                                $e // die "no [0-9]+\.git in `$uri'";
+                                $e => [ $uri->clone, $_ ];
+                        } @$epochs;
+                        ("$uri" =~ m!\A(.+/)git/[0-9]+\.git\z!) or
+                                die "BUG: `$uri' !~ m!/git/[0-9]+.git!";
+                        local $self->{cur_src} = $1;
+                        local $self->{cur_dst} = $self->{dst};
+                        if ($n > 1 && $uri->path =~ m!\A\Q$path_pfx$path\E/(.+)/
+                                                        git/[0-9]+\.git\z!x) {
+                                $self->{cur_dst} .= "/$1";
+                        }
+                        index($self->{cur_dst}, "\n") >= 0 and die <<EOM;
+E: `$self->{cur_dst}' must not contain newline
 EOM
-                my %v2_epochs = map {
-                        $uri->path($path_pfx.$_);
-                        my ($n) = ("$uri" =~ m!/([0-9]+)\.git\z!);
-                        $n => $uri->clone
-                } @v2_epochs;
-                clone_v2($self, \%v2_epochs, $m);
-        } elsif (defined $v1_path) {
-                clone_v1($self);
-        } else {
-                die "E: confused by <$uri>, possible matches:\n\t",
-                        join(', ', sort keys %$m), "\n";
+                        clone_v2_prep($self, \%v2_epochs, $m);
+                        return if !keep_going($self);
+                }
         }
-        if (delete $self->{-culled_manifest}) { # set by clone_v2
-                # write the smaller manifest if epochs were skipped so
-                # users won't have to delete manifest if they +w an
-                # epoch they no longer want to skip
-                my $json = PublicInbox::Config->json->encode($m);
-                gzip(\$json => $fn) or die "gzip: $GzipError";
+        if (my $v1 = delete $multi->{v1}) {
+                my $p = $path_pfx.$path;
+                chop($p) if substr($p, -1, 1) eq '/';
+                $uri->path($p);
+                for my $name (@$v1) {
+                        my $task = bless { %$self }, __PACKAGE__;
+                        $task->{-ent} = $m->{$name} //
+                                        die("BUG: no `$name' in manifest");
+                        $task->{cur_src} = "$uri";
+                        $task->{cur_dst} = $task->{dst};
+                        $task->{-key} = $name;
+                        if ($n > 1) {
+                                $task->{cur_dst} .= $name;
+                                $task->{cur_src} .= $name;
+                        }
+                        index($task->{cur_dst}, "\n") >= 0 and die <<EOM;
+E: `$task->{cur_dst}' must not contain newline
+EOM
+                        $task->{cur_src} .= '/';
+                        my $dep = $task->{-ent}->{reference} // '';
+                        push @{$TODO->{$dep}}, $task; # for clone_all
+                        $self->{any_want}->{$name} = 1;
+                }
+        }
+        delete local $lei->{opt}->{epoch} if defined($v2);
+        clone_all($self, $m);
+        return if !keep_going($self);
+
+        # set by clone_v2_prep/-I/--exclude
+        my $mis = delete $self->{chg}->{fp_mismatch};
+        if ($mis) {
+                my $t = fmt_localtime((stat($ft))[9]);
+                warn <<EOM;
+W: Fingerprints for the following repositories do not match
+W: $mf_url @ $t:
+W: These repositories may have updated since $t:
+EOM
+                warn "\t", $_, "\n" for @$mis;
+                warn <<EOM if !$self->{lei}->{opt}->{prune};
+W: The above fingerprints may never match without --prune
+EOM
+        }
+        if ((delete($self->{chg}->{manifest}) || $mis) && !$self->{dry_run}) {
+                dump_manifest($m => $ft);
         }
-        ft_rename($ft, "$self->{dst}/manifest.js.gz", 0666);
+        my $bad = delete $self->{chg}->{badlink};
+        warn(<<EOM, map { ("\t", $_, "\n") } @$bad) if $bad;
+W: The following exist and have not been converted to symlinks
+EOM
+        dump_project_list($self, $m);
+        ft_rename($ft, $manifest, 0666) if !$self->{dry_run};
+        !$self->{chg}->{nr_chg} && $lei->{opt}->{'exit-code'} and
+                $lei->child_error(127 << 8);
 }
 
 sub start_clone_url {
@@ -419,19 +1263,45 @@ sub start_clone_url {
         die "TODO: non-HTTP/HTTPS clone of $self->{src} not supported, yet";
 }
 
-sub do_mirror { # via wq_io_do
+sub do_mirror { # via wq_io_do or public-inbox-clone
         my ($self) = @_;
         my $lei = $self->{lei};
+        $self->{dry_run} = 1 if $lei->{opt}->{'dry-run'};
         umask($lei->{client_umask}) if defined $lei->{client_umask};
+        $self->{-initial_clone} = 1 if !-d $self->{dst};
+        local @PUH;
+        if (defined(my $puh = $lei->{opt}->{'post-update-hook'})) {
+                require Text::ParseWords;
+                @PUH = map { [ Text::ParseWords::shellwords($_) ] } @$puh;
+        }
         eval {
-                my $iv = $lei->{opt}->{'inbox-version'};
-                if (defined $iv) {
-                        return clone_v1($self) if $iv == 1;
-                        return try_scrape($self) if $iv == 2;
-                        die "bad --inbox-version=$iv\n";
+                my $ic = $lei->{opt}->{'inbox-config'} //= 'always';
+                $ic =~ /\A(?:v1|v2|always|never)\z/s or die <<"";
+--inbox-config must be one of `always', `v2', `v1', or `never'
+
+                # we support these switches with '' (empty string).
+                # defaults match example conf distributed with grokmirror
+                my @pairs = qw(objstore objstore manifest manifest.js.gz
+                                project-list projects.list);
+                while (@pairs) {
+                        my ($k, $default) = splice(@pairs, 0, 2);
+                        my $v = $lei->{opt}->{$k} // next;
+                        $v = $default if $v eq '';
+                        $v = "$self->{dst}/$v" if $v !~ m!\A\.{0,2}/!;
+                        $self->{"-$k"} = $v;
                 }
-                return start_clone_url($self) if $self->{src} =~ m!://!;
-                die "TODO: cloning local directories not supported, yet";
+
+                local $LIVE = {};
+                local $TODO = {};
+                local $FGRP_TODO = {};
+                my $iv = $lei->{opt}->{'inbox-version'} //
+                        return start_clone_url($self);
+                return clone_v1($self) if $iv == 1;
+                die "bad --inbox-version=$iv\n" if $iv != 2;
+                die <<EOM if $self->{src} !~ m!://!;
+cloning local v2 inboxes not supported
+EOM
+                try_scrape($self, 1);
         };
         $lei->fail($@) if $@;
 }
@@ -439,14 +1309,6 @@ sub do_mirror { # via wq_io_do
 sub start {
         my ($cls, $lei, $src, $dst) = @_;
         my $self = bless { src => $src, dst => $dst }, $cls;
-        if ($src =~ m!https?://!) {
-                require URI;
-                require PublicInbox::LeiCurl;
-        }
-        require PublicInbox::Lock;
-        require PublicInbox::Inbox;
-        require PublicInbox::Admin;
-        require PublicInbox::InboxWritable;
         $lei->request_umask;
         my ($op_c, $ops) = $lei->workers_start($self, 1);
         $lei->{wq1} = $self;
@@ -464,8 +1326,8 @@ sub ipc_atfork_child {
 sub write_makefile {
         my ($dir, $ibx_ver) = @_;
         my $f = "$dir/Makefile";
-        if (sysopen my $fh, $f, O_CREAT|O_EXCL|O_WRONLY) {
-                print $fh <<EOM or die "print($f) $!";
+        if (CORE::sysopen my $fh, $f, O_CREAT|O_EXCL|O_WRONLY) {
+                print $fh <<EOM;
 # This is a v$ibx_ver public-inbox, see the public-inbox-v$ibx_ver-format(5)
 # manpage for more information on the format.  This Makefile is
 # intended as a familiar wrapper for users unfamiliar with
@@ -479,7 +1341,7 @@ sub write_makefile {
 # so you may edit it freely with your own convenience targets
 # and notes.  public-inbox-fetch will recreate it if removed.
 EOM
-                print $fh <<'EOM' or die "print($f): $!";
+                print $fh <<'EOM';
 # the default target:
 help :
         @echo Common targets:
@@ -489,6 +1351,7 @@ help :
         @echo Rarely needed targets:
         @echo '    make reindex      - may be needed for new features/bugfixes'
         @echo '    make compact      - rewrite Xapian storage to save space'
+        @echo '    make index        - initial index after clone'
 
 fetch :
         public-inbox-fetch
@@ -505,14 +1368,16 @@ update :
                 echo 'public-inbox index not initialized'; \
                 echo 'see public-inbox-index(1) man page'; \
         fi
+index :
+        public-inbox-index
 reindex :
         public-inbox-index --reindex
 compact :
         public-inbox-compact
 
-.PHONY : help fetch update reindex compact
+.PHONY : help fetch update index reindex compact
 EOM
-                close $fh or die "close($f): $!";
+                close $fh;
         } else {
                 die "open($f): $!" unless $!{EEXIST};
         }
diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index db387633..8d900d0c 100644
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -27,13 +27,6 @@ sub flush_task { # PublicInbox::DS timer callback
         for my $lei (values %$todo) { flush_lei($lei) }
 }
 
-# sets a timer to flush
-sub note_event_arm_done ($) {
-        my ($lei) = @_;
-        PublicInbox::DS::add_uniq_timer('flush_timer', 5, \&flush_task);
-        $to_flush->{$lei->{cfg}->{'-f'}} //= $lei;
-}
-
 sub eml_event ($$$$) {
         my ($self, $eml, $vmd, $state) = @_;
         my $sto = $self->{lei}->{sto};
@@ -58,7 +51,7 @@ sub eml_event ($$$$) {
         }
 }
 
-sub maildir_event { # via wq_io_do
+sub maildir_event { # via wq_nonblock_do
         my ($self, $fn, $vmd, $state) = @_;
         if (my $eml = PublicInbox::InboxWritable::eml_from_path($fn)) {
                 eml_event($self, $eml, $vmd, $state);
@@ -67,6 +60,18 @@ sub maildir_event { # via wq_io_do
         } # else: eml_from_path already warns
 }
 
+sub _mh_cb { # mh_read_one cb
+        my ($dir, $bn, $kw, $eml, $self, $state) = @_;
+}
+
+sub mh_event { # via wq_nonblock_do
+        my ($self, $folder, $bn, $state) = @_;
+        my $dir = substr($folder, 3);
+        require PublicInbox::MHreader; # if we forked early
+        my $mhr = PublicInbox::MHreader->new($dir, $self->{lei}->{3});
+        $mhr->mh_read_one($bn, \&_mh_cb, $self, $state);
+}
+
 sub lei_note_event {
         my ($lei, $folder, $new_cur, $bn, $fn, @rest) = @_;
         die "BUG: unexpected: @rest" if @rest;
@@ -79,11 +84,14 @@ sub lei_note_event {
         $lms->arg2folder($lei, [ $folder ]);
         my $state = $cfg->get_1("watch.$folder.state") // 'tag-rw';
         return if $state eq 'pause';
-        return $lms->clear_src($folder, \$bn) if $new_cur eq '';
+        if ($new_cur eq '') {
+                my $id = $folder =~ /\Amaildir:/ ? \$bn : $bn + 0;
+                return $lms->clear_src($folder, $id);
+        }
         $lms->lms_pause;
         $lei->ale; # prepare
         $sto->write_prepare($lei);
-        require PublicInbox::MdirReader;
+        require PublicInbox::MHreader if $folder =~ /\Amh:/; # optimistic
         my $self = $cfg->{-lei_note_event} //= do {
                 my $wq = bless { lms => $lms }, __PACKAGE__;
                 # MUAs such as mutt can trigger massive rename() storms so
@@ -92,16 +100,21 @@ sub lei_note_event {
                 $jobs = 4 if $jobs > 4; # same default as V2Writable
                 my ($op_c, $ops) = $lei->workers_start($wq, $jobs);
                 $lei->wait_wq_events($op_c, $ops);
-                note_event_arm_done($lei);
+                PublicInbox::DS::add_uniq_timer('flush_timer', 5, \&flush_task);
+                $to_flush->{$lei->{cfg}->{'-f'}} //= $lei;
+                $wq->prepare_nonblock;
                 $lei->{lne} = $wq;
         };
         if ($folder =~ /\Amaildir:/i) {
+                require PublicInbox::MdirReader;
                 my $fl = PublicInbox::MdirReader::maildir_basename_flags($bn)
                         // return;
                 return if index($fl, 'T') >= 0;
                 my $kw = PublicInbox::MdirReader::flags2kw($fl);
                 my $vmd = { kw => $kw, sync_info => [ $folder, \$bn ] };
-                $self->wq_do('maildir_event', $fn, $vmd, $state);
+                $self->wq_nonblock_do('maildir_event', $fn, $vmd, $state);
+        } elsif ($folder =~ /\Amh:/) {
+                $self->wq_nonblock_do('mh_event', $folder, $bn, $state);
         } # else: TODO: imap
 }
 
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 2d3db9f4..0529bbe4 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -41,8 +41,8 @@ sub detect_fmt ($) {
         my ($dst) = @_;
         if ($dst =~ m!\A([:/]+://)!) {
                 die "$1 support not implemented, yet\n";
-        } elsif (!-e $dst || -d _) {
-                'maildir'; # the default TODO: MH?
+        } elsif (!-e $dst || -d _) { # maildir is the default TODO: MH
+                -e "$dst/inbox.lock" ? 'v2' : 'maildir';
         } elsif (-f _ || -p _) {
                 die "unable to determine mbox family of $dst\n";
         } else {
@@ -143,7 +143,7 @@ sub _unbless_smsg {
         $smsg->{dt} = iso8601(delete $smsg->{ds}); # JMAP UTCDate
         $smsg->{pct} = get_pct($mitem) if $mitem;
         if (my $r = delete $smsg->{references}) {
-                $smsg->{refs} = [ map { $_ } ($r =~ m/$MID_EXTRACT/go) ];
+                @{$smsg->{refs}} = ($r =~ m/$MID_EXTRACT/go);
         }
         if (my $m = delete($smsg->{mid})) {
                 $smsg->{'m'} = $m;
@@ -212,7 +212,8 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
                 sub {
                         my ($smsg, $mitem, $eml) = @_;
                         $smsg->{pct} = get_pct($mitem) if $mitem;
-                        $l2m->wq_io_do('write_mail', [], $smsg, $eml);
+                        eval { $l2m->wq_io_do('write_mail', [], $smsg, $eml) };
+                        $lei->fail($@) if $@ && !$!{ECONNRESET} && !$!{EPIPE};
                 }
         } elsif ($self->{fmt} =~ /\A(concat)?json\z/ && $lei->{opt}->{pretty}) {
                 my $EOR = ($1//'') eq 'concat' ? "\n}" : "\n},";
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 51ee3d9c..eadf811f 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -1,11 +1,10 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # handles "lei q" command and provides internals for
 # several other sub-commands (up, lcat, ...)
 package PublicInbox::LeiQuery;
-use strict;
-use v5.10.1;
+use v5.12;
 
 sub prep_ext { # externals_each callback
         my ($lxs, $exclude, $loc) = @_;
@@ -17,6 +16,8 @@ sub _start_query { # used by "lei q" and "lei up"
         require PublicInbox::LeiOverview;
         PublicInbox::LeiOverview->new($self) or return;
         my $opt = $self->{opt};
+        require PublicInbox::OverIdx; # loads DBI
+        PublicInbox::OverIdx::fork_ok($opt);
         my ($xj, $mj) = split(/,/, $opt->{jobs} // '');
         (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) and
                 die "`$xj' search jobs must be >= 1\n";
@@ -37,8 +38,11 @@ sub _start_query { # used by "lei q" and "lei up"
                         $lms->lms_write_prepare->lms_pause; # just create
                 }
         }
-        $l2m and $l2m->{-wq_nr_workers} //= $mj //
-                int($nproc * 0.75 + 0.5); # keep some CPU for git
+        $l2m and $l2m->{-wq_nr_workers} //= $mj // do {
+                # keep some CPU for git, and don't overload IMAP destinations
+                my $n = int($nproc * 0.75 + 0.5);
+                $self->{net} && $n > 4 ? 4 : $n;
+        };
 
         # descending docid order is cheapest, MUA controls sorting order
         $self->{mset_opt}->{relevance} //= -2 if $l2m || $opt->{threads};
@@ -55,18 +59,19 @@ sub _start_query { # used by "lei q" and "lei up"
         $lxs->do_query($self);
 }
 
-sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin
-        my ($lei) = @_; # $_[1] = $rbuf
-        $_[1] // $lei->fail("error reading stdin: $!");
-        return $lei->{mset_opt}->{qstr} .= $_[1] if $_[1] ne '';
-        eval {
-                $lei->fchdir;
-                $lei->{mset_opt}->{q_raw} = $lei->{mset_opt}->{qstr};
-                $lei->{lse}->query_approxidate($lei->{lse}->git,
-                                                $lei->{mset_opt}->{qstr});
-                _start_query($lei);
-        };
-        $lei->fail($@) if $@;
+sub do_qry { # do_env cb
+        my ($lei) = @_;
+        $lei->{mset_opt}->{q_raw} = $lei->{mset_opt}->{qstr}
+                                                = delete $lei->{stdin_buf};
+        $lei->{lse}->query_approxidate($lei->{lse}->git,
+                                        $lei->{mset_opt}->{qstr});
+        _start_query($lei);
+}
+
+# make the URI||PublicInbox::{Inbox,ExtSearch} a config-file friendly string
+sub cfg_ext ($) {
+        my ($x) = @_;
+        $x->isa('URI') ? "$x" : ($x->{inboxdir} // $x->{topdir});
 }
 
 sub lxs_prepare {
@@ -84,21 +89,32 @@ sub lxs_prepare {
                 $lxs->prepare_external($self->{lse});
         }
         if (@only) {
+                my $only;
                 for my $loc (@only) {
                         my @loc = $self->get_externals($loc) or return;
-                        $lxs->prepare_external($_) for @loc;
+                        for (@loc) {
+                                my $x = $lxs->prepare_external($_);
+                                push(@$only, cfg_ext($x)) if $x;
+                        }
                 }
+                $opt->{only} = $only if $only;
         } else {
-                my (@ilocals, @iremotes);
+                my (@ilocals, @iremotes, $incl);
                 for my $loc (@{$opt->{include} // []}) {
                         my @loc = $self->get_externals($loc) or return;
-                        $lxs->prepare_external($_) for @loc;
+                        for (@loc) {
+                                my $x = $lxs->prepare_external($_);
+                                push(@$incl, cfg_ext($x)) if $x;
+                        }
                         @ilocals = @{$lxs->{locals} // []};
                         @iremotes = @{$lxs->{remotes} // []};
                 }
+                $opt->{include} = $incl if $incl;
                 # --external is enabled by default, but allow --no-external
                 if ($opt->{external} //= 1) {
                         my $ex = $self->canonicalize_excludes($opt->{exclude});
+                        my @excl = keys %$ex;
+                        $opt->{exclude} = \@excl if scalar(@excl);
                         $self->externals_each(\&prep_ext, $lxs, $ex);
                         $opt->{remote} //= !($lxs->locals - $opt->{'local'});
                         $lxs->{locals} = \@ilocals if !$opt->{'local'};
@@ -137,9 +153,7 @@ sub lei_q {
                 return $self->fail(<<'') if @argv;
 no query allowed on command-line with --stdin
 
-                require PublicInbox::InputPipe;
-                PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self);
-                return;
+                return $self->slurp_stdin(\&do_qry);
         }
         chomp(@argv) and $self->qerr("# trailing `\\n' removed");
         $mset_opt{q_raw} = [ @argv ]; # copy
@@ -151,6 +165,8 @@ no query allowed on command-line with --stdin
 # shell completion helper called by lei__complete
 sub _complete_q {
         my ($self, @argv) = @_;
+        join('', @argv) =~ /\bL:\S*\z/ and
+                return eval { $self->_lei_store->search->all_terms('L') };
         my @cur;
         my $cb = $self->lazy_cb(qw(forget-external _complete_));
         while (@argv) {
@@ -185,7 +201,7 @@ sub _complete_q {
 # FIXME: Getopt::Long doesn't easily let us support support options with
 # '.' in them (e.g. --http1.1)
 # TODO: should we depend on "-c http.*" options for things which have
-# analogues in git(1)? that would reduce likelyhood of conflicts with
+# analogues in git(1)? that would reduce likelihood of conflicts with
 # our other CLI options
 # Note: some names are renamed to avoid potential conflicts,
 # see %lei2curl in lib/PublicInbox/LeiCurl.pm
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index c312d90f..35728330 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # The "lei rediff" sub-command, regenerates diffs with new options
@@ -7,7 +7,7 @@ use strict;
 use v5.10.1;
 use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
 use File::Temp 0.19 (); # 0.19 for ->newdir
-use PublicInbox::Spawn qw(spawn which);
+use PublicInbox::Spawn qw(run_wait popen_wr which);
 use PublicInbox::MsgIter qw(msg_part_text);
 use PublicInbox::ViewDiff;
 use PublicInbox::LeiBlob;
@@ -82,6 +82,7 @@ sub _lei_diff_prepare ($$) {
                         push @$cmd, $c ? "-$c" : "--$o";
                 }
         }
+        push(@$cmd, "-O$opt->{'order-file'}") if $opt->{'order-file'};
 }
 
 sub diff_ctxq ($$) {
@@ -113,65 +114,51 @@ EOM
         if (!$rw->{-tmp}) {
                 my $d = "$self->{rdtmp}/for_tree.git";
                 -d $d or PublicInbox::Import::init_bare($d);
-                my $f = "$d/objects/info/alternates"; # always overwrite
-                open my $fh, '>', $f or die "open $f: $!";
-                for my $git (@{$self->{gits}}) {
-                        print $fh $git->git_path('objects'),"\n";
-                }
-                close $fh or die "close $f: $!";
+                # always overwrite
+                PublicInbox::IO::write_file '>', "$d/objects/info/alternates",
+                        map { $_->git_path('objects')."\n" } @{$self->{gits}};
                 $rw = PublicInbox::Git->new($d);
         }
-        pipe(my ($r, $w)) or die "pipe: $!";
-        my $pid = spawn(['git', "--git-dir=$rw->{git_dir}",
+        my $w = popen_wr(['git', "--git-dir=$rw->{git_dir}",
                         qw(fast-import --quiet --done --date-format=raw)],
-                        $lei->{env}, { 2 => $lei->{2}, 0 => $r });
-        close $r or die "close r fast-import: $!";
+                        $lei->{env}, { 2 => $lei->{2} });
         print $w $ta, "\n", $tb, "\ndone\n" or die "print fast-import: $!";
-        close $w or die "close w fast-import: $!";
-        waitpid($pid, 0);
-        die "fast-import failed: \$?=$?" if $?;
+        $w->close or die "close w fast-import: \$?=$? \$!=$!";
 
         my $cmd = [ 'diff' ];
         _lei_diff_prepare($lei, $cmd);
         $lei->qerr("# git @$cmd");
         push @$cmd, qw(A B);
         unshift @$cmd, 'git', "--git-dir=$rw->{git_dir}";
-        $pid = spawn($cmd, $lei->{env}, { 2 => $lei->{2}, 1 => $lei->{1} });
-        waitpid($pid, 0);
-        $lei->child_error($?) if $?; # for git diff --exit-code
+        run_wait($cmd, $lei->{env}, { 2 => $lei->{2}, 1 => $lei->{1} }) and
+                $lei->child_error($?); # for git diff --exit-code
         undef;
 }
 
-sub wait_requote ($$$) { # OnDestroy callback
-        my ($lei, $pid, $old_1) = @_;
-        $lei->{1} = $old_1; # closes stdin of `perl -pE 's/^/> /'`
-        waitpid($pid, 0) == $pid or die "BUG(?) waitpid: \$!=$! \$?=$?";
-        $lei->child_error($?) if $?;
-}
+# awaitpid callback
+sub wait_requote { $_[1]->child_error($?) if $? }
 
-sub requote ($$) {
+sub requote ($$) { # '> ' prefix(es) lei->{1}
         my ($lei, $pfx) = @_;
-        pipe(my($r, $w)) or die "pipe: $!";
-        my $rdr = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2} };
+        my $opt = { 1 => $lei->{1}, 2 => $lei->{2} };
         # $^X (perl) is overkill, but maybe there's a weird system w/o sed
-        my $pid = spawn([$^X, '-pE', "s/^/$pfx/"], $lei->{env}, $rdr);
-        my $old_1 = $lei->{1};
-        $w->autoflush(1);
+        my $w = popen_wr([$^X, '-pe', "s/^/$pfx/"], $lei->{env}, $opt,
+                         \&wait_requote, $lei);
         binmode $w, ':utf8';
-        $lei->{1} = $w;
-        PublicInbox::OnDestroy->new(\&wait_requote, $lei, $pid, $old_1);
+        $w;
 }
 
 sub extract_oids { # Eml each_part callback
         my ($ary, $self) = @_;
+        my $lei = $self->{lei};
         my ($p, undef, $idx) = @$ary;
-        $self->{lei}->out($p->header_obj->as_string, "\n");
+        $lei->out($p->header_obj->as_string, "\n");
         my ($s, undef) = msg_part_text($p, $p->content_type || 'text/plain');
         defined $s or return;
-        my $rq;
-        if ($self->{dqre} && $s =~ s/$self->{dqre}//g) { # '> ' prefix(es)
-                $rq = requote($self->{lei}, $1) if $self->{lei}->{opt}->{drq};
-        }
+
+        $self->{dqre} && $s =~ s/$self->{dqre}//g && $lei->{opt}->{drq} and
+                local $lei->{1} = requote($lei, $1);
+
         my @top = split($PublicInbox::ViewDiff::EXTRACT_DIFFS, $s);
         undef $s;
         my $blobs = $self->{blobs}; # blobs to resolve
@@ -268,7 +255,7 @@ sub lei_rediff {
         if ($lxs->remotes) {
                 require PublicInbox::LeiRemote;
                 $lei->{curl} //= which('curl') or return
-                        $lei->fail('curl needed for', $lxs->remotes);
+                        $lei->fail('curl needed for '.join(', ',$lxs->remotes));
         }
         $lei->ale->refresh_externals($lxs, $lei);
         my $self = bless {
diff --git a/lib/PublicInbox/LeiReindex.pm b/lib/PublicInbox/LeiReindex.pm
new file mode 100644
index 00000000..3f109f33
--- /dev/null
+++ b/lib/PublicInbox/LeiReindex.pm
@@ -0,0 +1,49 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei reindex" command to reindex everything in lei/store
+package PublicInbox::LeiReindex;
+use v5.12;
+use parent qw(PublicInbox::IPC);
+
+sub reindex_full {
+        my ($lei) = @_;
+        my $sto = $lei->{sto};
+        my $max = $sto->search->over(1)->max;
+        $lei->qerr("# reindexing 1..$max");
+        $sto->wq_do('reindex_art', $_) for (1..$max);
+}
+
+sub reindex_store { # via wq_do
+        my ($self) = @_;
+        my ($lei, $argv) = delete @$self{qw(lei argv)};
+        if (!@$argv) {
+                reindex_full($lei);
+        }
+}
+
+sub lei_reindex {
+        my ($lei, @argv) = @_;
+        my $sto = $lei->_lei_store or return $lei->fail('nothing indexed');
+        $sto->write_prepare($lei);
+        my $self = bless { lei => $lei, argv => \@argv }, __PACKAGE__;
+        my ($op_c, $ops) = $lei->workers_start($self, 1);
+        $lei->{wq1} = $self;
+        $lei->wait_wq_events($op_c, $ops);
+        $self->wq_do('reindex_store');
+        $self->wq_close;
+}
+
+sub _lei_wq_eof { # EOF callback for main lei daemon
+        my ($lei) = @_;
+        $lei->{sto}->wq_do('reindex_done');
+        $lei->wq_eof;
+}
+
+sub ipc_atfork_child {
+        my ($self) = @_;
+        $self->{lei}->_lei_atfork_child;
+        $self->SUPER::ipc_atfork_child;
+}
+
+1;
diff --git a/lib/PublicInbox/LeiRemote.pm b/lib/PublicInbox/LeiRemote.pm
index 54750062..559fb8d5 100644
--- a/lib/PublicInbox/LeiRemote.pm
+++ b/lib/PublicInbox/LeiRemote.pm
@@ -12,7 +12,6 @@ use IO::Uncompress::Gunzip;
 use PublicInbox::MboxReader;
 use PublicInbox::Spawn qw(popen_rd);
 use PublicInbox::LeiCurl;
-use PublicInbox::AutoReap;
 use PublicInbox::ContentHash qw(git_sha);
 
 sub new {
@@ -22,7 +21,7 @@ sub new {
 
 sub isrch { $_[0] } # SolverGit expcets this
 
-sub _each_mboxrd_eml { # callback for MboxReader->mboxrd
+sub each_mboxrd_eml { # callback for MboxReader->mboxrd
         my ($eml, $self) = @_;
         my $lei = $self->{lei};
         my $xoids = $lei->{ale}->xoids_for($eml, 1);
@@ -47,14 +46,13 @@ sub mset {
         $uri->query_form(q => $qstr, x => 'm', r => 1); # r=1: relevance
         my $cmd = $curl->for_uri($self->{lei}, $uri);
         $self->{lei}->qerr("# $cmd");
-        my ($fh, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
-        my $ar = PublicInbox::AutoReap->new($pid);
         $self->{smsg} = [];
-        $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
-        PublicInbox::MboxReader->mboxrd($fh, \&_each_mboxrd_eml, $self);
+        my $fh = popen_rd($cmd, undef, { 2 => $lei->{2} });
+        $fh = IO::Uncompress::Gunzip->new($fh, MultiStream=>1, AutoClose=>1);
+        eval { PublicInbox::MboxReader->mboxrd($fh, \&each_mboxrd_eml, $self) };
+        my $err = $@ ? ": $@" : '';
         my $wait = $self->{lei}->{sto}->wq_do('done');
-        $ar->join;
-        $lei->child_error($?) if $?;
+        $lei->child_error($?, "@$cmd failed$err") if $err || $?;
         $self; # we are the mset (and $ibx, and $self)
 }
 
diff --git a/lib/PublicInbox/LeiRmWatch.pm b/lib/PublicInbox/LeiRmWatch.pm
index c0f336f0..19bee3ab 100644
--- a/lib/PublicInbox/LeiRmWatch.pm
+++ b/lib/PublicInbox/LeiRmWatch.pm
@@ -14,7 +14,7 @@ sub lei_rm_watch {
         my $self = bless { missing_ok => 1 }, __PACKAGE__;
         $self->prepare_inputs($lei, \@argv) or return;
         for my $w (@{$self->{inputs}}) {
-                $lei->_config('--remove-section', "watch.$w");
+                $lei->_config('--remove-section', "watch.$w") or return;
         }
         delete $lei->{cfg}; # force reload
         $lei->refresh_watches;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 1d13aef6..9ae9dcdb 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -1,10 +1,9 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # pretends to be like LeiDedupe and also PublicInbox::Inbox
 package PublicInbox::LeiSavedSearch;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::Lock);
 use PublicInbox::Git;
 use PublicInbox::OverIdx;
@@ -13,7 +12,9 @@ use PublicInbox::Config;
 use PublicInbox::Spawn qw(run_die);
 use PublicInbox::ContentHash qw(git_sha);
 use PublicInbox::MID qw(mids_for_index);
-use Digest::SHA qw(sha256_hex);
+use PublicInbox::SHA qw(sha256_hex);
+use File::Temp ();
+use IO::Handle ();
 our $LOCAL_PFX = qr!\A(?:maildir|mh|mbox.+|mmdf|v2):!i; # TODO: put in LeiToMail?
 
 # move this to PublicInbox::Config if other things use it:
@@ -76,20 +77,17 @@ sub list {
         my $lss_dir = $lei->share_path.'/saved-searches';
         return () unless -d $lss_dir;
         # TODO: persist the cache?  Use another format?
-        my $f = $lei->cache_dir."/saved-tmp.$$.".time.'.config';
-        open my $fh, '>', $f or die "open $f: $!";
+        my $fh = File::Temp->new(TEMPLATE => 'lss_list-XXXX', TMPDIR => 1) or
+                die "File::Temp->new: $!";
         print $fh "[include]\n";
         for my $p (glob("$lss_dir/*/lei.saved-search")) {
                 print $fh "\tpath = ", cquote_val($p), "\n";
         }
-        close $fh or die "close $f: $!";
-        my $cfg = $lei->cfg_dump($f);
-        unlink($f);
+        $fh->flush or die "flush: $fh";
+        my $cfg = $lei->cfg_dump($fh->filename);
         my $out = $cfg ? $cfg->get_all('lei.q.output') : [];
-        map {;
-                s!$LOCAL_PFX!!;
-                $_;
-        } @$out
+        s!$LOCAL_PFX!! for @$out;;
+        @$out;
 }
 
 sub translate_dedupe ($$) {
@@ -247,10 +245,10 @@ sub git { $_[0]->{git} //= PublicInbox::Git->new($_[0]->{ale}->git->{git_dir}) }
 
 sub pause_dedupe {
         my ($self) = @_;
-        git($self)->cleanup;
-        my $lockfh = delete $self->{lockfh}; # from lock_for_scope_fast;
-        my $oidx = delete($self->{oidx}) // return;
-        $oidx->commit_lazy;
+        my ($git, $oidx) = delete @$self{qw(git oidx)};
+        $git->cleanup if $git;
+        $oidx->commit_lazy if $oidx;
+        delete $self->{lockfh}; # from lock_for_scope_fast;
 }
 
 sub reset_dedupe {
@@ -299,7 +297,6 @@ no warnings 'once';
 *smsg_by_mid = \&PublicInbox::Inbox::smsg_by_mid;
 *msg_by_mid = \&PublicInbox::Inbox::msg_by_mid;
 *modified = \&PublicInbox::Inbox::modified;
-*recent = \&PublicInbox::Inbox::recent;
 *max_git_epoch = *nntp_usable = *msg_by_path = \&mm; # undef
 *isrch = *search = \&mm; # TODO
 *DESTROY = \&pause_dedupe;
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 936c2751..684668c5 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -9,6 +9,7 @@ use parent qw(PublicInbox::ExtSearch); # PublicInbox::Search->reopen
 use PublicInbox::Search qw(xap_terms);
 use PublicInbox::ContentHash qw(content_digest content_hash git_sha);
 use PublicInbox::MID qw(mids mids_for_index);
+use PublicInbox::Compat qw(uniqstr);
 use Carp qw(croak);
 
 sub _msg_kw { # retry_reopen callback
@@ -44,20 +45,16 @@ sub oidbin_keywords {
 sub _xsmsg_vmd { # retry_reopen
         my ($self, $smsg, $want_label) = @_;
         my $xdb = $self->xdb; # set {nshard};
-        my (%kw, %L, $doc, $x);
-        $kw{flagged} = 1 if delete($smsg->{lei_q_tt_flagged});
+        my (@kw, @L, $doc, $x);
+        @kw = qw(flagged) if delete($smsg->{lei_q_tt_flagged});
         my @num = $self->over->blob_exists($smsg->{blob});
         for my $num (@num) { # there should only be one...
                 $doc = $xdb->get_document($self->num2docid($num));
-                $x = xap_terms('K', $doc);
-                %kw = (%kw, %$x);
-                if ($want_label) { # JSON/JMAP only
-                        $x = xap_terms('L', $doc);
-                        %L = (%L, %$x);
-                }
+                push @kw, xap_terms('K', $doc);
+                push @L, xap_terms('L', $doc) if $want_label # JSON/JMAP only
         }
-        $smsg->{kw} = [ sort keys %kw ] if scalar(keys(%kw));
-        $smsg->{L} = [ sort keys %L ] if scalar(keys(%L));
+        @{$smsg->{kw}} = sort(uniqstr(@kw)) if @kw;
+        @{$smsg->{L}} = uniqstr(@L) if @L;
 }
 
 # lookup keywords+labels for external messages
@@ -106,6 +103,8 @@ sub xoids_for {
                 for my $o (@overs) {
                         my ($id, $prev);
                         while (my $cur = $o->next_by_mid($mid, \$id, \$prev)) {
+                                # {bytes} may be '' from old bug
+                                $cur->{bytes} = 1 if $cur->{bytes} eq '';
                                 next if $cur->{bytes} == 0 ||
                                         $xoids->{$cur->{blob}};
                                 $git->cat_async($cur->{blob}, \&_cmp_1st,
@@ -158,20 +157,6 @@ sub kw_changed {
         join("\0", @$new_kw_sorted) eq $cur_kw ? 0 : 1;
 }
 
-sub all_terms {
-        my ($self, $pfx) = @_;
-        my $xdb = $self->xdb;
-        my $cur = $xdb->allterms_begin($pfx);
-        my $end = $xdb->allterms_end($pfx);
-        my %ret;
-        for (; $cur != $end; $cur++) {
-                my $tn = $cur->get_termname;
-                index($tn, $pfx) == 0 and
-                        $ret{substr($tn, length($pfx))} = undef;
-        }
-        wantarray ? (sort keys %ret) : \%ret;
-}
-
 sub qparse_new {
         my ($self) = @_;
         my $qp = $self->SUPER::qparse_new; # PublicInbox::Search
diff --git a/lib/PublicInbox/LeiSelfSocket.pm b/lib/PublicInbox/LeiSelfSocket.pm
index 860020cb..0e15bc7c 100644
--- a/lib/PublicInbox/LeiSelfSocket.pm
+++ b/lib/PublicInbox/LeiSelfSocket.pm
@@ -5,31 +5,27 @@
 # This receives what script/lei receives, but isn't connected
 # to an interactive terminal so I'm not sure what to do with it...
 package PublicInbox::LeiSelfSocket;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::DS);
 use Data::Dumper;
 $Data::Dumper::Useqq = 1; # should've been the Perl default :P
 use PublicInbox::Syscall qw(EPOLLIN);
-use PublicInbox::Spawn;
-my $recv_cmd;
+use PublicInbox::IPC;
 
 sub new {
         my ($cls, $r) = @_;
-        my $self = bless { sock => $r }, $cls;
+        my $self = bless {}, $cls;
         $r->blocking(0);
-        no warnings 'once';
-        $recv_cmd = $PublicInbox::LEI::recv_cmd;
         $self->SUPER::new($r, EPOLLIN);
 }
 
 sub event_step {
         my ($self) = @_;
-        my (@fds) = $recv_cmd->($self->{sock}, my $buf, 4096 * 33);
+        my ($buf, @fds);
+        @fds = $PublicInbox::IPC::recv_cmd->($self->{sock}, $buf, 4096 * 33);
         if (scalar(@fds) == 1 && !defined($fds[0])) {
                 return if $!{EAGAIN};
                 die "recvmsg: $!" unless $!{ECONNRESET};
-                $buf = '';
         } else { # just in case open so perl can auto-close them:
                 for (@fds) { open my $fh, '+<&=', $_ };
         }
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 66049dfe..a752174d 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Local storage (cache/memo) for lei(1), suitable for personal/private
@@ -27,12 +27,15 @@ use PublicInbox::MDA;
 use PublicInbox::Spawn qw(spawn);
 use PublicInbox::MdirReader;
 use PublicInbox::LeiToMail;
-use File::Temp ();
+use PublicInbox::Compat qw(uniqstr);
+use File::Temp qw(tmpnam);
 use POSIX ();
 use IO::Handle (); # ->autoflush
 use Sys::Syslog qw(syslog openlog);
 use Errno qw(EEXIST ENOENT);
 use PublicInbox::Syscall qw(rename_noreplace);
+use PublicInbox::LeiStoreErr;
+use PublicInbox::DS qw(add_uniq_timer);
 
 sub new {
         my (undef, $dir, $opt) = @_;
@@ -107,10 +110,25 @@ sub search {
         PublicInbox::LeiSearch->new($_[0]->{priv_eidx}->{topdir});
 }
 
+sub cat_blob {
+        my ($self, $oid) = @_;
+        $self->{im} ? $self->{im}->cat_blob($oid) : undef;
+}
+
+sub schedule_commit {
+        my ($self, $sec) = @_;
+        add_uniq_timer($self->{priv_eidx}->{topdir}, $sec, \&done, $self);
+}
+
 # follows the stderr file
 sub _tail_err {
         my ($self) = @_;
-        print { $self->{-err_wr} } readline($self->{-tmp_err});
+        my $err = $self->{-tmp_err} // return;
+        $err->clearerr; # clear EOF marker
+        my @msg = readline($err);
+        PublicInbox::LeiStoreErr::emit($self->{-err_wr}, @msg) and return;
+        # syslog is the last resort if lei-daemon broke
+        syslog('warning', '%s', $_) for @msg;
 }
 
 sub eidx_init {
@@ -255,13 +273,13 @@ sub remove_eml_vmd { # remove just the VMD
 
 sub _lms_rw ($) { # it is important to have eidx processes open before lms
         my ($self) = @_;
-        my ($eidx, $tl) = eidx_init($self);
-        $self->{lms} //= do {
+        $self->{lms} // do {
                 require PublicInbox::LeiMailSync;
+                my ($eidx, $tl) = eidx_init($self);
                 my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3";
                 my $lms = PublicInbox::LeiMailSync->new($f);
                 $lms->lms_write_prepare;
-                $lms;
+                $self->{lms} = $lms;
         };
 }
 
@@ -324,15 +342,55 @@ sub _add_vmd ($$$$) {
 sub _docids_and_maybe_kw ($$) {
         my ($self, $docids) = @_;
         return $docids unless wantarray;
-        my $kw = {};
+        my (@kw, $idx, @tmp);
         for my $num (@$docids) { # likely only 1, unless ContentHash changes
                 # can't use ->search->msg_keywords on uncommitted docs
-                my $idx = $self->{priv_eidx}->idx_shard($num);
-                my $tmp = eval { $idx->ipc_do('get_terms', 'K', $num) };
-                if ($@) { warn "#$num get_terms: $@" }
-                else { @$kw{keys %$tmp} = values(%$tmp) };
+                $idx = $self->{priv_eidx}->idx_shard($num);
+                @tmp = eval { $idx->ipc_do('get_terms', 'K', $num) };
+                $@ ? warn("#$num get_terms: $@") : push(@kw, @tmp);
         }
-        ($docids, [ sort keys %$kw ]);
+        @kw = sort(uniqstr(@kw)) if @$docids > 1;
+        ($docids, \@kw);
+}
+
+sub _reindex_1 { # git->cat_async callback
+        my ($bref, $hex, $type, $size, $smsg) = @_;
+        my $self = delete $smsg->{-sto};
+        my ($eidx, $tl) = eidx_init($self);
+        $bref //= _lms_rw($self)->local_blob($hex, 1);
+        if ($bref) {
+                my $eml = PublicInbox::Eml->new($bref);
+                $smsg->{-merge_vmd} = 1; # preserve existing keywords
+                $eidx->idx_shard($smsg->{num})->index_eml($eml, $smsg);
+        } elsif ($type eq 'missing') {
+                # pre-release/buggy lei may've indexed external-only msgs,
+                # try to correct that, here
+                warn("E: missing $hex, culling (ancient lei artifact?)\n");
+                $smsg->{to} = $smsg->{cc} = $smsg->{from} = '';
+                $smsg->{bytes} = 0;
+                $eidx->{oidx}->update_blob($smsg, '');
+                my $eml = PublicInbox::Eml->new("\r\n\r\n");
+                $eidx->idx_shard($smsg->{num})->index_eml($eml, $smsg);
+        } else {
+                warn("E: $type $hex\n");
+        }
+}
+
+sub reindex_art {
+        my ($self, $art) = @_;
+        my ($eidx, $tl) = eidx_init($self);
+        my $smsg = $eidx->{oidx}->get_art($art) // return;
+        return if $smsg->{bytes} == 0; # external-only message
+        $smsg->{-sto} = $self;
+        $eidx->git->cat_async($smsg->{blob} // die("no blob (#$art)"),
+                                \&_reindex_1, $smsg);
+}
+
+sub reindex_done {
+        my ($self) = @_;
+        my ($eidx, $tl) = eidx_init($self);
+        $eidx->git->async_wait_all;
+        # ->done to be called via sto_done_request
 }
 
 sub add_eml {
@@ -347,8 +405,14 @@ sub add_eml {
                 _lms_rw($self)->set_src($smsg->oidbin, @{$vmd->{sync_info}});
         }
         unless ($im_mark) { # duplicate blob returns undef
-                return unless wantarray;
+                return unless wantarray || $vmd;
                 my @docids = $oidx->blob_exists($smsg->{blob});
+                if ($vmd) {
+                        for my $docid (@docids) {
+                                my $idx = $eidx->idx_shard($docid);
+                                _add_vmd($self, $idx, $docid, $vmd);
+                        }
+                }
                 return _docids_and_maybe_kw $self, \@docids;
         }
 
@@ -520,30 +584,30 @@ sub xchg_stderr {
         _tail_err($self) if $self->{-err_wr};
         my $dir = $self->{priv_eidx}->{topdir};
         return unless -e $dir;
-        my $old = delete $self->{-tmp_err};
-        my $pfx = POSIX::strftime('%Y%m%d%H%M%S', gmtime(time));
-        my $err = File::Temp->new(TEMPLATE => "$pfx.$$.err-XXXX",
-                                SUFFIX => '.err', DIR => $dir);
-        open STDERR, '>>', $err->filename or die "dup2: $!";
+        delete $self->{-tmp_err};
+        my ($err, $name) = tmpnam();
+        open STDERR, '>>', $name or die "dup2: $!";
+        unlink($name);
         STDERR->autoflush(1); # shared with shard subprocesses
         $self->{-tmp_err} = $err; # separate file description for RO access
         undef;
 }
 
 sub done {
-        my ($self, $sock_ref) = @_;
-        my $err = '';
+        my ($self) = @_;
+        my ($errfh, $lei_sock) = @$self{0, 1}; # via sto_done_request
+        my @err;
         if (my $im = delete($self->{im})) {
                 eval { $im->done };
-                if ($@) {
-                        $err .= "import done: $@\n";
-                        warn $err;
-                }
+                push(@err, "E: import done: $@\n") if $@;
         }
         delete $self->{lms};
-        $self->{priv_eidx}->done; # V2Writable::done
+        eval { $self->{priv_eidx}->done }; # V2Writable::done
+        push(@err, "E: priv_eidx done: $@\n") if $@;
+        print { $errfh // *STDERR{GLOB} } @err;
+        send($lei_sock, 'child_error 256', 0) if @err && $lei_sock;
         xchg_stderr($self);
-        die $err if $err;
+        die @err if @err;
 }
 
 sub ipc_atfork_child {
@@ -564,16 +628,15 @@ sub recv_and_run {
         $self->SUPER::recv_and_run(@args);
 }
 
-sub _sto_atexit { # dwaitpid callback
-        my ($args, $pid) = @_;
-        my $self = $args->[0];
+sub _sto_atexit { # awaitpid cb
+        my ($pid) = @_;
         warn "lei/store PID:$pid died \$?=$?\n" if $?;
 }
 
 sub write_prepare {
         my ($self, $lei) = @_;
         $lei // die 'BUG: $lei not passed';
-        unless ($self->{-ipc_req}) {
+        unless ($self->{-wq_s1}) {
                 my $dir = $lei->store_path;
                 substr($dir, -length('/lei/store'), 10, '');
                 pipe(my ($r, $w)) or die "pipe: $!";
@@ -581,13 +644,12 @@ sub write_prepare {
                 # Mail we import into lei are private, so headers filtered out
                 # by -mda for public mail are not appropriate
                 local @PublicInbox::MDA::BAD_HEADERS = ();
+                local $SIG{ALRM} = 'IGNORE';
                 $self->wq_workers_start("lei/store $dir", 1, $lei->oldset, {
                                         lei => $lei,
                                         -err_wr => $w,
                                         to_close => [ $r ],
-                                });
-                $self->wq_wait_async(\&_sto_atexit); # outlives $lei
-                require PublicInbox::LeiStoreErr;
+                                }, \&_sto_atexit);
                 PublicInbox::LeiStoreErr->new($r, $lei);
         }
         $lei->{sto} = $self;
diff --git a/lib/PublicInbox/LeiStoreErr.pm b/lib/PublicInbox/LeiStoreErr.pm
index cc085fdc..c8bc72b6 100644
--- a/lib/PublicInbox/LeiStoreErr.pm
+++ b/lib/PublicInbox/LeiStoreErr.pm
@@ -1,38 +1,60 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # forwards stderr from lei/store process to any lei clients using
 # the same store, falls back to syslog if no matching clients exist.
 package PublicInbox::LeiStoreErr;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+use PublicInbox::Syscall qw(EPOLLIN);
 use Sys::Syslog qw(openlog syslog closelog);
 use IO::Handle (); # ->blocking
+use Time::HiRes ();
+use autodie qw(open);
+our $err_wr;
+
+# We don't want blocked stderr on clients to block lei/store or lei-daemon.
+# We can't make stderr non-blocking since it can break MUAs or anything
+# lei might spawn.  So we setup a timer to wake us up after a second if
+# printing to a user's stderr hasn't completed, yet.  Unfortunately,
+# EINTR alone isn't enough since Perl auto-restarts writes on signals,
+# so to interrupt writes to clients with blocked stderr, we dup the
+# error output to $err_wr ahead-of-time and close $err_wr in the
+# SIGALRM handler to ensure `print' gets aborted:
+
+sub abort_err_wr { close($err_wr) if $err_wr; undef $err_wr }
+
+sub emit ($@) {
+        my ($efh, @msg) = @_;
+        open(local $err_wr, '>&', $efh); # fdopen(dup(fileno($efh)), "w")
+        local $SIG{ALRM} = \&abort_err_wr;
+        Time::HiRes::alarm(1.0, 0.1);
+        my $ret = print $err_wr @msg;
+        Time::HiRes::alarm(0);
+        $ret;
+}
 
 sub new {
         my ($cls, $rd, $lei) = @_;
         my $self = bless { sock => $rd, store_path => $lei->store_path }, $cls;
         $rd->blocking(0);
-        $self->SUPER::new($rd, EPOLLIN | EPOLLONESHOT);
+        $self->SUPER::new($rd, EPOLLIN); # level-trigger
 }
 
 sub event_step {
         my ($self) = @_;
-        my $rbuf = $self->{rbuf} // \(my $x = '');
-        $self->do_read($rbuf, 8192, length($$rbuf)) or return;
-        my $cb;
+        my $n = sysread($self->{sock}, my $buf, 8192);
+        return ($!{EAGAIN} ? 0 : $self->close) if !defined($n);
+        return $self->close if !$n;
         my $printed;
-        for my $lei (values %PublicInbox::DS::DescriptorMap) {
-                $cb = $lei->can('store_path') // next;
+        for my $lei (grep defined, @PublicInbox::DS::FD_MAP) {
+                my $cb = $lei->can('store_path') // next;
                 next if $cb->($lei) ne $self->{store_path};
-                my $err = $lei->{2} // next;
-                print $err $$rbuf and $printed = 1;
+                emit($lei->{2} // next, $buf) and $printed = 1;
         }
         if (!$printed) {
                 openlog('lei/store', 'pid,nowait,nofatal,ndelay', 'user');
-                for my $l (split(/\n/, $$rbuf)) { syslog('warning', '%s', $l) }
+                for my $l (split(/\n/, $buf)) { syslog('warning', '%s', $l) }
                 closelog(); # don't share across fork
         }
 }
diff --git a/lib/PublicInbox/LeiSucks.pm b/lib/PublicInbox/LeiSucks.pm
index 8e866fc9..ddb3faf7 100644
--- a/lib/PublicInbox/LeiSucks.pm
+++ b/lib/PublicInbox/LeiSucks.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Undocumented hidden command somebody might discover if they're
@@ -7,11 +7,12 @@
 package PublicInbox::LeiSucks;
 use strict;
 use v5.10.1;
-use Digest::SHA ();
+use PublicInbox::SHA qw(sha1_hex);
 use Config;
 use POSIX ();
 use PublicInbox::Config;
 use PublicInbox::IPC;
+use PublicInbox::IO qw(read_all);
 
 sub lei_sucks {
         my ($lei, @argv) = @_;
@@ -54,13 +55,13 @@ sub lei_sucks {
         } else {
                 push @out, "Xapian not available: $@\n";
         }
-        my $dig = Digest::SHA->new(1);
         push @out, "public-inbox blob OIDs of loaded features:\n";
         for my $m (grep(m{^PublicInbox/}, sort keys %INC)) {
                 my $f = $INC{$m} // next; # lazy require failed (missing dep)
-                $dig->add('blob '.(-s $f)."\0");
-                $dig->addfile($f);
-                push @out, '  '.$dig->hexdigest.' '.$m."\n";
+                open my $fh, '<', $f or do { warn "open($f): $!"; next };
+                my $size = -s $fh;
+                my $hex = sha1_hex("blob $size\0".read_all($fh, $size));
+                push @out, '  '.$hex.' '.$m."\n";
         }
         push @out, <<'EOM';
 Let us know how it sucks!  Please include the above and any other
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 8ce96a10..320b0355 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -13,9 +13,9 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
         if (my $xoids = $self->{lse}->xoids_for($eml) // # tries LeiMailSync
                         $self->{lei}->{ale}->xoids_for($eml)) {
                 $self->{lei}->{sto}->wq_do('update_xvmd', $xoids, $eml,
-                                                $self->{vmd_mod});
+                                                $self->{lei}->{vmd_mod});
         } else {
-                ++$self->{unimported};
+                ++$self->{-nr_unimported};
         }
 }
 
@@ -31,11 +31,8 @@ sub lei_tag { # the "lei tag" method
         my $sto = $lei->_lei_store(1)->write_prepare($lei);
         my $self = bless {}, __PACKAGE__;
         $lei->ale; # refresh and prepare
-        my $vmd_mod = $self->vmd_mod_extract(\@argv);
-        return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
-        $self->{vmd_mod} = $vmd_mod; # before LeiPmdir->new in prepare_inputs
         $self->prepare_inputs($lei, \@argv) or return;
-        grep(defined, @$vmd_mod{qw(+kw +L -L -kw)}) or
+        grep(defined, @{$lei->{vmd_mod}}{qw(+kw +L -L -kw)}) or
                 return $lei->fail('no keywords or labels specified');
         $lei->{-err_type} = 'non-fatal';
         $lei->wq1_start($self);
@@ -43,8 +40,8 @@ sub lei_tag { # the "lei tag" method
 
 sub note_unimported {
         my ($self) = @_;
-        my $n = $self->{unimported} or return;
-        $self->{lei}->{pkt_op_p}->pkt_do('incr', 'unimported', $n);
+        my $n = $self->{-nr_unimported} or return;
+        $self->{lei}->{pkt_op_p}->pkt_do('incr', -nr_unimported => $n);
 }
 
 sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 3c5e7e59..dfae29e9 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Writes PublicInbox::Eml objects atomically to a mbox variant or Maildir
@@ -7,12 +7,15 @@ use strict;
 use v5.10.1;
 use parent qw(PublicInbox::IPC);
 use PublicInbox::Eml;
-use PublicInbox::ProcessPipe;
+use PublicInbox::IO;
+use PublicInbox::Git;
 use PublicInbox::Spawn qw(spawn);
-use Symbol qw(gensym);
+use PublicInbox::Import;
 use IO::Handle; # ->autoflush
 use Fcntl qw(SEEK_SET SEEK_END O_CREAT O_EXCL O_WRONLY);
 use PublicInbox::Syscall qw(rename_noreplace);
+use autodie qw(open seek close);
+use Carp qw(croak);
 
 my %kw2char = ( # Maildir characters
         draft => 'D',
@@ -54,8 +57,7 @@ sub _mbox_hdr_buf ($$$) {
         }
         my $buf = delete $eml->{hdr};
 
-        # fixup old bug from import (pre-a0c07cba0e5d8b6a)
-        $$buf =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+        PublicInbox::Eml::strip_from($$buf);
         my $ident = $smsg->{blob} // 'lei';
         if (defined(my $pct = $smsg->{pct})) { $ident .= "=$pct" }
 
@@ -132,40 +134,41 @@ sub eml2mboxcl2 {
 }
 
 sub git_to_mail { # git->cat_async callback
-        my ($bref, $oid, $type, $size, $arg) = @_;
-        $type // return; # called by git->async_abort
-        my ($write_cb, $smsg) = @$arg;
-        if ($type eq 'missing' && $smsg->{-lms_rw}) {
-                if ($bref = $smsg->{-lms_rw}->local_blob($oid, 1)) {
+        my ($bref, $oid, $type, $size, $smsg) = @_;
+        $type // return; # called by PublicInbox::Git::close
+        return if $PublicInbox::Git::in_cleanup;
+        my $self = delete $smsg->{l2m} // croak "BUG: no l2m (type=$type)";
+        $self->{lei} // croak "BUG: no {lei} (type=$type)";
+        eval {
+                if ($type eq 'missing' &&
+                          ($bref = $self->{-lms_rw}->local_blob($oid, 1))) {
                         $type = 'blob';
                         $size = length($$bref);
                 }
-        }
-        return warn("W: $oid is $type (!= blob)\n") if $type ne 'blob';
-        return warn("E: $oid is empty\n") unless $size;
-        die "BUG: expected=$smsg->{blob} got=$oid" if $smsg->{blob} ne $oid;
-        $write_cb->($bref, $smsg);
+                $type eq 'blob' or return $self->{lei}->child_error(0,
+                                                "W: $oid is $type (!= blob)");
+                $size or return $self->{lei}->child_error(0,"E: $oid is empty");
+                $smsg->{blob} eq $oid or die "BUG: expected=$smsg->{blob}";
+                $smsg->{bytes} ||= $size;
+                $self->{wcb}->($bref, $smsg);
+        };
+        $self->{lei}->fail("$@ (oid=$oid)") if $@;
 }
 
-sub reap_compress { # dwaitpid callback
-        my ($lei, $pid) = @_;
-        my $cmd = delete $lei->{"pid.$pid"};
-        return if $? == 0;
-        $lei->fail("@$cmd failed", $? >> 8);
+sub reap_compress { # awaitpid callback
+        my ($pid, $lei, $cmd, $old_out) = @_;
+        $lei->{1} = $old_out;
+        $lei->fail($?, "@$cmd failed") if $?;
 }
 
-sub _post_augment_mbox { # open a compressor process from top-level process
+sub _post_augment_mbox { # open a compressor process from top-level lei-daemon
         my ($self, $lei) = @_;
         my $zsfx = $self->{zsfx} or return;
         my $cmd = PublicInbox::MboxReader::zsfx2cmd($zsfx, undef, $lei);
         my ($r, $w) = @{delete $lei->{zpipe}};
         my $rdr = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2}, pgid => 0 };
-        my $pid = spawn($cmd, undef, $rdr);
-        my $pp = gensym;
-        my $dup = bless { "pid.$pid" => $cmd }, ref($lei);
-        $dup->{$_} = $lei->{$_} for qw(2 sock);
-        tie *$pp, 'PublicInbox::ProcessPipe', $pid, $w, \&reap_compress, $dup;
-        $lei->{1} = $pp;
+        $lei->{1} = PublicInbox::IO::attach_pid($w, spawn($cmd, undef, $rdr),
+                                \&reap_compress, $lei, $cmd, $lei->{1});
 }
 
 # --augment existing output destination, with deduplication
@@ -197,6 +200,7 @@ sub _mbox_write_cb ($$) {
         sub { # for git_to_mail
                 my ($buf, $smsg, $eml) = @_;
                 $eml //= PublicInbox::Eml->new($buf);
+                ++$self->{-nr_seen};
                 return if $dedupe->is_dup($eml, $smsg);
                 $lse->xsmsg_vmd($smsg) if $lse;
                 $smsg->{-recent} = 1 if $set_recent;
@@ -207,7 +211,7 @@ sub _mbox_write_cb ($$) {
                         my $lk = $ovv->lock_for_scope;
                         $lei->out($$buf);
                 }
-                ++$lei->{-nr_write};
+                ++$self->{-nr_write};
         }
 }
 
@@ -258,7 +262,7 @@ sub _buf2maildir ($$$$) {
                 $tmp = $dst.'tmp/'.$rand.$common;
         } while (!($ok = sysopen($fh, $tmp, O_CREAT|O_EXCL|O_WRONLY)) &&
                 $!{EEXIST} && ($rand = _rand.','));
-        if ($ok && print $fh $$buf and close($fh)) {
+        if ($ok && print $fh $$buf and $fh->close) {
                 $dst .= $dir; # 'new/' or 'cur/'
                 $rand = '';
                 do {
@@ -291,6 +295,8 @@ sub _maildir_write_cb ($$) {
         sub { # for git_to_mail
                 my ($bref, $smsg, $eml) = @_;
                 $dst // return $lei->fail; # dst may be undef-ed in last run
+
+                ++$self->{-nr_seen};
                 return if $dedupe && $dedupe->is_dup($eml //
                                                 PublicInbox::Eml->new($$bref),
                                                 $smsg);
@@ -298,7 +304,7 @@ sub _maildir_write_cb ($$) {
                 my $n = _buf2maildir($dst, $bref // \($eml->as_string),
                                         $smsg, $dir);
                 $lms->set_src($smsg->oidbin, $out, $n) if $lms;
-                ++$lei->{-nr_write};
+                ++$self->{-nr_write};
         }
 }
 
@@ -307,8 +313,11 @@ sub _imap_write_cb ($$) {
         my $dedupe = $lei->{dedupe};
         $dedupe->prepare_dedupe if $dedupe;
         my $append = $lei->{net}->can('imap_append');
-        my $uri = $self->{uri};
-        my $mic = $lei->{net}->mic_get($uri);
+        my $uri = $self->{uri} // die 'BUG: no {uri}';
+        my $mic = $lei->{net}->mic_get($uri) // die <<EOM;
+E: $uri connection failed.
+E: Consider using `--jobs ,1' to limit IMAP connections
+EOM
         my $folder = $uri->mailbox;
         $uri->uidvalidity($mic->uidvalidity($folder));
         my $lse = $lei->{lse}; # may be undef
@@ -317,6 +326,8 @@ sub _imap_write_cb ($$) {
         sub { # for git_to_mail
                 my ($bref, $smsg, $eml) = @_;
                 $mic // return $lei->fail; # mic may be undef-ed in last run
+
+                ++$self->{-nr_seen};
                 return if $dedupe && $dedupe->is_dup($eml //
                                                 PublicInbox::Eml->new($$bref),
                                                 $smsg);
@@ -329,7 +340,7 @@ sub _imap_write_cb ($$) {
                 # imap_append returns UID if IMAP server has UIDPLUS extension
                 ($lms && $uid =~ /\A[0-9]+\z/) and
                         $lms->set_src($smsg->oidbin, $$uri, $uid + 0);
-                ++$lei->{-nr_write};
+                ++$self->{-nr_write};
         }
 }
 
@@ -357,12 +368,14 @@ sub _v2_write_cb ($$) {
         my ($self, $lei) = @_;
         my $dedupe = $lei->{dedupe};
         $dedupe->prepare_dedupe if $dedupe;
+        # only call in worker
+        $PublicInbox::Import::DROP_UNIQUE_UNSUB = $lei->{-drop_unique_unsub};
         sub { # for git_to_mail
                 my ($bref, $smsg, $eml) = @_;
                 $eml //= PublicInbox::Eml->new($bref);
+                ++$self->{-nr_seen};
                 return if $dedupe && $dedupe->is_dup($eml, $smsg);
-                $lei->{v2w}->wq_do('add', $eml); # V2Writable->add
-                ++$lei->{-nr_write};
+                $lei->{v2w}->add($eml) and ++$self->{-nr_write};
         }
 }
 
@@ -386,9 +399,16 @@ sub new {
                                 "$dst exists and is not a directory\n";
                 $lei->{ovv}->{dst} = $dst .= '/' if substr($dst, -1) ne '/';
                 $lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
+        } elsif ($fmt eq 'mh') {
+                -e $dst && !-d _ and die
+                                "$dst exists and is not a directory\n";
+                $lei->{ovv}->{dst} = $dst .= '/' if substr($dst, -1) ne '/';
+                $lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
         } elsif (substr($fmt, 0, 4) eq 'mbox') {
                 require PublicInbox::MboxReader;
-                $self->can("eml2$fmt") or die "bad mbox format: $fmt\n";
+                $self->can("eml2$fmt") or die <<EOM;
+E: bad mbox format: $fmt (did you mean: mboxrd, mboxo, mboxcl, or mboxcl2?)
+EOM
                 $self->{base_type} = 'mbox';
                 if ($lei->{cmd} eq 'q' &&
                                 (($lei->path_to_fd($dst) // -1) < 0) &&
@@ -422,7 +442,7 @@ sub new {
                         ($lei->{opt}->{dedupe}//'') eq 'oid';
                 $self->{base_type} = 'v2';
                 $self->{-wq_nr_workers} = 1; # v2 has shards
-                $lei->{opt}->{save} = \1;
+                $lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
                 $dst = $lei->{ovv}->{dst} = $lei->abs_path($dst);
                 @conflict = qw(mua sort);
         } else {
@@ -432,6 +452,8 @@ sub new {
                 (-d $dst || (-e _ && !-w _)) and die
                         "$dst exists and is not a writable file\n";
         }
+        $lei->{input_opt} and # lei_convert sets this
+                @conflict = grep { !$lei->{input_opt}->{$_} } @conflict;
         my @err = map { defined($lei->{opt}->{$_}) ? "--$_" : () } @conflict;
         die "@err incompatible with $fmt\n" if @err;
         $self->{dst} = $dst;
@@ -450,15 +472,10 @@ sub new {
 sub _pre_augment_maildir {
         my ($self, $lei) = @_;
         my $dst = $lei->{ovv}->{dst};
-        for my $x (qw(tmp new cur)) {
-                my $d = $dst.$x;
-                next if -d $d;
-                require File::Path;
-                File::Path::mkpath($d);
-                -d $d or die "$d is not a directory";
-        }
+        require File::Path;
+        File::Path::make_path(map { $dst.$_ } qw(tmp new cur));
         # for utime, so no opendir
-        open $self->{poke_dh}, '<', "${dst}cur" or die "open ${dst}cur: $!";
+        open $self->{poke_dh}, '<', "${dst}cur";
 }
 
 sub clobber_dst_prepare ($;$) {
@@ -538,11 +555,11 @@ sub _pre_augment_text {
                 $out = $lei->{$devfd};
         } else { # normal-looking path
                 if (-p $dst) {
-                        open $out, '>', $dst or die "open($dst): $!";
+                        open $out, '>', $dst;
                 } elsif (-f _ || !-e _) {
                         # text allows augment, HTML/Atom won't
                         my $mode = $lei->{opt}->{augment} ? '>>' : '>';
-                        open $out, $mode, $dst or die "open($mode, $dst): $!";
+                        open $out, $mode, $dst;
                 } else {
                         die "$dst is not a file or FIFO\n";
                 }
@@ -561,7 +578,7 @@ sub _pre_augment_mbox {
                 $out = $lei->{$devfd};
         } else { # normal-looking path
                 if (-p $dst) {
-                        open $out, '>', $dst or die "open($dst): $!";
+                        open $out, '>', $dst;
                 } elsif (-f _ || !-e _) {
                         require PublicInbox::MboxLock;
                         my $m = $lei->{opt}->{'lock'} //
@@ -574,7 +591,7 @@ sub _pre_augment_mbox {
                 $lei->{old_1} = $lei->{1}; # keep for spawning MUA
         }
         # Perl does SEEK_END even with O_APPEND :<
-        $self->{seekable} = seek($out, 0, SEEK_SET);
+        $self->{seekable} = $out->seek(0, SEEK_SET);
         if (!$self->{seekable} && !$!{ESPIPE} && !defined($devfd)) {
                 die "seek($dst): $!\n";
         }
@@ -600,6 +617,17 @@ sub _pre_augment_mbox {
         undef;
 }
 
+sub finish_output {
+        my ($self, $lei) = @_;
+        my $out = delete $lei->{1} // die 'BUG: no lei->{1}';
+        my $old = delete $lei->{old_1} or return; # path only
+        $lei->{1} = $old;
+        return if $out->close; # reaps gzip|pigz|xz|bzip2
+        my $msg = "E: Error closing $lei->{ovv}->{dst}";
+        $? ? $lei->child_error($?) : ($msg .= " ($!)");
+        die $msg;
+}
+
 sub _do_augment_mbox {
         my ($self, $lei) = @_;
         return unless $self->{seekable};
@@ -616,7 +644,7 @@ sub _do_augment_mbox {
         if (my $zsfx = $self->{zsfx}) {
                 $rd = PublicInbox::MboxReader::zsfxcat($out, $zsfx, $lei);
         } else {
-                open($rd, '+>>&', $out) or die "dup: $!";
+                open($rd, '+>>&', $out);
         }
         my $dedupe;
         if ($opt->{augment}) {
@@ -636,16 +664,10 @@ sub _do_augment_mbox {
                 PublicInbox::MboxReader->$fmt($rd, \&_augment, $lei);
         }
         # maybe some systems don't honor O_APPEND, Perl does this:
-        seek($out, 0, SEEK_END) or die "seek $dst: $!";
+        seek($out, 0, SEEK_END);
         $dedupe->pause_dedupe if $dedupe;
 }
 
-sub v2w_done_wait { # dwaitpid callback
-        my ($arg, $pid) = @_;
-        my ($v2w, $lei) = @$arg;
-        $lei->child_error($?, "error for $v2w->{ibx}->{inboxdir}") if $?;
-}
-
 sub _pre_augment_v2 {
         my ($self, $lei) = @_;
         my $dir = $self->{dst};
@@ -666,19 +688,21 @@ sub _pre_augment_v2 {
                 });
         }
         PublicInbox::InboxWritable->new($ibx, @creat);
+        local $PublicInbox::Import::DROP_UNIQUE_UNSUB; # only for workers
+        PublicInbox::Import::load_config(PublicInbox::Config->new, sub {
+                $lei->x_it(shift);
+                die "E: can't write v2 inbox with broken config\n";
+        });
+        $lei->{-drop_unique_unsub} = $PublicInbox::Import::DROP_UNIQUE_UNSUB;
         $ibx->init_inbox if @creat;
-        my $v2w = $ibx->importer;
-        $v2w->wq_workers_start("lei/v2w $dir", 1, $lei->oldset, {lei => $lei});
-        $v2w->wq_wait_async(\&v2w_done_wait, $lei);
-        $lei->{v2w} = $v2w;
+        $lei->{v2w} = $ibx->importer;
         return if !$lei->{opt}->{shared};
         my $d = "$lei->{ale}->{git}->{git_dir}/objects";
-        my $al = "$dir/git/0.git/objects/info/alternates";
-        open my $fh, '+>>', $al or die "open($al): $!";
-        seek($fh, 0, SEEK_SET) or die "seek($al): $!";
-        grep(/\A\Q$d\E\n/, <$fh>) and return;
-        print $fh "$d\n" or die "print($al): $!";
-        close $fh or die "close($al): $!";
+        open my $fh, '+>>', my $f = "$dir/git/0.git/objects/info/alternates";
+        seek($fh, 0, SEEK_SET); # Perl did SEEK_END when it saw '>>'
+        my $seen = grep /\A\Q$d\E\n/, PublicInbox::IO::read_all $fh;
+        print $fh "$d\n" if !$seen;
+        close $fh;
 }
 
 sub pre_augment { # fast (1 disk seek), runs in same process as post_augment
@@ -743,7 +767,8 @@ sub do_post_auth {
                 $au_peers->[1] = undef;
                 sysread($au_peers->[0], my $barrier1, 1);
         }
-        $self->{wcb} = $self->write_cb($lei);
+        eval { $self->{wcb} = $self->write_cb($lei) };
+        $lei->fail($@) if $@;
         if ($au_peers) { # wait for peer l2m to set write_cb
                 $au_peers->[3] = undef;
                 sysread($au_peers->[2], my $barrier2, 1);
@@ -780,21 +805,29 @@ sub poke_dst {
 
 sub write_mail { # via ->wq_io_do
         my ($self, $smsg, $eml) = @_;
-        return $self->{wcb}->(undef, $smsg, $eml) if $eml;
-        $smsg->{-lms_rw} = $self->{-lms_rw};
-        $self->{git}->cat_async($smsg->{blob}, \&git_to_mail,
-                                [$self->{wcb}, $smsg]);
+        if ($eml) {
+                eval { $self->{wcb}->(undef, $smsg, $eml) };
+                $self->{lei}->fail("blob=$smsg->{blob} $@") if $@;
+        } else {
+                $smsg->{l2m} = $self;
+                $self->{git}->cat_async($smsg->{blob}, \&git_to_mail, $smsg);
+        }
 }
 
 sub wq_atexit_child {
         my ($self) = @_;
         local $PublicInbox::DS::in_loop = 0; # waitpid synchronously
         my $lei = $self->{lei};
-        delete $self->{wcb};
         $lei->{ale}->git->async_wait_all;
-        my $nr = delete($lei->{-nr_write}) or return;
+        my ($nr_w, $nr_s) = delete(@$self{qw(-nr_write -nr_seen)});
+        if (my $v2w = delete $lei->{v2w}) {
+                eval { $v2w->done };
+                $lei->child_error($?, "E: $@ ($v2w->{ibx}->{inboxdir})") if $@;
+        }
+        delete $self->{wcb};
+        (($nr_w //= 0) + ($nr_s //= 0)) or return;
         return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
-        $lei->{pkt_op_p}->pkt_do('l2m_progress', $nr);
+        $lei->{pkt_op_p}->pkt_do('incr', -nr_write => $nr_w, -nr_seen => $nr_s)
 }
 
 # runs on a 1s timer in lei-daemon
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index b8a98360..9931f017 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -1,16 +1,16 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # "lei up" - updates the result of "lei q --save"
 package PublicInbox::LeiUp;
-use strict;
-use v5.10.1;
+use v5.12;
 # n.b. we use LeiInput to setup IMAP auth
 use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
-use PublicInbox::LeiSavedSearch;
+use PublicInbox::LeiSavedSearch; # OverIdx
 use PublicInbox::DS;
 use PublicInbox::PktOp;
 use PublicInbox::LeiFinmsg;
+use PublicInbox::LEI;
 my $REMOTE_RE = qr!\A(?:imap|http)s?://!i; # http(s) will be for JMAP
 
 sub up1 ($$) {
@@ -32,8 +32,10 @@ sub up1 ($$) {
         my $rawstr = $lss->{-cfg}->{'lei.internal.rawstr'} //
                 (scalar(@$q) == 1 && substr($q->[0], -1) eq "\n");
         if ($rawstr) {
-                scalar(@$q) > 1 and
-                        die "$f: lei.q has multiple values (@$q) (out=$out)\n";
+                die <<EOM if scalar(@$q) > 1;
+$f: lei.q has multiple values (@$q) (out=$out)
+$f: while lei.internal.rawstr is set
+EOM
                 $lse->query_approxidate($lse->git, $mset_opt->{qstr} = $q->[0]);
         } else {
                 $mset_opt->{qstr} = $lse->query_argv_to_string($lse->git, $q);
@@ -75,6 +77,7 @@ sub redispatch_all ($$) {
         my $upq = [ (@{$self->{o_local} // []}, @{$self->{o_remote} // []}) ];
         return up1($lei, $upq->[0]) if @$upq == 1; # just one, may start MUA
 
+        PublicInbox::OverIdx::fork_ok($lei->{opt});
         # FIXME: this is also used per-query, see lei->_start_query
         my $j = $lei->{opt}->{jobs} || do {
                 my $n = $self->detect_nproc // 1;
@@ -89,7 +92,6 @@ sub redispatch_all ($$) {
         $op_c->{ops} = { '' => [ $lei->can('dclose'), $lei ] };
         my @first_batch = splice(@$upq, 0, $j); # initial parallelism
         $lei->{-upq} = $upq;
-        $lei->{daemon_pid} = $$;
         $lei->event_step_init; # wait for client disconnects
         for my $out (@first_batch) {
                 PublicInbox::DS::requeue(
@@ -162,9 +164,8 @@ sub _complete_up { # lei__complete hook
         map { $match_cb->($_) } PublicInbox::LeiSavedSearch::list($lei);
 }
 
-sub _wq_done_wait { # dwaitpid callback
-        my ($arg, $pid) = @_;
-        my ($wq, $lei) = @$arg;
+sub _wq_done_wait { # awaitpid cb
+        my ($pid, $wq, $lei) = @_;
         $lei->child_error($?, 'auth failure') if $?
 }
 
@@ -172,8 +173,7 @@ no warnings 'once';
 *ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
 
 package PublicInbox::LeiUp1; # for redispatch_all
-use strict;
-use v5.10.1;
+use v5.12;
 
 sub nxt ($$$) {
         my ($lei, $out, $op_p) = @_;
@@ -210,8 +210,8 @@ sub event_step { # runs via PublicInbox::DS::requeue
 
 sub DESTROY {
         my ($self) = @_;
+        return if ($PublicInbox::LEI::daemon_pid // -1) != $$;
         my $lei = $self->{lei}; # the original, from lei_up
-        return if $lei->{daemon_pid} != $$;
         my $sock = delete $self->{unref_on_destroy};
         my $s = $lei->{-socks} // [];
         @$s = grep { $_ != $sock } @$s;
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index 53555467..c7d72c71 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -72,12 +72,11 @@ sub new {
         my $self = bless { %{$lei->{opt}}, -colored => \&uncolored }, $cls;
         $self->{-quote_reply} = 1 if $fmt eq 'reply';
         return $self unless $self->{color} //= -t $lei->{1};
-        my $cmd = [ qw(git config -z --includes -l) ];
-        my ($r, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
+        my @cmd = qw(git config -z --includes -l); # reuse normal git config
+        my $r = popen_rd(\@cmd, undef, { 2 => $lei->{2} });
         my $cfg = PublicInbox::Config::config_fh_parse($r, "\0", "\n");
-        waitpid($pid, 0);
-        if ($?) {
-                warn "# git-config failed, no color (non-fatal)\n";
+        if (!$r->close) {
+                warn "# @cmd failed, no color (non-fatal \$?=$?)\n";
                 return $self;
         }
         $self->{-colored} = \&my_colored;
diff --git a/lib/PublicInbox/LeiWatch.pm b/lib/PublicInbox/LeiWatch.pm
index 35267b58..b30e5152 100644
--- a/lib/PublicInbox/LeiWatch.pm
+++ b/lib/PublicInbox/LeiWatch.pm
@@ -1,13 +1,12 @@
 # Copyright all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# represents a Maildir or IMAP "watch" item
+# represents a Maildir, MH or IMAP "watch" item
 package PublicInbox::LeiWatch;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::IPC);
 
-# "url" may be something like "maildir:/path/to/dir"
+# "url" may be something like "maildir:/path/to/dir" or "mh:/path/to/dir"
 sub new { bless { url => $_[1] }, $_[0] }
 
 1;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 2958d3f9..fc95d401 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Combine any combination of PublicInbox::Search,
@@ -12,15 +12,16 @@ use PublicInbox::DS qw(now);
 use File::Temp 0.19 (); # 0.19 for ->newdir
 use File::Spec ();
 use PublicInbox::Search qw(xap_terms);
-use PublicInbox::Spawn qw(popen_rd spawn which);
+use PublicInbox::Spawn qw(popen_rd popen_wr which);
 use PublicInbox::MID qw(mids);
 use PublicInbox::Smsg;
-use PublicInbox::AutoReap;
 use PublicInbox::Eml;
 use PublicInbox::LEI;
 use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
 use PublicInbox::ContentHash qw(git_sha);
 use POSIX qw(strftime);
+use autodie qw(close open read seek truncate);
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
 
 sub new {
         my ($class) = @_;
@@ -103,13 +104,6 @@ sub smsg_for {
         $smsg;
 }
 
-sub recent {
-        my ($self, $qstr, $opt) = @_;
-        $opt //= {};
-        $opt->{relevance} //= -2;
-        $self->mset($qstr //= 'z:1..', $opt);
-}
-
 sub over {}
 
 sub _check_mset_limit ($$$) {
@@ -128,26 +122,16 @@ sub _mset_more ($$) {
         $size >= $mo->{limit} && (($mo->{offset} += $size) < $mo->{total});
 }
 
-# $startq will EOF when do_augment is done augmenting and allow
+# $startq will see `q' in do_post_augment -> start_mua if spawning MUA.
+# Otherwise $startq will EOF when do_augment is done augmenting and allow
 # query_combined_mset and query_thread_mset to proceed.
 sub wait_startq ($) {
         my ($lei) = @_;
-        my $startq = delete $lei->{startq} or return;
-        while (1) {
-                my $n = sysread($startq, my $do_augment_done, 1);
-                if (defined $n) {
-                        return if $n == 0; # no MUA
-                        if ($do_augment_done eq 'q') {
-                                $lei->{opt}->{quiet} = 1;
-                                delete $lei->{opt}->{verbose};
-                                delete $lei->{-progress};
-                        } else {
-                                die "BUG: do_augment_done=`$do_augment_done'";
-                        }
-                        return;
-                }
-                die "wait_startq: $!" unless $!{EINTR};
-        }
+        read(delete($lei->{startq}) // return, my $buf, 1) or return; # EOF
+        die "BUG: wrote `$buf' to au_done" if $buf ne 'q';
+        $lei->{opt}->{quiet} = 1;
+        delete $lei->{opt}->{verbose};
+        delete $lei->{-progress};
 }
 
 sub mset_progress {
@@ -162,70 +146,60 @@ sub mset_progress {
         }
 }
 
-sub l2m_progress {
-        my ($lei, $nr) = @_;
-        $lei->{-nr_write} += $nr;
-}
-
 sub query_one_mset { # for --threads and l2m w/o sort
         my ($self, $ibxish) = @_;
-        local $0 = "$0 query_one_mset";
         my $lei = $self->{lei};
         my ($srch, $over) = ($ibxish->search, $ibxish->over);
         my $dir = $ibxish->{inboxdir} // $ibxish->{topdir};
         return warn("$dir not indexed by Xapian\n") unless ($srch && $over);
         bless $srch, 'PublicInbox::LeiSearch'; # for ->qparse_new
         my $mo = { %{$lei->{mset_opt}} }; # copy
+        local $0 = "$0 1 $mo->{qstr}";
         my $mset;
         my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
         my $can_kw = !!$ibxish->can('msg_keywords');
         my $threads = $lei->{opt}->{threads} // 0;
         my $fl = $threads > 1 ? 1 : undef;
         my $lss = $lei->{lss};
-        my $maxk = "external.$dir.maxuid";
-        my $stop_at = $lss ? $lss->{-cfg}->{$maxk} : undef;
-        if (defined $stop_at) {
-                ref($stop_at) and
-                        return warn("$maxk=$stop_at has multiple values\n");
-                ($stop_at =~ /[^0-9]/) and
-                        return warn("$maxk=$stop_at not numeric\n");
-        }
+        my $maxk = "external.$dir.maxuid"; # max of previous, so our min
+        my $min = $lss ? ($lss->{-cfg}->{$maxk} // 0) : 0;
+        ref($min) and return warn("$maxk=$min has multiple values\n");
+        ($min =~ /[^0-9]/) and return warn("$maxk=$min not numeric\n");
         my $first_ids;
         do {
-                $mset = $srch->mset($mo->{qstr}, $mo);
+                $mset = eval { $srch->mset($mo->{qstr}, $mo) };
+                return $lei->child_error(22 << 8, "E: $@") if $@; # 22 from curl
                 mset_progress($lei, $dir, $mo->{offset} + $mset->size,
                                 $mset->get_matches_estimated);
                 wait_startq($lei); # wait for keyword updates
                 my $ids = $srch->mset_to_artnums($mset, $mo);
-                @$ids = grep { $_ > $stop_at } @$ids if defined($stop_at);
                 my $i = 0;
                 if ($threads) {
                         # copy $ids if $lss since over->expand_thread
                         # shifts @{$ctx->{ids}}
                         $first_ids = [ @$ids ] if $lss;
-                        my $ctx = { ids => $ids };
-                        my %n2item = map { ($ids->[$i++], $_) } $mset->items;
-                        while ($over->expand_thread($ctx)) {
-                                for my $n (@{$ctx->{xids}}) {
+                        my $ctx = { ids => $ids, min => $min };
+                        my %n2item = map { $ids->[$i++] => $_ } $mset->items;
+                        while ($over->expand_thread($ctx)) { # fills {xids}
+                                for my $n (@{delete $ctx->{xids}}) {
                                         my $smsg = $over->get_art($n) or next;
-                                        my $mitem = delete $n2item{$n};
+                                        my $mi = delete $n2item{$n};
                                         next if $smsg->{bytes} == 0;
-                                        if ($mitem && $can_kw) {
-                                                mitem_kw($srch, $smsg, $mitem,
-                                                        $fl);
-                                        } elsif ($mitem && $fl) {
+                                        if ($mi && $can_kw) {
+                                                mitem_kw($srch, $smsg, $mi, $fl)
+                                        } elsif ($mi && $fl) {
                                                 # call ->xsmsg_vmd, later
                                                 $smsg->{lei_q_tt_flagged} = 1;
                                         }
-                                        $each_smsg->($smsg, $mitem);
+                                        $each_smsg->($smsg, $mi);
                                 }
-                                @{$ctx->{xids}} = ();
                         }
                 } else {
                         $first_ids = $ids;
-                        my @items = $mset->items;
+                        my @items = $mset->items; # parallel with @$ids
                         for my $n (@$ids) {
                                 my $mitem = $items[$i++];
+                                next if $n <= $min;
                                 my $smsg = $over->get_art($n) or next;
                                 next if $smsg->{bytes} == 0;
                                 mitem_kw($srch, $smsg, $mitem, $fl) if $can_kw;
@@ -235,7 +209,6 @@ sub query_one_mset { # for --threads and l2m w/o sort
         } while (_mset_more($mset, $mo));
         _check_mset_limit($lei, $dir, $mset);
         if ($lss && scalar(@$first_ids)) {
-                undef $stop_at;
                 my $max = $first_ids->[0];
                 $lss->cfg_set($maxk, $max);
                 undef $lss;
@@ -246,16 +219,17 @@ sub query_one_mset { # for --threads and l2m w/o sort
 
 sub query_combined_mset { # non-parallel for non-"--threads" users
         my ($self) = @_;
-        local $0 = "$0 query_combined_mset";
         my $lei = $self->{lei};
         my $mo = { %{$lei->{mset_opt}} };
+        local $0 = "$0 C $mo->{qstr}";
         my $mset;
         for my $loc (locals($self)) {
                 attach_external($self, $loc);
         }
         my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
         do {
-                $mset = $self->mset($mo->{qstr}, $mo);
+                $mset = eval { $self->mset($mo->{qstr}, $mo) };
+                return $lei->child_error(22 << 8, "E: $@") if $@; # 22 from curl
                 mset_progress($lei, 'xsearch', $mo->{offset} + $mset->size,
                                 $mset->get_matches_estimated);
                 wait_startq($lei); # wait for keyword updates
@@ -285,7 +259,7 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
                 my ($res, $kw) = $self->{import_sto}->wq_do('add_eml', $eml);
                 if (ref($res) eq ref($smsg)) { # totally new message
                         $smsg = $res;
-                        $self->{-imported} = 1;
+                        $self->{-sto_imported} = 1;
                 }
                 $smsg->{kw} = $kw; # short-circuit xsmsg_vmd
         }
@@ -324,8 +298,9 @@ sub fudge_qstr_time ($$$) {
                 $rft = $diff;
         }
         $lr -= ($rft || (48 * 60 * 60));
+        require PublicInbox::Admin;
         $lei->qerr("# $uri limiting to ".
-                strftime('%Y-%m-%d %k:%M %z', localtime($lr)). ' and newer');
+                PublicInbox::Admin::fmt_localtime($lr).' and newer');
         # this should really be rt: (received-time), but no stable
         # public-inbox releases support it, yet.
         my $dt = 'dt:'.strftime('%Y%m%d%H%M%S', gmtime($lr)).'..';
@@ -338,26 +313,27 @@ sub fudge_qstr_time ($$$) {
 
 sub query_remote_mboxrd {
         my ($self, $uris) = @_;
-        local $0 = "$0 query_remote_mboxrd";
         local $SIG{TERM} = sub { exit(0) }; # for DESTROY (File::Temp, $reap)
         my $lei = $self->{lei};
         my $opt = $lei->{opt};
-        chomp(my $qstr = $lei->{mset_opt}->{qstr});
-        $qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly
+        my $qstr = $lei->{mset_opt}->{qstr};
+        local $0 = "$0 R $qstr";
         my @qform = (x => 'm');
         push(@qform, t => 1) if $opt->{threads};
-        my $verbose = $opt->{verbose};
-        my $reap_tail;
-        my $cerr = File::Temp->new(TEMPLATE => 'curl.err-XXXX', TMPDIR => 1);
-        fcntl($cerr, F_SETFL, O_APPEND|O_RDWR) or warn "set O_APPEND: $!";
+        open my $cerr, '+>', undef;
         my $rdr = { 2 => $cerr };
-        if ($verbose) {
-                # spawn a process to force line-buffering, otherwise curl
+        my @lbf_tee;
+        if ($opt->{verbose}) {
+                # spawn a line-buffered tee(1) script, otherwise curl
                 # will write 1 character at-a-time and parallel outputs
                 # mmmaaayyy llloookkk llliiikkkeee ttthhhiiisss
-                my $o = { 1 => $lei->{2}, 2 => $lei->{2} };
-                my $pid = spawn(['tail', '-f', $cerr->filename], undef, $o);
-                $reap_tail = PublicInbox::AutoReap->new($pid);
+                # (n.b. POSIX tee(1) cannot do any buffering)
+                my $o = { 1 => $cerr, 2 => $lei->{2} };
+                delete $rdr->{2};
+                @lbf_tee = ([ $^X, qw(-w -p -e), <<'' ], undef, $o);
+BEGIN { $| = 1; use IO::Handle; STDERR->autoflush(1); }
+print STDERR $_;
+
         }
         my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return;
         push @$curl, '-s', '-d', '';
@@ -365,40 +341,35 @@ sub query_remote_mboxrd {
         $self->{import_sto} = $lei->{sto} if $lei->{opt}->{'import-remote'};
         for my $uri (@$uris) {
                 $lei->{-current_url} = $uri->as_string;
-                $lei->{-nr_remote_eml} = 0;
                 my $start = time;
                 my ($q, $key) = fudge_qstr_time($lei, $uri, $qstr);
                 $uri->query_form(@qform, q => $q);
                 my $cmd = $curl->for_uri($lei, $uri);
                 $lei->qerr("# $cmd");
-                my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
-                my $reap_curl = PublicInbox::AutoReap->new($pid);
-                $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
-                PublicInbox::MboxReader->mboxrd($fh, \&each_remote_eml, $self,
-                                                $lei, $each_smsg);
-                if ($self->{import_sto} && delete($self->{-imported})) {
-                        my $wait = $self->{import_sto}->wq_do('done');
-                }
-                $reap_curl->join;
-                if ($? == 0) {
-                        # don't update if no results, maybe MTA is down
-                        my $nr = $lei->{-nr_remote_eml};
+                $rdr->{2} //= popen_wr(@lbf_tee) if @lbf_tee;
+                my $fh = popen_rd($cmd, undef, $rdr);
+                $fh = IO::Uncompress::Gunzip->new($fh,
+                                        MultiStream => 1, AutoClose => 1);
+                eval {
+                        PublicInbox::MboxReader->mboxrd($fh, \&each_remote_eml,
+                                                $self, $lei, $each_smsg);
+                };
+                my ($exc, $code) = ($@, $?);
+                $lei->sto_done_request if delete($self->{-sto_imported});
+                die "E: $exc" if $exc && !$code;
+                my $nr = delete $lei->{-nr_remote_eml} // 0;
+                if (!$code) { # don't update if no results, maybe MTA is down
                         $lei->{lss}->cfg_set($key, $start) if $key && $nr;
                         mset_progress($lei, $lei->{-current_url}, $nr, $nr);
                         next;
                 }
-                my $err;
-                if (-s $cerr) {
-                        seek($cerr, 0, SEEK_SET) //
-                                        warn "seek($cmd stderr): $!";
-                        $err = do { local $/; <$cerr> } //
-                                        warn "read($cmd stderr): $!";
-                        truncate($cerr, 0) // warn "truncate($cmd stderr): $!";
-                }
-                $err //= '';
-                next if (($? >> 8) == 22 && $err =~ /\b404\b/);
+                delete($rdr->{2})->close if @lbf_tee;
+                seek($cerr, 0, SEEK_SET);
+                read($cerr, my $err, -s $cerr);
+                truncate($cerr, 0);
+                next if (($code >> 8) == 22 && $err =~ /\b404\b/);
                 $uri->query_form(q => $qstr);
-                $lei->child_error($?, "E: <$uri> $err");
+                $lei->child_error($code, "E: <$uri> `$cmd` failed");
         }
         undef $each_smsg;
         $lei->{ovv}->ovv_atexit_child($lei);
@@ -406,9 +377,8 @@ sub query_remote_mboxrd {
 
 sub git { $_[0]->{git} // die 'BUG: git uninitialized' }
 
-sub xsearch_done_wait { # dwaitpid callback
-        my ($arg, $pid) = @_;
-        my ($wq, $lei) = @$arg;
+sub xsearch_done_wait { # awaitpid cb
+        my ($pid, $wq, $lei) = @_;
         return if !$?;
         my $s = $? & 127;
         return $lei->child_error($?) if $s == 13 || $s == 15;
@@ -417,72 +387,56 @@ sub xsearch_done_wait { # dwaitpid callback
 
 sub query_done { # EOF callback for main daemon
         my ($lei) = @_;
-        local $PublicInbox::LEI::current_lei = $lei;
-        eval {
-                my $l2m = delete $lei->{l2m};
-                delete $lei->{lxs};
-                ($lei->{opt}->{'mail-sync'} && !$lei->{sto}) and
-                        warn "BUG: {sto} missing with --mail-sync";
-                $lei->sto_done_request if $lei->{sto};
-                if (my $v2w = delete $lei->{v2w}) {
-                        my $wait = $v2w->wq_do('done'); # may die
-                        $v2w->wq_close;
+        my $l2m = delete $lei->{l2m};
+        delete $lei->{lxs};
+        ($lei->{opt}->{'mail-sync'} && !$lei->{sto}) and
+                warn "BUG: {sto} missing with --mail-sync";
+        $lei->sto_done_request;
+        $lei->{ovv}->ovv_end($lei);
+        if ($l2m) { # close() calls LeiToMail reap_compress
+                $l2m->finish_output($lei);
+                if ($l2m->lock_free) {
+                        $l2m->poke_dst;
+                        $lei->poke_mua;
+                } else { # mbox users
+                        delete $l2m->{mbl}; # drop dotlock
                 }
-                $lei->{ovv}->ovv_end($lei);
-                if ($l2m) { # close() calls LeiToMail reap_compress
-                        if (my $out = delete $lei->{old_1}) {
-                                if (my $mbout = $lei->{1}) {
-                                        close($mbout) or die <<"";
-Error closing $lei->{ovv}->{dst}: \$!=$! \$?=$?
-
-                                }
-                                $lei->{1} = $out;
-                        }
-                        if ($l2m->lock_free) {
-                                $l2m->poke_dst;
-                                $lei->poke_mua;
-                        } else { # mbox users
-                                delete $l2m->{mbl}; # drop dotlock
-                        }
-                }
-                if ($lei->{-progress}) {
-                        my $tot = $lei->{-mset_total} // 0;
-                        my $nr = $lei->{-nr_write} // 0;
-                        if ($l2m) {
-                                my $m = "# $nr written to " .
-                                        "$lei->{ovv}->{dst} ($tot matches)";
-                                $nr ? $lei->qfin($m) : $lei->qerr($m);
-                        } else {
-                                $lei->qerr("# $tot matches");
-                        }
+        }
+        my $nr_w = delete($lei->{-nr_write}) // 0;
+        my $nr_dup = (delete($lei->{-nr_seen}) // 0) - $nr_w;
+        if ($lei->{-progress}) {
+                my $tot = $lei->{-mset_total} // 0;
+                my $x = "$tot matches";
+                $x .= ", $nr_dup duplicates" if $nr_dup;
+                if ($l2m) {
+                        my $m = "# $nr_w written to " .
+                                "$lei->{ovv}->{dst} ($x)";
+                        $nr_w ? $lei->qfin($m) : $lei->qerr($m);
+                } else {
+                        $lei->qerr("# $x");
                 }
-                $lei->start_mua if $l2m && !$l2m->lock_free;
-                $lei->dclose;
-        };
-        $lei->fail($@) if $@;
+        }
+        $lei->start_mua if $l2m && !$l2m->lock_free;
+        $lei->dclose;
 }
 
 sub do_post_augment {
         my ($lei) = @_;
-        local $PublicInbox::LEI::current_lei = $lei;
         my $l2m = $lei->{l2m} or return; # client disconnected
-        eval {
-                $lei->fchdir;
-                $l2m->post_augment($lei);
-        };
+        eval { $l2m->post_augment($lei) };
         my $err = $@;
         if ($err) {
                 if (my $lxs = delete $lei->{lxs}) {
-                        $lxs->wq_kill('-TERM');
+                        $lxs->wq_kill(-POSIX::SIGTERM());
                         $lxs->wq_close;
                 }
                 $lei->fail("$err");
         }
         if (!$err && delete $lei->{early_mua}) { # non-augment case
-                eval { $lei->start_mua };
+                eval { $lei->start_mua }; # may trigger wait_startq
                 $lei->fail($@) if $@;
         }
-        close(delete $lei->{au_done}); # triggers wait_startq in lei_xsearch
+        close(delete $lei->{au_done}); # trigger wait_startq if start_mua didn't
 }
 
 sub incr_post_augment { # called whenever an l2m shard finishes augment
@@ -527,7 +481,7 @@ sub start_query ($$) { # always runs in main (lei-daemon) process
 }
 
 sub incr_start_query { # called whenever an l2m shard starts do_post_auth
-        my ($self, $lei) = @_;
+        my ($lei, $self) = @_;
         my $l2m = $lei->{l2m};
         return if ++$self->{nr_start_query} != $l2m->{-wq_nr_workers};
         start_query($self, $lei);
@@ -542,17 +496,20 @@ sub ipc_atfork_child {
 sub do_query {
         my ($self, $lei) = @_;
         my $l2m = $lei->{l2m};
+        my $qstr = \($lei->{mset_opt}->{qstr});
+        chomp $$qstr;
+        $$qstr =~ s/[ \n\t]+/ /sg; # make URLs and $0 less ugly
         my $ops = {
-                'sigpipe_handler' => [ $lei ],
-                'fail_handler' => [ $lei ],
-                'do_post_augment' => [ \&do_post_augment, $lei ],
-                'incr_post_augment' => [ \&incr_post_augment, $lei ],
+                sigpipe_handler => [ $lei ],
+                fail_handler => [ $lei ],
+                do_post_augment => [ \&do_post_augment, $lei ],
+                incr_post_augment => [ \&incr_post_augment, $lei ],
                 '' => [ \&query_done, $lei ],
-                'mset_progress' => [ \&mset_progress, $lei ],
-                'l2m_progress' => [ \&l2m_progress, $lei ],
-                'x_it' => [ $lei ],
-                'child_error' => [ $lei ],
-                'incr_start_query' => [ $self, $lei ],
+                mset_progress => [ \&mset_progress, $lei ],
+                incr => [ $lei ],
+                x_it => [ $lei ],
+                child_error => [ $lei ],
+                incr_start_query => [ \&incr_start_query, $lei, $self ],
         };
         $lei->{auth}->op_merge($ops, $l2m, $lei) if $l2m && $lei->{auth};
         my $end = $lei->pkt_op_pair;
@@ -565,7 +522,6 @@ sub do_query {
                 if ($lei->{opt}->{augment} && delete $lei->{early_mua}) {
                         $lei->start_mua;
                 }
-                my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
                 if ($l2m->{-wq_nr_workers} > 1 &&
                                 $l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
                         # setup two barriers to coordinate ->has_entries
@@ -577,15 +533,16 @@ sub do_query {
                         $l2m->{au_peers} = [ $a_r, $a_w, $b_r, $b_w ];
                 }
                 $l2m->wq_workers_start('lei2mail', undef,
-                                        $lei->oldset, { lei => $lei });
-                $l2m->wq_wait_async(\&xsearch_done_wait, $lei);
+                                        $lei->oldset, { lei => $lei },
+                                        \&xsearch_done_wait, $lei);
                 pipe($lei->{startq}, $lei->{au_done}) or die "pipe: $!";
                 fcntl($lei->{startq}, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
                 delete $l2m->{au_peers};
+                close(delete $l2m->{-wq_s2}); # share wq_s1 with lei_xsearch
         }
         $self->wq_workers_start('lei_xsearch', undef,
-                                $lei->oldset, { lei => $lei });
-        $self->wq_wait_async(\&xsearch_done_wait, $lei);
+                                $lei->oldset, { lei => $lei },
+                                \&xsearch_done_wait, $lei);
         my $op_c = delete $lei->{pkt_op_c};
         delete $lei->{pkt_op_p};
         @$end = ();
@@ -608,34 +565,40 @@ sub add_uri {
                 require IO::Uncompress::Gunzip;
                 require PublicInbox::LeiCurl;
                 push @{$self->{remotes}}, $uri;
+                $uri;
         } else {
                 warn "curl missing, ignoring $uri\n";
+                undef;
         }
 }
 
+# returns URI or PublicInbox::Inbox-like object
 sub prepare_external {
         my ($self, $loc, $boost) = @_; # n.b. already ordered by boost
         if (ref $loc) { # already a URI, or PublicInbox::Inbox-like object
                 return add_uri($self, $loc) if $loc->can('scheme');
+                # fall-through on Inbox-like objects
         } elsif ($loc =~ m!\Ahttps?://!) {
                 require URI;
                 return add_uri($self, URI->new($loc));
-        } elsif (-f "$loc/ei.lock") {
+        } elsif (-f "$loc/ei.lock" && -d "$loc/ALL.git/objects") {
                 require PublicInbox::ExtSearch;
                 die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
                 $loc = PublicInbox::ExtSearch->new($loc);
-        } elsif (-f "$loc/inbox.lock" || -d "$loc/public-inbox") {
+        } elsif ((-f "$loc/inbox.lock" && -d "$loc/all.git/objects") ||
+                        (-d "$loc/public-inbox" && -d "$loc/objects")) {
                 die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
                 require PublicInbox::Inbox; # v2, v1
                 $loc = bless { inboxdir => $loc }, 'PublicInbox::Inbox';
         } elsif (!-e $loc) {
                 warn "W: $loc gone, perhaps run: lei forget-external $loc\n";
-                return;
+                return undef;
         } else {
                 warn "W: $loc ignored, unable to determine external type\n";
-                return;
+                return undef;
         }
         push @{$self->{locals}}, $loc;
+        $loc;
 }
 
 sub _lcat_i { # LeiMailSync->each_src iterator callback
diff --git a/lib/PublicInbox/Limiter.pm b/lib/PublicInbox/Limiter.pm
new file mode 100644
index 00000000..a8d08fc3
--- /dev/null
+++ b/lib/PublicInbox/Limiter.pm
@@ -0,0 +1,50 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+package PublicInbox::Limiter;
+use v5.12;
+use PublicInbox::Spawn;
+
+sub new {
+        my ($class, $max) = @_;
+        bless {
+                # 32 is same as the git-daemon connection limit
+                max => $max || 32,
+                running => 0,
+                run_queue => [],
+                # RLIMIT_CPU => undef,
+                # RLIMIT_DATA => undef,
+                # RLIMIT_CORE => undef,
+        }, $class;
+}
+
+sub setup_rlimit {
+        my ($self, $name, $cfg) = @_;
+        for my $rlim (@PublicInbox::Spawn::RLIMITS) {
+                my $k = lc($rlim);
+                $k =~ tr/_//d;
+                $k = "publicinboxlimiter.$name.$k";
+                my $v = $cfg->{$k} // next;
+                my @rlimit = split(/\s*,\s*/, $v);
+                if (scalar(@rlimit) == 1) {
+                        push @rlimit, $rlimit[0];
+                } elsif (scalar(@rlimit) != 2) {
+                        warn "could not parse $k: $v\n";
+                }
+                my $inf = $v =~ /\binfinity\b/i ?
+                        $PublicInbox::Spawn::RLIMITS{RLIM_INFINITY} // eval {
+                                require BSD::Resource;
+                                BSD::Resource::RLIM_INFINITY();
+                        } // do {
+                                warn "BSD::Resource missing for $rlim";
+                                next;
+                        } : undef;
+                for my $i (0..$#rlimit) {
+                        next if $rlimit[$i] ne 'INFINITY';
+                        $rlimit[$i] = $inf;
+                }
+                $self->{$rlim} = \@rlimit;
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/Linkify.pm b/lib/PublicInbox/Linkify.pm
index 2ac74e2a..306a57e7 100644
--- a/lib/PublicInbox/Linkify.pm
+++ b/lib/PublicInbox/Linkify.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # two-step linkification.
@@ -11,8 +11,8 @@
 # Maybe this could be done more efficiently...
 package PublicInbox::Linkify;
 use strict;
-use warnings;
-use Digest::SHA qw/sha1_hex/;
+use v5.10.1;
+use PublicInbox::SHA qw(sha1_hex);
 use PublicInbox::Hval qw(ascii_html mid_href);
 use PublicInbox::MID qw($MID_EXTRACT);
 
@@ -68,23 +68,22 @@ sub linkify_1 {
                 # salt this, as this could be exploited to show
                 # links in the HTML which don't show up in the raw mail.
                 my $key = sha1_hex($url . $SALT);
-
+                $key =~ tr/0-9/A-J/; # no digits for YAML highlight
                 $_[0]->{$key} = $url;
-                $beg . 'PI-LINK-'. $key . $end;
+                $beg . 'LINKIFY' . $key . $end;
         ^geo;
         $_[1];
 }
 
 sub linkify_2 {
-        # Added "PI-LINK-" prefix to avoid false-positives on git commits
-        $_[1] =~ s!\bPI-LINK-([a-f0-9]{40})\b!
+        # Added "LINKIFY" prefix to avoid false-positives on git commits
+        $_[1] =~ s!\bLINKIFY([a-fA-J]{40})\b!
                 my $key = $1;
                 my $url = $_[0]->{$key};
                 if (defined $url) {
                         "<a\nhref=\"$url\">$url</a>";
-                } else {
-                        # false positive or somebody tried to mess with us
-                        $key;
+                } else { # false positive or somebody tried to mess with us
+                        'LINKIFY'.$key;
                 }
         !ge;
         $_[1];
@@ -102,20 +101,20 @@ sub linkify_mids {
                 # salt this, as this could be exploited to show
                 # links in the HTML which don't show up in the raw mail.
                 my $key = sha1_hex($html . $SALT);
+                $key =~ tr/0-9/A-J/;
                 my $repl = qq(&lt;<a\nhref="$pfx/$href/">$html</a>&gt;);
                 $repl .= qq{ (<a\nhref="$pfx/$href/raw">raw</a>)} if $raw;
                 $self->{$key} = $repl;
-                'PI-LINK-'. $key;
+                'LINKIFY'.$key;
                 !ge;
         $$str = ascii_html($$str);
-        $$str =~ s!\bPI-LINK-([a-f0-9]{40})\b!
+        $$str =~ s!\bLINKIFY([a-fA-J]{40})\b!
                 my $key = $1;
                 my $repl = $_[0]->{$key};
                 if (defined $repl) {
                         $repl;
-                } else {
-                        # false positive or somebody tried to mess with us
-                        $key;
+                } else { # false positive or somebody tried to mess with us
+                        'LINKIFY'.$key;
                 }
         !ge;
 }
diff --git a/lib/PublicInbox/Listener.pm b/lib/PublicInbox/Listener.pm
index 7cedc349..c83901b2 100644
--- a/lib/PublicInbox/Listener.pm
+++ b/lib/PublicInbox/Listener.pm
@@ -1,14 +1,15 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Used by -nntpd for listen sockets
 package PublicInbox::Listener;
-use strict;
+use v5.12;
 use parent 'PublicInbox::DS';
 use Socket qw(SOL_SOCKET SO_KEEPALIVE IPPROTO_TCP TCP_NODELAY);
 use IO::Handle;
 use PublicInbox::Syscall qw(EPOLLIN EPOLLEXCLUSIVE);
 use Errno qw(EAGAIN ECONNABORTED);
+our $MULTI_ACCEPT = 0;
 
 # Warn on transient errors, mostly resource limitations.
 # EINTR would indicate the failure to set NonBlocking in systemd or similar
@@ -16,37 +17,35 @@ my %ERR_WARN = map {;
         eval("Errno::$_()") => $_
 } qw(EMFILE ENFILE ENOBUFS ENOMEM EINTR);
 
-sub new ($$$) {
-        my ($class, $s, $cb) = @_;
+sub new {
+        my ($class, $s, $cb, $multi_accept) = @_;
         setsockopt($s, SOL_SOCKET, SO_KEEPALIVE, 1);
         setsockopt($s, IPPROTO_TCP, TCP_NODELAY, 1); # ignore errors on non-TCP
         listen($s, 2**31 - 1); # kernel will clamp
         my $self = bless { post_accept => $cb }, $class;
+        $self->{multi_accept} = $multi_accept //= $MULTI_ACCEPT;
         $self->SUPER::new($s, EPOLLIN|EPOLLEXCLUSIVE);
 }
 
 sub event_step {
         my ($self) = @_;
         my $sock = $self->{sock} or return;
-
-        # no loop here, we want to fairly distribute clients
-        # between multiple processes sharing the same socket
-        # XXX our event loop needs better granularity for
-        # a single accept() here to be, umm..., acceptable
-        # on high-traffic sites.
-        if (my $addr = accept(my $c, $sock)) {
-                IO::Handle::blocking($c, 0); # no accept4 :<
-                eval { $self->{post_accept}->($c, $addr, $sock) };
-                warn "E: $@\n" if $@;
-        } elsif ($! == EAGAIN || $! == ECONNABORTED) {
-                # EAGAIN is common and likely
-                # ECONNABORTED is common with bad connections
-                return;
-        } elsif (my $sym = $ERR_WARN{int($!)}) {
-                warn "W: accept(): $! ($sym)\n";
-        } else {
-                warn "BUG?: accept(): $!\n";
-        }
+        my $n = $self->{multi_accept};
+        do {
+                if (my $addr = accept(my $c, $sock)) {
+                        IO::Handle::blocking($c, 0); # no accept4 :<
+                        eval { $self->{post_accept}->($c, $addr, $sock) };
+                        warn "E: $@\n" if $@;
+                } elsif ($! == EAGAIN || $! == ECONNABORTED) {
+                        # EAGAIN is common and likely
+                        # ECONNABORTED is common with bad connections
+                        return;
+                } elsif (my $sym = $ERR_WARN{int($!)}) {
+                        return warn "W: accept(): $! ($sym)\n";
+                } else {
+                        return warn "BUG?: accept(): $!\n";
+                }
+        } while ($n--);
 }
 
 1;
diff --git a/lib/PublicInbox/Lock.pm b/lib/PublicInbox/Lock.pm
index 0ee2a8bd..ddaf3312 100644
--- a/lib/PublicInbox/Lock.pm
+++ b/lib/PublicInbox/Lock.pm
@@ -1,36 +1,42 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# Base class for per-inbox locking
+# Base class for per-inbox locking, subclassed by several
+# only uses {lock_path} and {lockfh} fields
 package PublicInbox::Lock;
-use strict;
-use v5.10.1;
-use Fcntl qw(:flock :DEFAULT);
+use v5.12;
+use Fcntl qw(LOCK_UN LOCK_EX O_RDWR O_CREAT);
 use Carp qw(croak);
 use PublicInbox::OnDestroy;
+use Errno qw(EINTR);
+use autodie qw(close sysopen syswrite);
+
+sub xflock ($$) {
+        until (flock($_[0], $_[1])) { return if $! != EINTR }
+        1;
+}
+
+sub new { bless { lock_path => $_[1] }, $_[0] }
 
 # we only acquire the flock if creating or reindexing;
 # PublicInbox::Import already has the lock on its own.
 sub lock_acquire {
         my ($self) = @_;
-        my $lock_path = $self->{lock_path};
-        croak 'already locked '.($lock_path // '(undef)') if $self->{lockfh};
-        return unless defined($lock_path);
-        sysopen(my $lockfh, $lock_path, O_RDWR|O_CREAT) or
-                croak "failed to open $lock_path: $!\n";
-        flock($lockfh, LOCK_EX) or croak "lock $lock_path failed: $!\n";
-        $self->{lockfh} = $lockfh;
+        my $fn = $self->{lock_path};
+        croak 'already locked '.($fn // '(undef)') if $self->{lockfh};
+        $fn // return;
+        sysopen(my $fh, $fn, O_RDWR|O_CREAT);
+        xflock($fh, LOCK_EX) or croak "LOCK_EX $fn: $!";
+        $self->{lockfh} = $fh;
 }
 
 sub lock_release {
         my ($self, $wake) = @_;
-        defined(my $lock_path = $self->{lock_path}) or return;
-        my $lockfh = delete $self->{lockfh} or croak "not locked: $lock_path";
-
-        syswrite($lockfh, '.') if $wake;
-
-        flock($lockfh, LOCK_UN) or croak "unlock $lock_path failed: $!\n";
-        close $lockfh or croak "close $lock_path failed: $!\n";
+        my $fn = $self->{lock_path} // return;
+        my $fh = delete $self->{lockfh} or croak "not locked: $fn";
+        syswrite($fh, '.') if $wake;
+        xflock($fh, LOCK_UN) or croak "LOCK_UN $fn: $!";
+        close $fh; # may detect errors
 }
 
 # caller must use return value
@@ -41,13 +47,13 @@ sub lock_for_scope {
 }
 
 sub lock_acquire_fast {
-        $_[0]->{lockfh} or return lock_acquire($_[0]);
-        flock($_[0]->{lockfh}, LOCK_EX) or croak "lock (fast) failed: $!";
+        my $fh = $_[0]->{lockfh} or return lock_acquire($_[0]);
+        xflock($fh, LOCK_EX) or croak "LOCK_EX $_[0]->{lock_path}: $!";
 }
 
 sub lock_release_fast {
-        flock($_[0]->{lockfh} // return, LOCK_UN) or
-                        croak "unlock (fast) $_[0]->{lock_path}: $!";
+        xflock($_[0]->{lockfh} // return, LOCK_UN) or
+                croak "LOCK_UN $_[0]->{lock_path}: $!"
 }
 
 # caller must use return value
diff --git a/lib/PublicInbox/MHreader.pm b/lib/PublicInbox/MHreader.pm
new file mode 100644
index 00000000..3e7bbd5c
--- /dev/null
+++ b/lib/PublicInbox/MHreader.pm
@@ -0,0 +1,104 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# MH reader, based on Lib/mailbox.py in cpython source
+package PublicInbox::MHreader;
+use v5.12;
+use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::OnDestroy;
+use PublicInbox::IO qw(try_cat);
+use PublicInbox::MdirSort;
+use Carp qw(carp);
+use autodie qw(chdir closedir opendir);
+
+my %FL2OFF = ( # mh_sequences key => our keyword
+        replied => 0,
+        flagged => 1,
+        unseen => 2, # negate
+);
+my @OFF2KW = qw(answered flagged); # [2] => unseen (negated)
+
+sub new {
+        my ($cls, $dir, $cwdfh) = @_;
+        if (substr($dir, -1) ne '/') { # TODO: do this earlier
+                carp "W: appending `/' to `$dir' (fix caller)\n";
+                $dir .= '/';
+        }
+        bless { dir => $dir, cwdfh => $cwdfh }, $cls;
+}
+
+sub read_mh_sequences ($) { # caller must chdir($self->{dir})
+        my ($self) = @_;
+        my ($fl, $off, @n);
+        my @seq = ('', '', '');
+        for (split /\n+/s, try_cat('.mh_sequences')) {
+                ($fl, @n) = split /[: \t]+/;
+                $off = $FL2OFF{$fl} // do { warn <<EOM;
+W: unknown `$fl' in $self->{dir}.mh_sequences (ignoring)
+EOM
+                        next;
+                };
+                @n = grep /\A[0-9]+\z/s, @n; # don't stat, yet
+                if (@n) {
+                        @n = sort { $b <=> $a } @n; # to avoid resize
+                        my $buf = '';
+                        vec($buf, $_, 1) = 1 for @n;
+                        $seq[$off] = $buf;
+                }
+        }
+        \@seq;
+}
+
+sub mh_each_file {
+        my ($self, $efcb, @arg) = @_;
+        opendir(my $dh, my $dir = $self->{dir});
+        my $restore = PublicInbox::OnDestroy->new($$, \&chdir, $self->{cwdfh});
+        chdir($dh);
+        my $sort = $self->{sort};
+        if (defined $sort && "@$sort" ne 'none') {
+                my @sort = map {
+                        my @tmp = $_ eq '' ? ('sequence') : split(/[, ]/);
+                        # sorting by name alphabetically makes no sense for MH:
+                        for my $k (@tmp) {
+                                s/\A(\-|\+|)(?:name|)\z/$1sequence/;
+                        }
+                        @tmp;
+                } @$sort;
+                my @n = grep /\A[0-9]+\z/s, readdir $dh;
+                mdir_sort \@n, \@sort;
+                $efcb->($dir, $_, $self, @arg) for @n;
+        } else {
+                while (readdir $dh) { # perl v5.12+ to set $_ on readdir
+                        $efcb->($dir, $_, $self, @arg) if /\A[0-9]+\z/s;
+                }
+        }
+        closedir $dh; # may die
+}
+
+sub kw_for ($$) {
+        my ($self, $n) = @_;
+        my $seq = $self->{mh_seq} //= read_mh_sequences($self);
+        my @kw = map { vec($seq->[$_], $n, 1) ? $OFF2KW[$_] : () } (0, 1);
+        vec($seq->[2], $n, 1) or push @kw, 'seen';
+        \@kw;
+}
+
+sub _file2eml { # mh_each_file / mh_read_one cb
+        my ($dir, $n, $self, $ucb, @arg) = @_;
+        my $eml = eml_from_path($n);
+        $ucb->($dir, $n, kw_for($self, $n), $eml, @arg) if $eml;
+}
+
+sub mh_each_eml {
+        my ($self, $ucb, @arg) = @_;
+        mh_each_file($self, \&_file2eml, $ucb, @arg);
+}
+
+sub mh_read_one {
+        my ($self, $n, $ucb, @arg) = @_;
+        my $restore = PublicInbox::OnDestroy->new($$, \&chdir, $self->{cwdfh});
+        chdir(my $dir = $self->{dir});
+        _file2eml($dir, $n, $self, $ucb, @arg);
+}
+
+1;
diff --git a/lib/PublicInbox/MID.pm b/lib/PublicInbox/MID.pm
index 35b517e0..36c05855 100644
--- a/lib/PublicInbox/MID.pm
+++ b/lib/PublicInbox/MID.pm
@@ -1,15 +1,15 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Various Message-ID-related functions.
 package PublicInbox::MID;
 use strict;
-use warnings;
-use base qw/Exporter/;
+use v5.10.1; # TODO: check unicode_strings compat for v5.12
+use parent qw(Exporter);
 our @EXPORT_OK = qw(mid_clean id_compress mid2path mid_escape MID_ESC
         mids references mids_for_index mids_in $MID_EXTRACT);
 use URI::Escape qw(uri_escape_utf8);
-use Digest::SHA qw/sha1_hex/;
+use PublicInbox::SHA qw(sha1_hex);
 require PublicInbox::Address;
 use constant {
         ID_MAX => 40, # SHA-1 hex length for HTML id anchors
@@ -92,8 +92,7 @@ sub references ($) {
         my ($hdr) = @_;
         my @mids;
         foreach my $f (qw(References In-Reply-To)) {
-                my @v = $hdr->header_raw($f);
-                foreach my $v (@v) {
+                for my $v ($hdr->header_raw($f)) {
                         push(@mids, ($v =~ /$MID_EXTRACT/g));
                 }
         }
@@ -104,8 +103,7 @@ sub references ($) {
         my %addr = ( y => 1, n => 1 );
 
         foreach my $f (qw(To From Cc)) {
-                my @v = $hdr->header_raw($f);
-                foreach my $v (@v) {
+                for my $v ($hdr->header_raw($f)) {
                         $addr{$_} = 1 for (PublicInbox::Address::emails($v));
                 }
         }
@@ -117,7 +115,7 @@ sub uniq_mids ($;$) {
         my @ret;
         $seen ||= {};
         foreach my $mid (@$mids) {
-                $mid =~ tr/\n\t\r//d;
+                $mid =~ tr/\n\t\r\0//d;
                 if (length($mid) > MAX_MID_SIZE) {
                         warn "Message-ID: <$mid> too long, truncating\n";
                         $mid = substr($mid, 0, MAX_MID_SIZE);
@@ -127,7 +125,7 @@ sub uniq_mids ($;$) {
         \@ret;
 }
 
-# RFC3986, section 3.3:
+# RFC3986, section 3.3 (pathnames only):
 sub MID_ESC () { '^A-Za-z0-9\-\._~!\$\&\'\(\)\*\+,;=:@' }
 sub mid_escape ($) { uri_escape_utf8($_[0], MID_ESC) }
 
diff --git a/lib/PublicInbox/MailDiff.pm b/lib/PublicInbox/MailDiff.pm
new file mode 100644
index 00000000..125360fe
--- /dev/null
+++ b/lib/PublicInbox/MailDiff.pm
@@ -0,0 +1,137 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+package PublicInbox::MailDiff;
+use v5.12;
+use File::Temp 0.19 (); # 0.19 for ->newdir
+use PublicInbox::ContentHash qw(content_digest);
+use PublicInbox::MsgIter qw(msg_part_text);
+use PublicInbox::ViewDiff qw(flush_diff);
+use PublicInbox::GitAsyncCat;
+use PublicInbox::ContentDigestDbg;
+use PublicInbox::Qspawn;
+use PublicInbox::IO qw(write_file);
+use autodie qw(close mkdir);
+
+sub write_part { # Eml->each_part callback
+        my ($ary, $self) = @_;
+        my ($part, $depth, $idx) = @$ary;
+        if ($idx ne '1' || $self->{-raw_hdr}) { # lei mail-diff --raw-header
+                write_file '>', "$self->{curdir}/$idx.hdr", ${$part->{hdr}};
+        }
+        my $ct = $part->content_type || 'text/plain';
+        my ($s, $err) = msg_part_text($part, $ct);
+        my $sfx = defined($s) ? 'txt' : 'bin';
+        $s //= $part->body;
+        $s =~ s/\r\n/\n/gs; # TODO: consider \r+\n to match View
+        $s =~ s/\s*\z//s;
+        write_file '>:utf8', "$self->{curdir}/$idx.$sfx", $s, "\n";
+}
+
+# public
+sub dump_eml ($$$) {
+        my ($self, $dir, $eml) = @_;
+        local $self->{curdir} = $dir;
+        mkdir $dir;
+        $eml->each_part(\&write_part, $self);
+        my $fh = write_file '>', "$dir/content_digest";
+        my $dig = PublicInbox::ContentDigestDbg->new($fh);
+        content_digest($eml, $dig);
+        say $fh "\n", $dig->hexdigest;
+        close $fh;
+}
+
+# public
+sub prep_a ($$) {
+        my ($self, $eml) = @_;
+        $self->{tmp} = File::Temp->newdir('mail-diff-XXXX', TMPDIR => 1);
+        dump_eml($self, "$self->{tmp}/a", $eml);
+}
+
+# WWW-specific stuff below (TODO: split out for non-lei)
+
+sub next_smsg ($) {
+        my ($self) = @_;
+        my $ctx = $self->{ctx};
+        my $over = $ctx->{ibx}->over;
+        $self->{smsg} = $over ? $over->next_by_mid(@{$self->{next_arg}})
+                        : $ctx->gone('over');
+        if (!$self->{smsg}) {
+                $ctx->write('</pre>', $ctx->_html_end);
+                return $ctx->close;
+        }
+        PublicInbox::DS::requeue($self) if $ctx->{env}->{'pi-httpd.async'};
+}
+
+sub emit_msg_diff {
+        my ($bref, $self) = @_; # bref is `git diff' output
+        require PublicInbox::Hval;
+        PublicInbox::Hval::utf8_maybe($$bref);
+
+        # will be escaped to `&#8226;' in HTML
+        $self->{ctx}->{ibx}->{obfuscate} and
+                PublicInbox::Hval::obfuscate_addrs($self->{ctx}->{ibx},
+                                                $$bref, "\x{2022}");
+        print { $self->{ctx}->{zfh} } '</pre><hr><pre>' if $self->{nr} > 1;
+        flush_diff($self->{ctx}, $bref);
+        next_smsg($self);
+}
+
+sub do_diff {
+        my ($self, $eml) = @_;
+        my $n = 'N'.(++$self->{nr});
+        my $dir = "$self->{tmp}/$n";
+        $self->dump_eml($dir, $eml);
+        my $cmd = [ qw(git diff --no-index --no-color -- a), $n ];
+        my $opt = { -C => "$self->{tmp}", quiet => 1 };
+        my $qsp = PublicInbox::Qspawn->new($cmd, undef, $opt);
+        $qsp->psgi_qx($self->{ctx}->{env}, undef, \&emit_msg_diff, $self);
+}
+
+sub diff_msg_i {
+        my ($self, $eml) = @_;
+        if ($eml) {
+                if ($self->{tmp}) { # 2nd..last message
+                        do_diff($self, $eml);
+                } else { # first message:
+                        prep_a($self, $eml);
+                        next_smsg($self);
+                }
+        } else {
+                warn "W: $self->{smsg}->{blob} missing\n";
+                next_smsg($self);
+        }
+}
+
+sub diff_msg_i_async {
+        my ($bref, $oid, $type, $size, $self) = @_;
+        diff_msg_i($self, $bref ? PublicInbox::Eml->new($bref) : undef);
+}
+
+sub event_step {
+        my ($self) = @_;
+        eval {
+                my $ctx = $self->{ctx};
+                if ($ctx->{env}->{'pi-httpd.async'}) {
+                        ibx_async_cat($ctx->{ibx}, $self->{smsg}->{blob},
+                                        \&diff_msg_i_async, $self);
+                } else {
+                        diff_msg_i($self, $ctx->{ibx}->smsg_eml($self->{smsg}));
+                }
+        };
+        if ($@) {
+                warn "E: $@";
+                delete $self->{smsg};
+                $self->{ctx}->close;
+        }
+}
+
+sub begin_mail_diff {
+        my ($self) = @_;
+        if ($self->{ctx}->{env}->{'pi-httpd.async'}) {
+                PublicInbox::DS::requeue($self);
+        } else {
+                event_step($self) while $self->{smsg};
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index d5048a96..1f739baa 100644
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -1,10 +1,10 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# generates manifest.js.gz for grokmirror(1)
+# generates manifest.js.gz for grokmirror(1) via PublicInbox::WWW
+# This doesn't parse manifest.js.gz (that happens in LeiMirror)
 package PublicInbox::ManifestJsGz;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::WwwListing);
 use PublicInbox::Config;
 use IO::Compress::Gzip qw(gzip);
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index b977308d..52f88ae3 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Streaming interface for mboxrd HTTP responses
@@ -19,13 +19,10 @@ sub getline {
         my $smsg = $ctx->{smsg} or return;
         my $ibx = $ctx->{ibx};
         my $eml = delete($ctx->{eml}) // $ibx->smsg_eml($smsg) // return;
-        my $n = $ctx->{smsg} = $ibx->over->next_by_mid(@{$ctx->{next_arg}});
-        $ctx->zmore(msg_hdr($ctx, $eml));
-        if ($n) {
-                $ctx->translate(msg_body($eml));
+        if (($ctx->{smsg} = $ibx->over->next_by_mid(@{$ctx->{next_arg}}))) {
+                $ctx->translate(msg_hdr($ctx, $eml), msg_body($eml));
         } else { # last message
-                $ctx->zmore(msg_body($eml));
-                $ctx->zflush;
+                $ctx->zflush(msg_hdr($ctx, $eml), msg_body($eml));
         }
 }
 
@@ -46,8 +43,7 @@ sub async_eml { # for async_blob_cb
         # next message
         $ctx->{smsg} = $ctx->{ibx}->over->next_by_mid(@{$ctx->{next_arg}});
         local $ctx->{eml} = $eml; # for mbox_hdr
-        $ctx->zmore(msg_hdr($ctx, $eml));
-        $ctx->write(msg_body($eml));
+        $ctx->write(msg_hdr($ctx, $eml), msg_body($eml));
 }
 
 sub mbox_hdr ($) {
@@ -83,7 +79,6 @@ sub no_over_raw ($) {
 # /$INBOX/$MESSAGE_ID/raw
 sub emit_raw {
         my ($ctx) = @_;
-        $ctx->{base_url} = $ctx->{ibx}->base_url($ctx->{env});
         my $over = $ctx->{ibx}->over or return no_over_raw($ctx);
         my ($id, $prev);
         my $mip = $ctx->{next_arg} = [ $ctx->{mid}, \$id, \$prev ];
@@ -94,17 +89,15 @@ sub emit_raw {
 
 sub msg_hdr ($$) {
         my ($ctx, $eml) = @_;
-        my $header_obj = $eml->header_obj;
 
-        # drop potentially confusing headers, ssoma already should've dropped
-        # Lines and Content-Length
-        foreach my $d (qw(Lines Bytes Content-Length Status)) {
-                $header_obj->header_set($d);
+        # drop potentially confusing headers, various importers should've
+        # already dropped these, but we can't trust stuff we've cloned
+        for my $d (qw(Lines Bytes Content-Length Status)) {
+                $eml->header_set($d);
         }
-        my $crlf = $header_obj->crlf;
-        my $buf = $header_obj->as_string;
-        # fixup old bug from import (pre-a0c07cba0e5d8b6a)
-        $buf =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+        my $crlf = $eml->crlf;
+        my $buf = $eml->header_obj->as_string;
+        PublicInbox::Eml::strip_from($buf);
         "From mboxrd\@z Thu Jan  1 00:00:00 1970" . $crlf . $buf . $crlf;
 }
 
@@ -230,10 +223,19 @@ sub mbox_all {
         return mbox_all_ids($ctx) if $q_string !~ /\S/;
         my $srch = $ctx->{ibx}->isrch or
                 return PublicInbox::WWW::need($ctx, 'Search');
-        my $over = $ctx->{ibx}->over or
-                return PublicInbox::WWW::need($ctx, 'Overview');
 
         my $qopts = $ctx->{qopts} = { relevance => -2 }; # ORDER BY docid DESC
+
+        # {threadid} limits results to a given thread
+        # {threads} collapses results from messages in the same thread,
+        # allowing us to use ->expand_thread w/o duplicates in our own code
+        if (defined($ctx->{mid})) {
+                my $over = ($ctx->{ibx}->{isrch} ?
+                                $ctx->{ibx}->{isrch}->{es}->over :
+                                $ctx->{ibx}->over) or
+                        return PublicInbox::WWW::need($ctx, 'Overview');
+                $qopts->{threadid} = $over->mid2tid($ctx->{mid});
+        }
         $qopts->{threads} = 1 if $q->{t};
         $srch->query_approxidate($ctx->{ibx}->git, $q_string);
         my $mset = $srch->mset($q_string, $qopts);
diff --git a/lib/PublicInbox/MboxGz.pm b/lib/PublicInbox/MboxGz.pm
index 3ed33867..533d2ff1 100644
--- a/lib/PublicInbox/MboxGz.pm
+++ b/lib/PublicInbox/MboxGz.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 package PublicInbox::MboxGz;
 use strict;
@@ -22,7 +22,6 @@ sub async_next ($) {
 sub mbox_gz {
         my ($self, $cb, $fn) = @_;
         $self->{cb} = $cb;
-        $self->{base_url} = $self->{ibx}->base_url($self->{env});
         $self->{gz} = PublicInbox::GzipFilter::gzip_or_die();
         $fn = to_filename($fn // '') // 'no-subject';
         # http://www.iana.org/assignments/media-types/application/gzip
@@ -38,8 +37,7 @@ sub getline {
         my $cb = $self->{cb} or return;
         while (my $smsg = $cb->($self)) {
                 my $eml = $self->{ibx}->smsg_eml($smsg) or next;
-                $self->zmore(msg_hdr($self, $eml));
-                return $self->translate(msg_body($eml));
+                return $self->translate(msg_hdr($self, $eml), msg_body($eml));
         }
         # signal that we're done and can return undef next call:
         delete $self->{cb};
diff --git a/lib/PublicInbox/MboxLock.pm b/lib/PublicInbox/MboxLock.pm
index 856b1e21..9d7d4a32 100644
--- a/lib/PublicInbox/MboxLock.pm
+++ b/lib/PublicInbox/MboxLock.pm
@@ -1,20 +1,24 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Various mbox locking methods
 package PublicInbox::MboxLock;
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::OnDestroy;
 use Fcntl qw(:flock F_SETLK F_SETLKW F_RDLCK F_WRLCK
                         O_CREAT O_EXCL O_WRONLY SEEK_SET);
 use Carp qw(croak);
 use PublicInbox::DS qw(now); # ugh...
+use autodie qw(chdir opendir unlink);
 
 our $TMPL = do {
-        if ($^O eq 'linux') { \'s @32' }
-        elsif ($^O =~ /bsd/) { \'@20 s @256' } # n.b. @32 may be enough...
-        else { eval { require File::FcntlLock; 1 } }
+        if ($^O eq 'linux') {
+                \'s @32'
+        } elsif ($^O =~ /bsd/ || $^O eq 'dragonfly') {
+                \'@20 s @256' # n.b. @32 may be enough...
+        } else {
+                 eval { require File::FcntlLock; 1 }
+        }
 };
 
 # This order matches Debian policy on Linux systems.
@@ -58,16 +62,13 @@ sub acq_dotlock {
                                         rand(0xffffffff), $pid, time);
                 if (sysopen(my $fh, $tmp, O_CREAT|O_EXCL|O_WRONLY)) {
                         if (link($tmp, $dot_lock)) {
-                                unlink($tmp) or die "unlink($tmp): $!";
+                                unlink($tmp);
                                 $self->{".lock$pid"} = $dot_lock;
-                                if (substr($dot_lock, 0, 1) ne '/') {
-                                        opendir(my $dh, '.') or
-                                                        die "opendir . $!";
-                                        $self->{dh} = $dh;
-                                }
+                                substr($dot_lock, 0, 1) eq '/' or
+                                        opendir($self->{dh}, '.');
                                 return;
                         }
-                        unlink($tmp) or die "unlink($tmp): $!";
+                        unlink($tmp);
                         select(undef, undef, undef, $self->{delay});
                 } else {
                         croak "open $tmp (for $dot_lock): $!" if !$!{EXIST};
@@ -83,18 +84,20 @@ sub acq_flock {
         my $end = now + $self->{timeout};
         do {
                 return if flock($self->{fh}, $op);
-                select(undef, undef, undef, $self->{delay});
+                if ($!{EWOULDBLOCK}) {
+                        select(undef, undef, undef, $self->{delay});
+                } elsif (!$!{EINTR}) {
+                        croak "flock($self->{f} ($self->{fh}): $!";
+                }
         } while (now < $end);
         die "flock timeout $self->{f}: $!\n";
 }
 
 sub acq {
         my ($cls, $f, $rw, $methods) = @_;
-        my $fh;
-        unless (open $fh, $rw ? '+>>' : '<', $f) {
-                croak "open($f): $!" if $rw || !$!{ENOENT};
-        }
-        my $self = bless { f => $f, fh => $fh, rw => $rw }, $cls;
+        my $self = bless { f => $f, rw => $rw }, $cls;
+        my $ok = open $self->{fh}, $rw ? '+>>' : '<', $f;
+        croak "open($f): $!" if !$ok && ($rw || !$!{ENOENT});
         my $m = "@$methods";
         if ($m ne 'none') {
                 my @m = map {
@@ -116,20 +119,16 @@ sub acq {
         $self;
 }
 
-sub _fchdir { chdir($_[0]) } # OnDestroy callback
-
 sub DESTROY {
         my ($self) = @_;
-        if (my $f = $self->{".lock$$"}) {
-                my $x;
-                if (my $dh = delete $self->{dh}) {
-                        opendir my $c, '.' or die "opendir . $!";
-                        $x = PublicInbox::OnDestroy->new(\&_fchdir, $c);
-                        chdir($dh) or die "chdir (for $f): $!";
-                }
-                unlink($f) or die "unlink($f): $! (lock stolen?)";
-                undef $x;
+        my $f = $self->{".lock$$"} or return;
+        my $x;
+        if (my $dh = delete $self->{dh}) {
+                opendir my $c, '.';
+                $x = PublicInbox::OnDestroy->new(\&chdir, $c);
+                chdir($dh);
         }
+        CORE::unlink($f) or die "unlink($f): $! (lock stolen?)";
 }
 
 1;
diff --git a/lib/PublicInbox/MboxReader.pm b/lib/PublicInbox/MboxReader.pm
index beffabe8..3d78ca23 100644
--- a/lib/PublicInbox/MboxReader.pm
+++ b/lib/PublicInbox/MboxReader.pm
@@ -1,10 +1,10 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# reader for mbox variants we support
+# reader for mbox variants we support (and also sets up commands for writing)
 package PublicInbox::MboxReader;
 use strict;
-use v5.10.1;
+use v5.10.1; # check regexps before v5.12
 use Data::Dumper;
 $Data::Dumper::Useqq = 1; # should've been the default, for bad data
 
@@ -29,7 +29,7 @@ sub _mbox_from {
         my @raw;
         while (defined(my $r = read($mbfh, $buf, 65536, length($buf)))) {
                 if ($r == 0) { # close here to check for "curl --fail"
-                        close($mbfh) or die "error closing mbox: \$?=$? $!";
+                        $mbfh->close or die "error closing mbox: \$?=$? $!";
                         @raw = ($buf);
                 } else {
                         @raw = split(/$from_strict/mos, $buf, -1);
@@ -88,12 +88,12 @@ sub _mbox_cl ($$$;@) {
         my $buf = '';
         while (defined(my $r = read($mbfh, $buf, 65536, length($buf)))) {
                 if ($r == 0) { # detect "curl --fail"
-                        close($mbfh) or
+                        $mbfh->close or
                                 die "error closing mboxcl/mboxcl2: \$?=$? $!";
                         undef $mbfh;
                 }
                 while (my $hdr = _extract_hdr(\$buf)) {
-                        $$hdr =~ s/\A[\r\n]*From [^\n]*\n//s or
+                        PublicInbox::Eml::strip_from($$hdr) or
                                 die "E: no 'From ' line in:\n", Dumper($hdr);
                         my $eml = PublicInbox::Eml->new($hdr);
                         next unless $eml->raw_size;
@@ -141,10 +141,9 @@ sub reads {
 
 # all of these support -c for stdout and -d for decompression,
 # mutt is commonly distributed with hooks for gz, bz2 and xz, at least
-# { foo => '' } means "--foo" is passed to the command-line,
-# otherwise { foo => '--bar' } passes "--bar"
+# { foo => '' } means "--foo" is passed to the command-line
 my %zsfx2cmd = (
-        gz => [ qw(GZIP pigz gzip) ],
+        gz => [ qw(GZIP pigz gzip), { rsyncable => '' } ],
         bz2 => [ 'bzip2', {} ],
         xz => [ 'xz', {} ],
         # don't add new entries here unless MUA support is widely available
@@ -173,28 +172,9 @@ sub zsfx2cmd ($$$) {
         }
         $cmd[0] // die join(' or ', @info)." missing for .$zsfx";
 
-        # not all gzip support --rsyncable, FreeBSD gzip doesn't even exit
-        # with an error code
-        if (!$decompress && $cmd[0] =~ m!/gzip\z! && !defined($cmd_opt)) {
-                pipe(my ($r, $w)) or die "pipe: $!";
-                open my $null, '+>', '/dev/null' or die "open: $!";
-                my $rdr = { 0 => $null, 1 => $null, 2 => $w };
-                my $tst = [ $cmd[0], '--rsyncable' ];
-                my $pid = PublicInbox::Spawn::spawn($tst, undef, $rdr);
-                close $w;
-                my $err = do { local $/; <$r> };
-                waitpid($pid, 0) == $pid or die "BUG: waitpid: $!";
-                $cmd_opt = $err ? {} : { rsyncable => '' };
-                push(@$x, $cmd_opt);
-        }
-        for my $bool (keys %$cmd_opt) {
-                my $switch = $cmd_opt->{$bool} // next;
-                push @cmd, '--'.($switch || $bool);
-        }
-        for my $key (qw(rsyncable)) { # support compression level?
-                my $switch = $cmd_opt->{$key} // next;
-                my $val = $lei->{opt}->{$key} // next;
-                push @cmd, $switch, $val;
+        # only for --rsyncable.  TODO: support compression level?
+        for my $key (keys %$cmd_opt) {
+                push @cmd, '--'.$key if $lei->{opt}->{$key};
         }
         \@cmd;
 }
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm
index dbb74d6d..2981b058 100644
--- a/lib/PublicInbox/MdirReader.pm
+++ b/lib/PublicInbox/MdirReader.pm
@@ -1,14 +1,14 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# Maildirs for now, MH eventually
+# Maildirs only (PublicInbox::MHreader exists, now)
 # ref: https://cr.yp.to/proto/maildir.html
 #        https://wiki2.dovecot.org/MailboxFormat/Maildir
 package PublicInbox::MdirReader;
 use strict;
 use v5.10.1;
 use PublicInbox::InboxWritable qw(eml_from_path);
-use Digest::SHA qw(sha256_hex);
+use PublicInbox::SHA qw(sha256_hex);
 
 # returns Maildir flags from a basename ('' for no flags, undef for invalid)
 sub maildir_basename_flags {
diff --git a/lib/PublicInbox/MdirSort.pm b/lib/PublicInbox/MdirSort.pm
new file mode 100644
index 00000000..6bd9fb6c
--- /dev/null
+++ b/lib/PublicInbox/MdirSort.pm
@@ -0,0 +1,46 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# used for sorting MH (and (TODO) Maildir) names
+# TODO: consider sort(1) to parallelize sorting of gigantic directories
+package PublicInbox::MdirSort;
+use v5.12;
+use Time::HiRes ();
+use parent qw(Exporter);
+use Fcntl qw(S_ISREG);
+our @EXPORT = qw(mdir_sort);
+my %ST = (sequence => 0, size => 1, atime => 2, mtime => 3, ctime => 4);
+
+sub mdir_sort ($$;$) {
+        my ($ent, $sort, $max) = @_;
+        my @st;
+        my @ent = map {
+                @st = Time::HiRes::stat $_;
+                # name, size, {a,m,c}time
+                S_ISREG($st[2]) ? [ $_, @st[7..10] ] : ();
+        } @$ent;
+        @ent = grep { $_->[1] <= $max } @ent if $max;
+        use sort 'stable';
+        for my $s (@$sort) {
+                if ($s =~ /\A(\-|\+|)name\z/) {
+                        if ($1 eq '-') {
+                                @ent = sort { $b->[0] cmp $a->[0] } @ent;
+                        } else {
+                                @ent = sort { $a->[0] cmp $b->[0] } @ent;
+                        }
+                } elsif ($s =~ /\A(\-|\+|)
+                                (sequence|size|ctime|mtime|atime)\z/x) {
+                        my $key = $ST{$2};
+                        if ($1 eq '-') {
+                                @ent = sort { $b->[$key] <=> $a->[$key] } @ent;
+                        } else {
+                                @ent = sort { $a->[$key] <=> $b->[$key] } @ent;
+                        }
+                } else {
+                        die "E: unrecognized sort parameter: `$s'";
+                }
+        }
+        @$ent = map { $_->[0] } @ent;
+}
+
+1;
diff --git a/lib/PublicInbox/MiscIdx.pm b/lib/PublicInbox/MiscIdx.pm
index 5faf5c66..6708527d 100644
--- a/lib/PublicInbox/MiscIdx.pm
+++ b/lib/PublicInbox/MiscIdx.pm
@@ -5,7 +5,7 @@
 # Things indexed include:
 # * inboxes themselves
 # * epoch information
-# * (maybe) git code repository information
+# * (maybe) git code repository information (not commits)
 # Expect ~100K-1M documents with no parallelism opportunities,
 # so no sharding, here.
 #
@@ -72,7 +72,7 @@ sub remove_eidx_key {
         }
         for my $docid (@docids) {
                 $xdb->delete_document($docid);
-                warn "I: remove inbox docid #$docid ($eidx_key)\n";
+                warn "# remove inbox docid #$docid ($eidx_key)\n";
         }
 }
 
@@ -108,12 +108,16 @@ EOF
         $doc->add_boolean_term('Q'.$eidx_key); # uniQue id
         $doc->add_boolean_term('T'.'inbox'); # Type
 
+        # force reread from disk, {description} could be loaded from {misc}
+        delete @$ibx{qw(-art_min -art_max description)};
         if (defined($ibx->{newsgroup}) && $ibx->nntp_usable) {
                 $doc->add_boolean_term('T'.'newsgroup'); # additional Type
+                my $n = $ibx->art_min;
+                add_val($doc, $PublicInbox::MiscSearch::ART_MIN, $n) if $n;
+                $n = $ibx->art_max;
+                add_val($doc, $PublicInbox::MiscSearch::ART_MAX, $n) if $n;
         }
 
-        # force reread from disk, {description} could be loaded from {misc}
-        delete $ibx->{description};
         my $desc = $ibx->description;
 
         # description = S/Subject (or title)
diff --git a/lib/PublicInbox/MiscSearch.pm b/lib/PublicInbox/MiscSearch.pm
index c6d2a062..5fb47d03 100644
--- a/lib/PublicInbox/MiscSearch.pm
+++ b/lib/PublicInbox/MiscSearch.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # read-only counterpart to MiscIdx
@@ -11,6 +11,8 @@ my $json;
 # Xapian value columns:
 our $MODIFIED = 0;
 our $UIDVALIDITY = 1; # (created time)
+our $ART_MIN = 2; # NNTP article number
+our $ART_MAX = 3; # NNTP article number
 
 # avoid conflicting with message Search::prob_prefix for UI/UX reasons
 my %PROB_PREFIX = (
@@ -87,14 +89,13 @@ sub ibx_data_once {
         my $term = 'Q'.$ibx->eidx_key; # may be {inboxdir}, so private
         my $head = $xdb->postlist_begin($term);
         my $tail = $xdb->postlist_end($term);
-        if ($head != $tail) {
-                my $doc = $xdb->get_document($head->get_docid);
-                $ibx->{uidvalidity} //= int_val($doc, $UIDVALIDITY);
-                $ibx->{-modified} = int_val($doc, $MODIFIED);
-                $doc->get_data;
-        } else {
-                undef;
-        }
+        return if $head == $tail;
+        my $doc = $xdb->get_document($head->get_docid);
+        $ibx->{uidvalidity} //= int_val($doc, $UIDVALIDITY);
+        $ibx->{-modified} = int_val($doc, $MODIFIED);
+        $ibx->{-art_min} = int_val($doc, $ART_MIN);
+        $ibx->{-art_max} = int_val($doc, $ART_MAX);
+        $doc->get_data;
 }
 
 sub doc2ibx_cache_ent { # @_ == ($self, $doc) OR ($doc)
@@ -109,6 +110,8 @@ sub doc2ibx_cache_ent { # @_ == ($self, $doc) OR ($doc)
         {
                 uidvalidity => int_val($doc, $UIDVALIDITY),
                 -modified => int_val($doc, $MODIFIED),
+                -art_min => int_val($doc, $ART_MIN), # may be undef
+                -art_max => int_val($doc, $ART_MAX), # may be undef
                 # extract description from manifest.js.gz epoch description
                 description => $d
         };
diff --git a/lib/PublicInbox/MsgTime.pm b/lib/PublicInbox/MsgTime.pm
index 5ee087fd..bbc9a007 100644
--- a/lib/PublicInbox/MsgTime.pm
+++ b/lib/PublicInbox/MsgTime.pm
@@ -1,11 +1,11 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Various date/time-related functions
 package PublicInbox::MsgTime;
+use v5.10.1; # unicode_strings in 5.12 may not work...
 use strict;
-use warnings;
-use base qw(Exporter);
+use parent qw(Exporter);
 our @EXPORT_OK = qw(msg_timestamp msg_datestamp);
 use Time::Local qw(timegm);
 my @MoY = qw(january february march april may june
@@ -125,10 +125,7 @@ sub str2date_zone ($) {
         # but we want to keep "git fsck" happy.
         # "-1200" is the furthest westermost zone offset,
         # but git fast-import is liberal so we use "-1400"
-        if ($zone >= 1400 || $zone <= -1400) {
-                warn "bogus TZ offset: $zone, ignoring and assuming +0000\n";
-                $zone = '+0000';
-        }
+        $zone = '+0000' if $zone >= 1400 || $zone <= -1400;
         [$ts, $zone];
 }
 
@@ -138,50 +135,38 @@ sub time_response ($) {
 }
 
 sub msg_received_at ($) {
-        my ($hdr) = @_; # PublicInbox::Eml
-        my @recvd = $hdr->header_raw('Received');
-        my ($ts);
-        foreach my $r (@recvd) {
+        my ($eml) = @_;
+        my $ts;
+        for my $r ($eml->header_raw('Received')) {
                 $r =~ /\s*([0-9]+\s+[a-zA-Z]+\s+[0-9]{2,4}\s+
                         [0-9]+[^0-9][0-9]+(?:[^0-9][0-9]+)
-                        \s+([\+\-][0-9]+))/sx or next;
+                        \s+(?:[\+\-][0-9]+))/sx or next;
                 $ts = eval { str2date_zone($1) } and return $ts;
-                my $mid = $hdr->header_raw('Message-ID');
-                warn "no date in $mid Received: $r\n";
         }
         undef;
 }
 
 sub msg_date_only ($) {
-        my ($hdr) = @_; # PublicInbox::Eml
-        my @date = $hdr->header_raw('Date');
-        my ($ts);
-        foreach my $d (@date) {
+        my ($eml) = @_;
+        my $ts;
+        for my $d ($eml->header_raw('Date')) {
                 $ts = eval { str2date_zone($d) } and return $ts;
-                if ($@) {
-                        my $mid = $hdr->header_raw('Message-ID');
-                        warn "bad Date: $d in $mid: $@\n";
-                }
         }
         undef;
 }
 
 # Favors Received header for sorting globally
 sub msg_timestamp ($;$) {
-        my ($hdr, $fallback) = @_; # PublicInbox::Eml
-        my $ret;
-        $ret = msg_received_at($hdr) and return time_response($ret);
-        $ret = msg_date_only($hdr) and return time_response($ret);
-        time_response([ $fallback // time, '+0000' ]);
+        my ($eml, $fallback) = @_;
+        time_response(msg_received_at($eml) // msg_date_only($eml) //
+                        [ $fallback // time, '+0000' ]);
 }
 
 # Favors the Date: header for display and sorting within a thread
 sub msg_datestamp ($;$) {
-        my ($hdr, $fallback) = @_; # PublicInbox::Eml
-        my $ret;
-        $ret = msg_date_only($hdr) and return time_response($ret);
-        $ret = msg_received_at($hdr) and return time_response($ret);
-        time_response([ $fallback // time, '+0000' ]);
+        my ($eml, $fallback) = @_; # PublicInbox::Eml
+        time_response(msg_date_only($eml) // msg_received_at($eml) //
+                        [ $fallback // time, '+0000' ]);
 }
 
 1;
diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index 1041cd17..cb4bb295 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -144,13 +144,17 @@ sub max {
         $sth->fetchrow_array // 0;
 }
 
-sub minmax {
-        # breaking MIN and MAX into separate queries speeds up from 250ms
-        # to around 700us with 2.7million messages.
+sub min {
         my $sth = $_[0]->{dbh}->prepare_cached('SELECT MIN(num) FROM msgmap',
                                                 undef, 1);
         $sth->execute;
-        ($sth->fetchrow_array // 0, max($_[0]));
+        $sth->fetchrow_array // 0;
+}
+
+sub minmax {
+        # breaking MIN and MAX into separate queries speeds up from 250ms
+        # to around 700us with 2.7million messages.
+        (min($_[0]), max($_[0]));
 }
 
 sub mid_delete {
diff --git a/lib/PublicInbox/MultiGit.pm b/lib/PublicInbox/MultiGit.pm
index 9429a00c..b7691806 100644
--- a/lib/PublicInbox/MultiGit.pm
+++ b/lib/PublicInbox/MultiGit.pm
@@ -5,10 +5,12 @@
 package PublicInbox::MultiGit;
 use strict;
 use v5.10.1;
-use PublicInbox::Spawn qw(run_die);
+use PublicInbox::Spawn qw(run_die run_qx);
 use PublicInbox::Import;
 use File::Temp 0.19;
 use List::Util qw(max);
+use PublicInbox::IO qw(read_all);
+use autodie qw(chmod close rename);
 
 sub new {
         my ($cls, $topdir, $all, $epfx) = @_;
@@ -31,7 +33,7 @@ sub read_alternates {
                         qr!\A\Q../../$self->{epfx}\E/([0-9]+)\.git/objects\z! :
                         undef;
                 $$moderef = (stat($fh))[2] & 07777;
-                for my $rel (split(/^/m, do { local $/; <$fh> })) {
+                for my $rel (split(/^/m, read_all($fh, -s _))) {
                         chomp(my $dir = $rel);
                         my $score;
                         if (defined($is_edir) && $dir =~ $is_edir) {
@@ -67,12 +69,10 @@ sub write_alternates {
         my $out = join('', sort { $alt->{$b} <=> $alt->{$a} } keys %$alt);
         my $info_dir = "$all_dir/objects/info";
         my $fh = File::Temp->new(TEMPLATE => 'alt-XXXX', DIR => $info_dir);
-        my $f = $fh->filename;
-        print $fh $out, @new or die "print($f): $!";
-        chmod($mode, $fh) or die "fchmod($f): $!";
-        close $fh or die "close($f): $!";
-        my $fn = "$info_dir/alternates";
-        rename($f, $fn) or die "rename($f, $fn): $!";
+        print $fh $out, @new;
+        chmod($mode, $fh);
+        close $fh;
+        rename($fh->filename, "$info_dir/alternates");
         $fh->unlink_on_destroy(0);
 }
 
@@ -108,8 +108,13 @@ sub fill_alternates {
 
 sub epoch_cfg_set {
         my ($self, $epoch_nr) = @_;
-        run_die([qw(git config -f), epoch_dir($self)."/$epoch_nr.git/config",
-                'include.path', "../../$self->{all}/config" ]);
+        my $f = epoch_dir($self)."/$epoch_nr.git/config";
+        my $v = "../../$self->{all}/config";
+        if (-r $f) {
+                chomp(my $x = run_qx([qw(git config -f), $f, 'include.path']));
+                return if $x eq $v;
+        }
+        run_die([qw(git config -f), $f, 'include.path', $v ]);
 }
 
 sub add_epoch {
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index b36722d7..603cf094 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Each instance of this represents a NNTP client socket
@@ -15,28 +15,26 @@ use PublicInbox::MID qw(mid_escape $MID_EXTRACT);
 use PublicInbox::Eml;
 use POSIX qw(strftime);
 use PublicInbox::DS qw(now);
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
 use Time::Local qw(timegm timelocal);
 use PublicInbox::GitAsyncCat;
 use PublicInbox::Address;
 
 use constant {
         LINE_MAX => 512, # RFC 977 section 2.3
-        r501 => '501 command syntax error',
-        r502 => '502 Command unavailable',
-        r221 => '221 Header follows',
-        r224 => '224 Overview information follows (multi-line)',
-        r225 =>        '225 Headers follow (multi-line)',
-        r430 => '430 No article with that message-id',
+        r501 => "501 command syntax error\r\n",
+        r502 => "502 Command unavailable\r\n",
+        r221 => "221 Header follows\r\n",
+        r225 =>        "225 Headers follow (multi-line)\r\n",
+        r430 => "430 No article with that message-id\r\n",
 };
-use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
 use Errno qw(EAGAIN);
 my $ONE_MSGID = qr/\A$MID_EXTRACT\z/;
 my @OVERVIEW = qw(Subject From Date Message-ID References);
 my $OVERVIEW_FMT = join(":\r\n", @OVERVIEW, qw(Bytes Lines), '') .
-                "Xref:full\r\n.";
+                "Xref:full\r\n.\r\n";
 my $LIST_HEADERS = join("\r\n", @OVERVIEW,
-                        qw(:bytes :lines Xref To Cc)) . "\r\n.";
+                        qw(:bytes :lines Xref To Cc)) . "\r\n.\r\n";
 my $CAPABILITIES = <<"";
 101 Capability list:\r
 VERSION 2\r
@@ -47,31 +45,17 @@ HDR\r
 OVER\r
 COMPRESS DEFLATE\r
 
-sub greet ($) { $_[0]->write($_[0]->{nntpd}->{greet}) };
-
-sub new ($$$) {
-        my ($class, $sock, $nntpd) = @_;
-        my $self = bless { nntpd => $nntpd }, $class;
-        my $ev = EPOLLIN;
-        my $wbuf;
-        if ($sock->can('accept_SSL') && !$sock->accept_SSL) {
-                return CORE::close($sock) if $! != EAGAIN;
-                $ev = PublicInbox::TLS::epollbit() or return CORE::close($sock);
-                $wbuf = [ \&PublicInbox::DS::accept_tls_step, \&greet ];
-        }
-        $self->SUPER::new($sock, $ev | EPOLLONESHOT);
-        if ($wbuf) {
-                $self->{wbuf} = $wbuf;
-        } else {
-                greet($self);
-        }
-        $self;
+sub do_greet ($) { $_[0]->write($_[0]->{nntpd}->{greet}) };
+
+sub new {
+        my ($cls, $sock, $nntpd) = @_;
+        (bless { nntpd => $nntpd }, $cls)->greet($sock)
 }
 
 sub args_ok ($$) {
         my ($cb, $argc) = @_;
         my $tot = prototype $cb;
-        my ($nreq, undef) = split(';', $tot);
+        my ($nreq, undef) = split(/;/, $tot);
         $nreq = ($nreq =~ tr/$//) - 1;
         $tot = ($tot =~ tr/$//) - 1;
         ($argc <= $tot && $argc >= $nreq);
@@ -82,19 +66,17 @@ sub process_line ($$) {
         my ($self, $l) = @_;
         my ($req, @args) = split(/[ \t]+/, $l);
         return 1 unless defined($req); # skip blank line
-        $req = $self->can('cmd_'.lc($req));
-        return res($self, '500 command not recognized') unless $req;
-        return res($self, r501) unless args_ok($req, scalar @args);
-
+        $req = $self->can('cmd_'.lc($req)) //
+                return $self->write(\"500 command not recognized\r\n");
+        return $self->write(\r501) unless args_ok($req, scalar @args);
         my $res = eval { $req->($self, @args) };
         my $err = $@;
         if ($err && $self->{sock}) {
-                local $/ = "\n";
-                chomp($l);
-                err($self, 'error from: %s (%s)', $l, $err);
-                $res = '503 program fault - command not performed';
+                $l =~ s/\r?\n//s;
+                warn("error from: $l ($err)\n");
+                $res = \"503 program fault - command not performed\r\n";
         }
-        defined($res) ? res($self, $res) : 0;
+        defined($res) ? $self->write($res) : 0;
 }
 
 # The keyword argument is not used (rfc3977 5.2.2)
@@ -102,22 +84,22 @@ sub cmd_capabilities ($;$) {
         my ($self, undef) = @_;
         my $res = $CAPABILITIES;
         if (!$self->{sock}->can('accept_SSL') &&
-                        $self->{nntpd}->{accept_tls}) {
+                        $self->{nntpd}->{ssl_ctx_opt}) {
                 $res .= "STARTTLS\r\n";
         }
-        $res .= '.';
+        $res .= ".\r\n";
 }
 
 sub cmd_mode ($$) {
         my ($self, $arg) = @_;
-        uc($arg) eq 'READER' ? '201 Posting prohibited' : r501;
+        uc($arg) eq 'READER' ? \"201 Posting prohibited\r\n" : \r501;
 }
 
-sub cmd_slave ($) { '202 slave status noted' }
+sub cmd_slave ($) { \"202 slave status noted\r\n" }
 
 sub cmd_xgtitle ($;$) {
         my ($self, $wildmat) = @_;
-        more($self, '282 list of groups and descriptions follows');
+        $self->msg_more("282 list of groups and descriptions follows\r\n");
         list_newsgroups($self, $wildmat);
 }
 
@@ -125,60 +107,63 @@ sub list_overview_fmt ($) { $OVERVIEW_FMT }
 
 sub list_headers ($;$) { $LIST_HEADERS }
 
-sub list_active_i { # "LIST ACTIVE" and also just "LIST" (no args)
-        my ($self, $groupnames) = @_;
-        my @window = splice(@$groupnames, 0, 100) or return 0;
-        my $ibx;
+sub names2ibx ($;$) {
+        my ($self, $names) = @_;
         my $groups = $self->{nntpd}->{pi_cfg}->{-by_newsgroup};
-        for my $ngname (@window) {
-                $ibx = $groups->{$ngname} and group_line($self, $ibx);
+        if ($names) { # modify arrayref in-place
+                $_ = $groups->{$_} for @$names;
+                $names; # now an arrayref of ibx
+        } else {
+                my @ret = map { $groups->{$_} } @{$self->{nntpd}->{groupnames}};
+                \@ret;
         }
-        scalar(@$groupnames); # continue if there's more
+}
+
+sub list_active_i { # "LIST ACTIVE" and also just "LIST" (no args)
+        my ($self, $ibxs) = @_;
+        my @window = splice(@$ibxs, 0, 1000);
+        emit_group_lines($self, \@window);
+        scalar @$ibxs; # continue if there's more
 }
 
 sub list_active ($;$) { # called by cmd_list
         my ($self, $wildmat) = @_;
         wildmat2re($wildmat);
-        long_response($self, \&list_active_i, [
-                grep(/$wildmat/, @{$self->{nntpd}->{groupnames}}) ]);
+        my @names = grep(/$wildmat/, @{$self->{nntpd}->{groupnames}});
+        $self->long_response(\&list_active_i, names2ibx($self, \@names));
 }
 
 sub list_active_times_i {
-        my ($self, $groupnames) = @_;
-        my @window = splice(@$groupnames, 0, 100) or return 0;
-        my $groups = $self->{nntpd}->{pi_cfg}->{-by_newsgroup};
-        for my $ngname (@window) {
-                my $ibx = $groups->{$ngname} or next;
-                my $c = eval { $ibx->uidvalidity } // time;
-                more($self, "$ngname $c <$ibx->{-primary_address}>");
-        }
-        scalar(@$groupnames); # continue if there's more
+        my ($self, $ibxs) = @_;
+        my @window = splice(@$ibxs, 0, 1000);
+        $self->msg_more(join('', map {
+                my $c = eval { $_->uidvalidity } // time;
+                "$_->{newsgroup} $c <$_->{-primary_address}>\r\n";
+        } @window));
+        scalar @$ibxs; # continue if there's more
 }
 
 sub list_active_times ($;$) { # called by cmd_list
         my ($self, $wildmat) = @_;
         wildmat2re($wildmat);
-        long_response($self, \&list_active_times_i, [
-                grep(/$wildmat/, @{$self->{nntpd}->{groupnames}}) ]);
+        my @names = grep(/$wildmat/, @{$self->{nntpd}->{groupnames}});
+        $self->long_response(\&list_active_times_i, names2ibx($self, \@names));
 }
 
 sub list_newsgroups_i {
-        my ($self, $groupnames) = @_;
-        my @window = splice(@$groupnames, 0, 100) or return 0;
-        my $groups = $self->{nntpd}->{pi_cfg}->{-by_newsgroup};
-        my $ibx;
-        for my $ngname (@window) {
-                $ibx = $groups->{$ngname} and
-                        more($self, "$ngname ".$ibx->description);
-        }
-        scalar(@$groupnames); # continue if there's more
+        my ($self, $ibxs) = @_;
+        my @window = splice(@$ibxs, 0, 1000);
+        $self->msg_more(join('', map {
+                "$_->{newsgroup} ".$_->description."\r\n"
+        } @window));
+        scalar @$ibxs; # continue if there's more
 }
 
 sub list_newsgroups ($;$) { # called by cmd_list
         my ($self, $wildmat) = @_;
         wildmat2re($wildmat);
-        long_response($self, \&list_newsgroups_i, [
-                grep(/$wildmat/, @{$self->{nntpd}->{groupnames}}) ]);
+        my @names = grep(/$wildmat/, @{$self->{nntpd}->{groupnames}});
+        $self->long_response(\&list_newsgroups_i, names2ibx($self, \@names));
 }
 
 # LIST SUBSCRIPTIONS, DISTRIB.PATS are not supported
@@ -191,12 +176,11 @@ sub cmd_list ($;$$) {
                 $arg = "list_$arg";
                 $arg = $self->can($arg);
                 return r501 unless $arg && args_ok($arg, scalar @args);
-                more($self, '215 information follows');
+                $self->msg_more("215 information follows\r\n");
                 $arg->($self, @args);
         } else {
-                more($self, '215 list of newsgroups follows');
-                long_response($self, \&list_active_i, [ # copy array
-                        @{$self->{nntpd}->{groupnames}} ]);
+                $self->msg_more("215 list of newsgroups follows\r\n");
+                $self->long_response(\&list_active_i, names2ibx($self));
         }
 }
 
@@ -212,7 +196,7 @@ sub listgroup_all_i {
         my ($self, $num) = @_;
         my $ary = $self->{ibx}->over(1)->ids_after($num);
         scalar(@$ary) or return;
-        more($self, join("\r\n", @$ary));
+        $self->msg_more(join("\r\n", @$ary, ''));
         1;
 }
 
@@ -220,16 +204,16 @@ sub cmd_listgroup ($;$$) {
         my ($self, $group, $range) = @_;
         if (defined $group) {
                 my $res = cmd_group($self, $group);
-                return $res if ($res !~ /\A211 /);
-                more($self, $res);
+                return $res if ref($res); # error if const strref
+                $self->msg_more($res);
         }
-        $self->{ibx} or return '412 no newsgroup selected';
+        $self->{ibx} or return \"412 no newsgroup selected\r\n";
         if (defined $range) {
                 my $r = get_range($self, $range);
                 return $r unless ref $r;
-                long_response($self, \&listgroup_range_i, @$r);
+                $self->long_response(\&listgroup_range_i, @$r);
         } else { # grab every article number
-                long_response($self, \&listgroup_all_i, \(my $num = 0));
+                $self->long_response(\&listgroup_all_i, \(my $num = 0));
         }
 }
 
@@ -259,23 +243,27 @@ sub parse_time ($$;$) {
         }
 }
 
-sub group_line ($$) {
-        my ($self, $ibx) = @_;
-        my ($min, $max) = $ibx->mm(1)->minmax;
-        more($self, "$ibx->{newsgroup} $max $min n");
+sub emit_group_lines {
+        my ($self, $ibxs) = @_;
+        my ($min, $max);
+        my $ALL = $self->{nntpd}->{pi_cfg}->ALL;
+        my $misc = $ALL->misc if $ALL;
+        my $buf = '';
+        for my $ibx (@$ibxs) {
+                $misc ? $misc->inbox_data($ibx) :
+                        delete(@$ibx{qw(-art_min -art_max)});
+                ($min, $max) = ($ibx->art_min, $ibx->art_max);
+                $buf .= "$ibx->{newsgroup} $max $min n\r\n";
+        }
+        $self->msg_more($buf);
 }
 
 sub newgroups_i {
-        my ($self, $ts, $i, $groupnames) = @_;
-        my $end = $$i + 100;
-        my $groups = $self->{nntpd}->{pi_cfg}->{-by_newsgroup};
-        while ($$i < $end) {
-                my $ngname = $groupnames->[$$i++] // return;
-                my $ibx = $groups->{$ngname} or next; # expired on reload
-                next unless (eval { $ibx->uidvalidity } // 0) > $ts;
-                group_line($self, $ibx);
-        }
-        1;
+        my ($self, $ts, $ibxs) = @_;
+        my @window = splice(@$ibxs, 0, 1000);
+        @window = grep { (eval { $_->uidvalidity } // 0) > $ts } @window;
+        emit_group_lines($self, \@window);
+        scalar @$ibxs; # any more?
 }
 
 sub cmd_newgroups ($$$;$$) {
@@ -284,9 +272,8 @@ sub cmd_newgroups ($$$;$$) {
         return r501 if $@;
 
         # TODO dists
-        more($self, '231 list of new newsgroups follows');
-        long_response($self, \&newgroups_i, $ts, \(my $i = 0),
-                                $self->{nntpd}->{groupnames});
+        $self->msg_more("231 list of new newsgroups follows\r\n");
+        $self->long_response(\&newgroups_i, $ts, names2ibx($self));
 }
 
 sub wildmat2re (;$) {
@@ -321,22 +308,19 @@ sub ngpat2re (;$) {
 }
 
 sub newnews_i {
-        my ($self, $names, $ts, $prev) = @_;
-        my $ngname = $names->[0];
-        if (my $ibx = $self->{nntpd}->{pi_cfg}->{-by_newsgroup}->{$ngname}) {
-                if (my $over = $ibx->over) {
-                        my $msgs = $over->query_ts($ts, $$prev);
-                        if (scalar @$msgs) {
-                                $self->msg_more(join('', map {
-                                                        "<$_->{mid}>\r\n";
-                                                } @$msgs));
-                                $$prev = $msgs->[-1]->{num};
-                                return 1; # continue on current group
-                        }
+        my ($self, $ibxs, $ts, $prev) = @_;
+        if (my $over = $ibxs->[0]->over) {
+                my $msgs = $over->query_ts($ts, $$prev);
+                if (scalar @$msgs) {
+                        $self->msg_more(join('', map {
+                                                "<$_->{mid}>\r\n";
+                                        } @$msgs));
+                        $$prev = $msgs->[-1]->{num};
+                        return 1; # continue on current group
                 }
         }
-        shift @$names;
-        if (@$names) { # continue onto next newsgroup
+        shift @$ibxs;
+        if (@$ibxs) { # continue onto next newsgroup
                 $$prev = 0;
                 1;
         } else { # all done, break out of the long_response
@@ -348,45 +332,45 @@ sub cmd_newnews ($$$$;$$) {
         my ($self, $newsgroups, $date, $time, $gmt, $dists) = @_;
         my $ts = eval { parse_time($date, $time, $gmt) };
         return r501 if $@;
-        more($self, '230 list of new articles by message-id follows');
-        my ($keep, $skip) = split('!', $newsgroups, 2);
+        $self->msg_more("230 list of new articles by message-id follows\r\n");
+        my ($keep, $skip) = split(/!/, $newsgroups, 2);
         ngpat2re($keep);
         ngpat2re($skip);
-        my @names = grep(!/$skip/, grep(/$keep/,
-                                @{$self->{nntpd}->{groupnames}}));
-        return '.' unless scalar(@names);
+        my @names = grep(/$keep/, @{$self->{nntpd}->{groupnames}});
+        @names = grep(!/$skip/, @names);
+        return \".\r\n" unless scalar(@names);
         my $prev = 0;
-        long_response($self, \&newnews_i, \@names, $ts, \$prev);
+        $self->long_response(\&newnews_i, names2ibx($self, \@names),
+                                $ts, \$prev);
 }
 
 sub cmd_group ($$) {
         my ($self, $group) = @_;
         my $nntpd = $self->{nntpd};
         my $ibx = $nntpd->{pi_cfg}->{-by_newsgroup}->{$group} or
-                return '411 no such news group';
+                return \"411 no such news group\r\n";
         $nntpd->idler_start;
 
         $self->{ibx} = $ibx;
         my ($min, $max) = $ibx->mm(1)->minmax;
         $self->{article} = $min;
         my $est_size = $max - $min;
-        "211 $est_size $min $max $group";
+        "211 $est_size $min $max $group\r\n";
 }
 
 sub article_adj ($$) {
         my ($self, $off) = @_;
-        my $ibx = $self->{ibx} or return '412 no newsgroup selected';
-
-        my $n = $self->{article};
-        defined $n or return '420 no current article has been selected';
+        my $ibx = $self->{ibx} // return \"412 no newsgroup selected\r\n";
+        my $n = $self->{article} //
+                return \"420 no current article has been selected\r\n";
 
         $n += $off;
         my $mid = $ibx->mm(1)->mid_for($n) // do {
                 $n = $off > 0 ? 'next' : 'previous';
-                return "421 no $n article in this group";
+                return "421 no $n article in this group\r\n";
         };
         $self->{article} = $n;
-        "223 $n <$mid> article retrieved - request text separately";
+        "223 $n <$mid> article retrieved - request text separately\r\n";
 }
 
 sub cmd_next ($) { article_adj($_[0], 1) }
@@ -397,13 +381,13 @@ sub cmd_last ($) { article_adj($_[0], -1) }
 sub cmd_post ($) {
         my ($self) = @_;
         my $ibx = $self->{ibx};
-        $ibx ? "440 mailto:$ibx->{-primary_address} to post"
-                : '440 posting not allowed'
+        $ibx ? "440 mailto:$ibx->{-primary_address} to post\r\n"
+                : \"440 posting not allowed\r\n"
 }
 
 sub cmd_quit ($) {
         my ($self) = @_;
-        res($self, '205 closing connection - goodbye!');
+        $self->write(\"205 closing connection - goodbye!\r\n");
         $self->shutdn;
         undef;
 }
@@ -486,22 +470,22 @@ sub art_lookup ($$$) {
         my $err;
         if (defined $art) {
                 if ($art =~ /\A[0-9]+\z/) {
-                        $err = '423 no such article number in this group';
+                        $err = \"423 no such article number in this group\r\n";
                         $n = int($art);
                         goto find_ibx;
                 } elsif ($art =~ $ONE_MSGID) {
                         ($ibx, $n) = mid_lookup($self, $1);
                         goto found if $ibx;
-                        return r430;
+                        return \r430;
                 } else {
-                        return r501;
+                        return \r501;
                 }
         } else {
-                $err = '420 no current article has been selected';
+                $err = \"420 no current article has been selected\r\n";
                 $n = $self->{article} // return $err;
 find_ibx:
                 $ibx = $self->{ibx} or
-                                return '412 no newsgroup has been selected';
+                        return \"412 no newsgroup has been selected\r\n";
         }
 found:
         my $smsg = $ibx->over(1)->get_art($n) or return $err;
@@ -509,7 +493,7 @@ found:
         if ($code == 223) { # STAT
                 set_art($self, $n);
                 "223 $n <$smsg->{mid}> article retrieved - " .
-                        "request text separately";
+                        "request text separately\r\n";
         } else { # HEAD | BODY | ARTICLE
                 $smsg->{nntp} = $self;
                 $smsg->{nntp_code} = $code;
@@ -525,7 +509,7 @@ sub msg_body_write ($$) {
         # these can momentarily double the memory consumption :<
         $$msg =~ s/^\./../smg;
         $$msg =~ s/(?<!\r)\n/\r\n/sg; # Alpine barfs without this
-        $$msg .= "\r\n" unless $$msg =~ /\r\n\z/s;
+        $$msg .= "\r\n" unless substr($$msg, -2, 2) eq "\r\n";
         $self->msg_more($$msg);
 }
 
@@ -539,8 +523,7 @@ sub msg_hdr_write ($$) {
         set_nntp_headers($eml, $smsg);
 
         my $hdr = $eml->{hdr} // \(my $x = '');
-        # fixup old bug from import (pre-a0c07cba0e5d8b6a)
-        $$hdr =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+        PublicInbox::Eml::strip_from($$hdr);
         $$hdr =~ s/(?<!\r)\n/\r\n/sg; # Alpine barfs without this
 
         # for leafnode compatibility, we need to ensure Message-ID headers
@@ -553,7 +536,11 @@ sub blob_cb { # called by git->cat_async via ibx_async_cat
         my ($bref, $oid, $type, $size, $smsg) = @_;
         my $self = $smsg->{nntp};
         my $code = $smsg->{nntp_code};
-        if (!defined($oid)) {
+        if (!defined($type)) {
+                warn "E: git aborted on $oid / $smsg->{blob} ".
+                        $self->{-ibx}->{inboxdir};
+                return $self->close;
+        } elsif ($type ne 'blob') {
                 # it's possible to have TOCTOU if an admin runs
                 # public-inbox-(edit|purge), just move onto the next message
                 warn "E: $smsg->{blob} missing in $smsg->{-ibx}->{inboxdir}\n";
@@ -565,21 +552,21 @@ sub blob_cb { # called by git->cat_async via ibx_async_cat
         my $r = "$code $smsg->{num} <$smsg->{mid}> article retrieved - ";
         my $eml = PublicInbox::Eml->new($bref);
         if ($code == 220) {
-                more($self, $r .= 'head and body follow');
+                $self->msg_more($r .= "head and body follow\r\n");
                 msg_hdr_write($eml, $smsg);
                 $self->msg_more("\r\n");
                 msg_body_write($self, $bref);
         } elsif ($code == 221) {
-                more($self, $r .= 'head follows');
+                $self->msg_more($r .= "head follows\r\n");
                 msg_hdr_write($eml, $smsg);
         } elsif ($code == 222) {
-                more($self, $r .= 'body follows');
+                $self->msg_more($r .= "body follows\r\n");
                 msg_body_write($self, $bref);
         } else {
                 $self->close;
                 die "BUG: bad code: $r";
         }
-        $self->write(\".\r\n"); # flushes (includes ->zflush)
+        $self->write(\".\r\n"); # flushes (includes ->dflush)
         $self->requeue;
 }
 
@@ -603,20 +590,18 @@ sub cmd_stat ($;$) {
         art_lookup($self, $art, 223); # art may be msgid
 }
 
-sub cmd_ihave ($) { '435 article not wanted - do not send it' }
+sub cmd_ihave ($) { \"435 article not wanted - do not send it\r\n" }
 
-sub cmd_date ($) { '111 '.strftime('%Y%m%d%H%M%S', gmtime(time)) }
+sub cmd_date ($) { '111 '.strftime('%Y%m%d%H%M%S', gmtime(time))."\r\n" }
 
-sub cmd_help ($) {
-        my ($self) = @_;
-        more($self, '100 help text follows');
-        '.'
-}
+sub cmd_help ($) { \"100 help text follows\r\n.\r\n" }
 
+# returns a ref on success
 sub get_range ($$) {
         my ($self, $range) = @_;
-        my $ibx = $self->{ibx} or return '412 no news group has been selected';
-        defined $range or return '420 No article(s) selected';
+        my $ibx = $self->{ibx} //
+                return "412 no news group has been selected\r\n";
+        $range // return "420 No article(s) selected\r\n";
         my ($beg, $end);
         my ($min, $max) = $ibx->mm(1)->minmax;
         if ($range =~ /\A([0-9]+)\z/) {
@@ -630,59 +615,10 @@ sub get_range ($$) {
         }
         $beg = $min if ($beg < $min);
         $end = $max if ($end > $max);
-        return '420 No article(s) selected' if ($beg > $end);
-        [ \$beg, $end ];
-}
-
-sub long_step {
-        my ($self) = @_;
-        # wbuf is unset or empty, here; {long} may add to it
-        my ($fd, $cb, $t0, @args) = @{$self->{long_cb}};
-        my $more = eval { $cb->($self, @args) };
-        if ($@ || !$self->{sock}) { # something bad happened...
-                delete $self->{long_cb};
-                my $elapsed = now() - $t0;
-                if ($@) {
-                        err($self,
-                            "%s during long response[$fd] - %0.6f",
-                            $@, $elapsed);
-                }
-                out($self, " deferred[$fd] aborted - %0.6f", $elapsed);
-                $self->close;
-        } elsif ($more) { # $self->{wbuf}:
-                # COMPRESS users all share the same DEFLATE context.
-                # Flush it here to ensure clients don't see
-                # each other's data
-                $self->zflush;
-
-                # no recursion, schedule another call ASAP, but only after
-                # all pending writes are done.  autovivify wbuf:
-                my $new_size = push(@{$self->{wbuf}}, \&long_step);
-
-                # wbuf may be populated by $cb, no need to rearm if so:
-                $self->requeue if $new_size == 1;
-        } else { # all done!
-                delete $self->{long_cb};
-                res($self, '.');
-                my $elapsed = now() - $t0;
-                my $fd = fileno($self->{sock});
-                out($self, " deferred[$fd] done - %0.6f", $elapsed);
-                my $wbuf = $self->{wbuf}; # do NOT autovivify
-                $self->requeue unless $wbuf && @$wbuf;
-        }
+        $beg > $end ? "420 No article(s) selected\r\n" : [ \$beg, $end ];
 }
 
-sub long_response ($$;@) {
-        my ($self, $cb, @args) = @_; # cb returns true if more, false if done
-
-        my $sock = $self->{sock} or return;
-        # make sure we disable reading during a long response,
-        # clients should not be sending us stuff and making us do more
-        # work while we are stream a response to them
-        $self->{long_cb} = [ fileno($sock), $cb, now(), @args ];
-        long_step($self); # kick off!
-        undef;
-}
+sub long_response_done { $_[0]->write(\".\r\n") } # overrides superclass
 
 sub hdr_msgid_range_i {
         my ($self, $beg, $end) = @_;
@@ -703,8 +639,8 @@ sub hdr_message_id ($$$) { # optimize XHDR Message-ID [range] for slrnpull.
                 $range = $self->{article} unless defined $range;
                 my $r = get_range($self, $range);
                 return $r unless ref $r;
-                more($self, $xhdr ? r221 : r225);
-                long_response($self, \&hdr_msgid_range_i, @$r);
+                $self->msg_more($xhdr ? r221 : r225);
+                $self->long_response(\&hdr_msgid_range_i, @$r);
         }
 }
 
@@ -775,8 +711,8 @@ sub hdr_xref ($$$) { # optimize XHDR Xref [range] for rtin
                 $range = $self->{article} unless defined $range;
                 my $r = get_range($self, $range);
                 return $r unless ref $r;
-                more($self, $xhdr ? r221 : r225);
-                long_response($self, \&xref_range_i, @$r);
+                $self->msg_more($xhdr ? r221 : r225);
+                $self->long_response(\&xref_range_i, @$r);
         }
 }
 
@@ -819,8 +755,8 @@ sub hdr_smsg ($$$$) {
                 $range = $self->{article} unless defined $range;
                 my $r = get_range($self, $range);
                 return $r unless ref $r;
-                more($self, $xhdr ? r221 : r225);
-                long_response($self, \&smsg_range_i, @$r, $field);
+                $self->msg_more($xhdr ? r221 : r225);
+                $self->long_response(\&smsg_range_i, @$r, $field);
         }
 }
 
@@ -837,7 +773,7 @@ sub do_hdr ($$$;$) {
         } elsif ($sub =~ /\A:(bytes|lines)\z/) {
                 hdr_smsg($self, $xhdr, $1, $range);
         } else {
-                $xhdr ? (r221 . "\r\n.") : "503 HDR not permitted on $header";
+                $xhdr ? (r221.".\r\n") : "503 HDR not permitted on $header\r\n";
         }
 }
 
@@ -867,37 +803,30 @@ sub hdr_mid_prefix ($$$$$) {
 
 sub hdr_mid_response ($$$$$$) {
         my ($self, $xhdr, $ibx, $n, $mid, $v) = @_;
-        my $res = '';
-        if ($xhdr) {
-                $res .= r221 . "\r\n";
-                $res .= "$mid $v\r\n";
-        } else {
-                $res .= r225 . "\r\n";
-                my $pfx = hdr_mid_prefix($self, $xhdr, $ibx, $n, $mid);
-                $res .= "$pfx $v\r\n";
-        }
-        res($self, $res .= '.');
+        $self->write(($xhdr ? r221.$mid :
+                   r225.hdr_mid_prefix($self, $xhdr, $ibx, $n, $mid)) .
+                " $v\r\n.\r\n");
         undef;
 }
 
 sub xrover_i {
         my ($self, $beg, $end) = @_;
         my $h = over_header_for($self->{ibx}, $$beg, 'references');
-        more($self, "$$beg $h") if defined($h);
+        $self->msg_more("$$beg $h\r\n") if defined($h);
         $$beg++ < $end;
 }
 
 sub cmd_xrover ($;$) {
         my ($self, $range) = @_;
-        my $ibx = $self->{ibx} or return '412 no newsgroup selected';
+        my $ibx = $self->{ibx} or return \"412 no newsgroup selected\r\n";
         (defined $range && $range =~ /[<>]/) and
-                return '420 No article(s) selected'; # no message IDs
+                return \"420 No article(s) selected\r\n"; # no message IDs
 
         $range = $self->{article} unless defined $range;
         my $r = get_range($self, $range);
         return $r unless ref $r;
-        more($self, '224 Overview information follows');
-        long_response($self, \&xrover_i, @$r);
+        $self->msg_more("224 Overview information follows\r\n");
+        $self->long_response(\&xrover_i, @$r);
 }
 
 sub over_line ($$$) {
@@ -923,7 +852,8 @@ sub cmd_over ($;$) {
                 my ($ibx, $n) = mid_lookup($self, $1);
                 defined $n or return r430;
                 my $smsg = $ibx->over(1)->get_art($n) or return r430;
-                more($self, '224 Overview information follows (multi-line)');
+                $self->msg_more(
+                        "224 Overview information follows (multi-line)\r\n");
 
                 # Only set article number column if it's the current group
                 # (RFC 3977 8.3.2)
@@ -933,8 +863,7 @@ sub cmd_over ($;$) {
                         $smsg->{-orig_num} = $smsg->{num};
                         $smsg->{num} = 0;
                 }
-                $self->msg_more(over_line($self, $ibx, $smsg));
-                '.';
+                over_line($self, $ibx, $smsg).".\r\n";
         } else {
                 cmd_xover($self, $range);
         }
@@ -959,21 +888,20 @@ sub cmd_xover ($;$) {
         my $r = get_range($self, $range);
         return $r unless ref $r;
         my ($beg, $end) = @$r;
-        more($self, "224 Overview information follows for $$beg to $end");
-        long_response($self, \&xover_i, @$r);
+        $self->msg_more(
+                "224 Overview information follows for $$beg to $end\r\n");
+        $self->long_response(\&xover_i, @$r);
 }
 
-sub compressed { undef }
-
 sub cmd_starttls ($) {
         my ($self) = @_;
-        my $sock = $self->{sock} or return;
         # RFC 4642 2.2.1
-        return r502 if ($sock->can('accept_SSL') || $self->compressed);
-        my $opt = $self->{nntpd}->{accept_tls} or
-                return '580 can not initiate TLS negotiation';
-        res($self, '382 Continue with TLS negotiation');
-        $self->{sock} = IO::Socket::SSL->start_SSL($sock, %$opt);
+        (($self->{sock} // return)->can('stop_SSL') || $self->compressed) and
+                return r502;
+        $self->{nntpd}->{ssl_ctx_opt} or
+                return \"580 can not initiate TLS negotiation\r\n";
+        $self->write(\"382 Continue with TLS negotiation\r\n");
+        PublicInbox::TLS::start($self->{sock}, $self->{nntpd});
         $self->requeue if PublicInbox::DS::accept_tls_step($self);
         undef;
 }
@@ -981,15 +909,15 @@ sub cmd_starttls ($) {
 # RFC 8054
 sub cmd_compress ($$) {
         my ($self, $alg) = @_;
-        return '503 Only DEFLATE is supported' if uc($alg) ne 'DEFLATE';
+        return "503 Only DEFLATE is supported\r\n" if uc($alg) ne 'DEFLATE';
         return r502 if $self->compressed;
-        PublicInbox::NNTPdeflate->enable($self);
+        PublicInbox::NNTPdeflate->enable($self) or return
+                                \"403 Unable to activate compression\r\n";
+        PublicInbox::DS::write($self, \"206 Compression active\r\n");
         $self->requeue;
         undef
 }
 
-sub zflush {} # overridden by NNTPdeflate
-
 sub cmd_xpath ($$) {
         my ($self, $mid) = @_;
         return r501 unless $mid =~ $ONE_MSGID;
@@ -1015,25 +943,8 @@ sub cmd_xpath ($$) {
                         push @paths, "$ibx->{newsgroup}/$n";
                 }
         }
-        return '430 no such article on server' unless @paths;
-        '223 '.join(' ', sort(@paths));
-}
-
-sub res ($$) { do_write($_[0], $_[1] . "\r\n") }
-
-sub more ($$) { $_[0]->msg_more($_[1] . "\r\n") }
-
-sub do_write ($$) {
-        my $self = $_[0];
-        my $done = $self->write(\($_[1]));
-        return 0 unless $self->{sock};
-
-        $done;
-}
-
-sub err ($$;@) {
-        my ($self, $fmt, @args) = @_;
-        printf { $self->{nntpd}->{err} } $fmt."\n", @args;
+        return \"430 no such article on server\r\n" unless @paths;
+        '223 '.join(' ', sort(@paths))."\r\n";
 }
 
 sub out ($$;@) {
@@ -1044,7 +955,7 @@ sub out ($$;@) {
 # callback used by PublicInbox::DS for any (e)poll (in/out/hup/err)
 sub event_step {
         my ($self) = @_;
-
+        local $SIG{__WARN__} = $self->{nntpd}->{warn_cb};
         return unless $self->flush_write && $self->{sock} && !$self->{long_cb};
 
         # only read more requests if we've drained the write buffer,
@@ -1079,4 +990,8 @@ sub busy { # for graceful shutdown in PublicInbox::Daemon:
         defined($self->{rbuf}) || defined($self->{wbuf})
 }
 
+package PublicInbox::NNTPdeflate;
+use PublicInbox::DSdeflate;
+our @ISA = qw(PublicInbox::DSdeflate PublicInbox::NNTP);
+
 1;
diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm
index 0350830b..4401a29b 100644
--- a/lib/PublicInbox/NNTPD.pm
+++ b/lib/PublicInbox/NNTPD.pm
@@ -9,33 +9,32 @@ use v5.10.1;
 use Sys::Hostname;
 use PublicInbox::Config;
 use PublicInbox::InboxIdle;
-use PublicInbox::NNTPdeflate; # loads PublicInbox::NNTP
+use PublicInbox::NNTP;
 
 sub new {
         my ($class) = @_;
-        my $pi_cfg = PublicInbox::Config->new;
-        my $name = $pi_cfg->{'publicinbox.nntpserver'};
-        if (!defined($name) or $name eq '') {
-                $name = hostname;
-        } elsif (ref($name) eq 'ARRAY') {
-                $name = $name->[0];
-        }
-
         bless {
-                groups => {},
                 err => \*STDERR,
                 out => \*STDOUT,
-                pi_cfg => $pi_cfg,
-                servername => $name,
-                greet => \"201 $name ready - post via email\r\n",
-                # accept_tls => { SSL_server => 1, ..., SSL_reuse_ctx => ... }
+                # pi_cfg => $pi_cfg,
+                # ssl_ctx_opt => { SSL_cert_file => ..., SSL_key_file => ... }
                 # idler => PublicInbox::InboxIdle
         }, $class;
 }
 
 sub refresh_groups {
         my ($self, $sig) = @_;
-        my $pi_cfg = $sig ? PublicInbox::Config->new : $self->{pi_cfg};
+        my $pi_cfg = PublicInbox::Config->new;
+        my $name = $pi_cfg->{'publicinbox.nntpserver'};
+        if (!defined($name) or $name eq '') {
+                $name = hostname;
+        } elsif (ref($name) eq 'ARRAY') {
+                $name = $name->[0];
+        }
+        if ($name ne ($self->{servername} // '')) {
+                $self->{servername} = $name;
+                $self->{greet} = \"201 $name ready - post via email\r\n";
+        }
         my $groups = $pi_cfg->{-by_newsgroup}; # filled during each_inbox
         my $cache = eval { $pi_cfg->ALL->misc->nntpd_cache_load } // {};
         $pi_cfg->each_inbox(sub {
@@ -46,16 +45,14 @@ sub refresh_groups {
                         # only valid if msgmap and over works
                         # preload to avoid fragmentation:
                         $ibx->description;
-                        $ibx->base_url;
                 } else {
                         delete $groups->{$ngname};
-                        delete $ibx->{newsgroup};
                         # Note: don't be tempted to delete more for memory
                         # savings just yet: NNTP, IMAP, and WWW may all
                         # run in the same process someday.
                 }
         });
-        $self->{groupnames} = [ sort(keys %$groups) ];
+        @{$self->{groupnames}} = sort(keys %$groups);
         # this will destroy old groups that got deleted
         $self->{pi_cfg} = $pi_cfg;
 }
diff --git a/lib/PublicInbox/NetNNTPSocks.pm b/lib/PublicInbox/NetNNTPSocks.pm
index 8495204a..d27efba1 100644
--- a/lib/PublicInbox/NetNNTPSocks.pm
+++ b/lib/PublicInbox/NetNNTPSocks.pm
@@ -1,14 +1,13 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# wrap Net::NNTP client with SOCKS support
+# wrap Net::NNTP client with SOCKS support.  Convoluted, but AFAIK this
+# is the only way to get SOCKS working with Net::NNTP w/o LD_PRELOAD.
 package PublicInbox::NetNNTPSocks;
-use strict;
-use v5.10.1;
+use v5.12;
 use Net::NNTP;
-our %OPT;
+our %OPT; # used to pass options between ->new_socks and our ->new
 our @ISA = qw(IO::Socket::Socks);
-my @SOCKS_KEYS = qw(ProxyAddr ProxyPort SocksVersion SocksDebug SocksResolve);
 
 # use this instead of Net::NNTP->new if using Proxy*
 sub new_socks {
@@ -17,17 +16,21 @@ sub new_socks {
         local @Net::NNTP::ISA = (qw(Net::Cmd), __PACKAGE__);
         local %OPT = map {;
                 defined($opt{$_}) ? ($_ => $opt{$_}) : ()
-        } @SOCKS_KEYS;
-        Net::NNTP->new(%opt); # this calls our new() below:
+        } qw(ProxyAddr ProxyPort SocksVersion SocksDebug SocksResolve);
+        no warnings 'uninitialized'; # needed for $SOCKS_ERROR
+        my $ret = Net::NNTP->new(%opt); # calls PublicInbox::NetNNTPSocks::new
+        return $ret if $ret || $!{EINTR};
+        $ret // die "errors: \$!=$! SOCKS=",
+                                eval('$IO::Socket::Socks::SOCKS_ERROR // ""'),
+                                ', SSL=',
+                                (eval('IO::Socket::SSL->errstr')  // ''), "\n";
 }
 
 # called by Net::NNTP->new
 sub new {
         my ($self, %opt) = @_;
         @OPT{qw(ConnectAddr ConnectPort)} = @opt{qw(PeerAddr PeerPort)};
-        my $ret = $self->SUPER::new(%OPT) or
-                die 'SOCKS error: '.eval('$IO::Socket::Socks::SOCKS_ERROR');
-        $ret;
+        $self->SUPER::new(%OPT);
 }
 
 1;
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index c1af03a3..ec18818b 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -3,8 +3,7 @@
 
 # common reader code for IMAP and NNTP (and maybe JMAP)
 package PublicInbox::NetReader;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(Exporter PublicInbox::IPC);
 use PublicInbox::Eml;
 use PublicInbox::Config;
@@ -15,7 +14,7 @@ our @EXPORT = qw(uri_section imap_uri nntp_uri);
 
 sub ndump {
         require Data::Dumper;
-        Data::Dumper->new(\@_)->Useqq(1)->Terse(1)->Dump;
+        Data::Dumper->new([ $_[-1] ])->Useqq(1)->Terse(1)->Dump;
 }
 
 # returns the git config section name, e.g [imap "imaps://user@example.com"]
@@ -41,10 +40,27 @@ EOM
         die "$val not understood (only socks5h:// is supported)\n";
 }
 
+# gives an arrayref suitable for the Mail::IMAPClient Ssl or Starttls arg
+sub mic_tls_opt ($$) {
+        my ($o, $hostname) = @_;
+        require IO::Socket::SSL;
+        $o = {} if !ref($o);
+        $o->{SSL_hostname} //= $hostname;
+        [ map { ($_, $o->{$_}) } keys %$o ];
+}
+
+sub set_ssl_verify_mode ($$) {
+        my ($o, $bool) = @_;
+        require IO::Socket::SSL;
+        $o->{SSL_verify_mode} = $bool ? IO::Socket::SSL::SSL_VERIFY_PEER() :
+                                        IO::Socket::SSL::SSL_VERIFY_NONE();
+}
+
 sub mic_new ($$$$) {
         my ($self, $mic_arg, $sec, $uri) = @_;
         my %mic_arg = (%$mic_arg, Keepalive => 1);
         my $sa = $self->{cfg_opt}->{$sec}->{-proxy_cfg} || $self->{-proxy_cli};
+        my ($mic, $s, $t);
         if ($sa) {
                 # this `require' needed for worker[1..Inf], since socks_args
                 # only got called in worker[0]
@@ -53,16 +69,37 @@ sub mic_new ($$$$) {
                 $opt{SocksDebug} = 1 if $mic_arg{Debug};
                 $opt{ConnectAddr} = delete $mic_arg{Server};
                 $opt{ConnectPort} = delete $mic_arg{Port};
-                my $s = IO::Socket::Socks->new(%opt) or die
-                        "E: <$uri> ".eval('$IO::Socket::Socks::SOCKS_ERROR');
-                if ($mic_arg->{Ssl}) { # for imaps://
-                        require IO::Socket::SSL;
-                        $s = IO::Socket::SSL->start_SSL($s) or die
-                                "E: <$uri> ".(IO::Socket::SSL->errstr // '');
-                }
+                do {
+                        $! = 0;
+                        $s = IO::Socket::Socks->new(%opt);
+                } until ($s || !$!{EINTR} || $self->{quit});
+                return if $self->{quit};
+                $s or die "E: <$uri> ".eval('$IO::Socket::Socks::SOCKS_ERROR');
                 $mic_arg{Socket} = $s;
+                if (my $o = delete $mic_arg{Ssl}) { # for imaps://
+                        $o = mic_tls_opt($o, $opt{ConnectAddr});
+                        do {
+                                $! = 0;
+                                $t = IO::Socket::SSL->start_SSL($s, @$o);
+                        } until ($t || !$!{EINTR} || $self->{quit});
+                        return if $self->{quit};
+                        $t or die "E: <$uri> ".(IO::Socket::SSL->errstr // '');
+                        $mic_arg{Socket} = $t;
+                } elsif ($o = $mic_arg{Starttls}) {
+                        # Mail::IMAPClient will use this:
+                        $mic_arg{Starttls} = mic_tls_opt($o, $opt{ConnectAddr});
+                }
+        } elsif ($mic_arg{Ssl} || $mic_arg{Starttls}) {
+                for my $f (qw(Ssl Starttls)) {
+                        my $o = $mic_arg{$f} or next;
+                        $mic_arg{$f} = mic_tls_opt($o, $mic_arg{Server});
+                }
         }
-        PublicInbox::IMAPClient->new(%mic_arg);
+        do {
+                $! = 0;
+                $mic = PublicInbox::IMAPClient->new(%mic_arg);
+        } until ($mic || !$!{EINTR} || $self->{quit});
+        $mic;
 }
 
 sub auth_anon_cb { '' }; # for Mail::IMAPClient::Authcallback
@@ -72,6 +109,7 @@ sub onion_hint ($$) {
         $uri->host =~ /\.onion\z/i or return "\n";
         my $t = $uri->isa('PublicInbox::URIimap') ? 'imap' : 'nntp';
         my $url = PublicInbox::Config::squote_maybe(uri_section($uri));
+        my $scheme = $uri->scheme;
         my $set_cfg = 'lei config';
         if (!$lei) { # public-inbox-watch
                 my $f = PublicInbox::Config::squote_maybe(
@@ -87,6 +125,10 @@ try configuring a socks5h:// proxy:
         url=$url
         $set_cfg $t.$dq\$url$dq.proxy socks5h://127.0.0.1:9050
 
+git 2.26+ users may instead rely on `*' to match all .onion URLs:
+
+        $set_cfg '$t.$scheme://*.onion.proxy' socks5h://127.0.0.1:9050
+
 ...before retrying your current command
 EOM
 }
@@ -122,9 +164,9 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient
                 Server => $host,
                 %$common, # may set Starttls, Compress, Debug ....
         };
-        $mic_arg->{Ssl} = 1 if $uri->scheme eq 'imaps';
         require PublicInbox::IMAPClient;
         my $mic = mic_new($self, $mic_arg, $sec, $uri);
+        return if $self->{quit};
         ($mic && $mic->IsConnected) or
                 die "E: <$uri> new: $@".onion_hint($lei, $uri);
 
@@ -175,17 +217,20 @@ sub mic_for ($$$$) { # mic = Mail::IMAPClient
         $mic;
 }
 
-sub nn_new ($$$) {
-        my ($nn_arg, $nntp_cfg, $uri) = @_;
+sub nn_new ($$$$) {
+        my ($self, $nn_arg, $nntp_cfg, $uri) = @_;
         my $nn;
+        my ($Net_NNTP, $new) = qw(Net::NNTP new);
         if (defined $nn_arg->{ProxyAddr}) {
                 require PublicInbox::NetNNTPSocks;
+                ($Net_NNTP, $new) = qw(PublicInbox::NetNNTPSocks new_socks);
                 $nn_arg->{SocksDebug} = 1 if $nn_arg->{Debug};
-                eval { $nn = PublicInbox::NetNNTPSocks->new_socks(%$nn_arg) };
-                die "E: <$uri> $@\n" if $@;
-        } else {
-                $nn = Net::NNTP->new(%$nn_arg) or return;
         }
+        do {
+                $! = 0;
+                $nn = $Net_NNTP->$new(%$nn_arg);
+        } until ($nn || !$!{EINTR} || $self->{quit});
+        $nn // return;
         setsockopt($nn, Socket::SOL_SOCKET(), Socket::SO_KEEPALIVE(), 1);
 
         # default to using STARTTLS if it's available, but allow
@@ -195,19 +240,19 @@ sub nn_new ($$$) {
                                 try_starttls($nn_arg->{Host})) {
                         # soft fail by default
                         $nn->starttls or warn <<"";
-W: <$uri> STARTTLS tried and failed (not requested)
+W: <$uri> STARTTLS tried and failed (not requested): ${\(ndump($nn->message))}
 
                 } elsif ($nntp_cfg->{starttls}) {
                         # hard fail if explicitly configured
                         $nn->starttls or die <<"";
-E: <$uri> STARTTLS requested and failed
+E: <$uri> STARTTLS requested and failed: ${\(ndump($nn->message))}
 
                 }
         } elsif ($nntp_cfg->{starttls}) {
                 $nn->can('starttls') or
                         die "E: <$uri> Net::NNTP too old for STARTTLS\n";
                 $nn->starttls or die <<"";
-E: <$uri> STARTTLS requested and failed
+E: <$uri> STARTTLS requested and failed: ${\(ndump($nn->message))}
 
         }
         $nn;
@@ -242,25 +287,32 @@ sub nn_for ($$$$) { # nn = Net::NNTP
         $nn_arg->{SSL} = 1 if $uri->secure; # snews == nntps
         my $sa = $self->{-proxy_cli};
         %$nn_arg = (%$nn_arg, %$sa) if $sa;
-        my $nn = nn_new($nn_arg, $nntp_cfg, $uri) or
-                die "E: <$uri> new: $@".onion_hint($lei, $uri);
+        my $nn = nn_new($self, $nn_arg, $nntp_cfg, $uri);
+        return if $self->{quit};
+        $nn // die "E: <$uri> new: $@".onion_hint($lei, $uri);
         if ($cred) {
-                $cred->fill($lei) unless defined($p); # may prompt user here
+                $p //= do {
+                        $cred->fill($lei); # may prompt user here
+                        $cred->{password};
+                };
                 if ($nn->authinfo($u, $p)) {
                         push @{$nntp_cfg->{-postconn}}, [ 'authinfo', $u, $p ];
                 } else {
-                        warn "E: <$uri> AUTHINFO $u XXXX failed\n";
+                        warn <<EOM;
+E: <$uri> AUTHINFO $u XXXX: ${\(ndump($nn->message))}
+EOM
                         $nn = undef;
                 }
         }
-
-        if ($nntp_cfg->{compress}) {
+        if ($nn && $nntp_cfg->{compress}) {
                 # https://rt.cpan.org/Ticket/Display.html?id=129967
                 if ($nn->can('compress')) {
                         if ($nn->compress) {
                                 push @{$nntp_cfg->{-postconn}}, [ 'compress' ];
                         } else {
-                                warn "W: <$uri> COMPRESS failed\n";
+                                warn <<EOM;
+W: <$uri> COMPRESS: ${\(ndump($nn->message))}
+EOM
                         }
                 } else {
                         delete $nntp_cfg->{compress};
@@ -304,14 +356,6 @@ sub cfg_intvl ($$$) {
         }
 }
 
-sub cfg_bool ($$$) {
-        my ($cfg, $key, $url) = @_;
-        my $orig = $cfg->urlmatch($key, $url) // return;
-        my $bool = $cfg->git_bool($orig);
-        warn "W: $key=$orig for $url is not boolean\n" unless defined($bool);
-        $bool;
-}
-
 # flesh out common IMAP-specific data structures
 sub imap_common_init ($;$) {
         my ($self, $lei) = @_;
@@ -329,11 +373,12 @@ sub imap_common_init ($;$) {
 
                 # knobs directly for Mail::IMAPClient->new
                 for my $k (qw(Starttls Debug Compress)) {
-                        my $bool = cfg_bool($cfg, "imap.$k", $$uri) // next;
-                        $mic_common->{$sec}->{$k} = $bool;
+                        my $v = $cfg->urlmatch('--bool', "imap.$k", $$uri);
+                        $mic_common->{$sec}->{$k} = $v if defined $v;
                 }
                 my $to = cfg_intvl($cfg, 'imap.timeout', $$uri);
                 $mic_common->{$sec}->{Timeout} = $to if $to;
+                $mic_common->{$sec}->{Ssl} = 1 if $uri->scheme eq 'imaps';
 
                 # knobs we use ourselves:
                 my $sa = socks_args($cfg->urlmatch('imap.Proxy', $$uri));
@@ -343,11 +388,18 @@ sub imap_common_init ($;$) {
                         $self->{cfg_opt}->{$sec}->{$k} = $to;
                 }
                 my $k = 'imap.fetchBatchSize';
-                my $bs = $cfg->urlmatch($k, $$uri) // next;
-                if ($bs =~ /\A([0-9]+)\z/ && $bs > 0) {
-                        $self->{cfg_opt}->{$sec}->{batch_size} = $bs;
-                } else {
-                        warn "$k=$bs is not a positive integer\n";
+                if (defined(my $bs = $cfg->urlmatch($k, $$uri))) {
+                        ($bs =~ /\A([0-9]+)\z/ && $bs > 0) ?
+                                ($self->{cfg_opt}->{$sec}->{batch_size} = $bs) :
+                                warn("$k=$bs is not a positive integer\n");
+                }
+                my $v = $cfg->urlmatch(qw(--bool imap.sslVerify), $$uri);
+                if (defined $v) {
+                        my $cur = $mic_common->{$sec} //= {};
+                        $cur->{Starttls} //= 1 if !$cur->{Ssl};
+                        for my $f (grep { $cur->{$_} } qw(Ssl Starttls)) {
+                                set_ssl_verify_mode($cur->{$f} = {}, $v);
+                        }
                 }
         }
         # make sure we can connect and cache the credentials in memory
@@ -356,8 +408,9 @@ sub imap_common_init ($;$) {
                 my $sec = uri_section($orig_uri);
                 my $uri = PublicInbox::URIimap->new("$sec/");
                 my $mic = $mics->{$sec} //=
-                                mic_for($self, $uri, $mic_common, $lei) //
-                                die "Unable to continue\n";
+                                mic_for($self, $uri, $mic_common, $lei);
+                return if $self->{quit};
+                $mic // die "Unable to continue\n";
                 next unless $self->isa('PublicInbox::NetWriter');
                 next if $self->{-skip_creat};
                 my $dst = $orig_uri->mailbox // next;
@@ -383,7 +436,7 @@ sub nntp_common_init ($;$) {
                 my $args = $nn_common->{$sec} //= {};
 
                 # Debug and Timeout are passed to Net::NNTP->new
-                my $v = cfg_bool($cfg, 'nntp.Debug', $$uri);
+                my $v = $cfg->urlmatch(qw(--bool nntp.Debug), $$uri);
                 $args->{Debug} = $v if defined $v;
                 my $to = cfg_intvl($cfg, 'nntp.Timeout', $$uri);
                 $args->{Timeout} = $to if $to;
@@ -392,9 +445,11 @@ sub nntp_common_init ($;$) {
 
                 # Net::NNTP post-connect commands
                 for my $k (qw(starttls compress)) {
-                        $v = cfg_bool($cfg, "nntp.$k", $$uri) // next;
-                        $self->{cfg_opt}->{$sec}->{$k} = $v;
+                        $v = $cfg->urlmatch('--bool', "nntp.$k", $$uri);
+                        $self->{cfg_opt}->{$sec}->{$k} = $v if defined $v;
                 }
+                $v = $cfg->urlmatch(qw(--bool nntp.sslVerify), $$uri);
+                set_ssl_verify_mode($args, $v) if defined $v;
 
                 # -watch internal option
                 for my $k (qw(pollInterval)) {
@@ -685,7 +740,13 @@ sub mic_get {
         }
         my $mic = mic_new($self, $mic_arg, $sec, $uri);
         $cached //= {}; # invalid placeholder if no cache enabled
-        $mic && $mic->IsConnected ? ($cached->{$sec} = $mic) : undef;
+        if ($mic && $mic->IsConnected) {
+                $cached->{$sec} = $mic;
+        } else {
+                warn 'IMAP LastError: ',$mic->LastError, "\n" if $mic;
+                warn "IMAP errno: $!\n" if $!;
+                undef;
+        }
 }
 
 sub imap_each {
@@ -717,7 +778,7 @@ sub nn_get {
         my $nn_arg = $self->{net_arg}->{$sec} or
                         die "BUG: no Net::NNTP->new arg for $sec";
         my $nntp_cfg = $self->{cfg_opt}->{$sec};
-        $nn = nn_new($nn_arg, $nntp_cfg, $uri) or return;
+        $nn = nn_new($self, $nn_arg, $nntp_cfg, $uri) or return;
         if (my $postconn = $nntp_cfg->{-postconn}) {
                 for my $m_arg (@$postconn) {
                         my ($method, @args) = @$m_arg;
@@ -759,14 +820,12 @@ sub _nntp_fetch_all ($$$) {
         $beg = $num_a if defined($num_a) && $num_a > $beg && $num_a <= $end;
         $end = $num_b if defined($num_b) && $num_b >= $beg && $num_b < $end;
         $end = $beg if defined($num_a) && !defined($num_b);
-        my ($err, $art, $last_art, $kw); # kw stays undef, no keywords in NNTP
-        unless ($self->{quiet}) {
-                warn "# $uri fetching ARTICLE $beg..$end\n";
-        }
+        my ($err, $last_art, $kw); # kw stays undef, no keywords in NNTP
+        warn "# $uri fetching ARTICLE $beg..$end\n" if !$self->{quiet};
         my $n = $self->{max_batch};
-        for ($beg..$end) {
+        for my $art ($beg..$end) {
                 last if $self->{quit};
-                $art = $_;
+                local $0 = "#$art $group $sec";
                 if (--$n < 0) {
                         run_commit_cb($self);
                         $itrk->update_last(0, $last_art) if $itrk;
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 4a1f34f6..7917ef89 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -1,10 +1,9 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # common writer code for IMAP (and later, JMAP)
 package PublicInbox::NetWriter;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::NetReader);
 use PublicInbox::Smsg;
 use PublicInbox::MsgTime qw(msg_timestamp);
diff --git a/lib/PublicInbox/OnDestroy.pm b/lib/PublicInbox/OnDestroy.pm
index 615bc450..d9a6cd24 100644
--- a/lib/PublicInbox/OnDestroy.pm
+++ b/lib/PublicInbox/OnDestroy.pm
@@ -1,13 +1,16 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 package PublicInbox::OnDestroy;
+use v5.12;
 
 sub new {
         shift; # ($class, $cb, @args)
         bless [ @_ ], __PACKAGE__;
 }
 
+sub cancel { @{$_[0]} = () }
+
 sub DESTROY {
         my ($cb, @args) = @{$_[0]};
         if (!ref($cb) && $cb) {
diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index 786f9d92..3b7d49f5 100644
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -12,6 +12,7 @@ use DBD::SQLite;
 use PublicInbox::Smsg;
 use Compress::Zlib qw(uncompress);
 use constant DEFAULT_LIMIT => 1000;
+use List::Util (); # for max
 
 sub dbh_new {
         my ($self, $rw) = @_;
@@ -81,7 +82,13 @@ sub dbh_close {
         }
 }
 
-sub dbh ($) { $_[0]->{dbh} //= $_[0]->dbh_new } # dbh_new may be subclassed
+sub dbh ($) {
+        my ($self) = @_;
+        $self->{dbh} // do {
+                my $dbh = $self->dbh_new; # dbh_new may be subclassed
+                $self->{dbh} = $dbh;
+        }
+}
 
 sub load_from_row ($;$) {
         my ($smsg, $cull) = @_;
@@ -193,15 +200,22 @@ ORDER BY $sort_col DESC
                 # TODO separate strict and loose matches here once --reindex
                 # is fixed to preserve `tid' properly
                 push @$msgs, @$loose;
+
+                # we wanted to retrieve the latest loose messages; but preserve
+                # chronological ordering for threading /$INBOX/$MSGID/[tT]/
+                $sort_col eq 'ds' and
+                        @$msgs = sort { $a->{ds} <=> $b->{ds} } @$msgs;
         }
         ($nr, $msgs);
 }
 
 # strict `tid' matches, only, for thread-expanded mbox.gz search results
-# and future CLI interface
+# and lei
 # returns true if we have IDs, undef if not
 sub expand_thread {
         my ($self, $ctx) = @_;
+        # previous maxuid for LeiSavedSearch is our min:
+        my $lss_min = $ctx->{min} // 0;
         my $dbh = dbh($self);
         do {
                 defined(my $num = $ctx->{ids}->[0]) or return;
@@ -214,7 +228,7 @@ SELECT num FROM over WHERE tid = ? AND num > ?
 ORDER BY num ASC LIMIT 1000
 
                         my $xids = $dbh->selectcol_arrayref($sql, undef, $tid,
-                                                        $ctx->{prev} // 0);
+                                List::Util::max($ctx->{prev} // 0, $lss_min));
                         if (scalar(@$xids)) {
                                 $ctx->{prev} = $xids->[-1];
                                 $ctx->{xids} = $xids;
@@ -253,9 +267,12 @@ SELECT ts,ds,ddd FROM over WHERE $s
 sub get_art {
         my ($self, $num) = @_;
         # caching $sth ourselves is faster than prepare_cached
-        my $sth = $self->{-get_art} //= dbh($self)->prepare(<<'');
+        my $sth = $self->{-get_art} // do {
+                my $sth = dbh($self)->prepare(<<'');
 SELECT num,tid,ds,ts,ddd FROM over WHERE num = ? LIMIT 1
 
+                $self->{-get_art} = $sth;
+        };
         $sth->execute($num);
         my $smsg = $sth->fetchrow_hashref;
         $smsg ? load_from_row($smsg) : undef;
@@ -273,13 +290,35 @@ SELECT ibx_id,xnum,oidbin FROM xref3 WHERE docid = ? ORDER BY ibx_id,xnum ASC
         my $eidx_key_sth = $dbh->prepare_cached(<<'', undef, 1);
 SELECT eidx_key FROM inboxes WHERE ibx_id = ?
 
-        [ map {
-                my $r = $_;
+        for my $r (@$rows) {
                 $eidx_key_sth->execute($r->[0]);
                 my $eidx_key = $eidx_key_sth->fetchrow_array;
                 $eidx_key //= "missing://ibx_id=$r->[0]";
-                "$eidx_key:$r->[1]:".unpack('H*', $r->[2]);
-        } @$rows ];
+                $r = "$eidx_key:$r->[1]:".unpack('H*', $r->[2]);
+        }
+        $rows;
+}
+
+sub mid2tid {
+        my ($self, $mid) = @_;
+        my $dbh = dbh($self);
+
+        my $sth = $dbh->prepare_cached(<<'', undef, 1);
+SELECT id FROM msgid WHERE mid = ? LIMIT 1
+
+        $sth->execute($mid);
+        my $id = $sth->fetchrow_array or return;
+        $sth = $dbh->prepare_cached(<<'', undef, 1);
+SELECT num FROM id2num WHERE id = ? AND num > ?
+ORDER BY num ASC LIMIT 1
+
+        $sth->execute($id, 0);
+        my $num = $sth->fetchrow_array or return;
+        $sth = $dbh->prepare(<<'');
+SELECT tid FROM over WHERE num = ? LIMIT 1
+
+        $sth->execute($num);
+        $sth->fetchrow_array;
 }
 
 sub next_by_mid {
@@ -288,7 +327,7 @@ sub next_by_mid {
 
         unless (defined $$id) {
                 my $sth = $dbh->prepare_cached(<<'', undef, 1);
-        SELECT id FROM msgid WHERE mid = ? LIMIT 1
+SELECT id FROM msgid WHERE mid = ? LIMIT 1
 
                 $sth->execute($mid);
                 $$id = $sth->fetchrow_array;
diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index e7c96e14..4f8533f7 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # for XOVER, OVER in NNTP, and feeds/homepage/threads in PSGI
@@ -17,6 +17,7 @@ use PublicInbox::MID qw/id_compress mids_for_index references/;
 use PublicInbox::Smsg qw(subject_normalized);
 use Compress::Zlib qw(compress);
 use Carp qw(croak);
+use bytes (); # length
 
 sub dbh_new {
         my ($self) = @_;
@@ -199,7 +200,7 @@ sub resolve_mid_to_tid {
         $tid // do { # create a new ghost
                 my $id = mid2id($self, $mid);
                 my $num = next_ghost_num($self);
-                $num < 0 or die "ghost num is non-negative: $num\n";
+                $num < 0 or croak "BUG: ghost num is non-negative: $num\n";
                 $tid = next_tid($self);
                 my $dbh = $self->{dbh};
                 $dbh->prepare_cached(<<'')->execute($num, $tid);
@@ -263,7 +264,10 @@ sub ddd_for ($) {
 
 sub add_overview {
         my ($self, $eml, $smsg) = @_;
-        $smsg->{lines} = $eml->body_raw =~ tr!\n!\n!;
+        my $raw = $eml->body_raw;
+        $smsg->{lines} = $raw =~ tr!\n!\n!;
+        $smsg->{bytes} //= bytes::length $raw;
+        undef $raw;
         my $mids = mids_for_index($eml);
         my $refs = $smsg->parse_references($eml, $mids);
         $mids->[0] //= do {
@@ -283,7 +287,7 @@ sub _add_over {
         my ($self, $smsg, $mid, $refs, $old_tid, $v) = @_;
         my $cur_tid = $smsg->{tid};
         my $n = $smsg->{num};
-        die "num must not be zero for $mid" if !$n;
+        croak "BUG: num must not be zero for $mid" if !$n;
         my $cur_valid = $cur_tid > $self->{min_tid};
 
         if ($n > 0) { # regular mail
@@ -454,7 +458,7 @@ sub rollback_lazy {
 
 sub dbh_close {
         my ($self) = @_;
-        die "in transaction" if $self->{txn};
+        Carp::confess('BUG: in transaction') if $self->{txn};
         $self->SUPER::dbh_close;
 }
 
@@ -509,18 +513,18 @@ EOF
                                 next;
                         }
                         $pr->(<<EOM) if $pr;
-I: ghost $r->{num} <$mid> THREADID=$r->{tid} culled
+# ghost $r->{num} <$mid> THREADID=$r->{tid} culled
 EOM
                 }
                 delete_by_num($self, $r->{num});
         }
-        $pr->("I: rethread culled $total ghosts\n") if $pr && $total;
+        $pr->("# rethread culled $total ghosts\n") if $pr && $total;
 }
 
 # used for cross-inbox search
 sub eidx_prep ($) {
         my ($self) = @_;
-        $self->{-eidx_prep} //= do {
+        $self->{-eidx_prep} // do {
                 my $dbh = $self->dbh;
                 $dbh->do(<<'');
 INSERT OR IGNORE INTO counter (key) VALUES ('eidx_docid')
@@ -565,7 +569,7 @@ CREATE TABLE IF NOT EXISTS eidx_meta (
                 $dbh->do(<<'');
 CREATE TABLE IF NOT EXISTS eidxq (docid INTEGER PRIMARY KEY NOT NULL)
 
-                1;
+                $self->{-eidx_prep} = 1;
         };
 }
 
@@ -670,4 +674,17 @@ sub vivify_xvmd {
         $smsg->{-vivify_xvmd} = \@vivify_xvmd;
 }
 
+sub fork_ok {
+        state $fork_ok = eval("v$DBD::SQLite::sqlite_version") ge v3.8.3;
+        return 1 if $fork_ok;
+        my ($opt) = @_;
+        my @j = split(/,/, $opt->{jobs} // '');
+        state $warned;
+        grep { $_ > 1 } @j and $warned //= warn(<<EOM);
+DBD::SQLite version is v$DBD::SQLite::sqlite_version, need >= v3.8.3 for --jobs > 1
+EOM
+        $opt->{jobs} = '1,1';
+        undef;
+}
+
 1;
diff --git a/lib/PublicInbox/POP3.pm b/lib/PublicInbox/POP3.pm
new file mode 100644
index 00000000..06772069
--- /dev/null
+++ b/lib/PublicInbox/POP3.pm
@@ -0,0 +1,428 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# Each instance of this represents a POP3 client connected to
+# public-inbox-{netd,pop3d}.  Much of this was taken from IMAP.pm and NNTP.pm
+#
+# POP3 is one mailbox per-user, so the "USER" command is like the
+# format of -imapd and is mapped to $NEWSGROUP.$SLICE (large inboxes
+# are sliced into 50K mailboxes in both POP3 and IMAP to avoid overloading
+# clients)
+#
+# Unlike IMAP, the "$NEWSGROUP" mailbox (without $SLICE) is a rolling
+# window of the latest messages.  We can do this for POP3 since the
+# typical POP3 session is short-lived while long-lived IMAP sessions
+# would cause slices to grow on the server side without bounds.
+#
+# Like IMAP, POP3 also has per-session message sequence numbers (MSN),
+# which require mapping to UIDs.  The offset of an entry into our
+# per-client cache is: (MSN-1)
+#
+# fields:
+# - uuid - 16-byte (binary) UUID representation (before successful login)
+# - cache - one-dimentional arrayref of (UID, bytesize, oidhex)
+# - nr_dele - number of deleted messages
+# - expire - string of packed unsigned short offsets
+# - user_id - user-ID mapped to UUID (on successful login + lock)
+# - txn_max_uid - for storing max deleted UID persistently
+# - ibx - PublicInbox::Inbox object
+# - slice - unsigned integer slice number (0..Inf), -1 => latest
+# - salt - pre-auth for APOP
+# - uid_dele - maximum deleted from previous session at login (NNTP ARTICLE)
+# - uid_base - base UID for mailbox slice (0-based) (same as IMAP)
+package PublicInbox::POP3;
+use v5.12;
+use parent qw(PublicInbox::DS);
+use PublicInbox::GitAsyncCat;
+use PublicInbox::DS qw(now);
+use Errno qw(EAGAIN);
+use Digest::MD5 qw(md5);
+use PublicInbox::IMAP; # for UID slice stuff
+
+use constant {
+        LINE_MAX => 512, # XXX unsure
+        UID_SLICE => PublicInbox::IMAP::UID_SLICE,
+};
+
+# XXX FIXME: duplicated stuff from NNTP.pm and IMAP.pm
+
+sub out ($$;@) {
+        my ($self, $fmt, @args) = @_;
+        printf { $self->{pop3d}->{out} } $fmt."\n", @args;
+}
+
+sub do_greet {
+        my ($self) = @_;
+        my $s = $self->{salt} = sprintf('%x.%x', int(rand(0x7fffffff)), time);
+        $self->write("+OK POP3 server ready <$s\@public-inbox>\r\n");
+}
+
+sub new {
+        my ($cls, $sock, $pop3d) = @_;
+        (bless { pop3d => $pop3d }, $cls)->greet($sock)
+}
+
+# POP user is $UUID1@$NEWSGROUP[.$SLICE][?QUERY_ARGS]
+sub cmd_user ($$) {
+        my ($self, $mailbox) = @_;
+        $self->{salt} // return \"-ERR already authed\r\n";
+        $mailbox =~ s/\A([a-f0-9\-]+)\@//i or
+                return \"-ERR no UUID@ in mailbox name\r\n";
+        my $user = $1;
+        $user =~ tr/-//d; # most have dashes, some (dbus-uuidgen) don't
+        $user =~ m!\A[a-f0-9]{32}\z!i or return \"-ERR user has no UUID\r\n";
+
+        my %l;
+        if ($mailbox =~ s/\?(.*)\z//) { # query args
+                for (split(/&+/, $1)) {
+                        /\A(initial_limit|limit)=([0-9]+)\z/ and $l{$1} = $2;
+                }
+                $self->{limits} = \%l;
+        }
+        my $slice = $mailbox =~ s/\.([0-9]+)\z// ? $1 + 0 : undef;
+
+        my $ibx = $self->{pop3d}->{pi_cfg}->lookup_newsgroup($mailbox) //
+                return \"-ERR $mailbox does not exist\r\n";
+        my $uidmax = $self->{uidmax} = $ibx->mm(1)->num_highwater // 0;
+        if (defined $slice) {
+                my $max = int($uidmax / UID_SLICE);
+                my $tip = "$mailbox.$max";
+                return \"-ERR $mailbox.$slice does not exist ($tip does)\r\n"
+                        if $slice > $max;
+                $self->{slice} = $slice;
+        } else { # latest messages:
+                $self->{slice} = -1;
+        }
+        $self->{ibx} = $ibx;
+        $self->{uuid} = pack('H*', $user); # deleted by _login_ok
+        $slice //= '(latest)';
+        \"+OK $ibx->{newsgroup} slice=$slice selected\r\n";
+}
+
+sub _login_ok ($) {
+        my ($self) = @_;
+        $self->{pop3d}->lock_mailbox($self) or
+                return \"-ERR [IN-USE] unable to lock maildrop\r\n";
+
+        my $l = delete $self->{limits};
+        $l = defined($self->{uid_dele}) ? $l->{limit}
+                                        : ($l->{initial_limit} // $l->{limit});
+        my $uidmax = delete $self->{uidmax};
+        if ($self->{slice} >= 0) {
+                $self->{uid_base} = $self->{slice} * UID_SLICE;
+                if (defined $l) { # n.b: the last slice is not full:
+                        my $max = int($uidmax/UID_SLICE) == $self->{slice} ?
+                                        ($uidmax % UID_SLICE) : UID_SLICE;
+                        my $off = $max - $l;
+                        $self->{uid_base} += $off if $off > 0;
+                }
+        } else { # latest $l messages, or 1k if unspecified
+                my $base = $uidmax - ($l // 1000);
+                $self->{uid_base} = $base < 0 ? 0 : $base;
+        }
+        $self->{uid_max} = $self->{ibx}->over(1)->max;
+        \"+OK logged in\r\n";
+}
+
+sub cmd_apop {
+        my ($self, $mailbox, $hex) = @_;
+        my $res = cmd_user($self, $mailbox); # sets {uuid}
+        return $res if substr($$res, 0, 1) eq '-';
+        my $s = delete($self->{salt}) // die 'BUG: salt missing';
+        return _login_ok($self) if md5("<$s\@public-inbox>anonymous") eq
+                                pack('H*', $hex);
+        $self->{salt} = $s;
+        \"-ERR APOP password mismatch\r\n";
+}
+
+sub cmd_pass {
+        my ($self, $pass) = @_;
+        $self->{ibx} // return \"-ERR mailbox unspecified\r\n";
+        my $s = delete($self->{salt}) // return \"-ERR already authed\r\n";
+        return _login_ok($self) if $pass eq 'anonymous';
+        $self->{salt} = $s;
+        \"-ERR password is not `anonymous'\r\n";
+}
+
+sub cmd_stls {
+        my ($self) = @_;
+        ($self->{sock} // return)->can('stop_SSL') and
+                return \"-ERR TLS already enabled\r\n";
+        $self->{pop3d}->{ssl_ctx_opt} or
+                return \"-ERR can't start TLS negotiation\r\n";
+        $self->write(\"+OK begin TLS negotiation now\r\n");
+        PublicInbox::TLS::start($self->{sock}, $self->{pop3d});
+        $self->requeue if PublicInbox::DS::accept_tls_step($self);
+        undef;
+}
+
+sub need_txn ($) {
+        exists($_[0]->{salt}) ? \"-ERR not in TRANSACTION\r\n" : undef;
+}
+
+sub _stat_cache ($) {
+        my ($self) = @_;
+        my ($beg, $end) = (($self->{uid_dele} // -1) + 1, $self->{uid_max});
+        PublicInbox::IMAP::uid_clamp($self, \$beg, \$end);
+        my (@cache, $m);
+        my $sth = $self->{ibx}->over(1)->dbh->prepare_cached(<<'', undef, 1);
+SELECT num,ddd FROM over WHERE num >= ? AND num <= ?
+ORDER BY num ASC
+
+        $sth->execute($beg, $end);
+        my $tot = 0;
+        while (defined($m = $sth->fetchall_arrayref({}, 1000))) {
+                for my $x (@$m) {
+                        PublicInbox::Over::load_from_row($x);
+                        push(@cache, $x->{num}, $x->{bytes} + 0, $x->{blob});
+                        undef $x; # saves ~1.5M memory w/ 50k messages
+                        $tot += $cache[-2];
+                }
+        }
+        $self->{total_bytes} = $tot;
+        $self->{cache} = \@cache;
+}
+
+sub cmd_stat {
+        my ($self) = @_;
+        my $err; $err = need_txn($self) and return $err;
+        my $cache = $self->{cache} // _stat_cache($self);
+        my $nr = @$cache / 3 - ($self->{nr_dele} // 0);
+        "+OK $nr $self->{total_bytes}\r\n";
+}
+
+# for LIST and UIDL
+sub _list {
+        my ($desc, $idx, $self, $msn) = @_;
+        my $err; $err = need_txn($self) and return $err;
+        my $cache = $self->{cache} // _stat_cache($self);
+        if (defined $msn) {
+                my $base_off = ($msn - 1) * 3;
+                my $val = $cache->[$base_off + $idx] //
+                                return \"-ERR no such message\r\n";
+                "+OK $desc listing follows\r\n$msn $val\r\n.\r\n";
+        } else { # always +OK, even if no messages
+                my $res = "+OK $desc listing follows\r\n";
+                my $msn = 0;
+                for (my $i = 0; $i < scalar(@$cache); $i += 3) {
+                        ++$msn;
+                        defined($cache->[$i]) and
+                                $res .= "$msn $cache->[$i + $idx]\r\n";
+                }
+                $res .= ".\r\n";
+        }
+}
+
+sub cmd_list { _list('scan', 1, @_) }
+sub cmd_uidl { _list('unique-id', 2, @_) }
+
+sub mark_dele ($$) {
+        my ($self, $off) = @_;
+        my $base_off = $off * 3;
+        my $cache = $self->{cache};
+        my $uid = $cache->[$base_off] // return; # already deleted
+
+        my $old = $self->{txn_max_uid} //= $uid;
+        $self->{txn_max_uid} = $uid if $uid > $old;
+
+        $self->{total_bytes} -= $cache->[$base_off + 1];
+        $cache->[$base_off] = undef; # clobber UID
+        $cache->[$base_off + 1] = undef; # clobber bytes
+        $cache->[$base_off + 2] = undef; # clobber oidhex
+        ++$self->{nr_dele};
+}
+
+sub retr_cb { # called by git->cat_async via ibx_async_cat
+        my ($bref, $oid, $type, $size, $args) = @_;
+        my ($self, $off, $top_nr) = @$args;
+        my $hex = $self->{cache}->[$off * 3 + 2] //
+                die "BUG: no hex (oid=$oid)";
+        if (!defined($type)) {
+                warn "E: git aborted on $oid / $hex $self->{ibx}->{inboxdir}";
+                return $self->close;
+        } elsif ($type ne 'blob') {
+                # it's possible to have TOCTOU if an admin runs
+                # public-inbox-(edit|purge), just move onto the next message
+                warn "E: $hex missing in $self->{ibx}->{inboxdir}\n";
+                $self->write(\"-ERR no such message\r\n");
+                return $self->requeue;
+        } elsif ($hex ne $oid) {
+                $self->close;
+                die "BUG: $hex != $oid";
+        }
+        PublicInbox::IMAP::to_crlf_full($bref);
+        if (defined $top_nr) {
+                my ($hdr, $bdy) = split(/\r\n\r\n/, $$bref, 2);
+                $bref = \$hdr;
+                $hdr .= "\r\n\r\n";
+                my @tmp = split(/^/m, $bdy);
+                $hdr .= join('', splice(@tmp, 0, $top_nr));
+        } elsif (exists $self->{expire}) {
+                $self->{expire} .= pack('S', $off);
+        }
+        $$bref =~ s/^\./../gms;
+        $$bref .= substr($$bref, -2, 2) eq "\r\n" ? ".\r\n" : "\r\n.\r\n";
+        $self->msg_more("+OK message follows\r\n");
+        $self->write($bref);
+        $self->requeue;
+}
+
+sub cmd_retr {
+        my ($self, $msn, $top_nr) = @_;
+        return \"-ERR lines must be a non-negative number\r\n" if
+                        (defined($top_nr) && $top_nr !~ /\A[0-9]+\z/);
+        my $err; $err = need_txn($self) and return $err;
+        my $cache = $self->{cache} // _stat_cache($self);
+        my $off = $msn - 1;
+        my $hex = $cache->[$off * 3 + 2] // return \"-ERR no such message\r\n";
+        ${ibx_async_cat($self->{ibx}, $hex, \&retr_cb,
+                        [ $self, $off, $top_nr ])};
+}
+
+sub cmd_noop { $_[0]->write(\"+OK\r\n") }
+
+sub cmd_rset {
+        my ($self) = @_;
+        my $err; $err = need_txn($self) and return $err;
+        delete $self->{cache};
+        delete $self->{txn_max_uid};
+        \"+OK\r\n";
+}
+
+sub cmd_dele {
+        my ($self, $msn) = @_;
+        my $err; $err = need_txn($self) and return $err;
+        $self->{cache} // _stat_cache($self);
+        $msn =~ /\A[1-9][0-9]*\z/ or return \"-ERR no such message\r\n";
+        mark_dele($self, $msn - 1) ? \"+OK\r\n" : \"-ERR no such message\r\n";
+}
+
+# RFC 2449
+sub cmd_capa {
+        my ($self) = @_;
+        my $STLS = !$self->{ibx} && !$self->{sock}->can('stop_SSL') &&
+                        $self->{pop3d}->{ssl_ctx_opt} ? "\nSTLS\r" : '';
+        $self->{expire} = ''; # "EXPIRE 0" allows clients to avoid DELE commands
+        <<EOM;
++OK Capability list follows\r
+TOP\r
+USER\r
+PIPELINING\r
+UIDL\r
+EXPIRE 0\r
+RESP-CODES\r$STLS
+.\r
+EOM
+}
+
+sub close {
+        my ($self) = @_;
+        $self->{pop3d}->unlock_mailbox($self);
+        $self->SUPER::close;
+}
+
+# must be called inside a state_dbh transaction with flock held
+sub __cleanup_state {
+        my ($self, $txn_id) = @_;
+        my $user_id = $self->{user_id} // die 'BUG: no {user_id}';
+        $self->{pop3d}->{-state_dbh}->prepare_cached(<<'')->execute($txn_id);
+DELETE FROM deletes WHERE txn_id = ? AND uid_dele = -1
+
+        my $sth = $self->{pop3d}->{-state_dbh}->prepare_cached(<<'', undef, 1);
+SELECT COUNT(*) FROM deletes WHERE user_id = ?
+
+        $sth->execute($user_id);
+        my $nr = $sth->fetchrow_array;
+        if ($nr == 0) {
+                $sth = $self->{pop3d}->{-state_dbh}->prepare_cached(<<'');
+DELETE FROM users WHERE user_id = ?
+
+                $sth->execute($user_id);
+        }
+        $nr;
+}
+
+sub cmd_quit {
+        my ($self) = @_;
+        if (defined(my $txn_id = $self->{txn_id})) {
+                my $user_id = $self->{user_id} // die 'BUG: no {user_id}';
+                if (my $exp = delete $self->{expire}) {
+                        mark_dele($self, $_) for unpack('S*', $exp);
+                }
+                my $keep = 1;
+                my $dbh = $self->{pop3d}->{-state_dbh};
+                my $lk = $self->{pop3d}->lock_for_scope;
+                $dbh->begin_work;
+
+                if (defined(my $max = $self->{txn_max_uid})) {
+                        $dbh->prepare_cached(<<'')->execute($max, $txn_id, $max)
+UPDATE deletes SET uid_dele = ? WHERE txn_id = ? AND uid_dele < ?
+
+                } else {
+                        $keep = $self->__cleanup_state($txn_id);
+                }
+                $dbh->prepare_cached(<<'')->execute(time, $user_id) if $keep;
+UPDATE users SET last_seen = ? WHERE user_id = ?
+
+                $dbh->commit;
+                # we MUST do txn_id F_UNLCK here inside ->lock_for_scope:
+                $self->{did_quit} = 1;
+                $self->{pop3d}->unlock_mailbox($self);
+        }
+        $self->write(\"+OK public-inbox POP3 server signing off\r\n");
+        $self->shutdn;
+        undef;
+}
+
+# returns 1 if we can continue, 0 if not due to buffered writes or disconnect
+sub process_line ($$) {
+        my ($self, $l) = @_;
+        my ($req, @args) = split(/[ \t]+/, $l);
+        return 1 unless defined($req); # skip blank line
+        $req = $self->can('cmd_'.lc($req));
+        my $res = $req ? eval { $req->($self, @args) } :
+                \"-ERR command not recognized\r\n";
+        my $err = $@;
+        if ($err && $self->{sock}) {
+                $l =~ s/\r?\n//s;
+                warn("error from: $l ($err)\n");
+                $res = \"-ERR program fault - command not performed\r\n";
+        }
+        defined($res) ? $self->write($res) : 0;
+}
+
+# callback used by PublicInbox::DS for any (e)poll (in/out/hup/err)
+sub event_step {
+        my ($self) = @_;
+        local $SIG{__WARN__} = $self->{pop3d}->{warn_cb};
+        return unless $self->flush_write && $self->{sock} && !$self->{long_cb};
+
+        # only read more requests if we've drained the write buffer,
+        # otherwise we can be buffering infinitely w/o backpressure
+        my $rbuf = $self->{rbuf} // \(my $x = '');
+        my $line = index($$rbuf, "\n");
+        while ($line < 0) {
+                return $self->close if length($$rbuf) >= LINE_MAX;
+                $self->do_read($rbuf, LINE_MAX, length($$rbuf)) or return;
+                $line = index($$rbuf, "\n");
+        }
+        $line = substr($$rbuf, 0, $line + 1, '');
+        $line =~ s/\r?\n\z//s;
+        return $self->close if $line =~ /[[:cntrl:]]/s;
+        my $t0 = now();
+        my $fd = fileno($self->{sock}); # may become invalid after process_line
+        my $r = eval { process_line($self, $line) };
+        my $pending = $self->{wbuf} ? ' pending' : '';
+        out($self, "[$fd] %s - %0.6f$pending - $r", $line, now() - $t0);
+        return $self->close if $r < 0;
+        $self->rbuf_idle($rbuf);
+
+        # maybe there's more pipelined data, or we'll have
+        # to register it for socket-readiness notifications
+        $self->requeue unless $pending;
+}
+
+no warnings 'once';
+*cmd_top = \&cmd_retr;
+
+1;
diff --git a/lib/PublicInbox/POP3D.pm b/lib/PublicInbox/POP3D.pm
new file mode 100644
index 00000000..bd440434
--- /dev/null
+++ b/lib/PublicInbox/POP3D.pm
@@ -0,0 +1,277 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# represents an POP3D
+package PublicInbox::POP3D;
+use v5.12;
+use parent qw(PublicInbox::Lock);
+use DBI qw(:sql_types); # SQL_BLOB
+use Carp ();
+use File::Temp 0.19 (); # 0.19 for ->newdir
+use PublicInbox::Config;
+use PublicInbox::POP3;
+use PublicInbox::Syscall;
+use File::Temp 0.19 (); # 0.19 for ->newdir
+use Fcntl qw(F_SETLK F_UNLCK F_WRLCK SEEK_SET);
+my ($FLOCK_TMPL, @FLOCK_ORDER);
+# are all BSDs the same "struct flock"? tested Free+Net+Open...
+if ($^O =~ /\A(?:linux|dragonfly)\z/ || $^O =~ /bsd/) {
+        require Config;
+        my $off_t;
+        my @LE_pad = ('', '');
+        my $sz = $Config::Config{lseeksize};
+        if ($sz == 8) {
+                if (eval('length(pack("q", 1)) == 8')) {
+                        $off_t = 'q';
+                } elsif ($Config::Config{byteorder} == 1234) { # OpenBSD i386
+                        $off_t = 'l';
+                        @LE_pad = ('@8', '@16');
+                } else { # I have no 32-bit BE machine to test on...
+                        warn <<EOM;
+Perl built with 64-bit file support but not 64-bit int (pack("q")) support.
+byteorder=$Config::Config{byteorder}
+EOM
+                }
+        } elsif ($sz == 4) {
+                $off_t = 'l';
+        } else {
+                warn "sizeof(off_t)=$sz requires File::FcntlLock\n"
+        }
+        if (defined($off_t)) {
+                if ($^O eq 'linux') {
+                        $FLOCK_TMPL = 'ss@8'.$off_t.$LE_pad[0].$off_t.'@32';
+                        @FLOCK_ORDER = qw(l_type l_whence l_start l_len);
+                } else { # *bsd including dragonfly
+                        $FLOCK_TMPL = $off_t.$LE_pad[0].$off_t.$LE_pad[1].
+                                        'lss@256';
+                        @FLOCK_ORDER = qw(l_start l_len l_pid l_type l_whence);
+                }
+        }
+}
+@FLOCK_ORDER or eval { require File::FcntlLock } or
+        die "File::FcntlLock required for POP3 on $^O: $@\n";
+
+sub new {
+        my ($cls) = @_;
+        bless {
+                err => \*STDERR,
+                out => \*STDOUT,
+                # pi_cfg => PublicInbox::Config
+                # lock_path => ...
+                # interprocess lock is the $pop3state/txn.locks file
+                # txn_locks => {}, # intraworker locks
+                # ssl_ctx_opt => { SSL_cert_file => ..., SSL_key_file => ... }
+        }, $cls;
+}
+
+sub refresh_groups { # PublicInbox::Daemon callback
+        my ($self, $sig) = @_;
+        # TODO share pi_cfg with nntpd/imapd inside -netd
+        my $new = PublicInbox::Config->new;
+        my $d = $new->{'publicinbox.pop3state'} //
+                die "publicinbox.pop3state undefined ($new->{-f})\n";
+        -d $d or do {
+                require File::Path;
+                File::Path::make_path($d, { mode => 0700 });
+                PublicInbox::Syscall::nodatacow_dir($d);
+        };
+        $self->{lock_path} //= "$d/db.lock";
+        if (my $old = $self->{pi_cfg}) {
+                my $s = 'publicinbox.pop3state';
+                $new->{$s} //= $old->{$s};
+                return warn <<EOM if $new->{$s} ne $old->{$s};
+$s changed: `$old->{$s}' => `$new->{$s}', config reload ignored
+EOM
+        }
+        $self->{pi_cfg} = $new;
+}
+
+# persistent tables
+sub create_state_tables ($$) {
+        my ($self, $dbh) = @_;
+
+        $dbh->do(<<''); # map publicinbox.<name>.newsgroup to integers
+CREATE TABLE IF NOT EXISTS newsgroups (
+        newsgroup_id INTEGER PRIMARY KEY NOT NULL,
+        newsgroup VARBINARY NOT NULL,
+        UNIQUE (newsgroup) )
+
+        # the $NEWSGROUP_NAME.$SLICE_INDEX is part of the POP3 username;
+        # POP3 has no concept of folders/mailboxes like IMAP/JMAP
+        $dbh->do(<<'');
+CREATE TABLE IF NOT EXISTS mailboxes (
+        mailbox_id INTEGER PRIMARY KEY NOT NULL,
+        newsgroup_id INTEGER NOT NULL REFERENCES newsgroups,
+        slice INTEGER NOT NULL, /* -1 for most recent slice */
+        UNIQUE (newsgroup_id, slice) )
+
+        $dbh->do(<<''); # actual users are differentiated by their UUID
+CREATE TABLE IF NOT EXISTS users (
+        user_id INTEGER PRIMARY KEY NOT NULL,
+        uuid VARBINARY NOT NULL,
+        last_seen INTEGER NOT NULL, /* to expire idle accounts */
+        UNIQUE (uuid) )
+
+        # we only track the highest-numbered deleted message per-UUID@mailbox
+        $dbh->do(<<'');
+CREATE TABLE IF NOT EXISTS deletes (
+        txn_id INTEGER PRIMARY KEY NOT NULL, /* -1 == txn lock offset */
+        user_id INTEGER NOT NULL REFERENCES users,
+        mailbox_id INTEGER NOT NULL REFERENCES mailboxes,
+        uid_dele INTEGER NOT NULL DEFAULT -1, /* IMAP UID, NNTP article */
+        UNIQUE(user_id, mailbox_id) )
+
+}
+
+sub state_dbh_new {
+        my ($self) = @_;
+        my $f = "$self->{pi_cfg}->{'publicinbox.pop3state'}/db.sqlite3";
+        my $creat = !-s $f;
+        if ($creat) {
+                open my $fh, '+>>', $f or Carp::croak "open($f): $!";
+                PublicInbox::Syscall::nodatacow_fh($fh);
+        }
+
+        my $dbh = DBI->connect("dbi:SQLite:dbname=$f",'','', {
+                AutoCommit => 1,
+                RaiseError => 1,
+                PrintError => 0,
+                sqlite_use_immediate_transaction => 1,
+                sqlite_see_if_its_a_number => 1,
+        });
+        $dbh->do('PRAGMA journal_mode = WAL') if $creat;
+        $dbh->do('PRAGMA foreign_keys = ON'); # don't forget this
+
+        # ensure the interprocess fcntl lock file exists
+        $f = "$self->{pi_cfg}->{'publicinbox.pop3state'}/txn.locks";
+        open my $fh, '+>>', $f or Carp::croak("open($f): $!");
+        $self->{txn_fh} = $fh;
+
+        create_state_tables($self, $dbh);
+        $dbh;
+}
+
+sub _setlk ($%) {
+        my ($self, %lk) = @_;
+        $lk{l_pid} = 0; # needed for *BSD
+        $lk{l_whence} = SEEK_SET;
+        if (@FLOCK_ORDER) {
+                fcntl($self->{txn_fh}, F_SETLK,
+                        pack($FLOCK_TMPL, @lk{@FLOCK_ORDER}));
+        } else {
+                my $fs = File::FcntlLock->new(%lk);
+                $fs->lock($self->{txn_fh}, F_SETLK);
+        }
+}
+
+sub lock_mailbox {
+        my ($self, $pop3) = @_; # pop3 - PublicInbox::POP3 client object
+        my $lk = $self->lock_for_scope; # lock the SQLite DB, only
+        my $dbh = $self->{-state_dbh} //= state_dbh_new($self);
+        my ($user_id, $ngid, $mbid, $txn_id);
+        my $uuid = delete $pop3->{uuid};
+        $dbh->begin_work;
+        my $creat = 0;
+
+        # 1. make sure the user exists, update `last_seen'
+        my $sth = $dbh->prepare_cached(<<'');
+INSERT OR IGNORE INTO users (uuid, last_seen) VALUES (?,?)
+
+        $sth->bind_param(1, $uuid, SQL_BLOB);
+        $sth->bind_param(2, time);
+        if ($sth->execute == 0) { # existing user
+                $sth = $dbh->prepare_cached(<<'', undef, 1);
+SELECT user_id FROM users WHERE uuid = ?
+
+                $sth->bind_param(1, $uuid, SQL_BLOB);
+                $sth->execute;
+                $user_id = $sth->fetchrow_array //
+                        die 'BUG: user '.unpack('H*', $uuid).' not found';
+                $sth = $dbh->prepare_cached(<<'');
+UPDATE users SET last_seen = ? WHERE user_id = ?
+
+                $sth->execute(time, $user_id);
+        } else { # new user
+                $user_id = $dbh->last_insert_id(undef, undef,
+                                                'users', 'user_id')
+        }
+
+        # 2. make sure the newsgroup has an integer ID
+        $sth = $dbh->prepare_cached(<<'');
+INSERT OR IGNORE INTO newsgroups (newsgroup) VALUES (?)
+
+        my $ng = $pop3->{ibx}->{newsgroup};
+        $sth->bind_param(1, $ng, SQL_BLOB);
+        if ($sth->execute == 0) {
+                $sth = $dbh->prepare_cached(<<'', undef, 1);
+SELECT newsgroup_id FROM newsgroups WHERE newsgroup = ?
+
+                $sth->bind_param(1, $ng, SQL_BLOB);
+                $sth->execute;
+                $ngid = $sth->fetchrow_array // die "BUG: `$ng' not found";
+        } else {
+                $ngid = $dbh->last_insert_id(undef, undef,
+                                                'newsgroups', 'newsgroup_id');
+        }
+
+        # 3. ensure the mailbox exists
+        $sth = $dbh->prepare_cached(<<'');
+INSERT OR IGNORE INTO mailboxes (newsgroup_id, slice) VALUES (?,?)
+
+        if ($sth->execute($ngid, $pop3->{slice}) == 0) {
+                $sth = $dbh->prepare_cached(<<'', undef, 1);
+SELECT mailbox_id FROM mailboxes WHERE newsgroup_id = ? AND slice = ?
+
+                $sth->execute($ngid, $pop3->{slice});
+                $mbid = $sth->fetchrow_array //
+                        die "BUG: mailbox_id for $ng.$pop3->{slice} not found";
+        } else {
+                $mbid = $dbh->last_insert_id(undef, undef,
+                                                'mailboxes', 'mailbox_id');
+        }
+
+        # 4. ensure the (max) deletes row exists for locking
+        $sth = $dbh->prepare_cached(<<'');
+INSERT OR IGNORE INTO deletes (user_id,mailbox_id) VALUES (?,?)
+
+        if ($sth->execute($user_id, $mbid) == 0) { # fetching into existing
+                $sth = $dbh->prepare_cached(<<'', undef, 1);
+SELECT txn_id,uid_dele FROM deletes WHERE user_id = ? AND mailbox_id = ?
+
+                $sth->execute($user_id, $mbid);
+                ($txn_id, $pop3->{uid_dele}) = $sth->fetchrow_array;
+        } else { # new user/mailbox combo
+                $txn_id = $dbh->last_insert_id(undef, undef,
+                                                'deletes', 'txn_id');
+        }
+        $dbh->commit;
+
+        # see if it's locked by the same worker:
+        return if $self->{txn_locks}->{$txn_id};
+
+        # see if it's locked by another worker:
+        _setlk($self, l_type => F_WRLCK, l_start => $txn_id - 1, l_len => 1)
+                or return;
+
+        $pop3->{user_id} = $user_id;
+        $pop3->{txn_id} = $txn_id;
+        $self->{txn_locks}->{$txn_id} = 1;
+}
+
+sub unlock_mailbox {
+        my ($self, $pop3) = @_;
+        my $txn_id = delete($pop3->{txn_id}) // return;
+        if (!$pop3->{did_quit}) { # deal with QUIT-less disconnects
+                my $lk = $self->lock_for_scope;
+                $self->{-state_dbh}->begin_work;
+                $pop3->__cleanup_state($txn_id);
+                $self->{-state_dbh}->commit;
+        }
+        delete $self->{txn_locks}->{$txn_id}; # same worker
+
+        # other workers
+        _setlk($self, l_type => F_UNLCK, l_start => $txn_id - 1, l_len => 1)
+                or die "F_UNLCK: $!";
+}
+
+1;
diff --git a/lib/PublicInbox/PktOp.pm b/lib/PublicInbox/PktOp.pm
index 4c434566..1bcdd799 100644
--- a/lib/PublicInbox/PktOp.pm
+++ b/lib/PublicInbox/PktOp.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # op dispatch socket, reads a message, runs a sub
@@ -6,12 +6,11 @@
 # Used for lei_xsearch and maybe other things
 # "command" => [ $sub, @fixed_operands ]
 package PublicInbox::PktOp;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::DS);
 use Errno qw(EAGAIN ECONNRESET);
 use PublicInbox::Syscall qw(EPOLLIN);
-use Socket qw(AF_UNIX MSG_EOR SOCK_SEQPACKET);
+use Socket qw(AF_UNIX SOCK_SEQPACKET);
 use PublicInbox::IPC qw(ipc_freeze ipc_thaw);
 use Scalar::Util qw(blessed);
 
@@ -32,7 +31,7 @@ sub pair {
 
 sub pkt_do { # for the producer to trigger event_step in consumer
         my ($self, $cmd, @args) = @_;
-        send($self->{op_p}, @args ? "$cmd\0".ipc_freeze(\@args) : $cmd, MSG_EOR)
+        send($self->{op_p}, @args ? "$cmd\0".ipc_freeze(\@args) : $cmd, 0)
 }
 
 sub event_step {
@@ -55,7 +54,15 @@ sub event_step {
         my $op = $self->{ops}->{$cmd //= $msg};
         if ($op) {
                 my ($obj, @args) = (@$op, @pargs);
-                blessed($obj) ? $obj->$cmd(@args) : $obj->(@args);
+                if (blessed($args[0]) && $args[0]->can('do_env')) {
+                        my $lei = shift @args;
+                        $lei->do_env($obj, @args);
+                } elsif (blessed($obj)) {
+                        $obj->can('do_env') ? $obj->do_env($cmd, @args)
+                                                : $obj->$cmd(@args);
+                } else {
+                        $obj->(@args);
+                }
         } elsif ($msg ne '') {
                 die "BUG: unknown message: `$cmd'";
         }
diff --git a/lib/PublicInbox/ProcessPipe.pm b/lib/PublicInbox/ProcessPipe.pm
deleted file mode 100644
index 97e9c268..00000000
--- a/lib/PublicInbox/ProcessPipe.pm
+++ /dev/null
@@ -1,70 +0,0 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-
-# a tied handle for auto reaping of children tied to a pipe, see perltie(1)
-package PublicInbox::ProcessPipe;
-use strict;
-use v5.10.1;
-use Carp qw(carp);
-
-sub TIEHANDLE {
-        my ($class, $pid, $fh, $cb, $arg) = @_;
-        bless { pid => $pid, fh => $fh, ppid => $$, cb => $cb, arg => $arg },
-                $class;
-}
-
-sub BINMODE { binmode(shift->{fh}) } # for IO::Uncompress::Gunzip
-
-sub READ { read($_[0]->{fh}, $_[1], $_[2], $_[3] || 0) }
-
-sub READLINE { readline($_[0]->{fh}) }
-
-sub WRITE {
-        use bytes qw(length);
-        syswrite($_[0]->{fh}, $_[1], $_[2] // length($_[1]), $_[3] // 0);
-}
-
-sub PRINT {
-        my $self = shift;
-        print { $self->{fh} } @_;
-}
-
-sub FILENO { fileno($_[0]->{fh}) }
-
-sub _close ($;$) {
-        my ($self, $wait) = @_;
-        my $fh = delete $self->{fh};
-        my $ret = defined($fh) ? close($fh) : '';
-        my ($pid, $cb, $arg) = delete @$self{qw(pid cb arg)};
-        return $ret unless defined($pid) && $self->{ppid} == $$;
-        if ($wait) { # caller cares about the exit status:
-                my $wp = waitpid($pid, 0);
-                if ($wp == $pid) {
-                        $ret = '' if $?;
-                        if ($cb) {
-                                eval { $cb->($arg, $pid) };
-                                carp "E: cb(arg, $pid): $@" if $@;
-                        }
-                } else {
-                        carp "waitpid($pid, 0) = $wp, \$!=$!, \$?=$?";
-                }
-        } else { # caller just undef-ed it, let event loop deal with it
-                require PublicInbox::DS;
-                PublicInbox::DS::dwaitpid($pid, $cb, $arg);
-        }
-        $ret;
-}
-
-# if caller uses close(), assume they want to check $? immediately so
-# we'll waitpid() synchronously.  n.b. wantarray doesn't seem to
-# propagate `undef' down to tied methods, otherwise I'd rely on that.
-sub CLOSE { _close($_[0], 1) }
-
-# if relying on DESTROY, assume the caller doesn't care about $? and
-# we can let the event loop call waitpid() whenever it gets SIGCHLD
-sub DESTROY {
-        _close($_[0]);
-        undef;
-}
-
-1;
diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
index 53d0ad55..0bf857c6 100644
--- a/lib/PublicInbox/Qspawn.pm
+++ b/lib/PublicInbox/Qspawn.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Like most Perl modules in public-inbox, this is internal and
@@ -25,9 +25,15 @@
 # processes such as git-apply(1).
 
 package PublicInbox::Qspawn;
-use strict;
+use v5.12;
 use PublicInbox::Spawn qw(popen_rd);
 use PublicInbox::GzipFilter;
+use Scalar::Util qw(blessed);
+use PublicInbox::Limiter;
+use PublicInbox::Aspawn qw(run_await);
+use PublicInbox::Syscall qw(EPOLLIN);
+use PublicInbox::InputPipe;
+use Carp qw(carp confess);
 
 # n.b.: we get EAGAIN with public-inbox-httpd, and EINTR on other PSGI servers
 use Errno qw(EAGAIN EINTR);
@@ -38,52 +44,45 @@ my $def_limiter;
 # $cmd is the command to spawn
 # $cmd_env is the environ for the child process (not PSGI env)
 # $opt can include redirects and perhaps other process spawning options
-sub new ($$$;) {
+# {qsp_err} is an optional error buffer callers may access themselves
+sub new {
         my ($class, $cmd, $cmd_env, $opt) = @_;
-        bless { args => [ $cmd, $cmd_env, $opt ] }, $class;
+        bless { args => [ $cmd, $cmd_env, $opt ? { %$opt } : {} ] }, $class;
 }
 
 sub _do_spawn {
         my ($self, $start_cb, $limiter) = @_;
-        my $err;
-        my ($cmd, $cmd_env, $opt) = @{delete $self->{args}};
+        my ($cmd, $cmd_env, $opt) = @{$self->{args}};
         my %o = %{$opt || {}};
         $self->{limiter} = $limiter;
-        foreach my $k (@PublicInbox::Spawn::RLIMITS) {
-                if (defined(my $rlimit = $limiter->{$k})) {
-                        $o{$k} = $rlimit;
-                }
+        for my $k (@PublicInbox::Spawn::RLIMITS) {
+                $opt->{$k} = $limiter->{$k} // next;
+        }
+        $self->{-quiet} = 1 if $o{quiet};
+        $limiter->{running}++;
+        if ($start_cb) {
+                eval { # popen_rd may die on EMFILE, ENFILE
+                        $self->{rpipe} = popen_rd($cmd, $cmd_env, $opt,
+                                                        \&waitpid_err, $self);
+                        $start_cb->($self); # EPOLL_CTL_ADD may ENOSPC/ENOMEM
+                };
+        } else {
+                eval { run_await($cmd, $cmd_env, $opt, \&wait_await, $self) };
+                warn "E: $@" if $@;
         }
-        $self->{cmd} = $o{quiet} ? undef : $cmd;
-        eval {
-                # popen_rd may die on EMFILE, ENFILE
-                $self->{rpipe} = popen_rd($cmd, $cmd_env, \%o);
-
-                die "E: $!" unless defined($self->{rpipe});
-
-                $limiter->{running}++;
-                $start_cb->($self); # EPOLL_CTL_ADD may ENOSPC/ENOMEM
-        };
         finish($self, $@) if $@;
 }
 
-sub child_err ($) {
-        my ($child_error) = @_; # typically $?
-        my $exitstatus = ($child_error >> 8) or return;
-        my $sig = $child_error & 127;
-        my $msg = "exit status=$exitstatus";
-        $msg .= " signal=$sig" if $sig;
-        $msg;
+sub psgi_status_err { # Qspawn itself is useful w/o PSGI
+        require PublicInbox::WwwStatic;
+        PublicInbox::WwwStatic::r($_[0] // 500);
 }
 
-sub finalize ($$) {
-        my ($self, $err) = @_;
-
-        my ($env, $qx_cb, $qx_arg, $qx_buf) =
-                delete @$self{qw(psgi_env qx_cb qx_arg qx_buf)};
+sub finalize ($) {
+        my ($self) = @_;
 
-        # done, spawn whatever's in the queue
-        my $limiter = $self->{limiter};
+        # process is done, spawn whatever's in the queue
+        my $limiter = delete $self->{limiter} or return;
         my $running = --$limiter->{running};
 
         if ($running < $limiter->{max}) {
@@ -91,34 +90,69 @@ sub finalize ($$) {
                         _do_spawn(@$next, $limiter);
                 }
         }
-
-        if ($err) {
-                if (defined $self->{err}) {
-                        $self->{err} .= "; $err";
-                } else {
-                        $self->{err} = $err;
-                }
-                if ($env && $self->{cmd}) {
-                        warn join(' ', @{$self->{cmd}}) . ": $err";
+        if (my $err = $self->{_err}) { # set by finish or waitpid_err
+                utf8::decode($err);
+                if (my $dst = $self->{qsp_err}) {
+                        $$dst .= $$dst ? " $err" : "; $err";
                 }
+                warn "E: @{$self->{args}->[0]}: $err\n" if !$self->{-quiet};
+        }
+
+        my ($env, $qx_cb_arg) = delete @$self{qw(psgi_env qx_cb_arg)};
+        if ($qx_cb_arg) {
+                my $cb = shift @$qx_cb_arg;
+                eval { $cb->($self->{args}->[2]->{1}, @$qx_cb_arg) };
+                return unless $@;
+                warn "E: $@"; # hope qspawn.wcb can handle it
         }
-        if ($qx_cb) {
-                eval { $qx_cb->($qx_buf, $qx_arg) };
-        } elsif (my $wcb = delete $env->{'qspawn.wcb'}) {
+        return if $self->{passed}; # another command chained it
+        if (my $wcb = delete $env->{'qspawn.wcb'}) {
                 # have we started writing, yet?
-                require PublicInbox::WwwStatic;
-                $wcb->(PublicInbox::WwwStatic::r(500));
+                $wcb->(psgi_status_err($env->{'qspawn.fallback'}));
         }
 }
 
-# callback for dwaitpid or ProcessPipe
-sub waitpid_err { finalize($_[0], child_err($?)) }
+sub waitpid_err { # callback for awaitpid
+        my (undef, $self) = @_; # $_[0]: pid
+        $self->{_err} = ''; # for defined check in ->finish
+        if ($?) { # XXX this may be redundant
+                my $status = $? >> 8;
+                my $sig = $? & 127;
+                $self->{_err} .= "exit status=$status";
+                $self->{_err} .= " signal=$sig" if $sig;
+        }
+        finalize($self) if !$self->{rpipe};
+}
+
+sub wait_await { # run_await cb
+        my ($pid, $cmd, $cmd_env, $opt, $self) = @_;
+        waitpid_err($pid, $self);
+}
+
+sub yield_chunk { # $_[-1] is sysread buffer (or undef)
+        my ($self, $ipipe) = @_;
+        if (!defined($_[-1])) {
+                warn "error reading body: $!";
+        } elsif ($_[-1] eq '') { # normal EOF
+                $self->finish;
+                $self->{qfh}->close;
+        } elsif (defined($self->{qfh}->write($_[-1]))) {
+                return; # continue while HTTP client is reading our writes
+        } # else { # HTTP client disconnected
+        delete $self->{rpipe};
+        $ipipe->close;
+}
 
 sub finish ($;$) {
         my ($self, $err) = @_;
-        my $tied_pp = delete($self->{rpipe}) or return finalize($self, $err);
-        my PublicInbox::ProcessPipe $pp = tied *$tied_pp;
-        @$pp{qw(cb arg)} = (\&waitpid_err, $self); # for ->DESTROY
+        $self->{_err} //= $err; # only for $@
+
+        # we can safely finalize if pipe was closed before, or if
+        # {_err} is defined by waitpid_err.  Deleting {rpipe} will
+        # trigger PublicInbox::IO::DESTROY -> waitpid_err,
+        # but it may not fire right away if inside the event loop.
+        my $closed_before = !delete($self->{rpipe});
+        finalize($self) if $closed_before || defined($self->{_err});
 }
 
 sub start ($$$) {
@@ -130,137 +164,92 @@ sub start ($$$) {
         }
 }
 
-sub psgi_qx_init_cb {
-        my ($self) = @_;
-        my $async = delete $self->{async};
-        my ($r, $buf);
-        my $qx_fh = $self->{qx_fh};
-reread:
-        $r = sysread($self->{rpipe}, $buf, 65536);
-        if ($async) {
-                $async->async_pass($self->{psgi_env}->{'psgix.io'},
-                                        $qx_fh, \$buf);
-        } elsif (defined $r) {
-                $r ? (print $qx_fh $buf) : event_step($self, undef);
-        } else {
-                return if $! == EAGAIN; # try again when notified
-                goto reread if $! == EINTR;
-                event_step($self, $!);
-        }
-}
-
-sub psgi_qx_start {
-        my ($self) = @_;
-        if (my $async = $self->{psgi_env}->{'pi-httpd.async'}) {
-                # PublicInbox::HTTPD::Async->new(rpipe, $cb, cb_arg, $end_obj)
-                $self->{async} = $async->($self->{rpipe},
-                                        \&psgi_qx_init_cb, $self, $self);
-                # init_cb will call ->async_pass or ->close
-        } else { # generic PSGI
-                psgi_qx_init_cb($self) while $self->{qx_fh};
-        }
-}
-
-# Similar to `backtick` or "qx" ("perldoc -f qx"), it calls $qx_cb with
+# Similar to `backtick` or "qx" ("perldoc -f qx"), it calls @qx_cb_arg with
 # the stdout of the given command when done; but respects the given limiter
 # $env is the PSGI env.  As with ``/qx; only use this when output is small
 # and safe to slurp.
 sub psgi_qx {
-        my ($self, $env, $limiter, $qx_cb, $qx_arg) = @_;
+        my ($self, $env, $limiter, @qx_cb_arg) = @_;
         $self->{psgi_env} = $env;
-        my $qx_buf = '';
-        open(my $qx_fh, '+>', \$qx_buf) or die; # PerlIO::scalar
-        $self->{qx_cb} = $qx_cb;
-        $self->{qx_arg} = $qx_arg;
-        $self->{qx_fh} = $qx_fh;
-        $self->{qx_buf} = \$qx_buf;
-        $limiter ||= $def_limiter ||= PublicInbox::Qspawn::Limiter->new(32);
-        start($self, $limiter, \&psgi_qx_start);
+        $self->{qx_cb_arg} = \@qx_cb_arg;
+        $limiter ||= $def_limiter ||= PublicInbox::Limiter->new(32);
+        start($self, $limiter, undef);
 }
 
-# this is called on pipe EOF to reap the process, may be called
-# via PublicInbox::DS event loop OR via GetlineBody for generic
-# PSGI servers.
-sub event_step {
-        my ($self, $err) = @_; # $err: $!
-        warn "psgi_{return,qx} $err" if defined($err);
-        finish($self);
-        my ($fh, $qx_fh) = delete(@$self{qw(fh qx_fh)});
-        $fh->close if $fh; # async-only (psgi_return)
-}
+sub yield_pass {
+        my ($self, $ipipe, $res) = @_; # $ipipe = InputPipe
+        my $env = $self->{psgi_env};
+        my $wcb = delete $env->{'qspawn.wcb'} // confess('BUG: no qspawn.wcb');
+        if (ref($res) eq 'CODE') { # chain another command
+                delete $self->{rpipe};
+                $ipipe->close if $ipipe;
+                $res->($wcb);
+                $self->{passed} = 1;
+                return; # all done
+        }
+        confess("BUG: $res unhandled") if ref($res) ne 'ARRAY';
 
-sub rd_hdr ($) {
-        my ($self) = @_;
-        # typically used for reading CGI headers
-        # We also need to check EINTR for generic PSGI servers.
-        my $ret;
-        my $total_rd = 0;
-        my $hdr_buf = $self->{hdr_buf};
-        my ($ph_cb, $ph_arg) = @{$self->{parse_hdr}};
-        do {
-                my $r = sysread($self->{rpipe}, $$hdr_buf, 4096,
-                                length($$hdr_buf));
-                if (defined($r)) {
-                        $total_rd += $r;
-                        eval { $ret = $ph_cb->($total_rd, $hdr_buf, $ph_arg) };
-                        if ($@) {
-                                warn "parse_hdr: $@";
-                                $ret = [ 500, [], [ "Internal error\n" ] ];
-                        }
-                } else {
-                        # caller should notify us when it's ready:
-                        return if $! == EAGAIN;
-                        next if $! == EINTR; # immediate retry
-                        warn "error reading header: $!";
-                        $ret = [ 500, [], [ "Internal error\n" ] ];
-                }
-        } until (defined $ret);
-        delete $self->{parse_hdr}; # done parsing headers
-        $ret;
+        my $filter = blessed($res->[2]) && $res->[2]->can('attach') ?
+                        pop(@$res) : delete($env->{'qspawn.filter'});
+        $filter //= PublicInbox::GzipFilter::qsp_maybe($res->[1], $env);
+
+        if (scalar(@$res) == 3) { # done early (likely error or static file)
+                delete $self->{rpipe};
+                $ipipe->close if $ipipe;
+                $wcb->($res); # all done
+                return;
+        }
+        scalar(@$res) == 2 or confess("BUG: scalar(res) != 2: @$res");
+        return ($wcb, $filter) if !$ipipe; # generic PSGI
+        # streaming response
+        my $qfh = $wcb->($res); # get PublicInbox::HTTP::(Chunked|Identity)
+        $qfh = $filter->attach($qfh) if $filter;
+        my ($bref) = @{delete $self->{yield_parse_hdr}};
+        $qfh->write($$bref) if $$bref ne '';
+        $self->{qfh} = $qfh; # keep $ipipe open
 }
 
-sub psgi_return_init_cb {
+sub parse_hdr_done ($$) {
         my ($self) = @_;
-        my $r = rd_hdr($self) or return;
-        my $env = $self->{psgi_env};
-        my $filter = delete $env->{'qspawn.filter'} //
-                PublicInbox::GzipFilter::qsp_maybe($r->[1], $env);
-
-        my $wcb = delete $env->{'qspawn.wcb'};
-        my $async = delete $self->{async};
-        if (scalar(@$r) == 3) { # error
-                if ($async) {
-                        # calls rpipe->close && ->event_step
-                        $async->close;
-                } else {
-                        $self->{rpipe}->close;
-                        event_step($self);
+        my ($ret, $err);
+        if (defined $_[-1]) {
+                my ($bref, $ph_cb, @ph_arg) = @{$self->{yield_parse_hdr}};
+                $$bref .= $_[-1];
+                $ret = eval { $ph_cb->(length($_[-1]), $bref, @ph_arg) };
+                if (($err = $@)) {
+                        $ret = psgi_status_err();
+                } elsif (!$ret && $_[-1] eq '') {
+                        $err = 'EOF';
+                        $ret = psgi_status_err();
                 }
-                $wcb->($r);
-        } elsif ($async) {
-                # done reading headers, handoff to read body
-                my $fh = $wcb->($r); # scalar @$r == 2
-                $fh = $filter->attach($fh) if $filter;
-                $self->{fh} = $fh;
-                $async->async_pass($env->{'psgix.io'}, $fh,
-                                        delete($self->{hdr_buf}));
-        } else { # for synchronous PSGI servers
-                require PublicInbox::GetlineBody;
-                $r->[2] = PublicInbox::GetlineBody->new($self->{rpipe},
-                                        \&event_step, $self,
-                                        ${$self->{hdr_buf}}, $filter);
-                $wcb->($r);
+        } else {
+                $err = "$!";
+                $ret = psgi_status_err();
         }
+        carp <<EOM if $err;
+E: $err @{$self->{args}->[0]} ($self->{psgi_env}->{REQUEST_URI})
+EOM
+        $ret; # undef if headers incomplete
+}
+
+sub ipipe_cb { # InputPipe callback
+        my ($ipipe, $self) = @_; # $_[-1] rbuf
+        if ($self->{qfh}) { # already streaming
+                yield_chunk($self, $ipipe, $_[-1]);
+        } elsif (my $res = parse_hdr_done($self, $_[-1])) {
+                yield_pass($self, $ipipe, $res);
+        } # else: headers incomplete, keep reading
 }
 
-sub psgi_return_start { # may run later, much later...
+sub _yield_start { # may run later, much later...
         my ($self) = @_;
-        if (my $async = $self->{psgi_env}->{'pi-httpd.async'}) {
-                # PublicInbox::HTTPD::Async->new(rpipe, $cb, $cb_arg, $end_obj)
-                $self->{async} = $async->($self->{rpipe},
-                                        \&psgi_return_init_cb, $self, $self);
-        } else { # generic PSGI
-                psgi_return_init_cb($self) while $self->{parse_hdr};
+        if ($self->{psgi_env}->{'pi-httpd.async'}) {
+                my $rpipe = $self->{rpipe};
+                $rpipe->blocking(0);
+                PublicInbox::InputPipe::consume($rpipe, \&ipipe_cb, $self);
+        } else {
+                require PublicInbox::GetlineResponse;
+                PublicInbox::GetlineResponse::response($self);
         }
 }
 
@@ -271,7 +260,7 @@ sub psgi_return_start { # may run later, much later...
 #   $env->{'qspawn.wcb'} - the write callback from the PSGI server
 #                          optional, use this if you've already
 #                          captured it elsewhere.  If not given,
-#                          psgi_return will return an anonymous
+#                          psgi_yield will return an anonymous
 #                          sub for the PSGI server to call
 #
 #   $env->{'qspawn.filter'} - filter object, responds to ->attach for
@@ -280,76 +269,33 @@ sub psgi_return_start { # may run later, much later...
 #
 # $limiter - the Limiter object to use (uses the def_limiter if not given)
 #
-# $parse_hdr - Initial read function; often for parsing CGI header output.
+# @parse_hdr_arg - Initial read cb+args; often for parsing CGI header output.
 #              It will be given the return value of sysread from the pipe
 #              and a string ref of the current buffer.  Returns an arrayref
 #              for PSGI responses.  2-element arrays in PSGI mean the
 #              body will be streamed, later, via writes (push-based) to
 #              psgix.io.  3-element arrays means the body is available
 #              immediately (or streamed via ->getline (pull-based)).
-sub psgi_return {
-        my ($self, $env, $limiter, $parse_hdr, $hdr_arg) = @_;
+
+sub psgi_yield {
+        my ($self, $env, $limiter, @parse_hdr_arg)= @_;
         $self->{psgi_env} = $env;
-        $self->{hdr_buf} = \(my $hdr_buf = '');
-        $self->{parse_hdr} = [ $parse_hdr, $hdr_arg ];
-        $limiter ||= $def_limiter ||= PublicInbox::Qspawn::Limiter->new(32);
+        $self->{yield_parse_hdr} = [ \(my $buf = ''), @parse_hdr_arg ];
+        $limiter ||= $def_limiter ||= PublicInbox::Limiter->new(32);
 
         # the caller already captured the PSGI write callback from
         # the PSGI server, so we can call ->start, here:
-        $env->{'qspawn.wcb'} and
-                return start($self, $limiter, \&psgi_return_start);
-
-        # the caller will return this sub to the PSGI server, so
-        # it can set the response callback (that is, for
-        # PublicInbox::HTTP, the chunked_wcb or identity_wcb callback),
-        # but other HTTP servers are supported:
-        sub {
+        $env->{'qspawn.wcb'} ? start($self, $limiter, \&_yield_start) : sub {
+                # the caller will return this sub to the PSGI server, so
+                # it can set the response callback (that is, for
+                # PublicInbox::HTTP, the chunked_wcb or identity_wcb callback),
+                # but other HTTP servers are supported:
                 $env->{'qspawn.wcb'} = $_[0];
-                start($self, $limiter, \&psgi_return_start);
+                start($self, $limiter, \&_yield_start);
         }
 }
 
-package PublicInbox::Qspawn::Limiter;
-use strict;
-use warnings;
-
-sub new {
-        my ($class, $max) = @_;
-        bless {
-                # 32 is same as the git-daemon connection limit
-                max => $max || 32,
-                running => 0,
-                run_queue => [],
-                # RLIMIT_CPU => undef,
-                # RLIMIT_DATA => undef,
-                # RLIMIT_CORE => undef,
-        }, $class;
-}
-
-sub setup_rlimit {
-        my ($self, $name, $cfg) = @_;
-        foreach my $rlim (@PublicInbox::Spawn::RLIMITS) {
-                my $k = lc($rlim);
-                $k =~ tr/_//d;
-                $k = "publicinboxlimiter.$name.$k";
-                defined(my $v = $cfg->{$k}) or next;
-                my @rlimit = split(/\s*,\s*/, $v);
-                if (scalar(@rlimit) == 1) {
-                        push @rlimit, $rlimit[0];
-                } elsif (scalar(@rlimit) != 2) {
-                        warn "could not parse $k: $v\n";
-                }
-                eval { require BSD::Resource };
-                if ($@) {
-                        warn "BSD::Resource missing for $rlim";
-                        next;
-                }
-                foreach my $i (0..$#rlimit) {
-                        next if $rlimit[$i] ne 'INFINITY';
-                        $rlimit[$i] = BSD::Resource::RLIM_INFINITY();
-                }
-                $self->{$rlim} = \@rlimit;
-        }
-}
+no warnings 'once';
+*DESTROY = \&finalize; # ->finalize is idempotent
 
 1;
diff --git a/lib/PublicInbox/Reply.pm b/lib/PublicInbox/Reply.pm
index 592dfb62..091f20bc 100644
--- a/lib/PublicInbox/Reply.pm
+++ b/lib/PublicInbox/Reply.pm
@@ -68,7 +68,6 @@ sub mailto_arg_link {
         my $obfs = $ibx->{obfuscate};
         my $subj = $hdr->header('Subject') || '';
         $subj = "Re: $subj" unless $subj =~ /\bRe:/i;
-        my $subj_raw = $subj;
         my $mid = $hdr->header_raw('Message-ID');
         push @arg, '--in-reply-to='.squote_maybe(mid_clean($mid));
         my $irt = mid_href($mid);
@@ -98,8 +97,6 @@ sub mailto_arg_link {
                 }
         }
 
-        push @arg, "--subject=".squote_maybe($subj_raw);
-
         # I'm not sure if address obfuscation and mailto: links can
         # be made compatible; and address obfuscation is misguided,
         # anyways.
diff --git a/lib/PublicInbox/RepoAtom.pm b/lib/PublicInbox/RepoAtom.pm
new file mode 100644
index 00000000..ab0f2fcc
--- /dev/null
+++ b/lib/PublicInbox/RepoAtom.pm
@@ -0,0 +1,128 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# git log => Atom feed (cgit-compatible: $REPO/atom/[PATH]?h=$tip
+package PublicInbox::RepoAtom;
+use v5.12;
+use parent qw(PublicInbox::GzipFilter);
+use POSIX qw(strftime);
+use URI::Escape qw(uri_escape);
+use Scalar::Util ();
+use PublicInbox::Hval qw(ascii_html utf8_maybe);
+
+# git for-each-ref and log use different format fields :<
+my $ATOM_FMT = '--pretty=tformat:'.join('%n',
+                                map { "%$_" } qw(H ct an ae at s b)).'%x00';
+
+my $EACH_REF_FMT = '--format='.join(';', map { "\$r{'$_'}=%($_)" } qw(
+        objectname refname:short creator contents:subject contents:body
+        *subject *body)).'%00';
+
+sub atom_ok { # parse_hdr for qspawn
+        my ($r, $bref, $ctx) = @_;
+        return [ 404, [], [ "Not Found\n"] ] if $r == 0;
+        bless $ctx, __PACKAGE__;
+        my $h = [ 'Content-Type' => 'application/atom+xml; charset=UTF-8' ];
+        $ctx->{gz} = $ctx->can('gz_or_noop')->($h, $ctx->{env});
+        my $title = ascii_html(delete $ctx->{-feed_title});
+        my $desc = ascii_html($ctx->{git}->description);
+        my $url = ascii_html($ctx->{git}->base_url($ctx->{env}));
+        $ctx->{-base_url} = $url;
+        $ctx->zmore(<<EOM);
+<?xml version="1.0"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+<title>$title</title><subtitle>$desc</subtitle><link
+rel="alternate" type="text/html" href="$url"/>
+EOM
+        [ 200, $h, $ctx ]; # [2] is qspawn.filter
+}
+
+# called by GzipFilter->close
+sub zflush { $_[0]->SUPER::zflush('</feed>') }
+
+# called by GzipFilter->write or GetlineResponse->getline
+sub translate {
+        my $self = shift;
+        $_[0] // return zflush($self); # getline caller
+        my @out;
+        my $lbuf = delete($self->{lbuf}) // shift;
+        $lbuf .= shift while @_;
+        my $is_tag = $self->{-is_tag};
+        my ($H, $ct, $an, $ae, $at, $s, $bdy);
+        while ($lbuf =~ s/\A([^\0]+)\0\n//s) {
+                utf8_maybe($bdy = $1);
+                if ($is_tag) {
+                        my %r;
+                        eval "$bdy";
+                        for (qw(contents:subject contents:body)) {
+                                $r{$_} =~ /\S/ or delete($r{$_})
+                        }
+                        $H = $r{objectname};
+                        $s = $r{'contents:subject'} // $r{'*subject'};
+                        $bdy = $r{'contents:body'} // $r{'*body'};
+                        $s .= " ($r{'refname:short'})";
+                        $_ = ascii_html($_) for ($s, $bdy, $r{creator});
+                        ($an, $ae, $at) = split(/\s*&[gl]t;\s*/, $r{creator});
+                        $at =~ s/ .*\z//; # no TZ
+                        $ct = $at = strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($at));
+                } else {
+                        $bdy = ascii_html($bdy);
+                        ($H, $ct, $an, $ae, $at, $s, $bdy) =
+                                                        split(/\n/, $bdy, 7);
+                        $at = strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($at));
+                        $ct = strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($ct));
+                }
+                $bdy //= '';
+                push @out, <<"";
+<entry><title>$s</title><updated>$ct</updated><author><name>$an</name>
+<email>$ae</email></author><published>$at</published><link
+rel="alternate" type="text/html" href="$self->{-base_url}$H/s/"
+/><id>$H</id>
+
+                push @out, <<'', $bdy, '</pre></div></content>' if $bdy ne '';
+<content type="xhtml"><div
+xmlns="http://www.w3.org/1999/xhtml"><pre style="white-space:pre-wrap">
+
+                push @out, '</entry>';
+        }
+        $self->{lbuf} = $lbuf;
+        chomp @out;
+        @out ? $self->SUPER::translate(@out) : ''; # not EOF, yet
+}
+
+# $REPO/tags.atom endpoint
+sub srv_tags_atom {
+        my ($ctx) = @_;
+        my $max = 50; # TODO configurable
+        my @cmd = ('git', "--git-dir=$ctx->{git}->{git_dir}",
+                        qw(for-each-ref --sort=-creatordate), "--count=$max",
+                        '--perl', $EACH_REF_FMT, 'refs/tags');
+        $ctx->{-feed_title} = "$ctx->{git}->{nick} tags";
+        my $qsp = PublicInbox::Qspawn->new(\@cmd);
+        $ctx->{-is_tag} = 1;
+        $qsp->psgi_yield($ctx->{env}, undef, \&atom_ok, $ctx);
+}
+
+sub srv_atom {
+        my ($ctx, $path) = @_;
+        return if index($path, '//') >= 0 || index($path, '/') == 0;
+        my $max = 50; # TODO configurable
+        my @cmd = ('git', "--git-dir=$ctx->{git}->{git_dir}",
+                        qw(log --no-notes --no-color --no-abbrev),
+                        $ATOM_FMT, "-$max");
+        my $tip = $ctx->{qp}->{h}; # same as cgit
+        $ctx->{-feed_title} = $ctx->{git}->{nick};
+        $ctx->{-feed_title} .= " $path" if $path ne '';
+        if (defined($tip)) {
+                push @cmd, $tip;
+                $ctx->{-feed_title} .= ", $tip";
+        }
+        # else: let git decide based on HEAD if $tip isn't defined
+        push @cmd, '--';
+        push @cmd, $path if $path ne '';
+        my $qsp = PublicInbox::Qspawn->new(\@cmd, undef,
+                                        { quiet => 1, 2 => $ctx->{lh} });
+        $qsp->psgi_yield($ctx->{env}, undef, \&atom_ok, $ctx);
+}
+
+1;
diff --git a/lib/PublicInbox/RepoList.pm b/lib/PublicInbox/RepoList.pm
new file mode 100644
index 00000000..39dc9c0b
--- /dev/null
+++ b/lib/PublicInbox/RepoList.pm
@@ -0,0 +1,39 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+package PublicInbox::RepoList;
+use v5.12;
+use parent qw(PublicInbox::WwwStream);
+use PublicInbox::Hval qw(ascii_html prurl fmt_ts);
+require PublicInbox::CodeSearch;
+
+sub html_top_fallback { # WwwStream->html_repo_top
+        my ($ctx) = @_;
+        my $title = delete($ctx->{-title_html}) //
+                ascii_html("$ctx->{env}->{PATH_INFO}*");
+        my $upfx = $ctx->{-upfx} // '';
+        "<html><head><title>$title</title>" .
+                $ctx->{www}->style($upfx) . '</head><body>';
+}
+
+sub html ($$$) {
+        my ($wcr, $ctx, $re) = @_;
+        my $cr = $wcr->{pi_cfg}->{-coderepos};
+        my @nicks = grep(m!$re!, keys %$cr) or return; # 404
+        __PACKAGE__->html_init($ctx);
+        my $zfh = $ctx->zfh;
+        print $zfh "<pre>matching coderepos\n";
+        my @recs = PublicInbox::CodeSearch::repos_sorted($wcr->{pi_cfg},
+                                                        @$cr{@nicks});
+        my $env = $ctx->{env};
+        for (@recs) {
+                my ($t, $git) = @$_;
+                my $nick = ascii_html("$git->{nick}");
+                for my $u ($git->pub_urls($env)) {
+                        $u = prurl($env, $u);
+                        print $zfh "\n".fmt_ts($t).qq{ <a\nhref="$u">$nick</a>}
+                }
+        }
+        $ctx->html_done('</pre>');
+}
+
+1;
diff --git a/lib/PublicInbox/RepoSnapshot.pm b/lib/PublicInbox/RepoSnapshot.pm
new file mode 100644
index 00000000..4c372569
--- /dev/null
+++ b/lib/PublicInbox/RepoSnapshot.pm
@@ -0,0 +1,87 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# cgit-compatible /snapshot/ endpoint for WWW coderepos
+package PublicInbox::RepoSnapshot;
+use v5.12;
+use PublicInbox::Qspawn;
+use PublicInbox::ViewVCS;
+use PublicInbox::WwwStatic qw(r);
+
+# Not using standard mime types since the compressed tarballs are
+# special or do not match my /etc/mime.types.  Choose what gitweb
+# and cgit agree on for compatibility.
+our %FMT_TYPES = (
+        'tar' => 'application/x-tar',
+        'tar.gz' => 'application/x-gzip',
+        'tar.bz2' => 'application/x-bzip2',
+        'tar.xz' => 'application/x-xz',
+        'zip' => 'application/x-zip',
+);
+
+our %FMT_CFG = (
+        'tar.xz' => 'xz -c',
+        'tar.bz2' => 'bzip2 -c',
+        # not supporting lz nor zstd for now to avoid format proliferation
+        # and increased cache overhead required to handle extra formats.
+);
+
+my $SUFFIX = join('|', map { quotemeta } keys %FMT_TYPES);
+
+# TODO deal with tagged blobs
+
+sub archive_hdr { # parse_hdr for Qspawn
+        my ($r, $bref, $ctx) = @_;
+        $r or return [500, [qw(Content-Type text/plain Content-Length 0)], []];
+        my $fn = "$ctx->{snap_pfx}.$ctx->{snap_fmt}";
+        my $type = $FMT_TYPES{$ctx->{snap_fmt}} //
+                                die "BUG: bad fmt: $ctx->{snap_fmt}";
+        [ 200, [ 'Content-Type', "$type; charset=UTF-8",
+                'Content-Disposition', qq(inline; filename="$fn"),
+                'ETag', qq("$ctx->{etag}") ] ];
+}
+
+sub ver_check { # git->check_async callback
+        my (undef, $oid, $type, $size, $ctx) = @_;
+        return if defined $ctx->{etag};
+        my $treeish = shift @{$ctx->{-try}} // die 'BUG: no {-try}';
+        if ($type eq 'missing') {
+                scalar(@{$ctx->{-try}}) or
+                        delete($ctx->{env}->{'qspawn.wcb'})->(r(404));
+        } else { # found, done:
+                $ctx->{etag} = $oid;
+                my @cfg;
+                if (my $cmd = $FMT_CFG{$ctx->{snap_fmt}}) {
+                        @cfg = ('-c', "tar.$ctx->{snap_fmt}.command=$cmd");
+                }
+                my $qsp = PublicInbox::Qspawn->new(['git', @cfg,
+                                "--git-dir=$ctx->{git}->{git_dir}", 'archive',
+                                "--prefix=$ctx->{snap_pfx}/",
+                                "--format=$ctx->{snap_fmt}", $treeish], undef,
+                                { quiet => 1 });
+                $qsp->psgi_yield($ctx->{env}, undef, \&archive_hdr, $ctx);
+        }
+}
+
+sub srv {
+        my ($ctx, $fn) = @_;
+        return if $fn =~ /["\s]/s;
+        my $fmt = $ctx->{wcr}->{snapshots}; # TODO per-repo snapshots
+        $fn =~ s/\.($SUFFIX)\z//o and $fmt->{$1} or return;
+        $ctx->{snap_fmt} = $1;
+        my $pfx = $ctx->{git}->local_nick // return;
+        $pfx =~ s/(?:\.git)?\z/-/;
+        ($pfx) = ($pfx =~ m!([^/]+)\z!);
+        substr($fn, 0, length($pfx)) eq $pfx or return;
+        $ctx->{snap_pfx} = $fn;
+        my $v = $ctx->{snap_ver} = substr($fn, length($pfx), length($fn));
+        # try without [vV] prefix, first
+        my @try = map { "$_$v" } ('', 'v', 'V'); # cf. cgit:ui-snapshot.c
+        @{$ctx->{-try}} = @try;
+        sub {
+                $ctx->{env}->{'qspawn.wcb'} = $_[0];
+                PublicInbox::ViewVCS::do_check_async($ctx, \&ver_check, @try);
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/RepoTree.pm b/lib/PublicInbox/RepoTree.pm
new file mode 100644
index 00000000..5c73531a
--- /dev/null
+++ b/lib/PublicInbox/RepoTree.pm
@@ -0,0 +1,99 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# cgit-compatible $REPO/tree/[PATH]?h=$tip redirector
+package PublicInbox::RepoTree;
+use v5.12;
+use PublicInbox::ViewDiff qw(uri_escape_path);
+use PublicInbox::WwwStatic qw(r);
+use PublicInbox::Qspawn;
+use PublicInbox::WwwStream qw(html_oneshot);
+use PublicInbox::Hval qw(ascii_html utf8_maybe);
+
+sub rd_404_log {
+        my ($bref, $ctx) = @_;
+        my $path = $ctx->{-q_value_html} = ascii_html($ctx->{-path});
+        my $tip = 'HEAD';
+        $tip = ascii_html($ctx->{qp}->{h}) if defined($ctx->{qp}->{h});
+        PublicInbox::WwwStream::html_init($ctx);
+        my $zfh = $ctx->{zfh};
+        print $zfh "<pre>\$ git log -1 $tip -- $path\n";
+        my $code = 200;
+        if ($$bref eq '') {
+                say $zfh "found no record of `$path' in git history in `$tip'";
+                $ctx->{-has_srch} and
+                        say $zfh 'perhaps try searching mail (above)';
+                $code = 404;
+        } else {
+                my ($H, $h, $s_as) = split(/ /, $$bref, 3);
+                utf8_maybe($s_as);
+                my $x = uri_escape_path($ctx->{-path});
+                $s_as = ascii_html($s_as);
+                print $zfh <<EOM;
+found last record of `$path' in the following commit:
+
+<a href="$ctx->{-upfx}$H/s/?b=$x">$h</a> $s_as
+EOM
+        }
+        my $res = $ctx->html_done;
+        $res->[0] = $code;
+        delete($ctx->{-wcb})->($res);
+}
+
+sub find_missing {
+        my ($ctx) = @_;
+        if ($ctx->{-path} eq '') {
+                my $tip = 'HEAD';
+                $tip = ascii_html($ctx->{qp}->{h}) if defined($ctx->{qp}->{h});
+                PublicInbox::WwwStream::html_init($ctx);
+                print { $ctx->{zfh} } "<pre>`$tip' ref not found</pre>";
+                my $res = $ctx->html_done;
+                $res->[0] = 404;
+                return delete($ctx->{-wcb})->($res);
+        }
+        my $cmd = ['git', "--git-dir=$ctx->{git}->{git_dir}",
+                qw(log --no-color -1), '--pretty=%H %h %s (%as)' ];
+        push @$cmd, $ctx->{qp}->{h} if defined($ctx->{qp}->{h});
+        push @$cmd, '--';
+        push @$cmd, $ctx->{-path};
+        my $qsp = PublicInbox::Qspawn->new($cmd, undef,
+                                        { quiet => 1, 2 => $ctx->{lh} });
+        $qsp->psgi_qx($ctx->{env}, undef, \&rd_404_log, $ctx);
+}
+
+sub tree_show { # git check_async callback
+        my (undef, $oid, $type, $size, $ctx) = @_;
+        return find_missing($ctx) if $type eq 'missing';
+
+        my $res = [ $ctx->{git}, $oid, $type, $size ];
+        my ($bn) = ($ctx->{-path} =~ m!/?([^/]+)\z!);
+        if ($type eq 'blob') {
+                my $obj = ascii_html($ctx->{-obj});
+                $ctx->{-q_value_html} = 'dfn:'.ascii_html($ctx->{-path}) .
+                        ' dfpost:'.substr($oid, 0, 7);
+                $ctx->{-paths} = [ $bn, qq[(<a
+href="$ctx->{-upfx}$oid/s/$bn">raw</a>)
+\$ git show $obj\t# shows this blob on the CLI] ];
+        }
+        PublicInbox::ViewVCS::solve_result($res, $ctx);
+}
+
+sub srv_tree {
+        my ($ctx, $path) = @_;
+        return if index($path, '//') >= 0 || index($path, '/') == 0;
+        my $tip = $ctx->{qp}->{h} // 'HEAD';
+        $ctx->{-upfx} = '../' x (($path =~ tr!/!/!) + 1);
+        $path =~ s!/\z!!;
+        my $obj = $ctx->{-obj} = "$tip:$path";
+        $ctx->{-path} = $path;
+
+        # "\n" breaks with `git cat-file --batch-check', and there's no
+        # legitimate use of "\n" in filenames anyways.
+        return if index($obj, "\n") >= 0;
+        sub {
+                $ctx->{-wcb} = $_[0]; # HTTP::{Chunked,Identity}
+                PublicInbox::ViewVCS::do_check_async($ctx, \&tree_show, $obj);
+        };
+}
+
+1;
diff --git a/lib/PublicInbox/SHA.pm b/lib/PublicInbox/SHA.pm
new file mode 100644
index 00000000..3fa8530e
--- /dev/null
+++ b/lib/PublicInbox/SHA.pm
@@ -0,0 +1,67 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# OpenSSL exception added in commit 22711f81f4e79da6b796820e37803a05cae14645
+# (README: add OpenSSL exception, 2015-10-05)
+
+# Replaces most uses of Digest::SHA with OpenSSL via Net::SSLeay if
+# possible.  OpenSSL SHA-256 is nearly twice as fast as Digest::SHA on
+# x86-64, and SHA-1 is a bit faster as well.
+# I don't think we can implement Digest::SHA->clone with what Net::SSLeay
+# gives us...  (maybe EVP_MD_CTX_copy+EVP_MD_CTX_copy_ex need to be added
+# to Net::SSLeay?)
+package PublicInbox::SHA;
+use v5.12;
+require Exporter;
+our @EXPORT_OK = qw(sha1_hex sha256_hex sha256 sha_all);
+use autodie qw(sysread);
+our @ISA;
+
+BEGIN {
+        push @ISA, 'Exporter';
+        unless (eval(<<'EOM')) {
+use Net::SSLeay 1.43;
+my %SHA = (
+        1 => Net::SSLeay::EVP_sha1(),
+        256 => Net::SSLeay::EVP_sha256(),
+);
+
+sub new {
+        my ($cls, $n) = @_;
+        my $mdctx = Net::SSLeay::EVP_MD_CTX_create();
+        Net::SSLeay::EVP_DigestInit($mdctx, $SHA{$n}) or
+                        die "EVP_DigestInit $n: $!";
+        bless \$mdctx, $cls;
+}
+
+sub add {
+        my $self = shift;
+        Net::SSLeay::EVP_DigestUpdate($$self, $_) for @_;
+        $self;
+}
+
+sub digest { Net::SSLeay::EVP_DigestFinal(${$_[0]}) };
+sub hexdigest { unpack('H*', Net::SSLeay::EVP_DigestFinal(${$_[0]})) }
+sub DESTROY { Net::SSLeay::EVP_MD_CTX_destroy(${$_[0]}) };
+
+sub sha1_hex { unpack('H*', Net::SSLeay::SHA1($_[0])) };
+sub sha256_hex { unpack('H*', Net::SSLeay::SHA256($_[0])) };
+*sha256 = \&Net::SSLeay::SHA256;
+# end of eval
+EOM
+        require Digest::SHA; # stdlib fallback
+        push @ISA, 'Digest::SHA';
+        *sha1_hex = \&Digest::SHA::sha1_hex;
+        *sha256_hex = \&Digest::SHA::sha256_hex;
+        *sha256 = \&Digest::SHA::sha256;
+}
+
+} # /BEGIN
+
+sub sha_all ($$) {
+        my ($n, $fh) = @_;
+        my ($dig, $buf) = (PublicInbox::SHA->new($n));
+        while (sysread($fh, $buf, 65536)) { $dig->add($buf) }
+        $dig
+}
+
+1;
diff --git a/lib/PublicInbox/SaPlugin/ListMirror.pm b/lib/PublicInbox/SaPlugin/ListMirror.pm
index 9acf86c0..06903cad 100644
--- a/lib/PublicInbox/SaPlugin/ListMirror.pm
+++ b/lib/PublicInbox/SaPlugin/ListMirror.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # SpamAssassin rules useful for running a mailing list mirror.  We want to:
@@ -39,7 +39,11 @@ sub check_list_mirror_received {
                 my $v = $pms->get($hdr) or next;
                 local $/ = "\n";
                 chomp $v;
-                next if $v ne $hval;
+                if (ref($hval)) {
+                        next if $v !~ $hval;
+                } else {
+                        next if $v ne $hval;
+                }
                 return 1 if $recvd !~ $host_re;
         }
 
@@ -91,6 +95,8 @@ sub config_list_mirror {
         $host_glob =~ s!(.)!$patmap{$1} || "\Q$1"!ge;
         my $host_re = qr/\A\s*from\s+$host_glob(?:\s|$)/si;
 
+        (lc($hdr) eq 'list-id' && $hval =~ /<([^>]+)>/) and
+                $hval = qr/\A<\Q$1\E>\z/;
         push @{$self->{list_mirror_check}}, [ $hdr, $hval, $host_re, $addr ];
 }
 
diff --git a/lib/PublicInbox/SaPlugin/ListMirror.pod b/lib/PublicInbox/SaPlugin/ListMirror.pod
index d931d762..e6a6c2ad 100644
--- a/lib/PublicInbox/SaPlugin/ListMirror.pod
+++ b/lib/PublicInbox/SaPlugin/ListMirror.pod
@@ -6,11 +6,11 @@ PublicInbox::SaPlugin::ListMirror - SpamAssassin plugin for mailing list mirrors
 
   loadplugin PublicInbox::SaPlugin::ListMirror
 
-Declare some mailing lists based on the expected List-Id value,
+Declare some mailing lists based on the expected List-ID value,
 expected servers, and mailing list address:
 
-  list_mirror List-Id <foo.example.com> *.example.com foo@example.com
-  list_mirror List-Id <bar.example.com> *.example.com bar@example.com
+  list_mirror List-ID <foo.example.com> *.example.com foo@example.com
+  list_mirror List-ID <bar.example.com> *.example.com bar@example.com
 
 Bump the score for messages which come from unexpected servers:
 
@@ -43,14 +43,25 @@ C<allow_user_rules 1>
 
 =item list_mirror HEADER HEADER_VALUE HOSTNAME_GLOB [LIST_ADDRESS]
 
-Declare a list based on an expected C<HEADER> matching C<HEADER_NAME>
-exactly coming from C<HOSTNAME_GLOB>.  C<LIST_ADDRESS> is optional,
+Declare a list based on an expected C<HEADER> matching C<HEADER_VALUE>
+coming from C<HOSTNAME_GLOB>.  C<LIST_ADDRESS> is optional,
 but may specify the address of the mailing list being mirrored.
 
-C<List-Id> or C<X-Mailing-List> are common values of C<HEADER>
+C<List-ID> is the recommended value of C<HEADER> as most
+mailing lists support it.
 
 An example of C<HEADER_VALUE> is C<E<lt>foo.example.orgE<gt>>
-if C<HEADER> is C<List-Id>.
+if C<HEADER> is C<List-ID>.
+
+As of public-inbox 2.0, using C<List-ID> as the C<HEADER> and a
+C<HEADER_VALUE> contained by angle brackets (E<lt>list-idE<gt>),
+matching is done in accordance with
+L<RFC 2919|https://tools.ietf.org/html/rfc2919>.  That is,
+C<HEADER_VALUE> will be a case-insensitive substring match
+and ignore the optional description C<phrase> as documented
+in RFC 2919.
+
+All other C<HEADER> values use exact matches for backwards-compatibility.
 
 C<HOSTNAME_GLOB> may be a wildcard match for machines where mail can
 come from or an exact match.
@@ -105,7 +116,7 @@ and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta
 
 =head1 COPYRIGHT
 
-Copyright (C) 2016-2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright (C) all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<http://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index b6141f68..678c8c5d 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -59,20 +59,46 @@ use PublicInbox::Smsg;
 use PublicInbox::Over;
 our $QP_FLAGS;
 our %X = map { $_ => 0 } qw(BoolWeight Database Enquire QueryParser Stem Query);
-our $Xap; # 'Search::Xapian' or 'Xapian'
+our $Xap; # 'Xapian' or 'Search::Xapian'
 our $NVRP; # '$Xap::'.('NumberValueRangeProcessor' or 'NumberRangeProcessor')
 
 # ENQ_DESCENDING and ENQ_ASCENDING weren't in SWIG Xapian.pm prior to 1.4.16,
 # let's hope the ABI is stable
 our $ENQ_DESCENDING = 0;
 our $ENQ_ASCENDING = 1;
+our @MAIL_VMAP = (
+        [ YYYYMMDD, 'd:'],
+        [ TS, 'rt:' ],
+        # these are undocumented for WWW, but lei and IMAP use them
+        [ DT, 'dt:' ],
+        [ BYTES, 'z:' ],
+        [ UID, 'uid:' ]
+);
+our @MAIL_NRP;
+
+# Getopt::Long spec, only short options for portability in C++ implementation
+our @XH_SPEC = (
+        'a', # ascending sort
+        'c', # code search
+        'd=s@', # shard dirs
+        'g=s', # git dir (with -c)
+        'k=i', # sort column (like sort(1))
+        'm=i', # maximum number of results
+        'o=i', # offset
+        'p', # show percent
+        'r', # 1=relevance then column
+        't', # collapse threads
+        'A=s@', # prefixes
+        'D', # emit docdata
+        'K=i', # timeout kill after i seconds
+        'O=s', # eidx_key
+        'T=i', # threadid
+);
 
 sub load_xapian () {
         return 1 if defined $Xap;
-        # n.b. PI_XAPIAN is intended for development use only.  We still
-        # favor Search::Xapian since that's what's available in current
-        # Debian stable (10.x) and derived distros.
-        for my $x (($ENV{PI_XAPIAN} // 'Search::Xapian'), 'Xapian') {
+        # n.b. PI_XAPIAN is intended for development use only
+        for my $x (($ENV{PI_XAPIAN} // 'Xapian'), 'Search::Xapian') {
                 eval "require $x";
                 next if $@;
 
@@ -85,8 +111,7 @@ sub load_xapian () {
 
                 # NumberRangeProcessor was added in Xapian 1.3.6,
                 # NumberValueRangeProcessor was removed for 1.5.0+,
-                # favor the older /Value/ variant since that's what our
-                # (currently) preferred Search::Xapian supports
+                # continue with the older /Value/ variant for now...
                 $NVRP = $x.'::'.($x eq 'Xapian' && $xver ge v1.5 ?
                         'NumberRangeProcessor' : 'NumberValueRangeProcessor');
                 $X{$_} = $Xap.'::'.$_ for (keys %X);
@@ -101,6 +126,7 @@ sub load_xapian () {
                 # or make indexlevel=medium as default
                 $QP_FLAGS = FLAG_PHRASE() | FLAG_BOOLEAN() | FLAG_LOVEHATE() |
                                 FLAG_WILDCARD();
+                @MAIL_NRP = map { $NVRP->new(@$_) } @MAIL_VMAP;
                 return 1;
         }
         undef;
@@ -110,43 +136,50 @@ sub load_xapian () {
 # a prefix common in patch emails
 our $LANG = 'english';
 
+our %PATCH_BOOL_COMMON = (
+        dfpre => 'XDFPRE',
+        dfpost => 'XDFPOST',
+        dfblob => 'XDFPRE XDFPOST',
+        patchid => 'XDFID',
+);
+
 # note: the non-X term prefix allocations are shared with
 # Xapian omega, see xapian-applications/omega/docs/termprefixes.rst
 my %bool_pfx_external = (
         mid => 'Q', # Message-ID (full/exact), this is mostly uniQue
         lid => 'G', # newsGroup (or similar entity), just inside <>
-        dfpre => 'XDFPRE',
-        dfpost => 'XDFPOST',
-        dfblob => 'XDFPRE XDFPOST',
-        patchid => 'XDFID',
+        %PATCH_BOOL_COMMON
 );
 
-my $non_quoted_body = 'XNQ XDFN XDFA XDFB XDFHH XDFCTX XDFPRE XDFPOST XDFID';
-my %prob_prefix = (
-        # for mairix compatibility
+# for mairix compatibility
+our $NON_QUOTED_BODY = 'XNQ XDFN XDFA XDFB XDFHH XDFCTX XDFPRE XDFPOST XDFID';
+our %PATCH_PROB_COMMON = (
         s => 'S',
-        m => 'XM', # 'mid:' (bool) is exact, 'm:' (prob) can do partial
-        l => 'XL', # 'lid:' (bool) is exact, 'l:' (prob) can do partial
         f => 'A',
-        t => 'XTO',
-        tc => 'XTO XCC',
-        c => 'XCC',
-        tcf => 'XTO XCC A',
-        a => 'XTO XCC A',
-        b => $non_quoted_body . ' XQUOT',
-        bs => $non_quoted_body . ' XQUOT S',
+        b => $NON_QUOTED_BODY . ' XQUOT',
+        bs => $NON_QUOTED_BODY . ' XQUOT S',
         n => 'XFN',
 
         q => 'XQUOT',
-        nq => $non_quoted_body,
+        nq => $NON_QUOTED_BODY,
         dfn => 'XDFN',
         dfa => 'XDFA',
         dfb => 'XDFB',
         dfhh => 'XDFHH',
         dfctx => 'XDFCTX',
+);
 
+my %prob_prefix = (
+        m => 'XM', # 'mid:' (bool) is exact, 'm:' (prob) can do partial
+        l => 'XL', # 'lid:' (bool) is exact, 'l:' (prob) can do partial
+        t => 'XTO',
+        tc => 'XTO XCC',
+        c => 'XCC',
+        tcf => 'XTO XCC A',
+        a => 'XTO XCC A',
+        %PATCH_PROB_COMMON,
         # default:
-        '' => 'XM S A XQUOT XFN ' . $non_quoted_body,
+        '' => 'XM S A XQUOT XFN ' . $NON_QUOTED_BODY,
 );
 
 # not documenting m: and mid: for now, the using the URLs works w/o Xapian
@@ -190,33 +223,37 @@ sub xdir ($;$) {
         my ($self, $rdonly) = @_;
         if ($rdonly || !defined($self->{shard})) {
                 $self->{xpfx};
-        } else { # v2 + extindex only:
+        } else { # v2, extindex, cindex only:
                 "$self->{xpfx}/$self->{shard}";
         }
 }
 
-# returns all shards as separate Xapian::Database objects w/o combining
-sub xdb_shards_flat ($) {
+# returns shard directories as an array of strings, does not verify existence
+sub shard_dirs ($) {
         my ($self) = @_;
         my $xpfx = $self->{xpfx};
-        my (@xdb, $slow_phrase);
-        load_xapian();
-        $self->{qp_flags} //= $QP_FLAGS;
-        if ($xpfx =~ m!/xapian[0-9]+\z!) {
-                @xdb = ($X{Database}->new($xpfx));
-                $self->{qp_flags} |= FLAG_PHRASE() if !-f "$xpfx/iamchert";
-        } else {
+        if ($xpfx =~ m!/xapian[0-9]+\z!) { # v1 inbox
+                ($xpfx);
+        } else { # v2 inbox, eidx, cidx
                 opendir(my $dh, $xpfx) or return (); # not initialized yet
                 # We need numeric sorting so shard[0] is first for reading
                 # Xapian metadata, if needed
                 my $last = max(grep(/\A[0-9]+\z/, readdir($dh))) // return ();
-                for (0..$last) {
-                        my $shard_dir = "$self->{xpfx}/$_";
-                        push @xdb, $X{Database}->new($shard_dir);
-                        $slow_phrase ||= -f "$shard_dir/iamchert";
-                }
-                $self->{qp_flags} |= FLAG_PHRASE() if !$slow_phrase;
+                map { "$xpfx/$_" } (0..$last);
         }
+}
+
+# returns all shards as separate Xapian::Database objects w/o combining
+sub xdb_shards_flat ($) {
+        my ($self) = @_;
+        load_xapian();
+        $self->{qp_flags} //= $QP_FLAGS;
+        my $slow_phrase;
+        my @xdb = map {
+                $slow_phrase ||= -f "$_/iamchert";
+                $X{Database}->new($_); # raises if missing
+        } shard_dirs($self);
+        $self->{qp_flags} |= FLAG_PHRASE() if !$slow_phrase;
         @xdb;
 }
 
@@ -229,6 +266,12 @@ sub mdocid {
         int(($docid - 1) / $nshard) + 1;
 }
 
+sub docids_to_artnums {
+        my $nshard = shift->{nshard};
+        # XXX does array vs arrayref make a difference in modern Perls?
+        map { int(($_ - 1) / $nshard) + 1 } @_;
+}
+
 sub mset_to_artnums {
         my ($self, $mset) = @_;
         my $nshard = $self->{nshard};
@@ -277,42 +320,18 @@ sub date_parse_prepare {
         my $end = $range =~ s/([\)\s]*)\z// ? $1 : '';
         my @r = split(/\.\./, $range, 2);
 
-        # expand "d:20101002" => "d:20101002..20101003" and like
+        # expand "dt:2010-10-02" => "dt:2010-10-02..2010-10-03" and like
         # n.b. git doesn't do YYYYMMDD w/o '-', it needs YYYY-MM-DD
-        # We upgrade "d:" to "dt:" to iff using approxidate
+        # We upgrade "d:" to "dt:" unconditionally
         if ($pfx eq 'd') {
-                my $fmt = "\0%Y%m%d";
-                if (!defined($r[1])) {
-                        if ($r[0] =~ /\A([0-9]{4})([0-9]{2})([0-9]{2})\z/) {
-                                push @$to_parse, "$1-$2-$3";
-                                # we could've handled as-is, but we need
-                                # to parse anyways for "d+" below
-                        } else {
-                                push @$to_parse, $r[0];
-                                if ($r[0] !~ /\A[0-9]{4}-[0-9]{2}-[0-9]{2}\z/) {
-                                        $pfx = 'dt';
-                                        $fmt = "\0%Y%m%d%H%M%S";
-                                }
-                        }
-                        $r[0] = "$fmt+$#$to_parse\0";
-                        $r[1] = "$fmt+\0";
-                } else {
-                        for my $x (@r) {
-                                next if $x eq '' || $x =~ /\A[0-9]{8}\z/;
-                                push @$to_parse, $x;
-                                if ($x !~ /\A[0-9]{4}-[0-9]{2}-[0-9]{2}\z/) {
-                                        $pfx = 'dt';
-                                }
-                                $x = "$fmt$#$to_parse\0";
-                        }
-                        if ($pfx eq 'dt') {
-                                for (@r) {
-                                        s/\0%Y%m%d/\0%Y%m%d%H%M%S/;
-                                        s/\A([0-9]{8})\z/${1}000000/;
-                                }
-                        }
-                }
-        } elsif ($pfx eq 'dt') {
+                $pfx = 'dt';
+                # upgrade YYYYMMDD to YYYYMMDDHHMMSS
+                $_ .= ' 00:00:00' for (grep(m!\A[0-9]{4}[^[:alnum:]]
+                                        [0-9]{2}[^[:alnum:]]
+                                        [0-9]{2}\z!x, @r));
+                $_ .= '000000' for (grep(m!\A[0-9]{8}\z!, @r));
+        }
+        if ($pfx eq 'dt') {
                 if (!defined($r[1])) { # git needs gaps and not /\d{14}/
                         if ($r[0] =~ /\A([0-9]{4})([0-9]{2})([0-9]{2})
                                         ([0-9]{2})([0-9]{2})([0-9]{2})\z/x) {
@@ -329,7 +348,7 @@ sub date_parse_prepare {
                                 $x = "\0%Y%m%d%H%M%S$#$to_parse\0";
                         }
                 }
-        } else { # "rt", let git interpret "YYYY", deal with Y10K later :P
+        } else { # (rt|ct), let git interpret "YYYY", deal with Y10K later :P
                 for my $x (@r) {
                         next if $x eq '' || $x =~ /\A[0-9]{5,}\z/;
                         push @$to_parse, $x;
@@ -388,13 +407,51 @@ sub query_approxidate {
         date_parse_finalize($git, $to_parse, $_[2]) if $to_parse;
 }
 
-# read-only
+# read-only, for mail only (codesearch has different rules)
 sub mset {
-        my ($self, $query_string, $opts) = @_;
-        $opts ||= {};
+        my ($self, $qry_str, $opt) = @_;
         my $qp = $self->{qp} //= $self->qparse_new;
-        my $query = $qp->parse_query($query_string, $self->{qp_flags});
-        _do_enquire($self, $query, $opts);
+        my $qry = $qp->parse_query($qry_str, $self->{qp_flags});
+        if (defined(my $eidx_key = $opt->{eidx_key})) {
+                $qry = $X{Query}->new(OP_FILTER(), $qry, 'O'.$eidx_key);
+        }
+        if (defined(my $uid_range = $opt->{uid_range})) {
+                my $range = $X{Query}->new(OP_VALUE_RANGE(), UID,
+                                        sortable_serialise($uid_range->[0]),
+                                        sortable_serialise($uid_range->[1]));
+                $qry = $X{Query}->new(OP_FILTER(), $qry, $range);
+        }
+        if (defined(my $tid = $opt->{threadid})) {
+                $tid = sortable_serialise($tid);
+                $qry = $X{Query}->new(OP_FILTER(), $qry,
+                        $X{Query}->new(OP_VALUE_RANGE(), THREADID, $tid, $tid));
+        }
+        do_enquire($self, $qry, $opt, TS);
+}
+
+sub do_enquire { # shared with CodeSearch
+        my ($self, $qry, $opt, $col) = @_;
+        my $enq = $X{Enquire}->new(xdb($self));
+        $enq->set_query($qry);
+        my $rel = $opt->{relevance} // 0;
+        if ($rel == -2) { # ORDER BY docid/UID (highest first)
+                $enq->set_weighting_scheme($X{BoolWeight}->new);
+                $enq->set_docid_order($ENQ_DESCENDING);
+        } elsif ($rel == -1) { # ORDER BY docid/UID (lowest first)
+                $enq->set_weighting_scheme($X{BoolWeight}->new);
+                $enq->set_docid_order($ENQ_ASCENDING);
+        } elsif ($rel == 0) {
+                $enq->set_sort_by_value_then_relevance($col, !$opt->{asc});
+        } else { # rel > 0
+                $enq->set_sort_by_relevance_then_value($col, !$opt->{asc});
+        }
+
+        # `lei q -t / --threads' or JMAP collapseThreads; but don't collapse
+        # on `-tt' ({threads} > 1) which sets the Flagged|Important keyword
+        (($opt->{threads} // 0) == 1 && has_threadid($self)) and
+                $enq->set_collapse_key(THREADID);
+        retry_reopen($self, \&enquire_once, $enq,
+                        $opt->{offset} || 0, $opt->{limit} || 50);
 }
 
 sub retry_reopen {
@@ -421,50 +478,15 @@ sub retry_reopen {
         Carp::croak("Too many Xapian database modifications in progress\n");
 }
 
-sub _do_enquire {
-        my ($self, $query, $opts) = @_;
-        retry_reopen($self, \&_enquire_once, $query, $opts);
-}
-
 # returns true if all docs have the THREADID value
 sub has_threadid ($) {
         my ($self) = @_;
         (xdb($self)->get_metadata('has_threadid') // '') eq '1';
 }
 
-sub _enquire_once { # retry_reopen callback
-        my ($self, $query, $opts) = @_;
-        my $xdb = xdb($self);
-        if (defined(my $eidx_key = $opts->{eidx_key})) {
-                $query = $X{Query}->new(OP_FILTER(), $query, 'O'.$eidx_key);
-        }
-        if (defined(my $uid_range = $opts->{uid_range})) {
-                my $range = $X{Query}->new(OP_VALUE_RANGE(), UID,
-                                        sortable_serialise($uid_range->[0]),
-                                        sortable_serialise($uid_range->[1]));
-                $query = $X{Query}->new(OP_FILTER(), $query, $range);
-        }
-        my $enquire = $X{Enquire}->new($xdb);
-        $enquire->set_query($query);
-        $opts ||= {};
-        my $rel = $opts->{relevance} // 0;
-        if ($rel == -2) { # ORDER BY docid/UID (highest first)
-                $enquire->set_weighting_scheme($X{BoolWeight}->new);
-                $enquire->set_docid_order($ENQ_DESCENDING);
-        } elsif ($rel == -1) { # ORDER BY docid/UID (lowest first)
-                $enquire->set_weighting_scheme($X{BoolWeight}->new);
-                $enquire->set_docid_order($ENQ_ASCENDING);
-        } elsif ($rel == 0) {
-                $enquire->set_sort_by_value_then_relevance(TS, !$opts->{asc});
-        } else { # rel > 0
-                $enquire->set_sort_by_relevance_then_value(TS, !$opts->{asc});
-        }
-
-        # `mairix -t / --threads' or JMAP collapseThreads
-        if ($opts->{threads} && has_threadid($self)) {
-                $enquire->set_collapse_key(THREADID);
-        }
-        $enquire->get_mset($opts->{offset} || 0, $opts->{limit} || 50);
+sub enquire_once { # retry_reopen callback
+        my (undef, $enq, $offset, $limit) = @_;
+        $enq->get_mset($offset, $limit);
 }
 
 sub mset_to_smsg {
@@ -481,29 +503,27 @@ sub mset_to_smsg {
 # read-write
 sub stemmer { $X{Stem}->new($LANG) }
 
-# read-only
-sub qparse_new {
+sub qp_init_common {
         my ($self) = @_;
-
-        my $xdb = xdb($self);
         my $qp = $X{QueryParser}->new;
         $qp->set_default_op(OP_AND());
-        $qp->set_database($xdb);
+        $qp->set_database(xdb($self));
         $qp->set_stemmer(stemmer($self));
         $qp->set_stemming_strategy(STEM_SOME());
         my $cb = $qp->can('set_max_wildcard_expansion') //
                 $qp->can('set_max_expansion'); # Xapian 1.5.0+
         $cb->($qp, 100);
-        $cb = $qp->can('add_valuerangeprocessor') //
-                $qp->can('add_rangeprocessor'); # Xapian 1.5.0+
-        $cb->($qp, $NVRP->new(YYYYMMDD, 'd:'));
-        $cb->($qp, $NVRP->new(DT, 'dt:'));
+        $qp;
+}
 
-        # for IMAP, undocumented for WWW and may be split off go away
-        $cb->($qp, $NVRP->new(BYTES, 'z:'));
-        $cb->($qp, $NVRP->new(TS, 'rt:'));
-        $cb->($qp, $NVRP->new(UID, 'uid:'));
+# read-only
+sub qparse_new {
+        my ($self) = @_;
+        my $qp = qp_init_common($self);
+        my $cb = $qp->can('add_valuerangeprocessor') //
+                $qp->can('add_rangeprocessor'); # Xapian 1.5.0+
 
+        $cb->($qp, $_) for @MAIL_NRP;
         while (my ($name, $prefix) = each %bool_pfx_external) {
                 $qp->add_boolean_prefix($name, $_) foreach split(/ /, $prefix);
         }
@@ -533,6 +553,40 @@ EOF
         $qp;
 }
 
+sub generate_cxx () { # generates snippet for xap_helper.h
+        my $ret = <<EOM;
+# line ${\__LINE__} "${\__FILE__}"
+static NRP *mail_nrp[${\scalar(@MAIL_VMAP)}];
+static void mail_nrp_init(void)
+{
+EOM
+        for (0..$#MAIL_VMAP) {
+                my $x = $MAIL_VMAP[$_];
+                $ret .= qq{\tmail_nrp[$_] = new NRP($x->[0], "$x->[1]");\n}
+        }
+$ret .= <<EOM;
+}
+
+# line ${\__LINE__} "${\__FILE__}"
+static void qp_init_mail_search(Xapian::QueryParser *qp)
+{
+        for (size_t i = 0; i < MY_ARRAY_SIZE(mail_nrp); i++)
+                qp->ADD_RP(mail_nrp[i]);
+EOM
+        for my $name (sort keys %bool_pfx_external) {
+                for (split(/ /, $bool_pfx_external{$name})) {
+                        $ret .= qq{\tqp->add_boolean_prefix("$name", "$_");\n}
+                }
+        }
+        # TODO: altid support
+        for my $name (sort keys %prob_prefix) {
+                for (split(/ /, $prob_prefix{$name})) {
+                        $ret .= qq{\tqp->add_prefix("$name", "$_");\n}
+                }
+        }
+        $ret .= "}\n";
+}
+
 sub help {
         my ($self) = @_;
         $self->{qp} //= $self->qparse_new; # parse altids
@@ -543,9 +597,10 @@ sub help {
         \@ret;
 }
 
+# always returns a scalar value
 sub int_val ($$) {
         my ($doc, $col) = @_;
-        my $val = $doc->get_value($col) or return; # undefined is '' in Xapian
+        my $val = $doc->get_value($col) or return undef; # undef is '' in Xapian
         sortable_unserialise($val) + 0; # PV => IV conversion
 }
 
@@ -559,24 +614,57 @@ sub get_pct ($) { # mset item
 
 sub xap_terms ($$;@) {
         my ($pfx, $xdb_or_doc, @docid) = @_; # @docid may be empty ()
-        my %ret;
         my $end = $xdb_or_doc->termlist_end(@docid);
         my $cur = $xdb_or_doc->termlist_begin(@docid);
+        $cur->skip_to($pfx);
+        my (@ret, $tn);
+        my $pfxlen = length($pfx);
         for (; $cur != $end; $cur++) {
-                $cur->skip_to($pfx);
-                last if $cur == $end;
-                my $tn = $cur->get_termname;
-                $ret{substr($tn, length($pfx))} = undef if !index($tn, $pfx);
+                $tn = $cur->get_termname;
+                index($tn, $pfx) ? last : push(@ret, substr($tn, $pfxlen));
         }
-        wantarray ? sort(keys(%ret)) : \%ret;
+        wantarray ? @ret : +{ map { $_ => undef } @ret };
 }
 
 # get combined docid from over.num:
-# (not generic Xapian, only works with our sharding scheme)
+# (not generic Xapian, only works with our sharding scheme for mail)
 sub num2docid ($$) {
         my ($self, $num) = @_;
         my $nshard = $self->{nshard};
         ($num - 1) * $nshard + $num % $nshard + 1;
 }
 
+sub all_terms {
+        my ($self, $pfx) = @_;
+        my $cur = xdb($self)->allterms_begin($pfx);
+        my $end = $self->{xdb}->allterms_end($pfx);
+        my $pfxlen = length($pfx);
+        my @ret;
+        for (; $cur != $end; $cur++) {
+                push @ret, substr($cur->get_termname, $pfxlen);
+        }
+        wantarray ? @ret : +{ map { $_ => undef } @ret };
+}
+
+sub xh_args { # prep getopt args to feed to xap_helper.h socket
+        map { ('-d', $_) } shard_dirs($_[0]);
+}
+
+sub docids_by_postlist ($$) {
+        my ($self, $q) = @_;
+        my $cur = $self->xdb->postlist_begin($q);
+        my $end = $self->{xdb}->postlist_end($q);
+        my @ids;
+        for (; $cur != $end; $cur++) { push(@ids, $cur->get_docid) };
+        @ids;
+}
+
+sub get_doc ($$) {
+        my ($self, $docid) = @_;
+        eval { $self->{xdb}->get_document($docid) } // do {
+                die $@ if $@ && ref($@) !~ /\bDocNotFoundError\b/;
+                undef;
+        }
+}
+
 1;
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index cbfe7816..1cbf6d23 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -9,7 +9,8 @@
 package PublicInbox::SearchIdx;
 use strict;
 use v5.10.1;
-use parent qw(PublicInbox::Search PublicInbox::Lock Exporter);
+use parent qw(PublicInbox::Search PublicInbox::Lock PublicInbox::Umask
+        Exporter);
 use PublicInbox::Eml;
 use PublicInbox::Search qw(xap_terms);
 use PublicInbox::InboxWritable;
@@ -21,7 +22,7 @@ use POSIX qw(strftime);
 use Fcntl qw(SEEK_SET);
 use Time::Local qw(timegm);
 use PublicInbox::OverIdx;
-use PublicInbox::Spawn qw(spawn);
+use PublicInbox::Spawn qw(run_wait popen_rd);
 use PublicInbox::Git qw(git_unquote);
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
 use PublicInbox::Address;
@@ -37,12 +38,13 @@ our $BATCH_BYTES = $ENV{XAPIAN_FLUSH_THRESHOLD} ? 0x7fffffff :
         # typical 32-bit system:
         (($Config{ptrsize} >= 8 ? 8192 : 1024) * 1024);
 use constant DEBUG => !!$ENV{DEBUG};
-my $BASE85 = qr/\A[a-zA-Z0-9\!\#\$\%\&\(\)\*\+\-;<=>\?\@\^_`\{\|\}\~]+\z/;
+my $BASE85 = qr/[a-zA-Z0-9\!\#\$\%\&\(\)\*\+\-;<=>\?\@\^_`\{\|\}\~]+/;
 my $xapianlevels = qr/\A(?:full|medium)\z/;
 my $hex = '[a-f0-9]';
 my $OID = $hex .'{40,}';
-my @VMD_MAP = (kw => 'K', L => 'L');
+my @VMD_MAP = (kw => 'K', L => 'L'); # value order matters
 our $INDEXLEVELS = qr/\A(?:full|medium|basic)\z/;
+our $PATCHID_BROKEN;
 
 sub new {
         my ($class, $ibx, $creat, $shard) = @_;
@@ -62,6 +64,7 @@ sub new {
                         die("Invalid indexlevel $ibx->{indexlevel}\n");
                 }
         }
+        undef $PATCHID_BROKEN; # retry on new instances in case of upgrades
         $ibx = PublicInbox::InboxWritable->new($ibx);
         my $self = PublicInbox::Search->new($ibx);
         bless $self, $class;
@@ -90,7 +93,7 @@ sub new {
         $self;
 }
 
-sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels }
+sub need_xapian ($) { ($_[0]->{indexlevel} // 'full') =~ $xapianlevels }
 
 sub idx_release {
         my ($self, $wake) = @_;
@@ -113,15 +116,15 @@ sub load_xapian_writable () {
         *sortable_serialise = $xap.'::sortable_serialise';
         $DB_CREATE_OR_OPEN = eval($xap.'::DB_CREATE_OR_OPEN()');
         $DB_OPEN = eval($xap.'::DB_OPEN()');
-        my $ver = (eval($xap.'::major_version()') << 16) |
-                (eval($xap.'::minor_version()') << 8) |
-                eval($xap.'::revision()');
-        if ($ver >= 0x10400) {
+        my $ver = eval 'v'.join('.', eval($xap.'::major_version()'),
+                                eval($xap.'::minor_version()'),
+                                eval($xap.'::revision()'));
+        if ($ver ge 1.4) { # new flags in Xapian 1.4
                 $DB_NO_SYNC = 0x4;
                 $DB_DANGEROUS = 0x10;
         }
         # Xapian v1.2.21..v1.2.24 were missing close-on-exec on OFD locks
-        $X->{CLOEXEC_UNSET} = 1 if $ver >= 0x010215 && $ver <= 0x010218;
+        $X->{CLOEXEC_UNSET} = 1 if $ver ge v1.2.21 && $ver le v1.2.24;
         1;
 }
 
@@ -134,6 +137,7 @@ sub idx_acquire {
                 load_xapian_writable();
                 $flag = $self->{creat} ? $DB_CREATE_OR_OPEN : $DB_OPEN;
         }
+        my $owner = $self->{ibx} // $self->{eidx} // $self;
         if ($self->{creat}) {
                 require File::Path;
                 $self->lock_acquire;
@@ -145,14 +149,13 @@ sub idx_acquire {
                         File::Path::mkpath($dir);
                         require PublicInbox::Syscall;
                         PublicInbox::Syscall::nodatacow_dir($dir);
-                        $self->{-set_has_threadid_once} = 1;
-                        if (($self->{ibx} // $self->{eidx})->{-dangerous}) {
-                                $flag |= $DB_DANGEROUS;
-                        }
+                        # owner == self for CodeSearchIdx
+                        $self->{-set_has_threadid_once} = 1 if $owner != $self;
+                        $flag |= $DB_DANGEROUS if $owner->{-dangerous};
                 }
         }
         return unless defined $flag;
-        $flag |= $DB_NO_SYNC if ($self->{ibx} // $self->{eidx})->{-no_fsync};
+        $flag |= $DB_NO_SYNC if $owner->{-no_fsync};
         my $xdb = eval { ($X->{WritableDatabase})->new($dir, $flag) };
         croak "Failed opening $dir: $@" if $@;
         $self->{xdb} = $xdb;
@@ -177,9 +180,8 @@ sub term_generator ($) { # write-only
 sub index_phrase ($$$$) {
         my ($self, $text, $wdf_inc, $prefix) = @_;
 
-        my $tg = term_generator($self);
-        $tg->index_text($text, $wdf_inc, $prefix);
-        $tg->increase_termpos;
+        term_generator($self)->index_text($text, $wdf_inc, $prefix);
+        $self->{term_generator}->increase_termpos;
 }
 
 sub index_text ($$$$) {
@@ -188,8 +190,8 @@ sub index_text ($$$$) {
         if ($self->{indexlevel} eq 'full') {
                 index_phrase($self, $text, $wdf_inc, $prefix);
         } else {
-                my $tg = term_generator($self);
-                $tg->index_text_without_positions($text, $wdf_inc, $prefix);
+                term_generator($self)->index_text_without_positions(
+                                        $text, $wdf_inc, $prefix);
         }
 }
 
@@ -263,14 +265,14 @@ sub index_diff ($$$) {
         while (defined($_ = shift @l)) {
                 if ($in_diff && /^GIT binary patch/) {
                         push @$xnq, $_;
-                        while (@l && $l[0] =~ /^literal /) {
+                        while (@l && $l[0] =~ /^(?:literal|delta) /) {
                                 # TODO allow searching by size range?
                                 # allows searching by exact size via:
-                                # "literal $SIZE"
+                                # "literal $SIZE" or "delta $SIZE"
                                 push @$xnq, shift(@l);
 
                                 # skip base85 and empty lines
-                                while (@l && ($l[0] =~ /$BASE85/o ||
+                                while (@l && ($l[0] =~ /\A$BASE85\h*\z/o ||
                                                 $l[0] !~ /\S/)) {
                                         shift @l;
                                 }
@@ -350,6 +352,52 @@ sub index_diff ($$$) {
         index_text($self, join("\n", @$xnq), 1, 'XNQ');
 }
 
+sub index_body_text {
+        my ($self, $doc, $sref) = @_;
+        my $rd;
+        # start patch-id in parallel
+        if ($$sref =~ /^(?:diff|---|\+\+\+) /ms && !$PATCHID_BROKEN) {
+                my $git = ($self->{ibx} // $self->{eidx} // $self)->git;
+                my $fh = PublicInbox::IO::write_file '+>:utf8', undef, $$sref;
+                $fh->flush or die "flush: $!";
+                sysseek($fh, 0, SEEK_SET);
+                $rd = popen_rd($git->cmd(qw(patch-id --stable)), undef,
+                                { 0 => $fh });
+        }
+
+        # split off quoted and unquoted blocks:
+        my @sections = PublicInbox::MsgIter::split_quotes($$sref);
+        undef $$sref; # free memory
+        for my $txt (@sections) {
+                if ($txt =~ /\A>/) {
+                        if ($txt =~ /^[>\t ]+GIT binary patch\r?/sm) {
+                                # get rid of Base-85 noise
+                                $txt =~ s/^([>\h]+(?:literal|delta)
+                                                \x20[0-9]+\r?\n)
+                                        (?:[>\h]+$BASE85\h*\r?\n)+/$1/gsmx;
+                        }
+                        index_text($self, $txt, 0, 'XQUOT');
+                } else { # does it look like a diff?
+                        if ($txt =~ /^(?:diff|---|\+\+\+) /ms) {
+                                index_diff($self, \$txt, $doc);
+                        } else {
+                                index_text($self, $txt, 1, 'XNQ');
+                        }
+                }
+                undef $txt; # free memory
+        }
+        if (defined $rd) { # reap `git patch-id'
+                (readline($rd) // '') =~ /\A([a-f0-9]{40,})/ and
+                        $doc->add_term('XDFID'.$1);
+                if (!$rd->close) {
+                        my $c = 'git patch-id --stable';
+                        $PATCHID_BROKEN = ($? >> 8) == 129;
+                        $PATCHID_BROKEN ? warn("W: $c requires git v2.1.0+\n")
+                                : warn("W: $c failed: \$?=$? (non-fatal)");
+                }
+        }
+}
+
 sub index_xapian { # msg_iter callback
         my $part = $_[0]->[0]; # ignore $depth and $idx
         my ($self, $doc) = @{$_[1]};
@@ -369,37 +417,7 @@ sub index_xapian { # msg_iter callback
         my ($s, undef) = msg_part_text($part, $ct);
         defined $s or return;
         $_[0]->[0] = $part = undef; # free memory
-
-        if ($s =~ /^(?:diff|---|\+\+\+) /ms) {
-                open(my $fh, '+>:utf8', undef) or die "open: $!";
-                open(my $eh, '+>', undef) or die "open: $!";
-                $fh->autoflush(1);
-                print $fh $s or die "print: $!";
-                sysseek($fh, 0, SEEK_SET) or die "sysseek: $!";
-                my $id = ($self->{ibx} // $self->{eidx})->git->qx(
-                                                [qw(patch-id --stable)],
-                                                {}, { 0 => $fh, 2 => $eh });
-                $id =~ /\A([a-f0-9]{40,})/ and $doc->add_term('XDFID'.$1);
-                seek($eh, 0, SEEK_SET) or die "seek: $!";
-                while (<$eh>) { warn $_ }
-        }
-
-        # split off quoted and unquoted blocks:
-        my @sections = PublicInbox::MsgIter::split_quotes($s);
-        undef $s; # free memory
-        for my $txt (@sections) {
-                if ($txt =~ /\A>/) {
-                        index_text($self, $txt, 0, 'XQUOT');
-                } else {
-                        # does it look like a diff?
-                        if ($txt =~ /^(?:diff|---|\+\+\+) /ms) {
-                                index_diff($self, \$txt, $doc);
-                        } else {
-                                index_text($self, $txt, 1, 'XNQ');
-                        }
-                }
-                undef $txt; # free memory
-        }
+        index_body_text($self, $doc, \$s);
 }
 
 sub index_list_id ($$$) {
@@ -407,6 +425,7 @@ sub index_list_id ($$$) {
         for my $l ($hdr->header_raw('List-Id')) {
                 $l =~ /<([^>]+)>/ or next;
                 my $lid = lc $1;
+                $lid =~ tr/\n\t\r\0//d; # same rules as Message-ID
                 $doc->add_boolean_term('G' . $lid);
                 index_phrase($self, $lid, 1, 'XL'); # probabilistic
         }
@@ -442,8 +461,7 @@ sub eml2doc ($$$;$) {
         add_val($doc, PublicInbox::Search::UID(), $smsg->{num});
         add_val($doc, PublicInbox::Search::THREADID, $smsg->{tid});
 
-        my $tg = term_generator($self);
-        $tg->set_document($doc);
+        term_generator($self)->set_document($doc);
         index_headers($self, $smsg);
 
         if (defined(my $eidx_key = $smsg->{eidx_key})) {
@@ -453,7 +471,7 @@ sub eml2doc ($$$;$) {
         index_ids($self, $doc, $eml, $mids);
 
         # by default, we maintain compatibility with v1.5.0 and earlier
-        # by writing to docdata.glass, users who never exect to downgrade can
+        # by writing to docdata.glass, users who never expect to downgrade can
         # use --skip-docdata
         if (!$self->{-skip_docdata}) {
                 # WWW doesn't need {to} or {cc}, only NNTP
@@ -540,9 +558,7 @@ sub add_message {
 
 sub _get_doc ($$) {
         my ($self, $docid) = @_;
-        my $doc = eval { $self->{xdb}->get_document($docid) };
-        $doc // do {
-                warn "E: $@\n" if $@;
+        $self->get_doc($docid) // do {
                 warn "E: #$docid missing in Xapian\n";
                 undef;
         }
@@ -600,17 +616,16 @@ sub set_vmd {
         my ($self, $docid, $vmd) = @_;
         begin_txn_lazy($self);
         my $doc = _get_doc($self, $docid) or return;
-        my ($end, @rm, @add);
+        my ($v, @rm, @add);
         my @x = @VMD_MAP;
+        my ($cur, $end) = ($doc->termlist_begin, $doc->termlist_end);
         while (my ($field, $pfx) = splice(@x, 0, 2)) {
                 my $set = $vmd->{$field} // next;
                 my %keep = map { $_ => 1 } @$set;
                 my %add = %keep;
-                $end //= $doc->termlist_end;
-                for (my $cur = $doc->termlist_begin; $cur != $end; $cur++) {
-                        $cur->skip_to($pfx);
-                        last if $cur == $end;
-                        my $v = $cur->get_termname;
+                $cur->skip_to($pfx); # works due to @VMD_MAP order
+                for (; $cur != $end; $cur++) {
+                        $v = $cur->get_termname;
                         $v =~ s/\A$pfx//s or next;
                         $keep{$v} ? delete($add{$v}) : push(@rm, $pfx.$v);
                 }
@@ -690,7 +705,7 @@ sub xdb_remove {
         my $xdb = $self->{xdb} // die 'BUG: missing {xdb}';
         for my $docid (@docids) {
                 eval { $xdb->delete_document($docid) };
-                warn "E: #$docid not in in Xapian? $@\n" if $@;
+                warn "E: #$docid not in Xapian? $@\n" if $@;
         }
 }
 
@@ -707,7 +722,6 @@ sub nr_quiet_rm { delete($_[0]->{-quiet_rm}) // 0 }
 sub index_git_blob_id {
         my ($doc, $pfx, $objid) = @_;
 
-        my $len = length($objid);
         for (my $len = length($objid); $len >= 7; ) {
                 $doc->add_term($pfx.$objid);
                 $objid = substr($objid, 0, --$len);
@@ -801,7 +815,8 @@ sub unindex_both { # git->cat_async callback
 
 sub with_umask {
         my $self = shift;
-        ($self->{ibx} // $self->{eidx})->with_umask(@_);
+        my $owner = $self->{ibx} // $self->{eidx};
+        $owner ? $owner->with_umask(@_) : $self->SUPER::with_umask(@_)
 }
 
 # called by public-inbox-index
@@ -819,10 +834,10 @@ sub index_sync {
 }
 
 sub check_size { # check_async cb for -index --max-size=...
-        my ($oid, $type, $size, $arg, $git) = @_;
-        (($type // '') eq 'blob') or die "E: bad $oid in $git->{git_dir}";
+        my (undef, $oid, $type, $size, $arg) = @_;
+        ($type // '') eq 'blob' or die "E: bad $oid in $arg->{git}->{git_dir}";
         if ($size <= $arg->{max_size}) {
-                $git->cat_async($oid, $arg->{index_oid}, $arg);
+                $arg->{git}->cat_async($oid, $arg->{index_oid}, $arg);
         } else {
                 warn "W: skipping $oid ($size > $arg->{max_size})\n";
         }
@@ -904,6 +919,7 @@ sub process_stack {
                         $arg->{autime} = $at;
                         $arg->{cotime} = $ct;
                         if ($sync->{max_size}) {
+                                $arg->{git} = $git;
                                 $git->check_async($oid, \&check_size, $arg);
                         } else {
                                 $git->cat_async($oid, \&index_both, $arg);
@@ -964,7 +980,7 @@ sub log2stack ($$$) {
                         $stk->push_rec('m', $at, $ct, $oid, $cmt);
                 }
         }
-        close $fh or die "git log failed: \$?=$?";
+        $fh->close or die "git log failed: \$?=$?";
         $stk //= PublicInbox::IdxStack->new;
         $stk->read_prepare;
 }
@@ -989,9 +1005,7 @@ sub is_ancestor ($$$) {
         return 0 unless $git->check($cur);
         my $cmd = [ 'git', "--git-dir=$git->{git_dir}",
                 qw(merge-base --is-ancestor), $cur, $tip ];
-        my $pid = spawn($cmd);
-        waitpid($pid, 0) == $pid or die join(' ', @$cmd) .' did not finish';
-        $? == 0;
+        run_wait($cmd) == 0;
 }
 
 sub need_update ($$$$) {
@@ -1052,7 +1066,11 @@ sub _index_sync {
         my $ibx = $self->{ibx};
         local $self->{current_info} = "$ibx->{inboxdir}";
         $self->{batch_bytes} = $opt->{batch_size} // $BATCH_BYTES;
-        $ibx->git->batch_prepare;
+
+        if ($X->{CLOEXEC_UNSET}) {
+                $ibx->git->cat_file($tip);
+                $ibx->git->check($tip);
+        }
         my $pr = $opt->{-progress};
         my $sync = { reindex => $opt->{reindex}, -opt => $opt, ibx => $ibx };
         my $quit = quit_cb($sync);
@@ -1088,8 +1106,10 @@ sub DESTROY {
         $_[0]->{lockfh} = undef;
 }
 
-sub _begin_txn {
+sub begin_txn_lazy {
         my ($self) = @_;
+        return if $self->{txn};
+        my $restore = $self->with_umask;
         my $xdb = $self->{xdb} || idx_acquire($self);
         $self->{oidx}->begin_lazy if $self->{oidx};
         $xdb->begin_transaction if $xdb;
@@ -1097,13 +1117,8 @@ sub _begin_txn {
         $xdb;
 }
 
-sub begin_txn_lazy {
-        my ($self) = @_;
-        $self->with_umask(\&_begin_txn, $self) if !$self->{txn};
-}
-
 # store 'indexlevel=medium' in v2 shard=0 and v1 (only one shard)
-# This metadata is read by Admin::detect_indexlevel:
+# This metadata is read by InboxWritable->detect_indexlevel:
 sub set_metadata_once {
         my ($self) = @_;
 
@@ -1125,8 +1140,10 @@ sub set_metadata_once {
         }
 }
 
-sub _commit_txn {
+sub commit_txn_lazy {
         my ($self) = @_;
+        return unless delete($self->{txn});
+        my $restore = $self->with_umask;
         if (my $eidx = $self->{eidx}) {
                 $eidx->git->async_wait_all;
                 $eidx->{transact_bytes} = 0;
@@ -1138,12 +1155,6 @@ sub _commit_txn {
         $self->{oidx}->commit_lazy if $self->{oidx};
 }
 
-sub commit_txn_lazy {
-        my ($self) = @_;
-        delete($self->{txn}) and
-                $self->with_umask(\&_commit_txn, $self);
-}
-
 sub eidx_shard_new {
         my ($class, $eidx, $shard) = @_;
         my $self = bless {
diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index 000abd94..1630eb4a 100644
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -1,13 +1,13 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Internal interface for a single Xapian shard in V2 inboxes.
 # See L<public-inbox-v2-format(5)> for more info on how we shard Xapian
 package PublicInbox::SearchIdxShard;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::SearchIdx PublicInbox::IPC);
 use PublicInbox::OnDestroy;
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
 
 sub new {
         my ($class, $v2w, $shard) = @_; # v2w may be ExtSearchIdx
@@ -21,24 +21,21 @@ sub new {
         if ($v2w->{parallel}) {
                 local $self->{-v2w_afc} = $v2w;
                 $self->ipc_worker_spawn("shard[$shard]");
-                # F_SETPIPE_SZ = 1031 on Linux; increasing the pipe size for
-                # inputs speeds V2Writable batch imports across 8 cores by
-                # nearly 20%.  Since any of our responses are small, make
-                # the response pipe as small as possible
-                if ($^O eq 'linux') {
-                        fcntl($self->{-ipc_req}, 1031, 1048576);
-                        fcntl($self->{-ipc_res}, 1031, 4096);
+                # Increasing the pipe size for requests speeds V2 batch imports
+                # across 8 cores by nearly 20%.  Since many of our responses
+                # are small, make the response pipe as small as possible
+                if ($F_SETPIPE_SZ) {
+                        fcntl($self->{-ipc_req}, $F_SETPIPE_SZ, 1048576);
+                        fcntl($self->{-ipc_res}, $F_SETPIPE_SZ, 4096);
                 }
         }
         $self;
 }
 
-sub _worker_done {
+sub _worker_done { # OnDestroy cb
         my ($self) = @_;
-        if ($self->need_xapian) {
-                die "$$ $0 xdb not released\n" if $self->{xdb};
-        }
-        die "$$ $0 still in transaction\n" if $self->{txn};
+        die "BUG: $$ $0 xdb active" if $self->need_xapian && $self->{xdb};
+        die "BUG: $$ $0 txn active" if $self->{txn};
 }
 
 sub ipc_atfork_child { # called automatically before ipc_worker_loop
@@ -47,7 +44,7 @@ sub ipc_atfork_child { # called automatically before ipc_worker_loop
         $v2w->atfork_child; # calls ipc_sibling_atfork_child on our siblings
         $v2w->{current_info} = "[$self->{shard}]"; # for $SIG{__WARN__}
         $self->begin_txn_lazy;
-        # caller must capture this:
+        # caller (ipc_worker_spawn) must capture this:
         PublicInbox::OnDestroy->new($$, \&_worker_done, $self);
 }
 
@@ -65,7 +62,7 @@ sub echo {
 
 sub idx_close {
         my ($self) = @_;
-        die "transaction in progress $self\n" if $self->{txn};
+        die "BUG: $$ $0 txn active" if $self->{txn};
         $self->idx_release if $self->{xdb};
 }
 
diff --git a/lib/PublicInbox/SearchQuery.pm b/lib/PublicInbox/SearchQuery.pm
index a6b7d843..747e3249 100644
--- a/lib/PublicInbox/SearchQuery.pm
+++ b/lib/PublicInbox/SearchQuery.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # used by PublicInbox::SearchView and PublicInbox::WwwListing
@@ -6,7 +6,7 @@ package PublicInbox::SearchQuery;
 use strict;
 use v5.10.1;
 use URI::Escape qw(uri_escape);
-use PublicInbox::MID qw(MID_ESC);
+use PublicInbox::Hval qw(ascii_html);
 our $LIM = 200;
 
 sub new {
@@ -16,10 +16,11 @@ sub new {
         my $t = $qp->{t}; # collapse threads
         my ($l) = (($qp->{l} || '') =~ /([0-9]+)/);
         $l = $LIM if !$l || $l > $LIM;
+        my ($o) = (($qp->{o} || '0') =~ /(-?[0-9]+)/);
         bless {
                 q => $qp->{'q'},
                 x => $qp->{x} || '',
-                o => (($qp->{o} || '0') =~ /(-?[0-9]+)/),
+                o => $o,
                 l => $l,
                 r => (defined $r && $r ne '0'),
                 t => (defined $t && $t ne '0'),
@@ -34,9 +35,13 @@ sub qs_html {
         }
         my $qs = '';
         if (defined(my $q = $self->{'q'})) {
-                $q = uri_escape($q, MID_ESC);
+                # not using MID_ESC since that's for the path component and
+                # this is for the query component.  Unlike MID_ESC,
+                # this disallows [\&\'\+=] and allows slash [/] for
+                # nicer looking dfn: queries
+                $q = uri_escape($q, '^A-Za-z0-9\-\._~!\$\(\)\*,;:@/');
                 $q =~ s/%20/+/g; # improve URL readability
-                $qs .= "q=$q";
+                $qs .= 'q='.ascii_html($q);
         }
         if (my $o = $self->{o}) { # ignore o == 0
                 $qs .= "&amp;o=$o";
diff --git a/lib/PublicInbox/SearchThread.pm b/lib/PublicInbox/SearchThread.pm
index cc8c90ce..00ae9fac 100644
--- a/lib/PublicInbox/SearchThread.pm
+++ b/lib/PublicInbox/SearchThread.pm
@@ -167,7 +167,7 @@ sub order_children {
         while (defined($cur = shift @q)) {
                 # the {children} hashref here...
                 my @c = grep { !$seen{$_}++ && visible($_, $ibx) }
-                        values %{$cur->{children}};
+                        values %{delete $cur->{children}};
                 $ordersub->(\@c) if scalar(@c) > 1;
                 $cur->{children} = \@c; # ...becomes an arrayref
                 push @q, @c;
diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm
index b1cdb480..2d3e942c 100644
--- a/lib/PublicInbox/SearchView.pm
+++ b/lib/PublicInbox/SearchView.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Displays search results for the web interface
@@ -34,7 +34,7 @@ sub sres_top_html {
                 return PublicInbox::WWW::need($ctx, 'Search');
         my $q = PublicInbox::SearchQuery->new($ctx->{qp});
         my $x = $q->{x};
-        my $o = $q->{o};
+        my $o = $q->{o} // 0;
         my $asc;
         if ($o < 0) {
                 $asc = 1;
@@ -82,7 +82,7 @@ retry:
                 mset_summary($ctx, $mset, $q); # appends to {-html_tip}
                 $html = '';
         }
-        html_oneshot($ctx, $code);
+        html_oneshot($ctx, $code, $html);
 }
 
 # display non-nested search results similar to what users expect from
@@ -134,7 +134,7 @@ sub mset_summary {
                 $q->{-min_pct} = $min;
                 $q->{-max_pct} = $max;
         }
-        $$res .= search_nav_bot($mset, $q);
+        $$res .= search_nav_bot($ctx, $mset, $q);
         undef;
 }
 
@@ -193,18 +193,24 @@ sub search_nav_top {
 
         my $x = $q->{x};
         my $pfx = "\t\t\t";
-        if ($x eq '') {
-                my $t = $q->qs_html(x => 't');
-                $rv .= qq{<b>summary</b>|<a\nhref="?$t">nested</a>}
-        } elsif ($x eq 't') {
+        if ($x eq 't') {
                 my $s = $q->qs_html(x => '');
                 $rv .= qq{<a\nhref="?$s">summary</a>|<b>nested</b>};
                 $pfx = "thread overview <a\nhref=#t>below</a> | ";
+        } else {
+                my $t = $q->qs_html(x => 't');
+                $rv .= qq{<b>summary</b>|<a\nhref="?$t">nested</a>}
         }
         my $A = $q->qs_html(x => 'A', r => undef);
-        $rv .= qq{|<a\nhref="?$A">Atom feed</a>]};
+        $rv .= qq{|<a\nhref="?$A">Atom feed</a>]\n};
+        $rv .= <<EOM if $x ne 't' && $q->{t};
+*** "t=1" collapses threads in summary, "full threads" requires mbox.gz ***
+EOM
+        $rv .= <<EOM if $x eq 'm';
+*** "x=m" ignored for GET requests, use download buttons below ***
+EOM
         if ($ctx->{ibx}->isrch->has_threadid) {
-                $rv .= qq{\n${pfx}download mbox.gz: } .
+                $rv .= qq{${pfx}download mbox.gz: } .
                         # we set name=z w/o using it since it seems required for
                         # lynx (but works fine for w3m).
                         qq{<input\ntype=submit\nname=z\n} .
@@ -212,14 +218,14 @@ sub search_nav_top {
                         qq{|<input\ntype=submit\nname=x\n} .
                                 q{value="full threads"/>};
         } else { # BOFH needs to --reindex
-                $rv .= qq{\n${pfx}download: } .
+                $rv .= qq{${pfx}download: } .
                         qq{<input\ntype=submit\nname=z\nvalue="mbox.gz"/>}
         }
         $rv .= qq{</pre></form><pre>};
 }
 
 sub search_nav_bot { # also used by WwwListing for searching extindex miscidx
-        my ($mset, $q) = @_;
+        my ($ctx, $mset, $q) = @_;
         my $total = $mset->get_matches_estimated;
         my $l = $q->{l};
         my $rv = '</pre><hr><pre id=t>';
@@ -268,9 +274,10 @@ sub search_nav_bot { # also used by WwwListing for searching extindex miscidx
         $rv .= qq{<a\nhref="?$prev"\nrel=prev>prev $pd</a>} if $prev;
 
         my $rev = $q->qs_html(o => $o < 0 ? 0 : -1);
-        $rv .= qq{ | <a\nhref="?$rev">reverse</a>} .
-                q{ | sort options + mbox downloads } .
-                q{<a href=#d>above</a></pre>};
+        $rv .= qq{ | <a\nhref="?$rev">reverse</a>};
+        exists($ctx->{ibx}) and
+                $rv .= q{ | options <a href=#d>above</a></pre>};
+        $rv;
 }
 
 sub sort_relevance {
@@ -295,9 +302,9 @@ sub mset_thread {
         my $rootset = PublicInbox::SearchThread::thread($msgs,
                 $r ? \&sort_relevance : \&PublicInbox::View::sort_ds,
                 $ctx);
-        my $skel = search_nav_bot($mset, $q).
-                "<pre>-- links below jump to the message on this page --\n";
-
+        my $skel = search_nav_bot($ctx, $mset, $q).'<pre>'. <<EOM;
+-- pct% links below jump to the message on this page, permalinks otherwise --
+EOM
         $ctx->{-upfx} = '';
         $ctx->{anchor_idx} = 1;
         $ctx->{cur_level} = 0;
@@ -315,20 +322,20 @@ sub mset_thread {
 
         # link $INBOX_DIR/description text to "recent" view around
         # the newest message in this result set:
-        $ctx->{-t_max} = max(map { delete $_->{ts} } @$msgs);
+        $ctx->{-t_max} = max(map { $_->{ts} } @$msgs);
 
         @$msgs = reverse @$msgs if $r;
         $ctx->{msgs} = $msgs;
-        PublicInbox::WwwStream::aresponse($ctx, 200, \&mset_thread_i);
+        PublicInbox::WwwStream::aresponse($ctx, \&mset_thread_i);
 }
 
 # callback for PublicInbox::WwwStream::getline
 sub mset_thread_i {
         my ($ctx, $eml) = @_;
-        $ctx->zmore($ctx->html_top) if exists $ctx->{-html_tip};
+        print { $ctx->zfh } $ctx->html_top if exists $ctx->{-html_tip};
         $eml and return PublicInbox::View::eml_entry($ctx, $eml);
         my $smsg = shift @{$ctx->{msgs}} or
-                $ctx->zmore(${delete($ctx->{skel})});
+                print { $ctx->zfh } ${delete($ctx->{skel})};
         $smsg;
 }
 
@@ -353,7 +360,7 @@ sub adump {
         my ($cb, $mset, $q, $ctx) = @_;
         $ctx->{ids} = $ctx->{ibx}->isrch->mset_to_artnums($mset);
         $ctx->{search_query} = $q; # used by WwwAtomStream::atom_header
-        PublicInbox::WwwAtomStream->response($ctx, 200, \&adump_i);
+        PublicInbox::WwwAtomStream->response($ctx, \&adump_i);
 }
 
 # callback for PublicInbox::WwwAtomStream::getline
diff --git a/lib/PublicInbox/Select.pm b/lib/PublicInbox/Select.pm
new file mode 100644
index 00000000..face8edc
--- /dev/null
+++ b/lib/PublicInbox/Select.pm
@@ -0,0 +1,43 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# This makes select(2) look like epoll to simplify the code in DS.pm.
+# Unlike IO::Select, it does NOT hold references to IO handles.
+# This is NOT meant to be an all encompassing emulation of epoll
+# via select, but only to support cases we care about.
+package PublicInbox::Select;
+use v5.12;
+use PublicInbox::Syscall qw(EPOLLONESHOT EPOLLIN EPOLLOUT);
+use Errno;
+
+sub new { bless {}, __PACKAGE__ } # fd => events
+
+sub ep_wait {
+        my ($self, $msec, $events) = @_;
+        my ($rvec, $wvec) = ('', ''); # we don't use EPOLLERR
+        while (my ($fd, $ev) = each %$self) {
+                vec($rvec, $fd, 1) = 1 if $ev & EPOLLIN;
+                vec($wvec, $fd, 1) = 1 if $ev & EPOLLOUT;
+        }
+        @$events = ();
+        my $to = $msec < 0 ? undef : ($msec/1000);
+        my $n = select $rvec, $wvec, undef, $to or return; # timeout expired
+        return if $n < 0 && $! == Errno::EINTR; # caller recalculates timeout
+        die "select: $!" if $n < 0;
+        while (my ($fd, $ev) = each %$self) {
+                if (vec($rvec, $fd, 1) || vec($wvec, $fd, 1)) {
+                        delete($self->{$fd}) if $ev & EPOLLONESHOT;
+                        push @$events, $fd;
+                }
+        }
+        $n == scalar(@$events) or
+                warn "BUG? select() returned $n, but got ".scalar(@$events);
+}
+
+sub ep_del { delete($_[0]->{fileno($_[1])}); 0 }
+sub ep_add { $_[0]->{fileno($_[1])} = $_[2]; 0 }
+
+no warnings 'once';
+*ep_mod = \&ep_add;
+
+1;
diff --git a/lib/PublicInbox/SharedKV.pm b/lib/PublicInbox/SharedKV.pm
index 90ccf2b4..89ab3f74 100644
--- a/lib/PublicInbox/SharedKV.pm
+++ b/lib/PublicInbox/SharedKV.pm
@@ -11,7 +11,7 @@ use parent qw(PublicInbox::Lock);
 use File::Temp qw(tempdir);
 use DBI qw(:sql_types); # SQL_BLOB
 use PublicInbox::Spawn;
-use File::Path qw(rmtree make_path);
+use File::Path qw(rmtree);
 
 sub dbh {
         my ($self, $lock) = @_;
@@ -43,7 +43,7 @@ CREATE TABLE IF NOT EXISTS kv (
 sub new {
         my ($cls, $dir, $base, $opt) = @_;
         my $self = bless { opt => $opt }, $cls;
-        make_path($dir) if defined($dir) && !-d $dir;
+        File::Path::mkpath($dir) if defined($dir);
         $dir //= $self->{"tmp$$.$self"} = tempdir("skv.$$-XXXX", TMPDIR => 1);
         $base //= '';
         my $f = $self->{filename} = "$dir/$base.sqlite3";
diff --git a/lib/PublicInbox/Sigfd.pm b/lib/PublicInbox/Sigfd.pm
index 81e5a1b1..b8a1ddfb 100644
--- a/lib/PublicInbox/Sigfd.pm
+++ b/lib/PublicInbox/Sigfd.pm
@@ -1,43 +1,35 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Wraps a signalfd (or similar) for PublicInbox::DS
 # fields: (sig: hashref similar to %SIG, but signal numbers as keys)
 package PublicInbox::Sigfd;
-use strict;
+use v5.12;
 use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(signalfd EPOLLIN EPOLLET);
+use PublicInbox::Syscall qw(signalfd EPOLLIN EPOLLET %SIGNUM);
 use POSIX ();
 
 # returns a coderef to unblock signals if neither signalfd or kqueue
 # are available.
 sub new {
-        my ($class, $sig, $nonblock) = @_;
+        my ($class, $sig) = @_;
         my %signo = map {;
-                my $cb = $sig->{$_};
-                # SIGWINCH is 28 on FreeBSD, NetBSD, OpenBSD
-                my $num = ($_ eq 'WINCH' && $^O =~ /linux|bsd/i) ? 28 : do {
-                        my $m = "SIG$_";
-                        POSIX->$m;
-                };
-                $num => $cb;
+                # $num => [ $cb, $signame ];
+                ($SIGNUM{$_} // POSIX->can("SIG$_")->()) => [ $sig->{$_}, $_ ]
         } keys %$sig;
         my $self = bless { sig => \%signo }, $class;
         my $io;
-        my $fd = signalfd([keys %signo], $nonblock);
+        my $fd = signalfd([keys %signo]);
         if (defined $fd && $fd >= 0) {
                 open($io, '+<&=', $fd) or die "open: $!";
         } elsif (eval { require PublicInbox::DSKQXS }) {
-                $io = PublicInbox::DSKQXS->signalfd([keys %signo], $nonblock);
+                $io = PublicInbox::DSKQXS->signalfd([keys %signo]);
         } else {
                 return; # wake up every second to check for signals
         }
-        if ($nonblock) { # it can go into the event loop
-                $self->SUPER::new($io, EPOLLIN | EPOLLET);
-        } else { # master main loop
-                $self->{sock} = $io;
-                $self;
-        }
+        $self->SUPER::new($io, EPOLLIN | EPOLLET);
+        $self->{is_kq} = 1 if tied(*$io);
+        $self;
 }
 
 # PublicInbox::Daemon in master main loop (blocking)
@@ -50,8 +42,8 @@ sub wait_once ($) {
                 for my $off (0..$nr) {
                         # the first uint32_t of signalfd_siginfo: ssi_signo
                         my $signo = unpack('L', substr($buf, 128 * $off, 4));
-                        my $cb = $self->{sig}->{$signo};
-                        $cb->($signo) if $cb ne 'IGNORE';
+                        my ($cb, $signame) = @{$self->{sig}->{$signo}};
+                        $cb->($signame) if $cb ne 'IGNORE';
                 }
         }
         $r;
diff --git a/lib/PublicInbox/Smsg.pm b/lib/PublicInbox/Smsg.pm
index 260ce6bb..b132381b 100644
--- a/lib/PublicInbox/Smsg.pm
+++ b/lib/PublicInbox/Smsg.pm
@@ -99,9 +99,6 @@ sub populate {
                 # to protect git and NNTP clients
                 $val =~ tr/\0\t\n/   /;
 
-                # rare: in case headers have wide chars (not RFC2047-encoded)
-                utf8::decode($val);
-
                 # lower-case fields for read-only stuff
                 $self->{lc($f)} = $val;
 
@@ -115,8 +112,10 @@ sub populate {
                 $self->{$f} = $val if $val ne '';
         }
         $sync //= {};
-        $self->{-ds} = [ my @ds = msg_datestamp($hdr, $sync->{autime}) ];
-        $self->{-ts} = [ my @ts = msg_timestamp($hdr, $sync->{cotime}) ];
+        my @ds = msg_datestamp($hdr, $sync->{autime} // $self->{ds});
+        my @ts = msg_timestamp($hdr, $sync->{cotime} // $self->{ts});
+        $self->{-ds} = \@ds;
+        $self->{-ts} = \@ts;
         $self->{ds} //= $ds[0]; # no zone
         $self->{ts} //= $ts[0];
         $self->{mid} //= mids($hdr)->[0];
diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm
index 62b5a343..296e7d17 100644
--- a/lib/PublicInbox/SolverGit.pm
+++ b/lib/PublicInbox/SolverGit.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # "Solve" blobs which don't exist in git code repositories by
@@ -11,13 +11,16 @@ package PublicInbox::SolverGit;
 use strict;
 use v5.10.1;
 use File::Temp 0.19 (); # 0.19 for ->newdir
+use autodie qw(mkdir);
 use Fcntl qw(SEEK_SET);
 use PublicInbox::Git qw(git_unquote git_quote);
+use PublicInbox::IO qw(write_file);
 use PublicInbox::MsgIter qw(msg_part_text);
 use PublicInbox::Qspawn;
 use PublicInbox::Tmpfile;
 use PublicInbox::GitAsyncCat;
 use PublicInbox::Eml;
+use PublicInbox::Compat qw(uniqstr);
 use URI::Escape qw(uri_escape_utf8);
 
 # POSIX requires _POSIX_ARG_MAX >= 4096, and xargs is required to
@@ -79,11 +82,13 @@ sub solve_existing ($$) {
         my $try = $want->{try_gits} //= [ @{$self->{gits}} ]; # array copy
         my $git = shift @$try or die 'BUG {try_gits} empty';
         my $oid_b = $want->{oid_b};
+
+        # can't use async_check due to last_check_err :<
         my ($oid_full, $type, $size) = $git->check($oid_b);
+        $git->schedule_cleanup if $self->{psgi_env}->{'pi-httpd.async'};
 
-        # other than {oid_b, try_gits, try_ibxs}
-        my $have_hints = scalar keys %$want > 3;
-        if (defined($type) && (!$have_hints || $type eq 'blob')) {
+        if ($oid_b eq ($oid_full // '') || (defined($type) &&
+                                (!$self->{have_hints} || $type eq 'blob'))) {
                 delete $want->{try_gits};
                 return [ $git, $oid_full, $type, int($size) ]; # done, success
         }
@@ -106,6 +111,11 @@ sub solve_existing ($$) {
         scalar(@$try);
 }
 
+sub _tmp {
+        $_[0]->{tmp} //=
+                File::Temp->newdir("solver.$_[0]->{oid_want}-XXXX", TMPDIR => 1);
+}
+
 sub extract_diff ($$) {
         my ($p, $arg) = @_;
         my ($self, $want, $smsg) = @$arg;
@@ -193,10 +203,8 @@ sub extract_diff ($$) {
 
         my $path = ++$self->{tot};
         $di->{n} = $path;
-        open(my $tmp, '>:utf8', $self->{tmp}->dirname . "/$path") or
-                die "open(tmp): $!";
-        print $tmp $di->{hdr_lines}, $patch or die "print(tmp): $!";
-        close $tmp or die "close(tmp): $!";
+        my $f = _tmp($self)->dirname."/$path";
+        write_file '>:utf8', $f, $di->{hdr_lines}, $patch;
 
         # for debugging/diagnostics:
         $di->{ibx} = $want->{cur_ibx};
@@ -242,14 +250,18 @@ sub find_smsgs ($$$) {
 
 sub update_index_result ($$) {
         my ($bref, $self) = @_;
-        my ($qsp, $msg) = delete @$self{qw(-qsp -msg)};
-        if (my $err = $qsp->{err}) {
-                ERR($self, "git update-index error: $err");
-        }
+        my ($qsp_err, $msg) = delete @$self{qw(-qsp_err -msg)};
+        ERR($self, "git update-index error:$qsp_err") if $qsp_err;
         dbg($self, $msg);
         next_step($self); # onto do_git_apply
 }
 
+sub qsp_qx ($$$) {
+        my ($self, $qsp, $cb) = @_;
+        $qsp->{qsp_err} = \($self->{-qsp_err} = '');
+        $qsp->psgi_qx($self->{psgi_env}, $self->{limiter}, $cb, $self);
+}
+
 sub prepare_index ($) {
         my ($self) = @_;
         my $patches = $self->{patches};
@@ -278,53 +290,39 @@ sub prepare_index ($) {
         my $cmd = [ qw(git update-index -z --index-info) ];
         my $qsp = PublicInbox::Qspawn->new($cmd, $self->{git_env}, $rdr);
         $path_a = git_quote($path_a);
-        $self->{-qsp} = $qsp;
         $self->{-msg} = "index prepared:\n$mode_a $oid_full\t$path_a";
-        $qsp->psgi_qx($self->{psgi_env}, undef, \&update_index_result, $self);
+        qsp_qx $self, $qsp, \&update_index_result;
 }
 
 # pure Perl "git init"
 sub do_git_init ($) {
         my ($self) = @_;
-        my $dir = $self->{tmp}->dirname;
-        my $git_dir = "$dir/git";
+        my $git_dir = _tmp($self)->dirname.'/git';
 
-        foreach ('', qw(objects refs objects/info refs/heads)) {
-                mkdir("$git_dir/$_") or die "mkdir $_: $!";
-        }
-        open my $fh, '>', "$git_dir/config" or die "open git/config: $!";
+        mkdir("$git_dir/$_") for ('', qw(objects refs objects/info refs/heads));
         my $first = $self->{gits}->[0];
         my $fmt = $first->object_format;
-        my $v = defined($$fmt) ? 1 : 0;
-        print $fh <<EOF or die "print git/config $!";
+        my ($v, @ext) = defined($$fmt) ? (1, <<EOM) : (0);
+[extensions]
+        objectformat = $$fmt
+EOM
+        write_file '>', "$git_dir/config", <<EOF, @ext;
 [core]
         repositoryFormatVersion = $v
         filemode = true
         bare = false
-        fsyncObjectfiles = false
         logAllRefUpdates = false
 EOF
-        print $fh <<EOM if defined($$fmt);
-[extensions]
-        objectformat = $$fmt
-EOM
-        close $fh or die "close git/config: $!";
-
-        open $fh, '>', "$git_dir/HEAD" or die "open git/HEAD: $!";
-        print $fh "ref: refs/heads/master\n" or die "print git/HEAD: $!";
-        close $fh or die "close git/HEAD: $!";
-
-        my $f = 'objects/info/alternates';
-        open $fh, '>', "$git_dir/$f" or die "open: $f: $!";
-        foreach my $git (@{$self->{gits}}) {
-                print $fh $git->git_path('objects'),"\n" or die "print $f: $!";
-        }
-        close $fh or die "close: $f: $!";
+        write_file '>', "$git_dir/HEAD", "ref: refs/heads/master\n";
+        write_file '>', "$git_dir/objects/info/alternates", map {
+                        $_->git_path('objects')."\n"
+                } @{$self->{gits}};
         my $tmp_git = $self->{tmp_git} = PublicInbox::Git->new($git_dir);
         $tmp_git->{-tmp} = $self->{tmp};
         $self->{git_env} = {
                 GIT_DIR => $git_dir,
                 GIT_INDEX_FILE => "$git_dir/index",
+                GIT_TEST_FSYNC => 0, # undocumented git env
         };
         prepare_index($self);
 }
@@ -384,12 +382,9 @@ sub event_step ($) {
 }
 
 sub next_step ($) {
-        my ($self) = @_;
         # if outside of public-inbox-httpd, caller is expected to be
         # looping event_step, anyways
-        my $async = $self->{psgi_env}->{'pi-httpd.async'} or return;
-        # PublicInbox::HTTPD::Async->new
-        $async->(undef, undef, $self);
+        PublicInbox::DS::requeue($_[0]) if $_[0]->{psgi_env}->{'pi-httpd.async'}
 }
 
 sub mark_found ($$$) {
@@ -405,21 +400,18 @@ sub mark_found ($$$) {
 
 sub parse_ls_files ($$) {
         my ($self, $bref) = @_;
-        my ($qsp, $di) = delete @$self{qw(-qsp -cur_di)};
-        if (my $err = $qsp->{err}) {
-                die "git ls-files error: $err";
-        }
+        my ($qsp_err, $di) = delete @$self{qw(-qsp_err -cur_di)};
+        die "git ls-files -s -z error:$qsp_err" if $qsp_err;
 
-        my ($line, @extra) = split(/\0/, $$bref);
+        my @ls = split(/\0/, $$bref);
+        my ($line, @extra) = grep(/\t\Q$di->{path_b}\E\z/, @ls);
         scalar(@extra) and die "BUG: extra files in index: <",
-                                join('> <', @extra), ">";
-
+                                join('> <', $line, @extra), ">";
+        $line // die "no \Q$di->{path_b}\E in <",join('> <', @ls), '>';
         my ($info, $file) = split(/\t/, $line, 2);
         my ($mode_b, $oid_b_full, $stage) = split(/ /, $info);
-        if ($file ne $di->{path_b}) {
-                die
+        $file eq $di->{path_b} or die
 "BUG: index mismatch: file=$file != path_b=$di->{path_b}";
-        }
 
         my $tmp_git = $self->{tmp_git} or die 'no git working tree';
         my (undef, undef, $size) = $tmp_git->check($oid_b_full);
@@ -454,50 +446,49 @@ sub skip_identical ($$$) {
         }
 }
 
-sub apply_result ($$) {
+sub apply_result ($$) { # qx_cb
         my ($bref, $self) = @_;
-        my ($qsp, $di) = delete @$self{qw(-qsp -cur_di)};
+        my ($qsp_err, $di) = delete @$self{qw(-qsp_err -cur_di)};
         dbg($self, $$bref);
         my $patches = $self->{patches};
-        if (my $err = $qsp->{err}) {
-                my $msg = "git apply error: $err";
+        if ($qsp_err) {
+                my $msg = "git apply error:$qsp_err";
                 my $nxt = $patches->[0];
                 if ($nxt && oids_same_ish($nxt->{oid_b}, $di->{oid_b})) {
                         dbg($self, $msg);
                         dbg($self, 'trying '.di_url($self, $nxt));
                         return do_git_apply($self);
                 } else {
-                        ERR($self, $msg);
+                        $msg .= " (no patches left to try for $di->{oid_b})\n";
+                        dbg($self, $msg);
+                        return done($self, undef);
                 }
         } else {
                 skip_identical($self, $patches, $di->{oid_b});
         }
 
         my @cmd = qw(git ls-files -s -z);
-        $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env});
+        my $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env});
         $self->{-cur_di} = $di;
-        $self->{-qsp} = $qsp;
-        $qsp->psgi_qx($self->{psgi_env}, undef, \&ls_files_result, $self);
+        qsp_qx $self, $qsp, \&ls_files_result;
 }
 
 sub do_git_apply ($) {
         my ($self) = @_;
-        my $dn = $self->{tmp}->dirname;
         my $patches = $self->{patches};
 
         # we need --ignore-whitespace because some patches are CRLF
         my @cmd = (qw(git apply --cached --ignore-whitespace
                         --unidiff-zero --whitespace=warn --verbose));
         my $len = length(join(' ', @cmd));
-        my $total = $self->{tot};
         my $di; # keep track of the last one for "git ls-files"
         my $prv_oid_b;
 
         do {
                 my $i = ++$self->{nr};
                 $di = shift @$patches;
-                dbg($self, "\napplying [$i/$total] " . di_url($self, $di) .
-                        "\n" . $di->{hdr_lines});
+                dbg($self, "\napplying [$i/$self->{nr_p}] " .
+                        di_url($self, $di) . "\n" . $di->{hdr_lines});
                 my $path = $di->{n};
                 $len += length($path) + 1;
                 push @cmd, $path;
@@ -505,11 +496,10 @@ sub do_git_apply ($) {
         } while (@$patches && $len < $ARG_SIZE_MAX &&
                  !oids_same_ish($patches->[0]->{oid_b}, $prv_oid_b));
 
-        my $opt = { 2 => 1, -C => $dn, quiet => 1 };
+        my $opt = { 2 => 1, -C => _tmp($self)->dirname, quiet => 1 };
         my $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env}, $opt);
         $self->{-cur_di} = $di;
-        $self->{-qsp} = $qsp;
-        $qsp->psgi_qx($self->{psgi_env}, undef, \&apply_result, $self);
+        qsp_qx $self, $qsp, \&apply_result;
 }
 
 sub di_url ($$) {
@@ -558,8 +548,9 @@ sub extract_diffs_done {
         my $diffs = delete $self->{tmp_diffs};
         if (scalar @$diffs) {
                 unshift @{$self->{patches}}, @$diffs;
-                dbg($self, "found $want->{oid_b} in " .  join(" ||\n\t",
-                        map { di_url($self, $_) } @$diffs));
+                my @u = uniqstr(map { di_url($self, $_) } @$diffs);
+                dbg($self, "found $want->{oid_b} in " .  join(" ||\n\t", @u));
+                ++$self->{nr_p};
 
                 # good, we can find a path to the oid we $want, now
                 # lets see if we need to apply more patches:
@@ -641,7 +632,7 @@ sub resolve_patch ($$) {
 
         # scan through inboxes to look for emails which results in
         # the oid we want:
-        my $ibx = shift(@{$want->{try_ibxs}}) or die 'BUG: {try_ibxs} empty';
+        my $ibx = shift(@{$want->{try_ibxs}}) or return done($self, undef);
         if (my $msgs = find_smsgs($self, $ibx, $want)) {
                 $want->{try_smsgs} = $msgs;
                 $want->{cur_ibx} = $ibx;
@@ -655,15 +646,19 @@ sub resolve_patch ($$) {
 # so user_cb never references the SolverGit object
 sub new {
         my ($class, $ibx, $user_cb, $uarg) = @_;
+        my $gits = $ibx ? $ibx->{-repo_objs} : undef;
+
+        # FIXME: cindex --join= is super-aggressive and may hit too many
+        $gits = [ @$gits[0..2] ] if $gits && @$gits > 3;
 
-        bless {
-                gits => $ibx->{-repo_objs},
+        bless { # $ibx is undef if coderepo only (see WwwCoderepo)
+                gits => $gits,
                 user_cb => $user_cb,
                 uarg => $uarg,
-                # -cur_di, -qsp, -msg => temporary fields for Qspawn callbacks
+                # -cur_di, -qsp_err, -msg => temp fields for Qspawn callbacks
 
                 # TODO: config option for searching related inboxes
-                inboxes => [ $ibx ],
+                inboxes => $ibx ? [ $ibx ] : [],
         }, $class;
 }
 
@@ -682,17 +677,16 @@ sub solve ($$$$$) {
         $self->{oid_want} = $oid_want;
         $self->{out} = $out;
         $self->{seen_oid} = {};
-        $self->{tot} = 0;
+        $self->{tot} = $self->{nr_p} = 0;
         $self->{psgi_env} = $env;
+        $self->{have_hints} = 1 if scalar keys %$hints;
         $self->{todo} = [ { %$hints, oid_b => $oid_want } ];
         $self->{patches} = []; # [ $di, $di, ... ]
         $self->{found} = {}; # { abbr => [ ::Git, oid, type, size, $di ] }
-        $self->{tmp} = File::Temp->newdir("solver.$oid_want-XXXX", TMPDIR => 1);
 
         dbg($self, "solving $oid_want ...");
-        if (my $async = $env->{'pi-httpd.async'}) {
-                # PublicInbox::HTTPD::Async->new
-                $async->(undef, undef, $self);
+        if ($env->{'pi-httpd.async'}) {
+                PublicInbox::DS::requeue($self);
         } else {
                 event_step($self) while $self->{user_cb};
         }
diff --git a/lib/PublicInbox/Spamcheck.pm b/lib/PublicInbox/Spamcheck.pm
index d8fa80c8..fbf9355d 100644
--- a/lib/PublicInbox/Spamcheck.pm
+++ b/lib/PublicInbox/Spamcheck.pm
@@ -1,21 +1,17 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Spamchecking used by -watch and -mda tools
 package PublicInbox::Spamcheck;
-use strict;
-use warnings;
+use v5.12;
 
 sub get {
         my ($cfg, $key, $default) = @_;
-        my $spamcheck = $cfg->{$key};
-        $spamcheck = $default unless $spamcheck;
+        my $spamcheck = $cfg->{$key} || $default;
 
         return if !$spamcheck || $spamcheck eq 'none';
 
-        if ($spamcheck eq 'spamc') {
-                $spamcheck = 'PublicInbox::Spamcheck::Spamc';
-        }
+        $spamcheck = 'PublicInbox::Spamcheck::Spamc' if $spamcheck eq 'spamc';
         if ($spamcheck =~ /::/) {
                 eval "require $spamcheck";
                 return $spamcheck->new;
diff --git a/lib/PublicInbox/Spamcheck/Spamc.pm b/lib/PublicInbox/Spamcheck/Spamc.pm
index d2b6429c..b4f95e2b 100644
--- a/lib/PublicInbox/Spamcheck/Spamc.pm
+++ b/lib/PublicInbox/Spamcheck/Spamc.pm
@@ -1,18 +1,17 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Default spam filter class for wrapping spamc(1)
 package PublicInbox::Spamcheck::Spamc;
-use strict;
-use warnings;
-use PublicInbox::Spawn qw(popen_rd spawn);
+use v5.12;
+use PublicInbox::Spawn qw(run_qx run_wait);
 use IO::Handle;
 use Fcntl qw(SEEK_SET);
 
 sub new {
         my ($class) = @_;
         bless {
-                checkcmd => [qw(spamc -E --headers)],
+                checkcmd => [qw(spamc -E)],
                 hamcmd => [qw(spamc -L ham)],
                 spamcmd => [qw(spamc -L spam)],
         }, $class;
@@ -21,15 +20,9 @@ sub new {
 sub spamcheck {
         my ($self, $msg, $out) = @_;
 
+        $out = \(my $buf = '') unless ref($out);
         my $rdr = { 0 => _msg_to_fh($self, $msg) };
-        my ($fh, $pid) = popen_rd($self->{checkcmd}, undef, $rdr);
-        unless (ref $out) {
-                my $buf = '';
-                $out = \$buf;
-        }
-        $$out = do { local $/; <$fh> };
-        close $fh or die "close failed: $!";
-        waitpid($pid, 0);
+        $$out = run_qx($self->{checkcmd}, undef, $rdr);
         ($? || $$out eq '') ? 0 : 1;
 }
 
@@ -49,9 +42,7 @@ sub _learn {
         $rdr->{0} = _msg_to_fh($self, $msg);
         $rdr->{1} ||= $self->_devnull;
         $rdr->{2} ||= $self->_devnull;
-        my $pid = spawn($self->{$field}, undef, $rdr);
-        waitpid($pid, 0);
-        !$?;
+        0 == run_wait($self->{$field}, undef, $rdr);
 }
 
 sub _devnull {
diff --git a/lib/PublicInbox/Spawn.pm b/lib/PublicInbox/Spawn.pm
index 3f69108a..e36659ce 100644
--- a/lib/PublicInbox/Spawn.pm
+++ b/lib/PublicInbox/Spawn.pm
@@ -6,10 +6,8 @@
 # is explicitly defined in the environment (and writable).
 # Under Linux, vfork can make a big difference in spawning performance
 # as process size increases (fork still needs to mark pages for CoW use).
-# Currently, we only use this for code intended for long running
-# daemons (inside the PSGI code (-httpd) and -nntpd).  The short-lived
-# scripts (-mda, -index, -learn, -init) either use IPC::run or standard
-# Perl routines.
+# None of this is intended to be thread-safe since Perl5 maintainers
+# officially discourage the use of threads.
 #
 # There'll probably be more OS-level C stuff here, down the line.
 # We don't want too many DSOs: https://udrepper.livejournal.com/8790.html
@@ -17,14 +15,17 @@
 package PublicInbox::Spawn;
 use v5.12;
 use parent qw(Exporter);
-use Symbol qw(gensym);
-use Fcntl qw(LOCK_EX SEEK_SET);
+use PublicInbox::Lock;
+use Fcntl qw(SEEK_SET);
 use IO::Handle ();
-use PublicInbox::ProcessPipe;
-our @EXPORT_OK = qw(which spawn popen_rd run_die);
-our @RLIMITS = qw(RLIMIT_CPU RLIMIT_CORE RLIMIT_DATA);
+use Carp qw(croak);
+use PublicInbox::IO;
+our @EXPORT_OK = qw(which spawn popen_rd popen_wr run_die run_wait run_qx);
+our (@RLIMITS, %RLIMITS);
+use autodie qw(close open pipe seek sysseek truncate);
 
 BEGIN {
+        @RLIMITS = qw(RLIMIT_CPU RLIMIT_CORE RLIMIT_DATA);
         my $all_libc = <<'ALL_LIBC'; # all *nix systems we support
 #include <sys/resource.h>
 #include <sys/socket.h>
@@ -37,12 +38,6 @@ BEGIN {
 #include <time.h>
 #include <stdio.h>
 #include <string.h>
-
-/* some platforms need alloca.h, but some don't */
-#if defined(__GNUC__) && !defined(alloca)
-#  define alloca(sz) __builtin_alloca(sz)
-#endif
-
 #include <signal.h>
 #include <assert.h>
 
@@ -54,11 +49,17 @@ BEGIN {
  *   This is unlike "sv_len", which returns what you would expect.
  */
 #define AV2C_COPY(dst, src) do { \
+        static size_t dst##__capa; \
         I32 i; \
         I32 top_index = av_len(src); \
         I32 real_len = top_index + 1; \
         I32 capa = real_len + 1; \
-        dst = alloca(capa * sizeof(char *)); \
+        if (capa > dst##__capa) { \
+                dst##__capa = 0; /* in case Newx croaks */ \
+                Safefree(dst); \
+                Newx(dst, capa, char *); \
+                dst##__capa = capa; \
+        } \
         for (i = 0; i < real_len; i++) { \
                 SV **sv = av_fetch(src, i, 0); \
                 dst[i] = SvPV_nolen(*sv); \
@@ -87,23 +88,28 @@ int pi_fork_exec(SV *redirref, SV *file, SV *cmdref, SV *envref, SV *rlimref,
         AV *env = (AV *)SvRV(envref);
         AV *rlim = (AV *)SvRV(rlimref);
         const char *filename = SvPV_nolen(file);
-        pid_t pid;
-        char **argv, **envp;
+        pid_t pid = -1;
+        static char **argv, **envp;
         sigset_t set, old;
         int ret, perrnum;
         volatile int cerrnum = 0; /* shared due to vfork */
-        int chld_is_member;
+        int chld_is_member; /* needed due to shared memory w/ vfork */
         I32 max_fd = av_len(redir);
 
         AV2C_COPY(argv, cmd);
         AV2C_COPY(envp, env);
 
-        if (sigfillset(&set)) return -1;
-        if (sigprocmask(SIG_SETMASK, &set, &old)) return -1;
+        if (sigfillset(&set)) goto out;
+        if (sigdelset(&set, SIGABRT)) goto out;
+        if (sigdelset(&set, SIGBUS)) goto out;
+        if (sigdelset(&set, SIGFPE)) goto out;
+        if (sigdelset(&set, SIGILL)) goto out;
+        if (sigdelset(&set, SIGSEGV)) goto out;
+        /* no XCPU/XFSZ here */
+        if (sigprocmask(SIG_SETMASK, &set, &old)) goto out;
         chld_is_member = sigismember(&old, SIGCHLD);
-        if (chld_is_member < 0) return -1;
-        if (chld_is_member > 0)
-                sigdelset(&old, SIGCHLD);
+        if (chld_is_member < 0) goto out;
+        if (chld_is_member > 0 && sigdelset(&old, SIGCHLD)) goto out;
 
         pid = vfork();
         if (pid == 0) {
@@ -122,8 +128,10 @@ int pi_fork_exec(SV *redirref, SV *file, SV *cmdref, SV *envref, SV *rlimref,
                         exit_err("setpgid", &cerrnum);
                 for (sig = 1; sig < NSIG; sig++)
                         signal(sig, SIG_DFL); /* ignore errors on signals */
-                if (*cd && chdir(cd) < 0)
-                        exit_err("chdir", &cerrnum);
+                if (*cd && chdir(cd) < 0) {
+                        write(2, "cd ", 3);
+                        exit_err(cd, &cerrnum);
+                }
 
                 max_rlim = av_len(rlim);
                 for (i = 0; i < max_rlim; i += 3) {
@@ -162,22 +170,26 @@ int pi_fork_exec(SV *redirref, SV *file, SV *cmdref, SV *envref, SV *rlimref,
         } else if (perrnum) {
                 errno = perrnum;
         }
+out:
+        if (pid < 0)
+                croak("E: fork_exec %s: %s\n", filename, strerror(errno));
         return (int)pid;
 }
 
-static int sleep_wait(unsigned *tries, int err)
+static int sendmsg_retry(unsigned *tries)
 {
         const struct timespec req = { 0, 100000000 }; /* 100ms */
+        int err = errno;
         switch (err) {
+        case EINTR: PERL_ASYNC_CHECK(); return 1;
         case ENOBUFS: case ENOMEM: case ETOOMANYREFS:
-                if (++*tries < 50) {
-                        fprintf(stderr, "sleeping on sendmsg: %s (#%u)\n",
-                                strerror(err), *tries);
-                        nanosleep(&req, NULL);
-                        return 1;
-                }
-        default:
-                return 0;
+                if (++*tries >= 50) return 0;
+                fprintf(stderr, "# sleeping on sendmsg: %s (#%u)\n",
+                        strerror(err), *tries);
+                nanosleep(&req, NULL);
+                PERL_ASYNC_CHECK();
+                return 1;
+        default: return 0;
         }
 }
 
@@ -229,7 +241,7 @@ SV *send_cmd4(PerlIO *s, SV *svfds, SV *data, int flags)
         }
         do {
                 sent = sendmsg(PerlIO_fileno(s), &msg, flags);
-        } while (sent < 0 && sleep_wait(&tries, errno));
+        } while (sent < 0 && sendmsg_retry(&tries));
         return sent >= 0 ? newSViv(sent) : &PL_sv_undef;
 }
 
@@ -251,58 +263,76 @@ void recv_cmd4(PerlIO *s, SV *buf, STRLEN n)
         msg.msg_control = &cmsg.hdr;
         msg.msg_controllen = CMSG_SPACE(SEND_FD_SPACE);
 
-        i = recvmsg(PerlIO_fileno(s), &msg, 0);
-        if (i < 0)
-                Inline_Stack_Push(&PL_sv_undef);
-        else
+        for (;;) {
+                i = recvmsg(PerlIO_fileno(s), &msg, 0);
+                if (i >= 0 || errno != EINTR) break;
+                PERL_ASYNC_CHECK();
+        }
+        if (i >= 0) {
                 SvCUR_set(buf, i);
-        if (i > 0 && cmsg.hdr.cmsg_level == SOL_SOCKET &&
-                        cmsg.hdr.cmsg_type == SCM_RIGHTS) {
-                size_t len = cmsg.hdr.cmsg_len;
-                int *fdp = (int *)CMSG_DATA(&cmsg.hdr);
-                for (i = 0; CMSG_LEN((i + 1) * sizeof(int)) <= len; i++)
-                        Inline_Stack_Push(sv_2mortal(newSViv(*fdp++)));
+                if (cmsg.hdr.cmsg_level == SOL_SOCKET &&
+                                cmsg.hdr.cmsg_type == SCM_RIGHTS) {
+                        size_t len = cmsg.hdr.cmsg_len;
+                        int *fdp = (int *)CMSG_DATA(&cmsg.hdr);
+                        for (i = 0; CMSG_LEN((i + 1) * sizeof(int)) <= len; i++)
+                                Inline_Stack_Push(sv_2mortal(newSViv(*fdp++)));
+                }
+        } else {
+                Inline_Stack_Push(&PL_sv_undef);
+                SvCUR_set(buf, 0);
         }
         Inline_Stack_Done;
 }
 #endif /* defined(CMSG_SPACE) && defined(CMSG_LEN) */
-ALL_LIBC
 
-        my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} //= (
+void rlimit_map()
+{
+        Inline_Stack_Vars;
+        Inline_Stack_Reset;
+ALL_LIBC
+        my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} // (
                         $ENV{XDG_CACHE_HOME} //
                         ( ($ENV{HOME} // '/nonexistent').'/.cache' )
                 ).'/public-inbox/inline-c';
-        warn "$inline_dir exists, not writable\n" if -e $inline_dir && !-w _;
-        $all_libc = undef unless -d _ && -w _;
+        undef $all_libc unless -d $inline_dir;
         if (defined $all_libc) {
-                my $f = "$inline_dir/.public-inbox.lock";
-                open my $oldout, '>&', \*STDOUT or die "dup(1): $!";
-                open my $olderr, '>&', \*STDERR or die "dup(2): $!";
-                open my $fh, '+>', $f or die "open($f): $!";
-                open STDOUT, '>&', $fh or die "1>$f: $!";
-                open STDERR, '>&', $fh or die "2>$f: $!";
+                for (@RLIMITS, 'RLIM_INFINITY') {
+                        $all_libc .= <<EOM;
+        Inline_Stack_Push(sv_2mortal(newSVpvs("$_")));
+        Inline_Stack_Push(sv_2mortal(newSViv($_)));
+EOM
+                }
+                $all_libc .= <<EOM;
+        Inline_Stack_Done;
+} // rlimit_map
+EOM
+                local $ENV{PERL_INLINE_DIRECTORY} = $inline_dir;
+                # CentOS 7.x ships Inline 0.53, 0.64+ has built-in locking
+                my $lk = PublicInbox::Lock->new($inline_dir.
+                                                '/.public-inbox.lock');
+                my $fh = $lk->lock_acquire;
+                open my $oldout, '>&', \*STDOUT;
+                open my $olderr, '>&', \*STDERR;
+                open STDOUT, '>&', $fh;
+                open STDERR, '>&', $fh;
                 STDERR->autoflush(1);
                 STDOUT->autoflush(1);
-
-                # CentOS 7.x ships Inline 0.53, 0.64+ has built-in locking
-                flock($fh, LOCK_EX) or die "LOCK_EX($f): $!";
-                eval <<'EOM';
-use Inline C => $all_libc, BUILD_NOISY => 1;
-EOM
+                eval 'use Inline C => $all_libc, BUILD_NOISY => 1';
                 my $err = $@;
-                my $ndc_err = '';
-                $err = $@;
-                open(STDERR, '>&', $olderr) or warn "restore stderr: $!";
-                open(STDOUT, '>&', $oldout) or warn "restore stdout: $!";
+                open(STDERR, '>&', $olderr);
+                open(STDOUT, '>&', $oldout);
                 if ($err) {
                         seek($fh, 0, SEEK_SET);
                         my @msg = <$fh>;
-                        warn "Inline::C build failed:\n",
-                                $ndc_err, $err, "\n", @msg;
+                        truncate($fh, 0);
+                        warn "Inline::C build failed:\n", $err, "\n", @msg;
                         $all_libc = undef;
                 }
         }
-        unless ($all_libc) {
+        if (defined $all_libc) { # set for Gcf2
+                $ENV{PERL_INLINE_DIRECTORY} = $inline_dir;
+                %RLIMITS = rlimit_map();
+        } else {
                 require PublicInbox::SpawnPP;
                 *pi_fork_exec = \&PublicInbox::SpawnPP::pi_fork_exec
         }
@@ -319,59 +349,94 @@ sub which ($) {
 }
 
 sub spawn ($;$$) {
-        my ($cmd, $env, $opts) = @_;
+        my ($cmd, $env, $opt) = @_;
         my $f = which($cmd->[0]) // die "$cmd->[0]: command not found\n";
-        my @env;
-        $opts ||= {};
+        my (@env, @rdr);
         my %env = (%ENV, $env ? %$env : ());
         while (my ($k, $v) = each %env) {
                 push @env, "$k=$v" if defined($v);
         }
-        my $redir = [];
         for my $child_fd (0..2) {
-                my $parent_fd = $opts->{$child_fd};
-                if (defined($parent_fd) && $parent_fd !~ /\A[0-9]+\z/) {
-                        my $fd = fileno($parent_fd) //
-                                        die "$parent_fd not an IO GLOB? $!";
-                        $parent_fd = $fd;
+                my $pfd = $opt->{$child_fd};
+                if ('SCALAR' eq ref($pfd)) {
+                        open my $fh, '+>', undef;
+                        $opt->{"fh.$child_fd"} = $fh; # for read_out_err
+                        if ($child_fd == 0) {
+                                print $fh $$pfd;
+                                $fh->flush or die "flush: $!";
+                                sysseek($fh, 0, SEEK_SET);
+                        }
+                        $pfd = fileno($fh);
+                } elsif (defined($pfd) && $pfd !~ /\A[0-9]+\z/) {
+                        my $fd = fileno($pfd) //
+                                        croak "BUG: $pfd not an IO GLOB? $!";
+                        $pfd = $fd;
                 }
-                $redir->[$child_fd] = $parent_fd // $child_fd;
+                $rdr[$child_fd] = $pfd // $child_fd;
         }
         my $rlim = [];
-
         foreach my $l (@RLIMITS) {
-                my $v = $opts->{$l} // next;
-                my $r = eval "require BSD::Resource; BSD::Resource::$l();";
-                unless (defined $r) {
-                        warn "$l undefined by BSD::Resource: $@\n";
-                        next;
-                }
+                my $v = $opt->{$l} // next;
+                my $r = $RLIMITS{$l} //
+                        eval "require BSD::Resource; BSD::Resource::$l();" //
+                        do {
+                                warn "$l undefined by BSD::Resource: $@\n";
+                                next;
+                        };
                 push @$rlim, $r, @$v;
         }
-        my $cd = $opts->{'-C'} // ''; # undef => NULL mapping doesn't work?
-        my $pgid = $opts->{pgid} // -1;
-        my $pid = pi_fork_exec($redir, $f, $cmd, \@env, $rlim, $cd, $pgid);
-        die "fork_exec @$cmd failed: $!\n" unless $pid > 0;
-        $pid;
+        my $cd = $opt->{'-C'} // ''; # undef => NULL mapping doesn't work?
+        my $pgid = $opt->{pgid} // -1;
+        pi_fork_exec(\@rdr, $f, $cmd, \@env, $rlim, $cd, $pgid);
 }
 
 sub popen_rd {
+        my ($cmd, $env, $opt, @cb_arg) = @_;
+        pipe(my $r, local $opt->{1});
+        PublicInbox::IO::attach_pid($r, spawn($cmd, $env, $opt), @cb_arg);
+}
+
+sub popen_wr {
+        my ($cmd, $env, $opt, @cb_arg) = @_;
+        pipe(local $opt->{0}, my $w);
+        $w->autoflush(1);
+        PublicInbox::IO::attach_pid($w, spawn($cmd, $env, $opt), @cb_arg);
+}
+
+sub read_out_err ($) {
+        my ($opt) = @_;
+        for my $fd (1, 2) { # read stdout/stderr
+                my $fh = delete($opt->{"fh.$fd"}) // next;
+                seek($fh, 0, SEEK_SET);
+                PublicInbox::IO::read_all $fh, undef, $opt->{$fd};
+        }
+}
+
+sub run_wait ($;$$) {
         my ($cmd, $env, $opt) = @_;
-        pipe(my ($r, $w)) or die "pipe: $!\n";
-        $opt ||= {};
-        $opt->{1} = fileno($w);
-        my $pid = spawn($cmd, $env, $opt);
-        return ($r, $pid) if wantarray;
-        my $ret = gensym;
-        tie *$ret, 'PublicInbox::ProcessPipe', $pid, $r, @$opt{qw(cb arg)};
-        $ret;
+        waitpid(spawn($cmd, $env, $opt), 0);
+        read_out_err($opt);
+        $?
 }
 
 sub run_die ($;$$) {
         my ($cmd, $env, $rdr) = @_;
-        my $pid = spawn($cmd, $env, $rdr);
-        waitpid($pid, 0) == $pid or die "@$cmd did not finish";
-        $? == 0 or die "@$cmd failed: \$?=$?\n";
+        run_wait($cmd, $env, $rdr) and croak "E: @$cmd failed: \$?=$?";
+}
+
+sub run_qx {
+        my ($cmd, $env, $opt) = @_;
+        my $fh = popen_rd($cmd, $env, $opt);
+        my @ret;
+        if (wantarray) {
+                @ret = <$fh>;
+        } else {
+                local $/;
+                $ret[0] = <$fh>;
+        }
+        $fh->close; # caller should check $?
+        read_out_err($opt);
+        wantarray ? @ret : $ret[0];
 }
 
 1;
diff --git a/lib/PublicInbox/SpawnPP.pm b/lib/PublicInbox/SpawnPP.pm
index 6d7e2c34..f89d37d4 100644
--- a/lib/PublicInbox/SpawnPP.pm
+++ b/lib/PublicInbox/SpawnPP.pm
@@ -1,27 +1,28 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Pure-Perl implementation of "spawn".  This can't take advantage
 # of vfork, so no speedups under Linux for spawning from large processes.
+# Do not require this directly, only use from PublicInbox::Spawn
 package PublicInbox::SpawnPP;
-use strict;
-use v5.10.1;
+use v5.12;
 use POSIX qw(dup2 _exit setpgid :signal_h);
+use autodie qw(chdir close fork pipe);
+# this is loaded by PublicInbox::Spawn, so we can't use/require it, here
 
 # Pure Perl implementation for folks that do not use Inline::C
 sub pi_fork_exec ($$$$$$$) {
         my ($redir, $f, $cmd, $env, $rlim, $cd, $pgid) = @_;
         my $old = POSIX::SigSet->new();
         my $set = POSIX::SigSet->new();
-        $set->fillset or die "fillset failed: $!";
-        sigprocmask(SIG_SETMASK, $set, $old) or die "can't block signals: $!";
-        my $syserr;
-        pipe(my ($r, $w));
-        my $pid = fork;
-        unless (defined $pid) { # compat with Inline::C version
-                $syserr = $!;
-                $pid = -1;
+        $set->fillset or die "sigfillset: $!";
+        for (POSIX::SIGABRT, POSIX::SIGBUS, POSIX::SIGFPE,
+                        POSIX::SIGILL, POSIX::SIGSEGV) {
+                $set->delset($_) or die "delset($_): $!";
         }
+        sigprocmask(SIG_SETMASK, $set, $old) or die "SIG_SETMASK(set): $!";
+        pipe(my $r, my $w);
+        my $pid = fork;
         if ($pid == 0) {
                 close $r;
                 $SIG{__DIE__} = sub {
@@ -38,36 +39,30 @@ sub pi_fork_exec ($$$$$$$) {
                 if ($pgid >= 0 && !defined(setpgid(0, $pgid))) {
                         die "setpgid(0, $pgid): $!";
                 }
-                for (keys %SIG) {
-                        $SIG{$_} = 'DEFAULT' if substr($_, 0, 1) ne '_';
-                }
-                if ($cd ne '') {
-                        chdir $cd or die "chdir $cd: $!";
-                }
+                $SIG{$_} = 'DEFAULT' for grep(!/\A__/, keys %SIG);
+                chdir($cd) if $cd ne '';
                 while (@$rlim) {
                         my ($r, $soft, $hard) = splice(@$rlim, 0, 3);
                         BSD::Resource::setrlimit($r, $soft, $hard) or
                                 die "setrlimit($r=[$soft,$hard]: $!)";
                 }
-                $old->delset(POSIX::SIGCHLD) or die "delset SIGCHLD: $!";
-                sigprocmask(SIG_SETMASK, $old) or die "SETMASK: ~SIGCHLD: $!";
+                $old->delset(POSIX::SIGCHLD) or die "sigdelset CHLD: $!";
+                sigprocmask(SIG_SETMASK, $old) or die "SIG_SETMASK ~CHLD: $!";
                 $cmd->[0] = $f;
                 if ($ENV{MOD_PERL}) {
-                        @$cmd = (which('env'), '-i', @$env, @$cmd);
+                        $f = PublicInbox::Spawn::which('env');
+                        @$cmd = ('env', '-i', @$env, @$cmd);
                 } else {
                         %ENV = map { split(/=/, $_, 2) } @$env;
                 }
-                undef $r;
                 exec { $f } @$cmd;
                 die "exec @$cmd failed: $!";
         }
         close $w;
-        sigprocmask(SIG_SETMASK, $old) or die "can't unblock signals: $!";
+        sigprocmask(SIG_SETMASK, $old) or die "SIG_SETMASK(old): $!";
         if (my $cerrnum = do { local $/, <$r> }) {
-                $pid = -1;
                 $! = $cerrnum;
-        } else {
-                $! = $syserr;
+                die "forked child $@: $!";
         }
         $pid;
 }
diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm
index 777c44d0..829cfa3c 100644
--- a/lib/PublicInbox/Syscall.pm
+++ b/lib/PublicInbox/Syscall.pm
@@ -2,9 +2,9 @@
 # specifically the Debian libsys-syscall-perl 0.25-6 version to
 # fix upstream regressions in 0.25.
 #
-# See devel/syscall-list in the public-inbox source tree for maintenance
+# See devel/sysdefs-list in the public-inbox source tree for maintenance
 # <https://80x24.org/public-inbox.git>, and machines from the GCC Farm:
-# <https://cfarm.tetaneutral.net/>
+# <https://portal.cfarm.net/>
 #
 # This license differs from the rest of public-inbox
 #
@@ -21,19 +21,15 @@ use parent qw(Exporter);
 use POSIX qw(ENOENT ENOSYS EINVAL O_NONBLOCK);
 use Socket qw(SOL_SOCKET SCM_RIGHTS);
 use Config;
+our %SIGNUM = (WINCH => 28); # most Linux, {Free,Net,Open}BSD, *Darwin
+our ($INOTIFY, %PACK);
 
 # $VERSION = '0.25'; # Sys::Syscall version
 our @EXPORT_OK = qw(epoll_ctl epoll_create epoll_wait
-                  EPOLLIN EPOLLOUT EPOLLET
-                  EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
-                  EPOLLONESHOT EPOLLEXCLUSIVE
-                  signalfd rename_noreplace);
-our %EXPORT_TAGS = (epoll => [qw(epoll_ctl epoll_create epoll_wait
-                             EPOLLIN EPOLLOUT
-                             EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
-                             EPOLLONESHOT EPOLLEXCLUSIVE)],
-                );
-
+                EPOLLIN EPOLLOUT EPOLLET
+                EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
+                EPOLLONESHOT EPOLLEXCLUSIVE
+                signalfd rename_noreplace %SIGNUM $F_SETPIPE_SZ);
 use constant {
         EPOLLIN => 1,
         EPOLLOUT => 4,
@@ -48,242 +44,283 @@ use constant {
         EPOLL_CTL_MOD => 3,
         SIZEOF_int => $Config{intsize},
         SIZEOF_size_t => $Config{sizesize},
+        SIZEOF_ptr => $Config{ptrsize},
         NUL => "\0",
 };
 
-use constant {
-        TMPL_size_t => SIZEOF_size_t == 8 ? 'Q' : 'L',
-        BYTES_4_hole => SIZEOF_size_t == 8 ? 'L' : '',
-        # cmsg_len, cmsg_level, cmsg_type
-        SIZEOF_cmsghdr => SIZEOF_int * 2 + SIZEOF_size_t,
-};
-
-my @BYTES_4_hole = BYTES_4_hole ? (0) : ();
-our $loaded_syscall = 0;
-
-sub _load_syscall {
-    # props to Gaal for this!
-    return if $loaded_syscall++;
-    my $clean = sub {
-        delete @INC{qw<syscall.ph asm/unistd.ph bits/syscall.ph
-                        _h2ph_pre.ph sys/syscall.ph>};
-    };
-    $clean->(); # don't trust modules before us
-    my $rv = eval { require 'syscall.ph'; 1 } || eval { require 'sys/syscall.ph'; 1 };
-    $clean->(); # don't require modules after us trust us
-    $rv;
-}
-
+use constant TMPL_size_t => SIZEOF_size_t == 8 ? 'Q' : 'L';
 
-our (
-     $SYS_epoll_create,
-     $SYS_epoll_ctl,
-     $SYS_epoll_wait,
-     $SYS_signalfd4,
-     $SYS_renameat2,
-     );
+our ($SYS_epoll_create,
+        $SYS_epoll_ctl,
+        $SYS_epoll_wait,
+        $SYS_signalfd4,
+        $SYS_renameat2,
+        $F_SETPIPE_SZ,
+        $SYS_sendmsg,
+        $SYS_recvmsg);
 
-my ($SYS_sendmsg, $SYS_recvmsg);
 my $SYS_fstatfs; # don't need fstatfs64, just statfs.f_type
 my ($FS_IOC_GETFLAGS, $FS_IOC_SETFLAGS);
 my $SFD_CLOEXEC = 02000000; # Perl does not expose O_CLOEXEC
 our $no_deprecated = 0;
 
 if ($^O eq "linux") {
-    my (undef, undef, $release, undef, $machine) = POSIX::uname();
-    my ($maj, $min) = ($release =~ /\A([0-9]+)\.([0-9]+)/);
-    $SYS_renameat2 = 0 if "$maj.$min" < 3.15;
-    # whether the machine requires 64-bit numbers to be on 8-byte
-    # boundaries.
-    my $u64_mod_8 = 0;
-
-    # if we're running on an x86_64 kernel, but a 32-bit process,
-    # we need to use the x32 or i386 syscall numbers.
-    if ($machine eq "x86_64" && $Config{ptrsize} == 4) {
-        $machine = $Config{cppsymbols} =~ /\b__ILP32__=1\b/ ? 'x32' : 'i386';
-    }
-
-    # Similarly for mips64 vs mips
-    if ($machine eq "mips64" && $Config{ptrsize} == 4) {
-        $machine = "mips";
-    }
-
-    if ($machine =~ m/^i[3456]86$/) {
-        $SYS_epoll_create = 254;
-        $SYS_epoll_ctl    = 255;
-        $SYS_epoll_wait   = 256;
-        $SYS_signalfd4 = 327;
-        $SYS_renameat2 //= 353;
-        $SYS_fstatfs = 100;
-        $SYS_sendmsg = 370;
-        $SYS_recvmsg = 372;
-        $FS_IOC_GETFLAGS = 0x80046601;
-        $FS_IOC_SETFLAGS = 0x40046602;
-    } elsif ($machine eq "x86_64") {
-        $SYS_epoll_create = 213;
-        $SYS_epoll_ctl    = 233;
-        $SYS_epoll_wait   = 232;
-        $SYS_signalfd4 = 289;
-        $SYS_renameat2 //= 316;
-        $SYS_fstatfs = 138;
-        $SYS_sendmsg = 46;
-        $SYS_recvmsg = 47;
-        $FS_IOC_GETFLAGS = 0x80086601;
-        $FS_IOC_SETFLAGS = 0x40086602;
-    } elsif ($machine eq 'x32') {
-        $SYS_epoll_create = 1073742037;
-        $SYS_epoll_ctl = 1073742057;
-        $SYS_epoll_wait = 1073742056;
-        $SYS_signalfd4 = 1073742113;
-        $SYS_renameat2 //= 0x40000000 + 316;
-        $SYS_fstatfs = 138;
-        $SYS_sendmsg = 0x40000206;
-        $SYS_recvmsg = 0x40000207;
-        $FS_IOC_GETFLAGS = 0x80046601;
-        $FS_IOC_SETFLAGS = 0x40046602;
-    } elsif ($machine eq 'sparc64') {
-        $SYS_epoll_create = 193;
-        $SYS_epoll_ctl = 194;
-        $SYS_epoll_wait = 195;
-        $u64_mod_8 = 1;
-        $SYS_signalfd4 = 317;
-        $SYS_renameat2 //= 345;
-        $SFD_CLOEXEC = 020000000;
-        $SYS_fstatfs = 158;
-        $SYS_sendmsg = 114;
-        $SYS_recvmsg = 113;
-        $FS_IOC_GETFLAGS = 0x40086601;
-        $FS_IOC_SETFLAGS = 0x80086602;
-    } elsif ($machine =~ m/^parisc/) {
-        $SYS_epoll_create = 224;
-        $SYS_epoll_ctl    = 225;
-        $SYS_epoll_wait   = 226;
-        $u64_mod_8        = 1;
-        $SYS_signalfd4 = 309;
-    } elsif ($machine =~ m/^ppc64/) {
-        $SYS_epoll_create = 236;
-        $SYS_epoll_ctl    = 237;
-        $SYS_epoll_wait   = 238;
-        $u64_mod_8        = 1;
-        $SYS_signalfd4 = 313;
-        $SYS_renameat2 //= 357;
-        $SYS_fstatfs = 100;
-        $SYS_sendmsg = 341;
-        $SYS_recvmsg = 342;
-        $FS_IOC_GETFLAGS = 0x40086601;
-        $FS_IOC_SETFLAGS = 0x80086602;
-    } elsif ($machine eq "ppc") {
-        $SYS_epoll_create = 236;
-        $SYS_epoll_ctl    = 237;
-        $SYS_epoll_wait   = 238;
-        $u64_mod_8        = 1;
-        $SYS_signalfd4 = 313;
-        $SYS_renameat2 //= 357;
-        $SYS_fstatfs = 100;
-        $FS_IOC_GETFLAGS = 0x40086601;
-        $FS_IOC_SETFLAGS = 0x80086602;
-    } elsif ($machine =~ m/^s390/) { # untested, no machine on cfarm
-        $SYS_epoll_create = 249;
-        $SYS_epoll_ctl    = 250;
-        $SYS_epoll_wait   = 251;
-        $u64_mod_8        = 1;
-        $SYS_signalfd4 = 322;
-        $SYS_renameat2 //= 347;
-        $SYS_fstatfs = 100;
-        $SYS_sendmsg = 370;
-        $SYS_recvmsg = 372;
-    } elsif ($machine eq 'ia64') { # untested, no machine on cfarm
-        $SYS_epoll_create = 1243;
-        $SYS_epoll_ctl    = 1244;
-        $SYS_epoll_wait   = 1245;
-        $u64_mod_8        = 1;
-        $SYS_signalfd4 = 289;
-    } elsif ($machine eq "alpha") { # untested, no machine on cfarm
-        # natural alignment, ints are 32-bits
-        $SYS_epoll_create = 407;
-        $SYS_epoll_ctl    = 408;
-        $SYS_epoll_wait   = 409;
-        $u64_mod_8        = 1;
-        $SYS_signalfd4 = 484;
-        $SFD_CLOEXEC = 010000000;
-    } elsif ($machine eq 'aarch64' || $machine eq 'loongarch64') {
-        $SYS_epoll_create = 20;  # (sys_epoll_create1)
-        $SYS_epoll_ctl    = 21;
-        $SYS_epoll_wait   = 22;  # (sys_epoll_pwait)
-        $u64_mod_8        = 1;
-        $no_deprecated    = 1;
-        $SYS_signalfd4 = 74;
-        $SYS_renameat2 //= 276;
-        $SYS_fstatfs = 44;
-        $SYS_sendmsg = 211;
-        $SYS_recvmsg = 212;
-        $FS_IOC_GETFLAGS = 0x80086601;
-        $FS_IOC_SETFLAGS = 0x40086602;
-    } elsif ($machine =~ m/arm(v\d+)?.*l/) { # ARM OABI (untested on cfarm)
-        $SYS_epoll_create = 250;
-        $SYS_epoll_ctl    = 251;
-        $SYS_epoll_wait   = 252;
-        $u64_mod_8        = 1;
-        $SYS_signalfd4 = 355;
-        $SYS_renameat2 //= 382;
-        $SYS_fstatfs = 100;
-        $SYS_sendmsg = 296;
-        $SYS_recvmsg = 297;
-    } elsif ($machine =~ m/^mips64/) { # cfarm only has 32-bit userspace
-        $SYS_epoll_create = 5207;
-        $SYS_epoll_ctl    = 5208;
-        $SYS_epoll_wait   = 5209;
-        $u64_mod_8        = 1;
-        $SYS_signalfd4 = 5283;
-        $SYS_renameat2 //= 5311;
-        $SYS_fstatfs = 5135;
-        $SYS_sendmsg = 5045;
-        $SYS_recvmsg = 5046;
-        $FS_IOC_GETFLAGS = 0x40046601;
-        $FS_IOC_SETFLAGS = 0x80046602;
-    } elsif ($machine =~ m/^mips/) { # 32-bit, tested on mips64 cfarm machine
-        $SYS_epoll_create = 4248;
-        $SYS_epoll_ctl    = 4249;
-        $SYS_epoll_wait   = 4250;
-        $u64_mod_8        = 1;
-        $SYS_signalfd4 = 4324;
-        $SYS_renameat2 //= 4351;
-        $SYS_fstatfs = 4100;
-        $SYS_sendmsg = 4179;
-        $SYS_recvmsg = 4177;
-        $FS_IOC_GETFLAGS = 0x40046601;
-        $FS_IOC_SETFLAGS = 0x80046602;
-    } else {
-        # as a last resort, try using the *.ph files which may not
-        # exist or may be wrong
-        _load_syscall();
-        $SYS_epoll_create = eval { &SYS_epoll_create; } || 0;
-        $SYS_epoll_ctl    = eval { &SYS_epoll_ctl;    } || 0;
-        $SYS_epoll_wait   = eval { &SYS_epoll_wait;   } || 0;
-
-        # Note: do NOT add new syscalls to depend on *.ph, here.
-        # Better to miss syscalls (so we can fallback to IO::Poll)
-        # than to use wrong ones, since the names are not stable
-        # (at least not on FreeBSD), if the actual numbers are.
-    }
-
-    if ($u64_mod_8) {
-        *epoll_wait = \&epoll_wait_mod8;
-        *epoll_ctl = \&epoll_ctl_mod8;
-    } else {
-        *epoll_wait = \&epoll_wait_mod4;
-        *epoll_ctl = \&epoll_ctl_mod4;
-    }
+        $F_SETPIPE_SZ = 1031;
+        my (undef, undef, $release, undef, $machine) = POSIX::uname();
+        my ($maj, $min) = ($release =~ /\A([0-9]+)\.([0-9]+)/);
+        $SYS_renameat2 = 0 if "$maj.$min" < 3.15;
+        # whether the machine requires 64-bit numbers to be on 8-byte
+        # boundaries.
+        my $u64_mod_8 = 0;
+
+        if (SIZEOF_ptr == 4) {
+                # if we're running on an x86_64 kernel, but a 32-bit process,
+                # we need to use the x32 or i386 syscall numbers.
+                if ($machine eq 'x86_64') {
+                        my $s = $Config{cppsymbols};
+                        $machine = ($s =~ /\b__ILP32__=1\b/ &&
+                                        $s =~ /\b__x86_64__=1\b/) ?
+                                'x32' : 'i386'
+                } elsif ($machine eq 'mips64') { # similarly for mips64 vs mips
+                        $machine = 'mips';
+                }
+        }
+        if ($machine =~ m/^i[3456]86$/) {
+                $SYS_epoll_create = 254;
+                $SYS_epoll_ctl = 255;
+                $SYS_epoll_wait = 256;
+                $SYS_signalfd4 = 327;
+                $SYS_renameat2 //= 353;
+                $SYS_fstatfs = 100;
+                $SYS_sendmsg = 370;
+                $SYS_recvmsg = 372;
+                $INOTIFY = { # usage: `use constant $PublicInbox::Syscall::INOTIFY'
+                        SYS_inotify_init1 => 332,
+                        SYS_inotify_add_watch => 292,
+                        SYS_inotify_rm_watch => 293,
+                };
+                $FS_IOC_GETFLAGS = 0x80046601;
+                $FS_IOC_SETFLAGS = 0x40046602;
+        } elsif ($machine eq "x86_64") {
+                $SYS_epoll_create = 213;
+                $SYS_epoll_ctl = 233;
+                $SYS_epoll_wait = 232;
+                $SYS_signalfd4 = 289;
+                $SYS_renameat2 //= 316;
+                $SYS_fstatfs = 138;
+                $SYS_sendmsg = 46;
+                $SYS_recvmsg = 47;
+                $INOTIFY = {
+                        SYS_inotify_init1 => 294,
+                        SYS_inotify_add_watch => 254,
+                        SYS_inotify_rm_watch => 255,
+                };
+                $FS_IOC_GETFLAGS = 0x80086601;
+                $FS_IOC_SETFLAGS = 0x40086602;
+        } elsif ($machine eq 'x32') {
+                $SYS_epoll_create = 1073742037;
+                $SYS_epoll_ctl = 1073742057;
+                $SYS_epoll_wait = 1073742056;
+                $SYS_signalfd4 = 1073742113;
+                $SYS_renameat2 //= 0x40000000 + 316;
+                $SYS_fstatfs = 138;
+                $SYS_sendmsg = 0x40000206;
+                $SYS_recvmsg = 0x40000207;
+                $FS_IOC_GETFLAGS = 0x80046601;
+                $FS_IOC_SETFLAGS = 0x40046602;
+                $INOTIFY = {
+                        SYS_inotify_init1 => 1073742118,
+                        SYS_inotify_add_watch => 1073742078,
+                        SYS_inotify_rm_watch => 1073742079,
+                };
+        } elsif ($machine eq 'sparc64') {
+                $SYS_epoll_create = 193;
+                $SYS_epoll_ctl = 194;
+                $SYS_epoll_wait = 195;
+                $u64_mod_8 = 1;
+                $SYS_signalfd4 = 317;
+                $SYS_renameat2 //= 345;
+                $SFD_CLOEXEC = 020000000;
+                $SYS_fstatfs = 158;
+                $SYS_sendmsg = 114;
+                $SYS_recvmsg = 113;
+                $FS_IOC_GETFLAGS = 0x40086601;
+                $FS_IOC_SETFLAGS = 0x80086602;
+        } elsif ($machine =~ m/^parisc/) { # untested, no machine on cfarm
+                $SYS_epoll_create = 224;
+                $SYS_epoll_ctl = 225;
+                $SYS_epoll_wait = 226;
+                $u64_mod_8 = 1;
+                $SYS_signalfd4 = 309;
+                $SIGNUM{WINCH} = 23;
+        } elsif ($machine =~ m/^ppc64/) {
+                $SYS_epoll_create = 236;
+                $SYS_epoll_ctl = 237;
+                $SYS_epoll_wait = 238;
+                $u64_mod_8 = 1;
+                $SYS_signalfd4 = 313;
+                $SYS_renameat2 //= 357;
+                $SYS_fstatfs = 100;
+                $SYS_sendmsg = 341;
+                $SYS_recvmsg = 342;
+                $FS_IOC_GETFLAGS = 0x40086601;
+                $FS_IOC_SETFLAGS = 0x80086602;
+                $INOTIFY = {
+                        SYS_inotify_init1 => 318,
+                        SYS_inotify_add_watch => 276,
+                        SYS_inotify_rm_watch => 277,
+                };
+        } elsif ($machine eq "ppc") { # untested, no machine on cfarm
+                $SYS_epoll_create = 236;
+                $SYS_epoll_ctl = 237;
+                $SYS_epoll_wait = 238;
+                $u64_mod_8 = 1;
+                $SYS_signalfd4 = 313;
+                $SYS_renameat2 //= 357;
+                $SYS_fstatfs = 100;
+                $FS_IOC_GETFLAGS = 0x40086601;
+                $FS_IOC_SETFLAGS = 0x80086602;
+        } elsif ($machine =~ m/^s390/) { # untested, no machine on cfarm
+                $SYS_epoll_create = 249;
+                $SYS_epoll_ctl = 250;
+                $SYS_epoll_wait = 251;
+                $u64_mod_8 = 1;
+                $SYS_signalfd4 = 322;
+                $SYS_renameat2 //= 347;
+                $SYS_fstatfs = 100;
+                $SYS_sendmsg = 370;
+                $SYS_recvmsg = 372;
+        } elsif ($machine eq 'ia64') { # untested, no machine on cfarm
+                $SYS_epoll_create = 1243;
+                $SYS_epoll_ctl = 1244;
+                $SYS_epoll_wait = 1245;
+                $u64_mod_8 = 1;
+                $SYS_signalfd4 = 289;
+        } elsif ($machine eq "alpha") { # untested, no machine on cfarm
+                # natural alignment, ints are 32-bits
+                $SYS_epoll_create = 407;
+                $SYS_epoll_ctl = 408;
+                $SYS_epoll_wait = 409;
+                $u64_mod_8 = 1;
+                $SYS_signalfd4 = 484;
+                $SFD_CLOEXEC = 010000000;
+        } elsif ($machine =~ /\A(?:loong|a)arch64\z/ || $machine eq 'riscv64') {
+                $SYS_epoll_create = 20; # (sys_epoll_create1)
+                $SYS_epoll_ctl = 21;
+                $SYS_epoll_wait = 22; # (sys_epoll_pwait)
+                $u64_mod_8 = 1;
+                $no_deprecated = 1;
+                $SYS_signalfd4 = 74;
+                $SYS_renameat2 //= 276;
+                $SYS_fstatfs = 44;
+                $SYS_sendmsg = 211;
+                $SYS_recvmsg = 212;
+                $INOTIFY = {
+                        SYS_inotify_init1 => 26,
+                        SYS_inotify_add_watch => 27,
+                        SYS_inotify_rm_watch => 28,
+                };
+                $FS_IOC_GETFLAGS = 0x80086601;
+                $FS_IOC_SETFLAGS = 0x40086602;
+        } elsif ($machine =~ m/arm(v\d+)?.*l/) { # ARM OABI (untested on cfarm)
+                $SYS_epoll_create = 250;
+                $SYS_epoll_ctl = 251;
+                $SYS_epoll_wait = 252;
+                $u64_mod_8 = 1;
+                $SYS_signalfd4 = 355;
+                $SYS_renameat2 //= 382;
+                $SYS_fstatfs = 100;
+                $SYS_sendmsg = 296;
+                $SYS_recvmsg = 297;
+        } elsif ($machine =~ m/^mips64/) { # cfarm only has 32-bit userspace
+                $SYS_epoll_create = 5207;
+                $SYS_epoll_ctl = 5208;
+                $SYS_epoll_wait = 5209;
+                $u64_mod_8 = 1;
+                $SYS_signalfd4 = 5283;
+                $SYS_renameat2 //= 5311;
+                $SYS_fstatfs = 5135;
+                $SYS_sendmsg = 5045;
+                $SYS_recvmsg = 5046;
+                $FS_IOC_GETFLAGS = 0x40046601;
+                $FS_IOC_SETFLAGS = 0x80046602;
+        } elsif ($machine =~ m/^mips/) { # 32-bit, tested on mips64 cfarm host
+                $SYS_epoll_create = 4248;
+                $SYS_epoll_ctl = 4249;
+                $SYS_epoll_wait = 4250;
+                $u64_mod_8 = 1;
+                $SYS_signalfd4 = 4324;
+                $SYS_renameat2 //= 4351;
+                $SYS_fstatfs = 4100;
+                $SYS_sendmsg = 4179;
+                $SYS_recvmsg = 4177;
+                $FS_IOC_GETFLAGS = 0x40046601;
+                $FS_IOC_SETFLAGS = 0x80046602;
+                $SIGNUM{WINCH} = 20;
+                $INOTIFY = {
+                        SYS_inotify_init1 => 4329,
+                        SYS_inotify_add_watch => 4285,
+                        SYS_inotify_rm_watch => 4286,
+                };
+        } else {
+                warn <<EOM;
+machine=$machine ptrsize=$Config{ptrsize} has no syscall definitions
+git clone https://80x24.org/public-inbox.git and
+Send the output of ./devel/sysdefs-list to meta\@public-inbox.org
+EOM
+        }
+        if ($u64_mod_8) {
+                *epoll_wait = \&epoll_wait_mod8;
+                *epoll_ctl = \&epoll_ctl_mod8;
+        } else {
+                *epoll_wait = \&epoll_wait_mod4;
+                *epoll_ctl = \&epoll_ctl_mod4;
+        }
+} elsif ($^O =~ /\A(?:freebsd|openbsd|netbsd|dragonfly)\z/) {
+# don't use syscall.ph here, name => number mappings are not stable on *BSD
+# but the actual numbers are.
+# OpenBSD perl redirects syscall perlop to libc functions
+# https://cvsweb.openbsd.org/src/gnu/usr.bin/perl/gen_syscall_emulator.pl
+# https://www.netbsd.org/docs/internals/en/chap-processes.html#syscall_versioning
+# https://wiki.freebsd.org/AddingSyscalls#Backward_compatibily
+# (I'm assuming Dragonfly copies FreeBSD, here, too)
+        $SYS_recvmsg = 27;
+        $SYS_sendmsg = 28;
 }
-# use Inline::C for *BSD-only or general POSIX stuff.
-# Linux guarantees stable syscall numbering, BSDs only offer a stable libc
-# use scripts/syscall-list on Linux to detect new syscall numbers
 
-############################################################################
-# epoll functions
-############################################################################
+BEGIN {
+        if ($^O eq 'linux') {
+                %PACK = (
+                        TMPL_cmsg_len => TMPL_size_t,
+                        # cmsg_len, cmsg_level, cmsg_type
+                        SIZEOF_cmsghdr => SIZEOF_int * 2 + SIZEOF_size_t,
+                        CMSG_DATA_off => '',
+                        TMPL_msghdr => 'PL' . # msg_name, msg_namelen
+                                '@'.(2 * SIZEOF_ptr).'P'. # msg_iov
+                                'i'. # msg_iovlen
+                                '@'.(4 * SIZEOF_ptr).'P'. # msg_control
+                                'L'. # msg_controllen (socklen_t)
+                                'i', # msg_flags
+                );
+        } elsif ($^O =~ /\A(?:freebsd|openbsd|netbsd|dragonfly)\z/) {
+                %PACK = (
+                        TMPL_cmsg_len => 'L', # socklen_t
+                        SIZEOF_cmsghdr => SIZEOF_int * 3,
+                        CMSG_DATA_off => SIZEOF_ptr == 8 ? '@16' : '',
+                        TMPL_msghdr => 'PL' . # msg_name, msg_namelen
+                                '@'.(2 * SIZEOF_ptr).'P'. # msg_iov
+                                TMPL_size_t. # msg_iovlen
+                                '@'.(4 * SIZEOF_ptr).'P'. # msg_control
+                                TMPL_size_t. # msg_controllen
+                                'i', # msg_flags
+
+                )
+        }
+        $PACK{CMSG_ALIGN_size} = SIZEOF_size_t;
+}
 
-sub epoll_defined { $SYS_epoll_create ? 1 : 0; }
+# SFD_CLOEXEC is arch-dependent, so IN_CLOEXEC may be, too
+$INOTIFY->{IN_CLOEXEC} //= 0x80000 if $INOTIFY;
 
 sub epoll_create {
         syscall($SYS_epoll_create, $no_deprecated ? 0 : 100);
@@ -292,10 +329,13 @@ sub epoll_create {
 # epoll_ctl wrapper
 # ARGS: (epfd, op, fd, events_mask)
 sub epoll_ctl_mod4 {
-    syscall($SYS_epoll_ctl, $_[0]+0, $_[1]+0, $_[2]+0, pack("LLL", $_[3], $_[2], 0));
+        syscall($SYS_epoll_ctl, $_[0]+0, $_[1]+0, $_[2]+0,
+                pack("LLL", $_[3], $_[2], 0));
 }
+
 sub epoll_ctl_mod8 {
-    syscall($SYS_epoll_ctl, $_[0]+0, $_[1]+0, $_[2]+0, pack("LLLL", $_[3], 0, $_[2], 0));
+        syscall($SYS_epoll_ctl, $_[0]+0, $_[1]+0, $_[2]+0,
+                pack("LLLL", $_[3], 0, $_[2], 0));
 }
 
 # epoll_wait wrapper
@@ -308,7 +348,7 @@ sub epoll_wait_mod4 {
         # resize our static buffer if maxevents bigger than we've ever done
         if ($maxevents > $epoll_wait_size) {
                 $epoll_wait_size = $maxevents;
-                vec($epoll_wait_events, $maxevents * 12 * 8 - 1, 1) = 0;
+                vec($epoll_wait_events, $maxevents * 12 - 1, 8) = 0;
         }
         @$events = ();
         my $ct = syscall($SYS_epoll_wait, $epfd, $epoll_wait_events,
@@ -329,7 +369,7 @@ sub epoll_wait_mod8 {
         # resize our static buffer if maxevents bigger than we've ever done
         if ($maxevents > $epoll_wait_size) {
                 $epoll_wait_size = $maxevents;
-                vec($epoll_wait_events, $maxevents * 16 * 8 - 1, 1) = 0;
+                vec($epoll_wait_events, $maxevents * 16 - 1, 8) = 0;
         }
         @$events = ();
         my $ct = syscall($SYS_epoll_wait, $epfd, $epoll_wait_events,
@@ -346,15 +386,15 @@ sub epoll_wait_mod8 {
         }
 }
 
-sub signalfd ($$) {
-        my ($signos, $nonblock) = @_;
+sub signalfd ($) {
+        my ($signos) = @_;
         if ($SYS_signalfd4) {
                 my $set = POSIX::SigSet->new(@$signos);
                 syscall($SYS_signalfd4, -1, "$$set",
                         # $Config{sig_count} is NSIG, so this is NSIG/8:
                         int($Config{sig_count}/8),
                         # SFD_NONBLOCK == O_NONBLOCK for every architecture
-                        ($nonblock ? O_NONBLOCK : 0) |$SFD_CLOEXEC);
+                        O_NONBLOCK|$SFD_CLOEXEC);
         } else {
                 $! = ENOSYS;
                 undef;
@@ -411,70 +451,70 @@ sub nodatacow_dir {
         if (open my $fh, '<', $_[0]) { nodatacow_fh($fh) }
 }
 
-sub CMSG_ALIGN ($) { ($_[0] + SIZEOF_size_t - 1) & ~(SIZEOF_size_t - 1) }
+use constant \%PACK;
+sub CMSG_ALIGN ($) { ($_[0] + CMSG_ALIGN_size - 1) & ~(CMSG_ALIGN_size - 1) }
 use constant CMSG_ALIGN_SIZEOF_cmsghdr => CMSG_ALIGN(SIZEOF_cmsghdr);
 sub CMSG_SPACE ($) { CMSG_ALIGN($_[0]) + CMSG_ALIGN_SIZEOF_cmsghdr }
 sub CMSG_LEN ($) { CMSG_ALIGN_SIZEOF_cmsghdr + $_[0] }
-use constant msg_controllen => CMSG_SPACE(10 * SIZEOF_int) + 16; # 10 FDs
+use constant msg_controllen_max =>
+        CMSG_SPACE(10 * SIZEOF_int) + SIZEOF_cmsghdr; # space for 10 FDs
 
 if (defined($SYS_sendmsg) && defined($SYS_recvmsg)) {
 no warnings 'once';
+require PublicInbox::CmdIPC4;
+
 *send_cmd4 = sub ($$$$) {
         my ($sock, $fds, undef, $flags) = @_;
         my $iov = pack('P'.TMPL_size_t,
                         $_[2] // NUL, length($_[2] // NUL) || 1);
-        my $cmsghdr = pack(TMPL_size_t . # cmsg_len
+        my $fd_space = scalar(@$fds) * SIZEOF_int;
+        my $msg_controllen = CMSG_SPACE($fd_space);
+        my $cmsghdr = pack(TMPL_cmsg_len .
                         'LL' .  # cmsg_level, cmsg_type,
-                        ('i' x scalar(@$fds)),
-                        CMSG_LEN(scalar(@$fds) * SIZEOF_int), # cmsg_len
+                        CMSG_DATA_off.('i' x scalar(@$fds)). # CMSG_DATA
+                        '@'.($msg_controllen - 1).'x1', # pad to space, not len
+                        CMSG_LEN($fd_space), # cmsg_len
                         SOL_SOCKET, SCM_RIGHTS, # cmsg_{level,type}
                         @$fds); # CMSG_DATA
-        my $mh = pack('PL' . # msg_name, msg_namelen (socklen_t (U32))
-                        BYTES_4_hole . # 4-byte padding on 64-bit
-                        'P'.TMPL_size_t . # msg_iov, msg_iovlen,
-                        'P'.TMPL_size_t . # msg_control, msg_controllen,
-                        'i', # msg_flags
-                        NUL, 0, # msg_name, msg_namelen (unused)
-                        @BYTES_4_hole,
+        my $mh = pack(TMPL_msghdr,
+                        undef, 0, # msg_name, msg_namelen (unused)
                         $iov, 1, # msg_iov, msg_iovlen
                         $cmsghdr, # msg_control
-                        CMSG_SPACE(scalar(@$fds) * SIZEOF_int), # msg_controllen
+                        $msg_controllen,
                         0); # msg_flags
-        my $sent;
+        my $s;
         my $try = 0;
         do {
-                $sent = syscall($SYS_sendmsg, fileno($sock), $mh, $flags);
-        } while ($sent < 0 &&
-                        ($!{ENOBUFS} || $!{ENOMEM} || $!{ETOOMANYREFS}) &&
-                        (++$try < 50) &&
-                        warn "sleeping on sendmsg: $! (#$try)\n" &&
-                        select(undef, undef, undef, 0.1) == 0);
-        $sent >= 0 ? $sent : undef;
+                $s = syscall($SYS_sendmsg, fileno($sock), $mh, $flags);
+        } while ($s < 0 && PublicInbox::CmdIPC4::sendmsg_retry($try));
+        $s >= 0 ? $s : undef;
 };
 
 *recv_cmd4 = sub ($$$) {
         my ($sock, undef, $len) = @_;
-        vec($_[1], ($len + 1) * 8, 1) = 0;
-        my $cmsghdr = "\0" x msg_controllen; # 10 * sizeof(int)
+        vec($_[1] //= '', $len - 1, 8) = 0;
+        my $cmsghdr = "\0" x msg_controllen_max; # 10 * sizeof(int)
         my $iov = pack('P'.TMPL_size_t, $_[1], $len);
-        my $mh = pack('PL' . # msg_name, msg_namelen (socklen_t (U32))
-                        BYTES_4_hole . # 4-byte padding on 64-bit
-                        'P'.TMPL_size_t . # msg_iov, msg_iovlen,
-                        'P'.TMPL_size_t . # msg_control, msg_controllen,
-                        'i', # msg_flags
-                        NUL, 0, # msg_name, msg_namelen (unused)
-                        @BYTES_4_hole,
+        my $mh = pack(TMPL_msghdr,
+                        undef, 0, # msg_name, msg_namelen (unused)
                         $iov, 1, # msg_iov, msg_iovlen
                         $cmsghdr, # msg_control
-                        msg_controllen,
+                        msg_controllen_max,
                         0); # msg_flags
-        my $r = syscall($SYS_recvmsg, fileno($sock), $mh, 0);
-        return (undef) if $r < 0; # $! set
+        my $r;
+        do {
+                $r = syscall($SYS_recvmsg, fileno($sock), $mh, 0);
+        } while ($r < 0 && $!{EINTR});
+        if ($r < 0) {
+                $_[1] = '';
+                return (undef);
+        }
         substr($_[1], $r, length($_[1]), '');
         my @ret;
         if ($r > 0) {
-                my ($len, $lvl, $type, @fds) = unpack(TMPL_size_t . # cmsg_len
-                                        'LLi*', # cmsg_level, cmsg_type, @fds
+                my ($len, $lvl, $type, @fds) = unpack(TMPL_cmsg_len.
+                                        'LL'. # cmsg_level, cmsg_type
+                                        CMSG_DATA_off.'i*', # @fds
                                         $cmsghdr);
                 if ($lvl == SOL_SOCKET && $type == SCM_RIGHTS) {
                         $len -= CMSG_ALIGN_SIZEOF_cmsghdr;
diff --git a/lib/PublicInbox/TLS.pm b/lib/PublicInbox/TLS.pm
index 3fe16a62..3ce57f1b 100644
--- a/lib/PublicInbox/TLS.pm
+++ b/lib/PublicInbox/TLS.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # IO::Socket::SSL support code
@@ -6,7 +6,7 @@ package PublicInbox::TLS;
 use strict;
 use IO::Socket::SSL;
 use PublicInbox::Syscall qw(EPOLLIN EPOLLOUT);
-use Carp qw(carp);
+use Carp qw(carp croak);
 
 sub err () { $SSL_ERROR }
 
@@ -18,4 +18,28 @@ sub epollbit () {
         undef;
 }
 
+sub _ctx_new ($) {
+        my ($tlsd) = @_;
+        my $ctx = IO::Socket::SSL::SSL_Context->new(
+                                @{$tlsd->{ssl_ctx_opt}}, SSL_server => 1) or
+                croak "SSL_Context->new: $SSL_ERROR";
+
+        # save ~34K per idle connection (cf. SSL_CTX_set_mode(3ssl))
+        # RSS goes from 346MB to 171MB with 10K idle NNTPS clients on amd64
+        # cf. https://rt.cpan.org/Ticket/Display.html?id=129463
+        my $mode = eval { Net::SSLeay::MODE_RELEASE_BUFFERS() };
+        if ($mode && $ctx->{context}) {
+                eval { Net::SSLeay::CTX_set_mode($ctx->{context}, $mode) };
+                warn "W: $@ (setting SSL_MODE_RELEASE_BUFFERS)\n" if $@;
+        }
+        $ctx;
+}
+
+sub start {
+        my ($io, $tlsd) = @_;
+        IO::Socket::SSL->start_SSL($io, SSL_server => 1,
+                SSL_reuse_ctx => ($tlsd->{ssl_ctx} //= _ctx_new($tlsd)),
+                SSL_startHandshake => 0);
+}
+
 1;
diff --git a/lib/PublicInbox/TailNotify.pm b/lib/PublicInbox/TailNotify.pm
new file mode 100644
index 00000000..84340a35
--- /dev/null
+++ b/lib/PublicInbox/TailNotify.pm
@@ -0,0 +1,97 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# only used for tests at the moment...
+package PublicInbox::TailNotify;
+use v5.12;
+use parent qw(PublicInbox::DirIdle); # not optimal, maybe..
+use PublicInbox::DS qw(now);
+
+my ($TAIL_MOD, $ino_cls);
+if ($^O eq 'linux' && eval { require PublicInbox::Inotify; 1 }) {
+        $TAIL_MOD = PublicInbox::Inotify::IN_MOVED_TO() |
+                PublicInbox::Inotify::IN_CREATE() |
+                PublicInbox::Inotify::IN_MODIFY();
+        $ino_cls = 'PublicInbox::Inotify';
+} elsif (eval { require PublicInbox::KQNotify }) {
+        $TAIL_MOD = PublicInbox::KQNotify::MOVED_TO_OR_CREATE() |
+                IO::KQueue::NOTE_DELETE() | IO::KQueue::NOTE_RENAME();
+        $ino_cls = 'PublicInbox::KQNotify';
+} else {
+        require PublicInbox::FakeInotify;
+        $TAIL_MOD = PublicInbox::FakeInotify::MOVED_TO_OR_CREATE() |
+                PublicInbox::FakeInotify::IN_MODIFY() |
+                PublicInbox::FakeInotify::IN_DELETE();
+}
+require IO::Poll if $ino_cls;
+
+sub reopen_file ($) {
+        my ($self) = @_;
+
+        open my $fh, '<', $self->{fn} or return undef;
+        my @st = stat $fh or die "fstat($self->{fn}): $!";
+        $self->{ino_dev} = "@st[0, 1]";
+        $self->{inot}->watch($self->{fn}, $TAIL_MOD);
+        $self->{watch_fh} = $fh; # return value
+}
+
+sub new {
+        my ($cls, $fn) = @_;
+        my $self = bless { fn => $fn }, $cls;
+        if ($ino_cls) {
+                $self->{inot} = $ino_cls->new or die "E: $ino_cls->new: $!";
+                $self->{inot}->blocking(0);
+                my ($dn) = ($fn =~ m!\A(.+)/+[^/]+\z!);
+                $self->{inot}->watch($dn // '.', $TAIL_MOD);
+        } else {
+                $self->{inot} = PublicInbox::FakeInotify->new;
+        }
+        reopen_file($self);
+        $self->{inot}->watch($fn, $TAIL_MOD);
+        $self;
+}
+
+sub delete_self {
+        for (@_) { return 1 if $_->IN_DELETE_SELF }
+        undef;
+}
+
+sub getlines {
+        my ($self, $timeo) = @_;
+        my ($fh, $buf, $rfds, @ret, @events);
+        my $end = defined($timeo) ? now + $timeo : undef;
+again:
+        while (1) {
+                @events = $self->{inot}->read; # Linux::Inotify2::read
+                last if @events;
+                return () if defined($timeo) && (!$timeo || (now > $end));
+                my $wait = 0.1;
+                if ($ino_cls) {
+                        vec($rfds = '', $self->{inot}->fileno, 1) = 1;
+                        if (defined $end) {
+                                $wait = $end - now;
+                                $wait = 0 if $wait < 0;
+                        } else {
+                                undef $wait;
+                        }
+                }
+                select($rfds, undef, undef, $wait);
+        }
+        if ($fh = $self->{watch_fh}) {
+                sysread($fh, $buf, -s $fh) and
+                        push @ret, split(/^/sm, $buf);
+                my @st = stat($self->{fn});
+                if (!@st || "@st[0, 1]" ne $self->{ino_dev} ||
+                                delete_self(@events)) {
+                        delete @$self{qw(ino_dev watch_fh)};
+                }
+        }
+        if ($fh = $self->{watch_fh} // reopen_file($self)) {
+                sysread($fh, $buf, -s $fh) and
+                        push @ret, split(/^/sm, $buf);
+        }
+        goto again if (!@ret && (!defined($end) || now < $end));
+        @ret;
+}
+
+1;
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 943dd2fa..5f159683 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -6,21 +6,31 @@ package PublicInbox::TestCommon;
 use strict;
 use parent qw(Exporter);
 use v5.10.1;
-use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD :seek);
+use Fcntl qw(F_SETFD F_GETFD FD_CLOEXEC :seek);
 use POSIX qw(dup2);
 use IO::Socket::INET;
 use File::Spec;
+use Scalar::Util qw(isvstring);
+use Carp ();
 our @EXPORT;
 my $lei_loud = $ENV{TEST_LEI_ERR_LOUD};
-my $tail_cmd = $ENV{TAIL};
-our ($lei_opt, $lei_out, $lei_err, $lei_cwdfh);
+our $tail_cmd = $ENV{TAIL};
+our ($lei_opt, $lei_out, $lei_err);
+use autodie qw(chdir close fcntl mkdir open opendir seek unlink);
+
+$_ = File::Spec->rel2abs($_) for (grep(!m!^/!, @INC));
+
 BEGIN {
         @EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods
                 run_script start_script key2sub xsys xsys_e xqx eml_load tick
                 have_xapian_compact json_utf8 setup_public_inboxes create_inbox
+                create_dir
+                create_coderepo require_bsd kernel_version check_broken_tmpfs
+                quit_waiter_pipe wait_for_eof require_git_http_backend
                 tcp_host_port test_lei lei lei_ok $lei_out $lei_err $lei_opt
                 test_httpd xbail require_cmd is_xdeeply tail_f
-                ignore_inline_c_missing);
+                ignore_inline_c_missing no_pollerfd no_coredump cfg_new
+                strace strace_inject lsof_pid oct_is);
         require Test::More;
         my @methods = grep(!/\W/, @Test::More::EXPORT);
         eval(join('', map { "*$_=\\&Test::More::$_;" } @methods));
@@ -28,23 +38,56 @@ BEGIN {
         push @EXPORT, @methods;
 }
 
+sub kernel_version () {
+        state $version = do {
+                require POSIX;
+                my @u = POSIX::uname();
+                if ($u[2] =~ /\A([0-9]+(?:\.[0-9]+)+)/) {
+                        eval "v$1";
+                } else {
+                        local $" = "', `";
+                        diag "Unable to get kernel version from: `@u'";
+                        undef;
+                }
+        };
+}
+
+sub check_broken_tmpfs () {
+        return if $^O ne 'dragonfly' || kernel_version ge v6.5;
+        diag 'EVFILT_VNODE + tmpfs is broken on dragonfly <= 6.4 (have: '.
+                sprintf('%vd', kernel_version).')';
+        1;
+}
+
+sub require_bsd (;$) {
+        state $ok = ($^O =~ m!\A(?:free|net|open)bsd\z! ||
+                        $^O eq 'dragonfly');
+        return 1 if $ok;
+        return if defined(wantarray);
+        my $m = "$0 is BSD-only (\$^O=$^O)";
+        @_ ? skip($m, 1) : plan(skip_all => $m);
+}
+
 sub xbail (@) { BAIL_OUT join(' ', map { ref() ? (explain($_)) : ($_) } @_) }
 
+sub read_all ($;$$$) {
+        require PublicInbox::IO;
+        PublicInbox::IO::read_all($_[0], $_[1], $_[2], $_[3])
+}
+
 sub eml_load ($) {
         my ($path, $cb) = @_;
-        open(my $fh, '<', $path) or die "open $path: $!";
+        open(my $fh, '<', $path);
         require PublicInbox::Eml;
-        PublicInbox::Eml->new(\(do { local $/; <$fh> }));
+        PublicInbox::Eml->new(\(scalar read_all $fh));
 }
 
 sub tmpdir (;$) {
         my ($base) = @_;
         require File::Temp;
-        unless (defined $base) {
-                ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!);
-        }
+        ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!) unless defined $base;
         my $tmpdir = File::Temp->newdir("pi-$base-$$-XXXX", TMPDIR => 1);
-        ($tmpdir->dirname, $tmpdir);
+        wantarray ? ($tmpdir->dirname, $tmpdir) : $tmpdir;
 }
 
 sub tcp_server () {
@@ -57,8 +100,12 @@ sub tcp_server () {
         );
         eval {
                 die 'IPv4-only' if $ENV{TEST_IPV4_ONLY};
-                require IO::Socket::INET6;
-                IO::Socket::INET6->new(%opt, LocalAddr => '[::1]')
+                my $pkg;
+                for (qw(IO::Socket::IP IO::Socket::INET6)) {
+                        eval "require $_" or next;
+                        $pkg = $_ and last;
+                }
+                $pkg->new(%opt, LocalAddr => '[::1]');
         } || eval {
                 die 'IPv6-only' if $ENV{TEST_IPV6_ONLY};
                 IO::Socket::INET->new(%opt, LocalAddr => '127.0.0.1')
@@ -90,31 +137,65 @@ sub tcp_connect {
 }
 
 sub require_cmd ($;$) {
-        my ($cmd, $maybe) = @_;
+        my ($cmd, $nr) = @_;
         require PublicInbox::Spawn;
-        my $bin = PublicInbox::Spawn::which($cmd);
+        state %CACHE;
+        my $bin = $CACHE{$cmd} //= PublicInbox::Spawn::which($cmd);
         return $bin if $bin;
-        $maybe ? 0 : plan(skip_all => "$cmd missing from PATH for $0");
+        return plan(skip_all => "$cmd missing from PATH for $0") if !$nr;
+        defined(wantarray) ? undef : skip("$cmd missing", $nr);
 }
 
-sub have_xapian_compact () {
-        require_cmd($ENV{XAPIAN_COMPACT} || 'xapian-compact', 1);
+sub have_xapian_compact (;$) {
+        require_cmd($ENV{XAPIAN_COMPACT} || 'xapian-compact', @_ ? $_[0] : ());
 }
 
 sub require_git ($;$) {
-        my ($req, $maybe) = @_;
-        my ($req_maj, $req_min, $req_sub) = split(/\./, $req);
-        my ($cur_maj, $cur_min, $cur_sub) = (xqx([qw(git --version)])
-                        =~ /version (\d+)\.(\d+)(?:\.(\d+))?/);
-
-        my $req_int = ($req_maj << 24) | ($req_min << 16) | ($req_sub // 0);
-        my $cur_int = ($cur_maj << 24) | ($cur_min << 16) | ($cur_sub // 0);
-        if ($cur_int < $req_int) {
-                return 0 if $maybe;
-                plan skip_all =>
-                        "git $req+ required, have $cur_maj.$cur_min.$cur_sub";
+        my ($req, $nr) = @_;
+        require PublicInbox::Git;
+        state $cur_vstr = PublicInbox::Git::git_version();
+        $req = eval("v$req") unless isvstring($req);
+
+        return 1 if $cur_vstr ge $req;
+        state $cur_ver = sprintf('%vd', $cur_vstr);
+        return plan skip_all => "git $req+ required, have $cur_ver" if !$nr;
+        defined(wantarray) ? undef :
+                skip("git $req+ required (have $cur_ver)", $nr)
+}
+
+sub require_git_http_backend (;$) {
+        my ($nr) = @_;
+        state $ok = do {
+                require PublicInbox::Git;
+                my $git = PublicInbox::Git::check_git_exe() or plan
+                        skip_all => 'nothing in public-inbox works w/o git';
+                my $rdr = { 1 => \my $out, 2 => \my $err };
+                xsys([$git, qw(http-backend)], undef, $rdr);
+                $out =~ /^Status:/ism;
+        };
+        if (!$ok) {
+                my $msg = "`git http-backend' not available";
+                defined($nr) ? skip $msg, $nr : plan skip_all => $msg;
         }
-        1;
+        $ok;
+}
+
+my %IPv6_VERSION = (
+        'Net::NNTP' => 3.00,
+        'Mail::IMAPClient' => 3.40,
+        'HTTP::Tiny' => 0.042,
+        'Net::POP3' => 2.32,
+);
+
+sub need_accept_filter ($) {
+        my ($af) = @_;
+        return if $^O eq 'netbsd'; # since NetBSD 5.0, no kldstat needed
+        $^O =~ /\A(?:freebsd|dragonfly)\z/ or
+                skip 'SO_ACCEPTFILTER is FreeBSD/NetBSD/Dragonfly-only so far',
+                        1;
+        state $tried = {};
+        ($tried->{$af} //= system("kldstat -m $af >/dev/null")) and
+                skip "$af not loaded: kldload $af", 1;
 }
 
 sub require_mods {
@@ -124,7 +205,7 @@ sub require_mods {
         while (my $mod = shift(@mods)) {
                 if ($mod eq 'lei') {
                         require_git(2.6, $maybe ? $maybe : ());
-                        push @mods, qw(DBD::SQLite Search::Xapian);
+                        push @mods, qw(DBD::SQLite Xapian +SCM_RIGHTS);
                         $mod = 'json'; # fall-through
                 }
                 if ($mod eq 'json') {
@@ -133,18 +214,30 @@ sub require_mods {
                         push @mods, qw(Plack::Builder Plack::Util);
                         next;
                 } elsif ($mod eq '-imapd') {
-                        push @mods, qw(Parse::RecDescent DBD::SQLite
-                                        Email::Address::XS||Mail::Address);
+                        push @mods, qw(Parse::RecDescent DBD::SQLite);
                         next;
-                } elsif ($mod eq '-nntpd') {
+                } elsif ($mod eq '-nntpd' || $mod eq 'v2') {
                         push @mods, qw(DBD::SQLite);
                         next;
                 }
-                if ($mod eq 'Search::Xapian') {
+                if ($mod eq 'Xapian') {
                         if (eval { require PublicInbox::Search } &&
                                 PublicInbox::Search::load_xapian()) {
                                 next;
                         }
+                } elsif ($mod eq '+SCM_RIGHTS') {
+                        if (my $msg = need_scm_rights()) {
+                                push @need, $msg;
+                                next;
+                        }
+                } elsif ($mod eq ':fcntl_lock') {
+                        next if $^O eq 'linux' || require_bsd;
+                        diag "untested platform: $^O, ".
+                                "requiring File::FcntlLock...";
+                        push @mods, 'File::FcntlLock';
+                } elsif ($mod =~ /\A\+(accf_.*)\z/) {
+                        need_accept_filter($1);
+                        next
                 } elsif (index($mod, '||') >= 0) { # "Foo||Bar"
                         my $ok;
                         for my $m (split(/\Q||\E/, $mod)) {
@@ -167,9 +260,13 @@ sub require_mods {
                                 !eval{ IO::Socket::SSL->VERSION(2.007); 1 }) {
                         push @need, $@;
                 }
+                if (defined(my $v = $IPv6_VERSION{$mod})) {
+                        $ENV{TEST_IPV4_ONLY} = 1 if !eval { $mod->VERSION($v) };
+                }
         }
         return unless @need;
         my $m = join(', ', @need)." missing for $0";
+        $m =~ s/\bEmail::MIME\b/Email::MIME (development purposes only)/;
         skip($m, $maybe) if $maybe;
         plan(skip_all => $m)
 }
@@ -192,9 +289,9 @@ sub _prepare_redirects ($) {
         for (my $fd = 0; $fd <= $#io_mode; $fd++) {
                 my $fh = $fhref->[$fd] or next;
                 my ($oldfh, $mode) = @{$io_mode[$fd]};
-                open my $orig, $mode, $oldfh or die "$oldfh $mode stash: $!";
+                open(my $orig, $mode, $oldfh);
                 $orig_io->[$fd] = $orig;
-                open $oldfh, $mode, $fh or die "$oldfh $mode redirect: $!";
+                open $oldfh, $mode, $fh;
         }
         $orig_io;
 }
@@ -204,7 +301,7 @@ sub _undo_redirects ($) {
         for (my $fd = 0; $fd <= $#io_mode; $fd++) {
                 my $fh = $orig_io->[$fd] or next;
                 my ($oldfh, $mode) = @{$io_mode[$fd]};
-                open $oldfh, $mode, $fh or die "$$oldfh $mode redirect: $!";
+                open $oldfh, $mode, $fh;
         }
 }
 
@@ -230,8 +327,8 @@ sub key2sub ($) {
         my ($key) = @_;
         $cached_scripts{$key} //= do {
                 my $f = key2script($key);
-                open my $fh, '<', $f or die "open $f: $!";
-                my $str = do { local $/; <$fh> };
+                open my $fh, '<', $f;
+                my $str = read_all($fh);
                 my $pkg = (split(m!/!, $f))[-1];
                 $pkg =~ s/([a-z])([a-z0-9]+)(\.t)?\z/\U$1\E$2/;
                 $pkg .= "_T" if $3;
@@ -246,7 +343,7 @@ use subs qw(exit);
 sub main {
 # the below "line" directive is a magic comment, see perlsyn(1) manpage
 # line 1 "$f"
-$str
+{ $str }
         0;
 }
 1;
@@ -275,20 +372,46 @@ sub _run_sub ($$$) {
         }
 }
 
+sub no_coredump (@) {
+        my @dirs = @_;
+        my $cwdfh;
+        opendir($cwdfh, '.') if @dirs;
+        my @found;
+        for (@dirs, '.') {
+                chdir $_;
+                my @cores = glob('core.* *.core');
+                push @cores, 'core' if -d 'core';
+                push(@found, "@cores found in $_") if @cores;
+                chdir $cwdfh if $cwdfh;
+        }
+        return if !@found; # keep it quiet.
+        is(scalar(@found), 0, 'no core dumps found');
+        diag(join("\n", @found) . Carp::longmess());
+        if (-t STDIN) {
+                diag 'press ENTER to continue, (q) to quit';
+                chomp(my $line = <STDIN>);
+                xbail 'user quit' if $line =~ /\Aq/;
+        }
+}
+
 sub run_script ($;$$) {
         my ($cmd, $env, $opt) = @_;
+        no_coredump($opt->{-C} ? ($opt->{-C}) : ());
         my ($key, @argv) = @$cmd;
         my $run_mode = $ENV{TEST_RUN_MODE} // $opt->{run_mode} // 1;
         my $sub = $run_mode == 0 ? undef : key2sub($key);
         my $fhref = [];
         my $spawn_opt = {};
         my @tail_paths;
+        local $tail_cmd = $tail_cmd;
         for my $fd (0..2) {
                 my $redir = $opt->{$fd};
                 my $ref = ref($redir);
                 if ($ref eq 'SCALAR') {
                         my $fh;
-                        if ($tail_cmd && $ENV{TAIL_ALL} && $fd > 0) {
+                        if ($ENV{TAIL_ALL} && $fd > 0) {
+                                # tail -F is better, but not portable :<
+                                $tail_cmd //= 'tail -f';
                                 require File::Temp;
                                 $fh = File::Temp->new("fd.$fd-XXXX", TMPDIR=>1);
                                 push @tail_paths, $fh->filename;
@@ -301,15 +424,15 @@ sub run_script ($;$$) {
                         next if $fd > 0;
                         $fh->autoflush(1);
                         print $fh $$redir or die "print: $!";
-                        seek($fh, 0, SEEK_SET) or die "seek: $!";
+                        seek($fh, 0, SEEK_SET);
                 } elsif ($ref eq 'GLOB') {
                         $spawn_opt->{$fd} = $fhref->[$fd] = $redir;
                 } elsif ($ref) {
                         die "unable to deal with $ref $redir";
                 }
         }
-        my $tail = @tail_paths ? tail_f(@tail_paths) : undef;
-        if ($key =~ /-(index|convert|extindex|convert|xcpdb)\z/) {
+        my $tail = @tail_paths ? tail_f(@tail_paths, $opt) : undef;
+        if ($key =~ /-(index|cindex|extindex|convert|xcpdb)\z/) {
                 unshift @argv, '--no-fsync';
         }
         if ($run_mode == 0) {
@@ -320,11 +443,7 @@ sub run_script ($;$$) {
                         $cmd->[0] = File::Spec->rel2abs($cmd->[0]);
                         $spawn_opt->{'-C'} = $d;
                 }
-                my $pid = PublicInbox::Spawn::spawn($cmd, $env, $spawn_opt);
-                if (defined $pid) {
-                        my $r = waitpid($pid, 0) // die "waitpid: $!";
-                        $r == $pid or die "waitpid: expected $pid, got $r";
-                }
+                PublicInbox::Spawn::run_wait($cmd, $env, $spawn_opt);
         } else { # localize and run everything in the same process:
                 # note: "local *STDIN = *STDIN;" and so forth did not work in
                 # old versions of perl
@@ -334,16 +453,12 @@ sub run_script ($;$$) {
                 local $SIG{FPE} = 'IGNORE'; # Perl default
                 local $0 = join(' ', @$cmd);
                 my $orig_io = _prepare_redirects($fhref);
-                my $cwdfh = $lei_cwdfh;
-                if (my $d = $opt->{'-C'}) {
-                        unless ($cwdfh) {
-                                opendir $cwdfh, '.' or die "opendir .: $!";
-                        }
-                        chdir $d or die "chdir $d: $!";
-                }
+                opendir(my $cwdfh, '.');
+                chdir $opt->{-C} if defined $opt->{-C};
                 _run_sub($sub, $key, \@argv);
-                eval { PublicInbox::Inbox::cleanup_task() };
-                die "fchdir(restore): $!" if $cwdfh && !chdir($cwdfh);
+                # n.b. all our uses of PublicInbox::DS should be fine
+                # with this and we can't Reset here.
+                chdir($cwdfh);
                 _undo_redirects($orig_io);
                 select STDOUT;
                 umask($umask);
@@ -354,11 +469,10 @@ sub run_script ($;$$) {
         for my $fd (1..2) {
                 my $fh = $fhref->[$fd] or next;
                 next unless -f $fh;
-                seek($fh, 0, SEEK_SET) or die "seek: $!";
-                my $redir = $opt->{$fd};
-                local $/;
-                $$redir = <$fh>;
+                seek($fh, 0, SEEK_SET);
+                ${$opt->{$fd}} = read_all($fh);
         }
+        no_coredump($opt->{-C} ? ($opt->{-C}) : ());
         $? == 0;
 }
 
@@ -378,7 +492,7 @@ sub wait_for_tail {
                 my @ino;
                 do {
                         @ino = grep {
-                                readlink($_) =~ /\binotify\b/
+                                (readlink($_) // '') =~ /\binotify\b/
                         } glob("/proc/$tail_pid/fd/*");
                 } while (!@ino && time <= $end and tick);
                 return if !@ino;
@@ -386,7 +500,7 @@ sub wait_for_tail {
                 $ino[0] =~ s!/fd/!/fdinfo/!;
                 my @info;
                 do {
-                        if (open my $fh, '<', $ino[0]) {
+                        if (CORE::open(my $fh, '<', $ino[0])) {
                                 local $/ = "\n";
                                 @info = grep(/^inotify wd:/, <$fh>);
                         }
@@ -424,12 +538,23 @@ sub xqx {
 }
 
 sub tail_f (@) {
+        my @f = grep(defined, @_);
         $tail_cmd or return; # "tail -F" or "tail -f"
-        for (@_) { open(my $fh, '>>', $_) or die $! };
-        my $cmd = [ split(/ /, $tail_cmd), @_ ];
+        my $opt = (ref($f[-1]) eq 'HASH') ? pop(@f) : {};
+        my $clofork = $opt->{-CLOFORK} // [];
+        my @cfmap = map {
+                my $fl = fcntl($_, F_GETFD, 0);
+                fcntl($_, F_SETFD, $fl | FD_CLOEXEC) unless $fl & FD_CLOEXEC;
+                ($_, $fl);
+        } @$clofork;
+        for (@f) { open(my $fh, '>>', $_) };
+        my $cmd = [ split(/ /, $tail_cmd), @f ];
         require PublicInbox::Spawn;
         my $pid = PublicInbox::Spawn::spawn($cmd, undef, { 1 => 2 });
-        wait_for_tail($pid, scalar @_);
+        while (my ($io, $fl) = splice(@cfmap, 0, 2)) {
+                fcntl($io, F_SETFD, $fl);
+        }
+        wait_for_tail($pid, scalar @f);
         require PublicInbox::AutoReap;
         PublicInbox::AutoReap->new($pid, \&wait_for_tail);
 }
@@ -458,24 +583,27 @@ sub start_script {
                                 }
                         }
                 }
-                $tail = tail_f(@paths);
+                $tail = tail_f(@paths, $opt);
         }
-        my $pid = fork // die "fork: $!\n";
+        require PublicInbox::DS;
+        my $oset = PublicInbox::DS::block_signals();
+        require PublicInbox::OnDestroy;
+        my $tmp_mask = PublicInbox::OnDestroy->new(
+                                        \&PublicInbox::DS::sig_setmask, $oset);
+        my $pid = PublicInbox::DS::do_fork();
         if ($pid == 0) {
-                eval { PublicInbox::DS->Reset };
+                close($_) for (@{delete($opt->{-CLOFORK}) // []});
                 # pretend to be systemd (cf. sd_listen_fds(3))
                 # 3 == SD_LISTEN_FDS_START
                 my $fd;
-                for ($fd = 0; 1; $fd++) {
-                        my $s = $opt->{$fd};
-                        last if $fd >= 3 && !defined($s);
-                        next unless $s;
-                        my $fl = fcntl($s, F_GETFD, 0);
-                        if (($fl & FD_CLOEXEC) != FD_CLOEXEC) {
-                                warn "got FD:".fileno($s)." w/o CLOEXEC\n";
+                for ($fd = 0; $fd < 3 || defined($opt->{$fd}); $fd++) {
+                        my $io = $opt->{$fd} // next;
+                        my $old = fileno($io);
+                        if ($old == $fd) {
+                                fcntl($io, F_SETFD, 0);
+                        } else {
+                                dup2($old, $fd) // die "dup2($old, $fd): $!";
                         }
-                        fcntl($s, F_SETFD, $fl &= ~FD_CLOEXEC);
-                        dup2(fileno($s), $fd) or die "dup2 failed: $!\n";
                 }
                 %ENV = (%ENV, %$env) if $env;
                 my $fds = $fd - 3;
@@ -483,10 +611,12 @@ sub start_script {
                         $ENV{LISTEN_PID} = $$;
                         $ENV{LISTEN_FDS} = $fds;
                 }
-                if ($opt->{-C}) { chdir($opt->{-C}) or die "chdir: $!" }
+                if ($opt->{-C}) { chdir($opt->{-C}) }
                 $0 = join(' ', @$cmd);
+                local @SIG{keys %SIG} = map { undef } values %SIG;
+                local $SIG{FPE} = 'IGNORE'; # Perl default
+                undef $tmp_mask;
                 if ($sub) {
-                        eval { PublicInbox::DS->Reset };
                         _run_sub($sub, $key, \@argv);
                         POSIX::_exit($? >> 8);
                 } else {
@@ -494,6 +624,7 @@ sub start_script {
                         die "FAIL: ",join(' ', $key, @argv), ": $!\n";
                 }
         }
+        undef $tmp_mask;
         require PublicInbox::AutoReap;
         my $td = PublicInbox::AutoReap->new($pid);
         $td->{-extra} = $tail;
@@ -528,7 +659,7 @@ sub lei_ok (@) {
         my @msg = ref($_[0]) eq 'ARRAY' ? @{$_[0]} : @_;
         if (!$lei_loud) {
                 for (@msg) {
-                        s!\A([a-z0-9]+://)[^/]+/!$1\$HOST_PORT/!;
+                        s!(127\.0\.0\.1|\[::1\]):(?:\d+)!$1:\$PORT!g;
                         s!$tmpdir\b/(?:[^/]+/)?!\$TMPDIR/!g;
                         s!\Q$PWD\E\b!\$PWD!g;
                 }
@@ -553,13 +684,35 @@ sub ignore_inline_c_missing {
                 grep(!/\bInline\b/, split(/^/m, $_[0])))));
 }
 
+sub need_scm_rights () {
+        state $ok = PublicInbox::Spawn->can('send_cmd4') || do {
+                        require PublicInbox::Syscall;
+                        PublicInbox::Syscall->can('send_cmd4'); # Linux only
+                } || eval { require Socket::MsgHdr; 1 };
+        return if $ok;
+        'need SCM_RIGHTS support: Inline::C unconfigured/missing '.
+        '(mkdir -p ~/.cache/public-inbox/inline-c) OR Socket::MsgHdr missing';
+}
+
+# returns a pipe with FD_CLOEXEC disabled on the write-end
+sub quit_waiter_pipe () {
+        pipe(my $r, my $w);
+        fcntl($w, F_SETFD, fcntl($w, F_GETFD, 0) & ~FD_CLOEXEC);
+        ($r, $w);
+}
+
+sub wait_for_eof ($$;$) {
+        my ($io, $msg, $sec) = @_;
+        vec(my $rset = '', fileno($io), 1) = 1;
+        ok(select($rset, undef, undef, $sec // 9), "$msg (select)");
+        is(my $line = <$io>, undef, "$msg EOF");
+}
+
 sub test_lei {
 SKIP: {
         my ($cb) = pop @_;
         my $test_opt = shift // {};
-        local $lei_cwdfh;
-        opendir $lei_cwdfh, '.' or xbail "opendir .: $!";
-        require_git(2.6, 1) or skip('git 2.6+ required for lei test', 2);
+        require_git(2.6, 1);
         my $mods = $test_opt->{mods} // [ 'lei' ];
         require_mods(@$mods, 2);
 
@@ -577,40 +730,35 @@ SKIP: {
         $ENV{LANG} = $ENV{LC_ALL} = 'C';
         my (undef, $fn, $lineno) = caller(0);
         my $t = "$fn:$lineno";
-        state $lei_daemon = PublicInbox::Spawn->can('send_cmd4') || do {
-                        require PublicInbox::Syscall;
-                        PublicInbox::Syscall->can('send_cmd4');
-                } || eval { require Socket::MsgHdr; 1 };
-        unless ($lei_daemon) {
-                skip('Inline::C unconfigured/missing '.
-'(mkdir -p ~/.cache/public-inbox/inline-c) OR Socket::MsgHdr missing',
-                        1);
-        }
         $lei_opt = { 1 => \$lei_out, 2 => \$lei_err };
         my ($daemon_pid, $for_destroy, $daemon_xrd);
         my $tmpdir = $test_opt->{tmpdir};
-        File::Path::mkpath($tmpdir) if (defined $tmpdir && !-d $tmpdir);
+        File::Path::mkpath($tmpdir) if defined $tmpdir;
         ($tmpdir, $for_destroy) = tmpdir unless $tmpdir;
+        my ($dead_r, $dead_w);
         state $persist_xrd = $ENV{TEST_LEI_DAEMON_PERSIST_DIR};
         SKIP: {
                 $ENV{TEST_LEI_ONESHOT} and
                         xbail 'TEST_LEI_ONESHOT no longer supported';
                 my $home = "$tmpdir/lei-daemon";
-                mkdir($home, 0700) or BAIL_OUT "mkdir: $!";
+                mkdir($home, 0700);
                 local $ENV{HOME} = $home;
                 my $persist;
                 if ($persist_xrd && !$test_opt->{daemon_only}) {
                         $persist = $daemon_xrd = $persist_xrd;
                 } else {
                         $daemon_xrd = "$home/xdg_run";
-                        mkdir($daemon_xrd, 0700) or BAIL_OUT "mkdir: $!";
+                        mkdir($daemon_xrd, 0700);
+                        ($dead_r, $dead_w) = quit_waiter_pipe;
                 }
                 local $ENV{XDG_RUNTIME_DIR} = $daemon_xrd;
-                $cb->();
+                $cb->(); # likely shares $dead_w with lei-daemon
+                undef $dead_w; # so select() wakes up when daemon dies
                 if ($persist) { # remove before ~/.local gets removed
                         File::Path::rmtree([glob("$home/*")]);
                         File::Path::rmtree("$home/.config");
                 } else {
+                        no_coredump $tmpdir;
                         lei_ok(qw(daemon-pid), \"daemon-pid after $t");
                         chomp($daemon_pid = $lei_out);
                         if (!$daemon_pid) {
@@ -622,13 +770,10 @@ SKIP: {
                 }
         }; # SKIP for lei_daemon
         if ($daemon_pid) {
-                for (0..10) {
-                        kill(0, $daemon_pid) or last;
-                        tick;
-                }
-                ok(!kill(0, $daemon_pid), "$t daemon stopped");
+                wait_for_eof($dead_r, 'daemon quit pipe');
+                no_coredump $tmpdir;
                 my $f = "$daemon_xrd/lei/errors.log";
-                open my $fh, '<', $f or BAIL_OUT "$f: $!";
+                open my $fh, '<', $f;
                 my @l = <$fh>;
                 is_xdeeply(\@l, [],
                         "$t daemon XDG_RUNTIME_DIR/lei/errors.log empty");
@@ -646,8 +791,7 @@ sub setup_public_inboxes () {
         return @ret if -f $stamp;
 
         require PublicInbox::Lock;
-        my $lk = bless { lock_path => "$test_home/setup.lock" },
-                        'PublicInbox::Lock';
+        my $lk = PublicInbox::Lock->new("$test_home/setup.lock");
         my $end = $lk->lock_for_scope;
         return @ret if -f $stamp;
 
@@ -657,7 +801,7 @@ sub setup_public_inboxes () {
                                 '--newsgroup', "t.v$V", "t$V",
                                 "$test_home/t$V", "http://example.com/t$V",
                                 "t$V\@example.com" ]) or xbail "init v$V";
-                unlink "$test_home/t$V/description" or xbail "unlink $!";
+                unlink "$test_home/t$V/description";
         }
         require PublicInbox::Config;
         require PublicInbox::InboxWritable;
@@ -677,11 +821,63 @@ sub setup_public_inboxes () {
                 $im->done;
         });
         $seen or BAIL_OUT 'no imports';
-        open my $fh, '>', $stamp or BAIL_OUT "open $stamp: $!";
+        open my $fh, '>', $stamp;
         @ret;
 }
 
-sub create_inbox ($$;@) {
+our %COMMIT_ENV = (
+        GIT_AUTHOR_NAME => 'A U Thor',
+        GIT_COMMITTER_NAME => 'C O Mitter',
+        GIT_AUTHOR_EMAIL => 'a@example.com',
+        GIT_COMMITTER_EMAIL => 'c@example.com',
+);
+
+# for memoizing based on coderefs and various create_* params
+sub my_sum {
+        require PublicInbox::SHA;
+        require Data::Dumper;
+        my $d = Data::Dumper->new(\@_);
+        $d->$_(1) for qw(Deparse Sortkeys Terse);
+        my @l = split /\n/s, $d->Dump;
+        @l = grep !/\$\^H\{.+?[A-Z]+\(0x[0-9a-f]+\)/, @l; # autodie addresses
+        my @addr = grep /[A-Za-z]+\(0x[0-9a-f]+\)/, @l;
+        xbail 'undumpable addresses: ', \@addr if @addr;
+        substr PublicInbox::SHA::sha256_hex(join('', @l)), 0, 8;
+}
+
+sub create_dir (@) {
+        my ($ident, $cb) = (shift, pop);
+        my %opt = @_;
+        require PublicInbox::Lock;
+        require PublicInbox::Import;
+        my $tmpdir = delete $opt{tmpdir};
+        my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!);
+        my $dir = "t/data-gen/$base.$ident-".my_sum($cb, \%opt);
+        require File::Path;
+        my $new = File::Path::make_path($dir);
+        my $lk = PublicInbox::Lock->new("$dir/creat.lock");
+        my $scope = $lk->lock_for_scope;
+        if (!-f "$dir/creat.stamp") {
+                opendir(my $cwd, '.');
+                chdir($dir);
+                local %ENV = (%ENV, %COMMIT_ENV);
+                $cb->($dir);
+                chdir($cwd); # some $cb chdir around
+                open my $s, '>', "$dir/creat.stamp";
+        }
+        return $dir if !defined($tmpdir);
+        xsys_e([qw(/bin/cp -Rp), $dir, $tmpdir]);
+        $tmpdir;
+}
+
+sub create_coderepo (@) {
+        my $ident = shift;
+        require PublicInbox::Import;
+        my ($db) = (PublicInbox::Import::default_branch() =~ m!([^/]+)\z!);
+        create_dir "$ident-$db", @_;
+}
+
+sub create_inbox ($;@) {
         my $ident = shift;
         my $cb = pop;
         my %opt = @_;
@@ -690,13 +886,11 @@ sub create_inbox ($$;@) {
         require PublicInbox::Import;
         my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!);
         my ($db) = (PublicInbox::Import::default_branch() =~ m!([^/]+)\z!);
-        my $dir = "t/data-gen/$base.$ident-$db";
-        my $new = !-d $dir;
-        if ($new) {
-                mkdir $dir; # may race
-                -d $dir or BAIL_OUT "$dir could not be created: $!";
-        }
-        my $lk = bless { lock_path => "$dir/creat.lock" }, 'PublicInbox::Lock';
+        my $tmpdir = delete $opt{tmpdir};
+        my $dir = "t/data-gen/$base.$ident-".my_sum($db, $cb, \%opt);
+        require File::Path;
+        my $new = File::Path::make_path($dir);
+        my $lk = PublicInbox::Lock->new("$dir/creat.lock");
         $opt{inboxdir} = File::Spec->rel2abs($dir);
         $opt{name} //= $ident;
         my $scope = $lk->lock_for_scope;
@@ -704,7 +898,6 @@ sub create_inbox ($$;@) {
         $pre_cb->($dir) if $pre_cb && $new;
         $opt{-no_fsync} = 1;
         my $no_gc = delete $opt{-no_gc};
-        my $tmpdir = delete $opt{tmpdir};
         my $addr = $opt{address} // [];
         $opt{-primary_address} //= $addr->[0] // "$ident\@example.com";
         my $parallel = delete($opt{importer_parallel}) // 0;
@@ -721,8 +914,7 @@ sub create_inbox ($$;@) {
                                 xsys_e([ qw(git gc -q) ], { GIT_DIR => $dir });
                         }
                 }
-                open my $s, '>', "$dir/creat.stamp" or
-                        BAIL_OUT "error creating $dir/creat.stamp: $!";
+                open my $s, '>', "$dir/creat.stamp";
         }
         if ($tmpdir) {
                 undef $ibx;
@@ -734,23 +926,31 @@ sub create_inbox ($$;@) {
         $ibx;
 }
 
-sub test_httpd ($$;$) {
-        my ($env, $client, $skip) = @_;
-        for (qw(PI_CONFIG TMPDIR)) {
-                $env->{$_} or BAIL_OUT "$_ unset";
-        }
+sub test_httpd ($$;$$) {
+        my ($env, $client, $skip, $cb) = @_;
+        my ($tmpdir, $for_destroy);
+        $env->{TMPDIR} //= do {
+                ($tmpdir, $for_destroy) = tmpdir();
+                $tmpdir;
+        };
+        for (qw(PI_CONFIG)) { $env->{$_} or BAIL_OUT "$_ unset" }
         SKIP: {
-                require_mods(qw(Plack::Test::ExternalServer), $skip // 1);
+                require_mods(qw(Plack::Test::ExternalServer LWP::UserAgent),
+                                $skip // 1);
                 my $sock = tcp_server() or die;
                 my ($out, $err) = map { "$env->{TMPDIR}/std$_.log" } qw(out err);
                 my $cmd = [ qw(-httpd -W0), "--stdout=$out", "--stderr=$err" ];
                 my $td = start_script($cmd, $env, { 3 => $sock });
                 my ($h, $p) = tcp_host_port($sock);
                 local $ENV{PLACK_TEST_EXTERNALSERVER_URI} = "http://$h:$p";
-                Plack::Test::ExternalServer::test_psgi(client => $client);
+                my $ua = LWP::UserAgent->new;
+                $ua->max_redirect(0);
+                Plack::Test::ExternalServer::test_psgi(client => $client,
+                                                        ua => $ua);
+                $cb->() if $cb;
                 $td->join('TERM');
-                open my $fh, '<', $err or BAIL_OUT $!;
-                my $e = do { local $/; <$fh> };
+                open my $fh, '<', $err;
+                my $e = read_all($fh);
                 if ($e =~ s/^Plack::Middleware::ReverseProxy missing,\n//gms) {
                         $e =~ s/^URL generation for redirects .*\n//gms;
                 }
@@ -758,6 +958,93 @@ sub test_httpd ($$;$) {
         }
 };
 
+# TODO: support fstat(1) on OpenBSD, lsof already works on FreeBSD + Linux
+# don't use this for deleted file checks, we only check that on Linux atm
+# and we can readlink /proc/PID/fd/* directly
+sub lsof_pid ($;$) {
+        my ($pid, $rdr) = @_;
+        state $lsof = require_cmd('lsof', 1);
+        $lsof or skip 'lsof missing/broken', 1;
+        my @out = xqx([$lsof, '-p', $pid], undef, $rdr);
+        if ($?) {
+                undef $lsof;
+                skip "lsof -p PID broken \$?=$?", 1;
+        }
+        my @cols = split ' ', $out[0];
+        if (($cols[7] // '') eq 'NODE') { # normal lsof
+                @out;
+        } else { # busybox lsof ignores -p, so we DIY it
+                grep /\b$pid\b/, @out;
+        }
+}
+
+sub no_pollerfd ($) {
+        my ($pid) = @_;
+        my ($re, @cmd);
+        $^O eq 'linux' and
+                ($re, @cmd) = (qr/\Q[eventpoll]\E/, qw(lsof -p), $pid);
+        # n.b. *BSDs uses kqueue to emulate signalfd and/or inotify,
+        # and we can't distinguish which is which easily.
+        SKIP: {
+                (@cmd && $re) or
+                        skip 'open poller test is Linux-only', 1;
+                my $bin = require_cmd($cmd[0], 1) or skip "$cmd[0] missing", 1;
+                $cmd[0] = $bin;
+                my @of = xqx(\@cmd, {}, {2 => \(my $e)});
+                my $err = $?;
+                skip "$bin broken? (\$?=$err) ($e)", 1 if $err;
+                @of = grep /\b$pid\b/, @of; # busybox lsof ignores -p
+                is(grep(/$re/, @of), 0, "no $re FDs") or diag explain(\@of);
+        }
+}
+
+sub cfg_new ($;@) {
+        my ($tmpdir, @body) = @_;
+        require PublicInbox::Config;
+        my $f = "$tmpdir/tmp_cfg";
+        open my $fh, '>', $f;
+        print $fh @body;
+        close $fh;
+        PublicInbox::Config->new($f);
+}
+
+our $strace_cmd;
+sub strace (@) {
+        my ($for_daemon) = @_;
+        skip 'linux only test', 1 if $^O ne 'linux';
+        if ($for_daemon) {
+                my $f = '/proc/sys/kernel/yama/ptrace_scope';
+                # TODO: we could fiddle with prctl in the daemon to make
+                # things work, but I'm not sure it's worth it...
+                state $ps = do {
+                        my $fh;
+                        CORE::open($fh, '<', $f) ? readline($fh) : 0;
+                };
+                chomp $ps;
+                skip "strace unusable on daemons\n$f is `$ps' (!= 0)", 1 if $ps;
+        }
+        require_cmd('strace', 1) or skip 'strace not available', 1;
+}
+
+sub strace_inject (;$) {
+        my $cmd = strace(@_);
+        state $ver = do {
+                require PublicInbox::Spawn;
+                my $v = PublicInbox::Spawn::run_qx([$cmd, '-V']);
+                $v =~ m!version\s+([1-9]+\.[0-9]+)! or
+                                xbail "no strace -V: $v";
+                eval("v$1");
+        };
+        $ver ge v4.16 or skip "$cmd too old for syscall injection (".
+                                sprintf('v%vd', $ver). ' < v4.16)', 1;
+        $cmd
+}
+
+sub oct_is ($$$) {
+        my ($got, $exp, $msg) = @_;
+        @_ = (sprintf('0%03o', $got), sprintf('0%03o', $exp), $msg);
+        goto &is; # tail recursion to get lineno from callers on failure
+}
 
 package PublicInbox::TestCommon::InboxWakeup;
 use strict;
diff --git a/lib/PublicInbox/Tmpfile.pm b/lib/PublicInbox/Tmpfile.pm
index 3040dd77..72dd9d24 100644
--- a/lib/PublicInbox/Tmpfile.pm
+++ b/lib/PublicInbox/Tmpfile.pm
@@ -1,9 +1,9 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 package PublicInbox::Tmpfile;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(Exporter);
+use autodie qw(unlink);
 our @EXPORT = qw(tmpfile);
 use Fcntl qw(:DEFAULT);
 use Errno qw(EEXIST);
@@ -21,7 +21,7 @@ sub tmpfile ($;$$) {
         if (defined $sock) {
                 # add the socket inode number so we can figure out which
                 # socket it belongs to
-                my @st = stat($sock);
+                my @st = stat($sock) or die "stat($sock): $!";
                 $id .= '-ino:'.$st[1];
         }
         $id =~ tr!/!^!;
@@ -31,7 +31,7 @@ sub tmpfile ($;$$) {
         do {
                 my $fn = File::Spec->tmpdir . "/$id-".time.'-'.rand;
                 if (sysopen(my $fh, $fn, $fl, 0600)) { # likely
-                        unlink($fn) or warn "unlink($fn): $!"; # FS broken
+                        unlink($fn);
                         return $fh; # success
                 }
         } while ($! == EEXIST);
diff --git a/lib/PublicInbox/URIimap.pm b/lib/PublicInbox/URIimap.pm
index 81644914..41c2842a 100644
--- a/lib/PublicInbox/URIimap.pm
+++ b/lib/PublicInbox/URIimap.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # cf. RFC 5092, which the `URI' package doesn't support
 #
@@ -11,8 +11,7 @@
 #
 # RFC 2192 also describes ";TYPE=<list_type>"
 package PublicInbox::URIimap;
-use strict;
-use v5.10.1;
+use v5.12;
 use URI::Split qw(uri_split uri_join); # part of URI
 use URI::Escape qw(uri_unescape uri_escape);
 use overload '""' => \&as_string;
diff --git a/lib/PublicInbox/URInntps.pm b/lib/PublicInbox/URInntps.pm
index 231b247b..88c8d641 100644
--- a/lib/PublicInbox/URInntps.pm
+++ b/lib/PublicInbox/URInntps.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # deal with the lack of URI::nntps in upstream URI.
@@ -6,7 +6,7 @@
 # cf. https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983419
 # Fixed in URI 5.08, we can drop this by 2035 when LTS distros all have it
 package PublicInbox::URInntps;
-use strict;
+use v5.12;
 use parent qw(URI::snews);
 use URI;
 
diff --git a/lib/PublicInbox/Umask.pm b/lib/PublicInbox/Umask.pm
new file mode 100644
index 00000000..00772ce5
--- /dev/null
+++ b/lib/PublicInbox/Umask.pm
@@ -0,0 +1,70 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# base class to ensures Xapian||SQLite files respect core.sharedRepository
+# of git repos
+package PublicInbox::Umask;
+use v5.12;
+use PublicInbox::OnDestroy;
+
+use constant {
+        PERM_UMASK => 0,
+        OLD_PERM_GROUP => 1,
+        OLD_PERM_EVERYBODY => 2,
+        PERM_GROUP => 0660,
+        PERM_EVERYBODY => 0664,
+};
+
+sub _read_git_config_perm {
+        my ($self) = @_;
+        chomp(my $perm = $self->git->qx('config', 'core.sharedRepository'));
+        $perm;
+}
+
+sub _git_config_perm {
+        my $self = shift;
+        my $perm = scalar @_ ? $_[0] : _read_git_config_perm($self);
+        $perm //= '';
+        return PERM_UMASK if $perm eq '' || $perm eq 'umask';
+        return PERM_GROUP if $perm eq 'group';
+        return PERM_EVERYBODY if $perm =~ /\A(?:all|world|everybody)\z/;
+        return PERM_GROUP if ($perm =~ /\A(?:true|yes|on|1)\z/);
+        return PERM_UMASK if ($perm =~ /\A(?:false|no|off|0)\z/);
+
+        my $i = oct($perm);
+        return PERM_UMASK if $i == PERM_UMASK;
+        return PERM_GROUP if $i == OLD_PERM_GROUP;
+        return PERM_EVERYBODY if $i == OLD_PERM_EVERYBODY;
+
+        if (($i & 0600) != 0600) {
+                die "core.sharedRepository mode invalid: ".
+                    sprintf('%.3o', $i) . "\nOwner must have permissions\n";
+        }
+        ($i & 0666);
+}
+
+sub _umask_for {
+        my ($perm) = @_; # _git_config_perm return value
+        my $rv = $perm;
+        return umask if $rv == 0;
+
+        # set +x bit if +r or +w were set
+        $rv |= 0100 if ($rv & 0600);
+        $rv |= 0010 if ($rv & 0060);
+        $rv |= 0001 if ($rv & 0006);
+        (~$rv & 0777);
+}
+
+sub with_umask {
+        my ($self, $cb, @arg) = @_;
+        my $old = umask($self->{umask} //= umask_prepare($self));
+        my $restore = PublicInbox::OnDestroy->new($$, \&CORE::umask, $old);
+        $cb ? $cb->(@arg) : $restore;
+}
+
+sub umask_prepare {
+        my ($self) = @_;
+        _umask_for(_git_config_perm($self));
+}
+
+1;
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index ed5182ae..fb259396 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # This interface wraps and mimics PublicInbox::Import
@@ -8,7 +8,7 @@ use strict;
 use v5.10.1;
 use parent qw(PublicInbox::Lock PublicInbox::IPC);
 use PublicInbox::SearchIdxShard;
-use PublicInbox::IPC;
+use PublicInbox::IPC qw(nproc_shards);
 use PublicInbox::Eml;
 use PublicInbox::Git;
 use PublicInbox::Import;
@@ -22,37 +22,12 @@ use PublicInbox::Spawn qw(spawn popen_rd run_die);
 use PublicInbox::Search;
 use PublicInbox::SearchIdx qw(log2stack is_ancestor check_size is_bad_blob);
 use IO::Handle; # ->autoflush
-use File::Temp ();
 use POSIX ();
 
 my $OID = qr/[a-f0-9]{40,}/;
 # an estimate of the post-packed size to the raw uncompressed size
 our $PACKING_FACTOR = 0.4;
 
-# SATA storage lags behind what CPUs are capable of, so relying on
-# nproc(1) can be misleading and having extra Xapian shards is a
-# waste of FDs and space.  It can also lead to excessive IO latency
-# and slow things down.  Users on NVME or other fast storage can
-# use the NPROC env or switches in our script/public-inbox-* programs
-# to increase Xapian shards
-our $NPROC_MAX_DEFAULT = 4;
-
-sub nproc_shards ($) {
-        my ($creat_opt) = @_;
-        my $n = $creat_opt->{nproc} if ref($creat_opt) eq 'HASH';
-        $n //= $ENV{NPROC};
-        if (!$n) {
-                # assume 2 cores if not detectable or zero
-                state $NPROC_DETECTED = PublicInbox::IPC::detect_nproc() || 2;
-                $n = $NPROC_DETECTED;
-                $n = $NPROC_MAX_DEFAULT if $n > $NPROC_MAX_DEFAULT;
-        }
-
-        # subtract for the main process and git-fast-import
-        $n -= 1;
-        $n < 1 ? 1 : $n;
-}
-
 sub count_shards ($) {
         my ($self) = @_;
         # always load existing shards in case core count changes:
@@ -113,13 +88,6 @@ sub init_inbox {
         $self->done;
 }
 
-# returns undef on duplicate or spam
-# mimics Import::add and wraps it for v2
-sub add {
-        my ($self, $eml, $check_cb) = @_;
-        $self->{ibx}->with_umask(\&_add, $self, $eml, $check_cb);
-}
-
 sub idx_shard ($$) {
         my ($self, $num) = @_;
         $self->{idx_shards}->[$num % scalar(@{$self->{idx_shards}})];
@@ -137,8 +105,11 @@ sub do_idx ($$$) {
         $n >= $self->{batch_bytes};
 }
 
-sub _add {
+# returns undef on duplicate or spam
+# mimics Import::add and wraps it for v2
+sub add {
         my ($self, $mime, $check_cb) = @_;
+        my $restore = $self->{ibx}->with_umask;
 
         # spam check:
         if ($check_cb) {
@@ -164,7 +135,6 @@ sub _add {
         if (do_idx($self, $mime, $smsg)) {
                 $self->checkpoint;
         }
-
         $cmt;
 }
 
@@ -415,17 +385,16 @@ sub rewrite_internal ($$;$$$) {
 # (retval[2]) is not part of the stable API shared with Import->remove
 sub remove {
         my ($self, $eml, $cmt_msg) = @_;
-        my $r = $self->{ibx}->with_umask(\&rewrite_internal,
-                                                $self, $eml, $cmt_msg);
+        my $restore = $self->{ibx}->with_umask;
+        my $r = rewrite_internal($self, $eml, $cmt_msg);
         defined($r) && defined($r->[0]) ? @$r: undef;
 }
 
 sub _replace ($$;$$) {
         my ($self, $old_eml, $new_eml, $sref) = @_;
-        my $arg = [ $self, $old_eml, undef, $new_eml, $sref ];
-        my $rewritten = $self->{ibx}->with_umask(\&rewrite_internal,
-                        $self, $old_eml, undef, $new_eml, $sref) or return;
-
+        my $restore = $self->{ibx}->with_umask;
+        my $rewritten = rewrite_internal($self, $old_eml, undef,
+                                        $new_eml, $sref) or return;
         my $rewrites = $rewritten->{rewrites};
         # ->done is called if there are rewrites since we gc+prune from git
         $self->idx_init if @$rewrites;
@@ -689,23 +658,6 @@ sub import_init {
         $im;
 }
 
-# XXX experimental
-sub diff ($$$) {
-        my ($mid, $cur, $new) = @_;
-
-        my $ah = File::Temp->new(TEMPLATE => 'email-cur-XXXX', TMPDIR => 1);
-        print $ah $cur->as_string or die "print: $!";
-        $ah->flush or die "flush: $!";
-        PublicInbox::Import::drop_unwanted_headers($new);
-        my $bh = File::Temp->new(TEMPLATE => 'email-new-XXXX', TMPDIR => 1);
-        print $bh $new->as_string or die "print: $!";
-        $bh->flush or die "flush: $!";
-        my $cmd = [ qw(diff -u), $ah->filename, $bh->filename ];
-        print STDERR "# MID conflict <$mid>\n";
-        my $pid = spawn($cmd, undef, { 1 => 2 });
-        waitpid($pid, 0) == $pid or die "diff did not finish";
-}
-
 sub get_blob ($$) {
         my ($self, $smsg) = @_;
         if (my $im = $self->{im}) {
@@ -729,9 +681,6 @@ sub content_exists ($$$) {
                 }
                 my $cur = PublicInbox::Eml->new($msg);
                 return 1 if content_matches($chashes, $cur);
-
-                # XXX DEBUG_DIFF is experimental and may be removed
-                diff($mid, $cur, $mime) if $ENV{DEBUG_DIFF};
         }
         undef;
 }
@@ -1120,7 +1069,7 @@ sub unindex_todo ($$$) {
                 /\A:\d{6} 100644 $OID ($OID) [AM]\tm$/o or next;
                 $self->git->cat_async($1, $unindex_oid, { %$sync, oid => $1 });
         }
-        close $fh or die "git log failed: \$?=$?";
+        $fh->close or die "git log failed: \$?=$?";
         $self->git->async_wait_all;
 
         return unless $sync->{-opt}->{prune};
@@ -1210,6 +1159,7 @@ sub index_todo ($$$) {
                 };
                 if ($f eq 'm') {
                         if ($sync->{max_size}) {
+                                $req->{git} = $all;
                                 $all->check_async($oid, \&check_size, $req);
                         } else {
                                 $all->cat_async($oid, $index_oid, $req);
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index b90cb08b..44e1f2a8 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Used for displaying the HTML web interface.
@@ -7,6 +7,7 @@ package PublicInbox::View;
 use strict;
 use v5.10.1;
 use List::Util qw(max);
+use Text::Wrap qw(wrap); # stdlib, we need Perl 5.6+ for $huge
 use PublicInbox::MsgTime qw(msg_datestamp);
 use PublicInbox::Hval qw(ascii_html obfuscate_addrs prurl mid_href
                         ts2str fmt_ts);
@@ -19,6 +20,7 @@ use PublicInbox::WwwStream qw(html_oneshot);
 use PublicInbox::Reply;
 use PublicInbox::ViewDiff qw(flush_diff);
 use PublicInbox::Eml;
+use POSIX qw(strftime);
 use Time::Local qw(timegm);
 use PublicInbox::Smsg qw(subject_normalized);
 use PublicInbox::ContentHash qw(content_hash);
@@ -36,14 +38,12 @@ sub msg_page_i {
                                 : $ctx->gone('over');
                 $ctx->{mhref} = ($ctx->{nr} || $ctx->{smsg}) ?
                                 "../${\mid_href($smsg->{mid})}/" : '';
-                my $obuf = $ctx->{obuf} = _msg_page_prepare_obuf($eml, $ctx);
-                if (length($$obuf)) {
-                        multipart_text_as_html($eml, $ctx);
-                        $$obuf .= '</pre><hr>';
+                if (_msg_page_prepare($eml, $ctx, $smsg->{ts})) {
+                        $eml->each_part(\&add_text_body, $ctx, 1);
+                        print { $ctx->{zfh} } '</pre><hr>';
                 }
-                delete $ctx->{obuf};
-                $$obuf .= html_footer($ctx, $ctx->{first_hdr}) if !$ctx->{smsg};
-                $$obuf;
+                html_footer($ctx, $ctx->{first_hdr}) if !$ctx->{smsg};
+                ''; # XXX TODO cleanup
         } else { # called by WwwStream::async_next or getline
                 $ctx->{smsg}; # may be undef
         }
@@ -56,14 +56,12 @@ sub no_over_html ($) {
         my $eml = PublicInbox::Eml->new($bref);
         $ctx->{mhref} = '';
         PublicInbox::WwwStream::init($ctx);
-        my $obuf = $ctx->{obuf} = _msg_page_prepare_obuf($eml, $ctx);
-        if (length($$obuf)) {
-                multipart_text_as_html($eml, $ctx);
-                $$obuf .= '</pre><hr>';
+        if (_msg_page_prepare($eml, $ctx)) { # sets {-title_html}
+                $eml->each_part(\&add_text_body, $ctx, 1);
+                print { $ctx->{zfh} } '</pre><hr>';
         }
-        delete $ctx->{obuf};
-        eval { $$obuf .= html_footer($ctx, $eml) };
-        html_oneshot($ctx, 200, $obuf);
+        html_footer($ctx, $eml);
+        $ctx->html_done;
 }
 
 # public functions: (unstable)
@@ -82,7 +80,8 @@ sub msg_page {
         # allow user to easily browse the range around this message if
         # they have ->over
         $ctx->{-t_max} = $smsg->{ts};
-        PublicInbox::WwwStream::aresponse($ctx, 200, \&msg_page_i);
+        $ctx->{-spfx} = '../' if $ibx->{-repo_objs};
+        PublicInbox::WwwStream::aresponse($ctx, \&msg_page_i);
 }
 
 # /$INBOX/$MESSAGE_ID/#R
@@ -142,6 +141,9 @@ $info
   <a
 href="$se_url">$se_url</a>
 $link</pre>
+
+  Be sure your reply has a <b>Subject:</b> header at the top and a blank line
+  before the message body.
 EOF
 }
 
@@ -181,6 +183,59 @@ sub nr_to_s ($$$) {
         $nr == 1 ? "$nr $singular" : "$nr $plural";
 }
 
+sub addr2urlmap ($) {
+        my ($ctx) = @_;
+        # cache makes a huge difference with /[tT] and large threads
+        my $key = PublicInbox::Git::host_prefix_url($ctx->{env}, '');
+        my $ent = $ctx->{www}->{pi_cfg}->{-addr2urlmap}->{$key} // do {
+                my $by_addr = $ctx->{www}->{pi_cfg}->{-by_addr};
+                my (%addr2url, $url);
+                while (my ($addr, $ibx) = each %$by_addr) {
+                        $url = $ibx->base_url // $ibx->base_url($ctx->{env});
+                        $addr2url{$addr} = ascii_html($url) if defined $url;
+                }
+                # don't allow attackers to randomly change Host: headers
+                # and OOM us if the server handles all hostnames:
+                my $tmp = $ctx->{www}->{pi_cfg}->{-addr2urlmap};
+                my @k = keys %$tmp; # random order
+                delete @$tmp{@k[0..3]} if scalar(@k) > 7;
+                my $re = join('|', map { quotemeta } keys %addr2url);
+                $tmp->{$key} = [ qr/\b($re)\b/i, \%addr2url ];
+        };
+        @$ent;
+}
+
+sub to_cc_html ($$$$) {
+        my ($ctx, $eml, $field, $t) = @_;
+        my @vals = $eml->header($field) or return ('', 0);
+        my (undef, $addr2url) = addr2urlmap($ctx);
+        my $pairs = PublicInbox::Address::pairs(join(', ', @vals));
+        my ($len, $line_len, $html) = (0, 0, '');
+        my ($pair, $url);
+        my ($cur_ibx, $env) = @$ctx{qw(ibx env)};
+        # avoid excessive ascii_html calls (already hot in profiles):
+        my @html = split /\n/, ascii_html(join("\n", map {
+                $_->[0] // (split(/\@/, $_->[1]))[0]; # addr user if no name
+        } @$pairs));
+        for my $n (@html) {
+                $pair = shift @$pairs;
+                if ($line_len) { # 9 = display width of ",\t":
+                        if ($line_len + length($n) > COLS - 9) {
+                                $html .= ",\n\t";
+                                $len += $line_len;
+                                $line_len = 0;
+                        } else {
+                                $html .= ', ';
+                                $line_len += 2;
+                        }
+                }
+                $line_len += length($n);
+                $url = $addr2url->{lc($pair->[1] // '')};
+                $html .= $url ? qq(<a\nhref="$url$t">$n</a>) : $n;
+        }
+        ($html, $len + $line_len);
+}
+
 # Displays the text of of the message for /$INBOX/$MSGID/[Tt]/ endpoint
 # this is already inside a <pre>
 sub eml_entry {
@@ -205,7 +260,8 @@ sub eml_entry {
         my $ds = delete $smsg->{ds}; # for v1 non-Xapian/SQLite users
 
         # Deleting these fields saves about 400K as we iterate across 1K msgs
-        delete @$smsg{qw(ts blob)};
+        my ($t, undef) = delete @$smsg{qw(ts blob)};
+        $t = $t ? '?t='.ts2str($t) : '';
 
         my $from = _hdr_names_html($eml, 'From');
         obfuscate_addrs($obfs_ibx, $from) if $obfs_ibx;
@@ -214,9 +270,8 @@ sub eml_entry {
         my $mhref = $upfx . mid_href($mid_raw) . '/';
         $rv .= qq{ (<a\nhref="$mhref">permalink</a> / };
         $rv .= qq{<a\nhref="${mhref}raw">raw</a>)\n};
-        my $to = fold_addresses(_hdr_names_html($eml, 'To'));
-        my $cc = fold_addresses(_hdr_names_html($eml, 'Cc'));
-        my ($tlen, $clen) = (length($to), length($cc));
+        my ($to, $tlen) = to_cc_html($ctx, $eml, 'To', $t);
+        my ($cc, $clen) = to_cc_html($ctx, $eml, 'Cc', $t);
         my $to_cc = '';
         if (($tlen + $clen) > COLS) {
                 $to_cc .= '  To: '.$to."\n" if $tlen;
@@ -239,20 +294,22 @@ sub eml_entry {
                 my $html = ascii_html($irt);
                 $rv .= qq(In-Reply-To: &lt;<a\nhref="$href">$html</a>&gt;\n)
         }
-        $rv .= "\n";
+        say { $ctx->zfh } $rv;
 
         # scan through all parts, looking for displayable text
         $ctx->{mhref} = $mhref;
-        $ctx->{obuf} = \$rv;
-        $eml->each_part(\&add_text_body, $ctx, 1);
-        delete $ctx->{obuf};
+        $ctx->{changed_href} = "#e$id"; # for diffstat "files? changed,"
+        $eml->each_part(\&add_text_body, $ctx, 1); # expensive
 
         # add the footer
-        $rv .= "\n<a\nhref=#$id_m\nid=e$id>^</a> ".
+        $rv = "\n<a\nhref=#$id_m\nid=e$id>^</a> ".
                 "<a\nhref=\"$mhref\">permalink</a>" .
                 " <a\nhref=\"${mhref}raw\">raw</a>" .
                 " <a\nhref=\"${mhref}#R\">reply</a>";
 
+        delete($ctx->{-qry}) and
+                $rv .= qq[ <a\nhref="${mhref}#related">related</a>];
+
         my $hr;
         if (defined(my $pct = $smsg->{pct})) { # used by SearchView.pm
                 $rv .= "\t[relevance $pct%]";
@@ -297,8 +354,7 @@ sub _th_index_lite {
         my $rv = '';
         my $mapping = $ctx->{mapping} or return $rv;
         my $pad = '  ';
-        my $mid_map = $mapping->{$mid_raw};
-        defined $mid_map or
+        my $mid_map = $mapping->{$mid_raw} //
                 return 'public-inbox BUG: '.ascii_html($mid_raw).' not mapped';
         my ($attr, $node, $idx, $level) = @$mid_map;
         my $children = $node->{children};
@@ -330,10 +386,10 @@ sub _th_index_lite {
         }
         my $s_s = nr_to_s($nr_s, 'sibling', 'siblings');
         my $s_c = nr_to_s($nr_c, 'reply', 'replies');
-        $attr =~ s!\n\z!</b>\n!s;
+        chop $attr; # remove "\n"
         $attr =~ s!<a\nhref.*</a> (?:&#34; )?!!s; # no point in dup subject
         $attr =~ s!<a\nhref=[^>]+>([^<]+)</a>!$1!s; # no point linking to self
-        $rv .= "<b>@ $attr";
+        $rv .= "<b>@ $attr</b>\n";
         if ($nr_c) {
                 my $cmid = $children->[0] ? $children->[0]->{mid} : undef;
                 $rv .= $pad . _skel_hdr($mapping, $cmid);
@@ -383,7 +439,9 @@ sub pre_thread  { # walk_thread callback
 sub thread_eml_entry {
         my ($ctx, $eml) = @_;
         my ($beg, $end) = thread_adj_level($ctx, $ctx->{level});
-        $beg . '<pre>' . eml_entry($ctx, $eml) . '</pre>' . $end;
+        print { $ctx->zfh } $beg, '<pre>';
+        print { $ctx->{zfh} } eml_entry($ctx, $eml), '</pre>';
+        $end;
 }
 
 sub next_in_queue ($$) {
@@ -410,15 +468,15 @@ sub stream_thread_i { # PublicInbox::WwwStream::getline callback
                                 if (!$ghost_ok) { # first non-ghost
                                         $ctx->{-title_html} =
                                                 ascii_html($smsg->{subject});
-                                        $ctx->zmore($ctx->html_top);
+                                        print { $ctx->zfh } $ctx->html_top;
                                 }
                                 return $smsg;
                         }
                         # buffer the ghost entry and loop
-                        $ctx->zmore(ghost_index_entry($ctx, $lvl, $smsg));
+                        print { $ctx->zfh } ghost_index_entry($ctx, $lvl, $smsg)
                 } else { # all done
-                        $ctx->zmore(join('', thread_adj_level($ctx, 0)));
-                        $ctx->zmore(${delete($ctx->{skel})});
+                        print { $ctx->zfh } thread_adj_level($ctx, 0),
+                                                ${delete($ctx->{skel})};
                         return;
                 }
         }
@@ -426,8 +484,8 @@ sub stream_thread_i { # PublicInbox::WwwStream::getline callback
 
 sub stream_thread ($$) {
         my ($rootset, $ctx) = @_;
-        $ctx->{-queue} = [ map { (0, $_) } @$rootset ];
-        PublicInbox::WwwStream::aresponse($ctx, 200, \&stream_thread_i);
+        @{$ctx->{-queue}} = map { (0, $_) } @$rootset;
+        PublicInbox::WwwStream::aresponse($ctx, \&stream_thread_i);
 }
 
 # /$INBOX/$MSGID/t/ and /$INBOX/$MSGID/T/
@@ -438,10 +496,11 @@ sub thread_html {
         my $ibx = $ctx->{ibx};
         my ($nr, $msgs) = $ibx->over->get_thread($mid);
         return missing_thread($ctx) if $nr == 0;
+        $ctx->{-spfx} = '../../' if $ibx->{-repo_objs};
 
         # link $INBOX_DIR/description text to "index_topics" view around
         # the newest message in this thread
-        my $t = ts2str($ctx->{-t_max} = max(map { delete $_->{ts} } @$msgs));
+        my $t = ts2str($ctx->{-t_max} = max(map { $_->{ts} } @$msgs));
         my $t_fmt = fmt_ts($ctx->{-t_max});
 
         my $skel = '<hr><pre>';
@@ -478,7 +537,7 @@ EOF
         # flat display: lazy load the full message from smsg
         $ctx->{msgs} = $msgs;
         $ctx->{-html_tip} = '<pre>';
-        PublicInbox::WwwStream::aresponse($ctx, 200, \&thread_html_i);
+        PublicInbox::WwwStream::aresponse($ctx, \&thread_html_i);
 }
 
 sub thread_html_i { # PublicInbox::WwwStream::getline callback
@@ -487,7 +546,7 @@ sub thread_html_i { # PublicInbox::WwwStream::getline callback
                 my $smsg = $ctx->{smsg};
                 if (exists $ctx->{-html_tip}) {
                         $ctx->{-title_html} = ascii_html($smsg->{subject});
-                        $ctx->zmore($ctx->html_top);
+                        print { $ctx->zfh } $ctx->html_top;
                 }
                 return eml_entry($ctx, $eml);
         } else {
@@ -495,31 +554,19 @@ sub thread_html_i { # PublicInbox::WwwStream::getline callback
                         return $smsg if exists($smsg->{blob});
                 }
                 my $skel = delete($ctx->{skel}) or return; # all done
-                $ctx->zmore($$skel);
+                print { $ctx->zfh } $$skel;
                 undef;
         }
 }
 
-sub multipart_text_as_html {
-        # ($mime, $ctx) = @_; # each_part may do "$_[0] = undef"
-
-        # scan through all parts, looking for displayable text
-        $_[0]->each_part(\&add_text_body, $_[1], 1);
-}
-
 sub submsg_hdr ($$) {
         my ($ctx, $eml) = @_;
-        my $obfs_ibx = $ctx->{-obfs_ibx};
-        my $rv = $ctx->{obuf};
-        $$rv .= "\n";
+        my $s = "\n";
         for my $h (qw(From To Cc Subject Date Message-ID X-Alt-Message-ID)) {
-                my @v = $eml->header($h);
-                for my $v (@v) {
-                        obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx;
-                        $v = ascii_html($v);
-                        $$rv .= "$h: $v\n";
-                }
+                $s .= "$h: $_\n" for $eml->header($h);
         }
+        obfuscate_addrs($ctx->{-obfs_ibx}, $s) if $ctx->{-obfs_ibx};
+        ascii_html($s);
 }
 
 sub attach_link ($$$$;$) {
@@ -530,7 +577,6 @@ sub attach_link ($$$$;$) {
         # downloads for 0-byte multipart attachments
         return unless $part->{bdy};
 
-        my $nl = $idx eq '1' ? '' : "\n"; # like join("\n", ...)
         my $size = length($part->body);
         delete $part->{bdy}; # save memory
 
@@ -546,23 +592,17 @@ sub attach_link ($$$$;$) {
         } else {
                 $sfn = 'a.bin';
         }
-        my $rv = $ctx->{obuf};
-        $$rv .= qq($nl<a\nhref="$ctx->{mhref}$idx-$sfn">);
-        if ($err) {
-                $$rv .= <<EOF;
+        my $rv = $idx eq '1' ? '' : "\n"; # like join("\n", ...)
+        $rv .= qq(<a\nhref="$ctx->{mhref}$idx-$sfn">);
+        $rv .= <<EOF if $err;
 [-- Warning: decoded text below may be mangled, UTF-8 assumed --]
 EOF
-        }
-        $$rv .= "[-- Attachment #$idx: ";
-        my $ts = "Type: $ct, Size: $size bytes";
+        $rv .= "[-- Attachment #$idx: ";
         my $desc = $part->header('Content-Description') // $fn // '';
-        $desc = ascii_html($desc);
-        $$rv .= ($desc eq '') ? "$ts --]" : "$desc --]\n[-- $ts --]";
-        $$rv .= "</a>\n";
-
-        submsg_hdr($ctx, $part) if $part->{is_submsg};
-
-        undef;
+        $rv .= ascii_html($desc)." --]\n[-- " if $desc ne '';
+        $rv .= "Type: $ct, Size: $size bytes --]</a>\n";
+        $rv .= submsg_hdr($ctx, $part) if $part->{is_submsg};
+        $rv;
 }
 
 sub add_text_body { # callback for each_part
@@ -575,13 +615,9 @@ sub add_text_body { # callback for each_part
         my $ct = $part->content_type || 'text/plain';
         my $fn = $part->filename;
         my ($s, $err) = msg_part_text($part, $ct);
-        return attach_link($ctx, $ct, $p, $fn) unless defined $s;
-
-        my $rv = $ctx->{obuf};
-        if ($part->{is_submsg}) {
-                submsg_hdr($ctx, $part);
-                $$rv .= "\n";
-        }
+        my $zfh = $ctx->zfh;
+        $s // return print $zfh (attach_link($ctx, $ct, $p, $fn) // '');
+        say $zfh submsg_hdr($ctx, $part) if $part->{is_submsg};
 
         # makes no difference to browsers, and don't screw up filename
         # link generation in diffs with the extra '%0D'
@@ -604,24 +640,6 @@ sub add_text_body { # callback for each_part
                 $ctx->{-anchors} = {} if $s =~ /^diff --git /sm;
                 $diff = 1;
                 delete $ctx->{-long_path};
-                my $spfx;
-                # absolute URL (Atom feeds)
-                if ($ibx->{coderepo}) {
-                        if (index($upfx, '//') >= 0) {
-                                $spfx = $upfx;
-                                $spfx =~ s!/([^/]*)/\z!/!;
-                        } else {
-                                my $n_slash = $upfx =~ tr!/!/!;
-                                if ($n_slash == 0) {
-                                        $spfx = '../';
-                                } elsif ($n_slash == 1) {
-                                        $spfx = '';
-                                } else { # nslash == 2
-                                        $spfx = '../../';
-                                }
-                        }
-                }
-                $ctx->{-spfx} = $spfx;
         };
 
         # split off quoted and unquoted blocks:
@@ -629,110 +647,122 @@ sub add_text_body { # callback for each_part
         undef $s; # free memory
         if (defined($fn) || ($depth > 0 && !$part->{is_submsg}) || $err) {
                 # badly-encoded message with $err? tell the world about it!
-                attach_link($ctx, $ct, $p, $fn, $err);
-                $$rv .= "\n";
+                say $zfh attach_link($ctx, $ct, $p, $fn, $err);
         }
         delete $part->{bdy}; # save memory
-        foreach my $cur (@sections) {
+        for my $cur (@sections) { # $cur may be huge
                 if ($cur =~ /\A>/) {
                         # we use a <span> here to allow users to specify
                         # their own color for quoted text
-                        $$rv .= qq(<span\nclass="q">);
-                        $$rv .= $l->to_html($cur);
-                        $$rv .= '</span>';
+                        print $zfh qq(<span\nclass="q">),
+                                        $l->to_html($cur), '</span>';
                 } elsif ($diff) {
                         flush_diff($ctx, \$cur);
-                } else {
-                        # regular lines, OK
-                        $$rv .= $l->to_html($cur);
+                } else { # regular lines, OK
+                        print $zfh $l->to_html($cur);
                 }
                 undef $cur; # free memory
         }
 }
 
-sub _msg_page_prepare_obuf {
-        my ($eml, $ctx) = @_;
-        my $over = $ctx->{ibx}->over;
-        my $obfs_ibx = $ctx->{-obfs_ibx};
-        my $rv = '';
+sub _msg_page_prepare {
+        my ($eml, $ctx, $ts) = @_;
+        my $have_over = !!$ctx->{ibx}->over;
         my $mids = mids_for_index($eml);
         my $nr = $ctx->{nr}++;
         if ($nr) { # unlikely
                 if ($ctx->{chash} eq content_hash($eml)) {
                         warn "W: BUG? @$mids not deduplicated properly\n";
-                        return \$rv;
+                        return;
                 }
-                $rv .=
-"<pre>WARNING: multiple messages have this Message-ID\n</pre>";
-                $rv .= '<pre>';
+                $ctx->{-html_tip} =
+qq[<pre>WARNING: multiple messages have this Message-ID (<a
+href="d/">diff</a>)</pre><pre>];
         } else {
                 $ctx->{first_hdr} = $eml->header_obj;
                 $ctx->{chash} = content_hash($eml) if $ctx->{smsg}; # reused MID
-                $rv .= "<pre\nid=b>"; # anchor for body start
+                $ctx->{-html_tip} = "<pre\nid=b>"; # anchor for body start
         }
-        $ctx->{-upfx} = '../' if $over;
+        $ctx->{-upfx} = '../';
         my @title; # (Subject[0], From[0])
+        my $hbuf = '';
         for my $v ($eml->header('From')) {
                 my @n = PublicInbox::Address::names($v);
-                $v = ascii_html($v);
-                $title[1] //= ascii_html(join(', ', @n));
-                if ($obfs_ibx) {
-                        obfuscate_addrs($obfs_ibx, $v);
-                        obfuscate_addrs($obfs_ibx, $title[1]);
-                }
-                $rv .= "From: $v\n" if $v ne '';
+                $title[1] //= join(', ', @n);
+                $hbuf .= "From: $v\n" if $v ne '';
         }
-        foreach my $h (qw(To Cc)) {
+        for my $h (qw(To Cc)) {
                 for my $v ($eml->header($h)) {
                         fold_addresses($v);
-                        $v = ascii_html($v);
-                        obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx;
-                        $rv .= "$h: $v\n" if $v ne '';
+                        $hbuf .= "$h: $v\n" if $v ne '';
                 }
         }
         my @subj = $eml->header('Subject');
-        if (@subj) {
-                my $v = ascii_html(shift @subj);
-                obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx;
-                $rv .= 'Subject: ';
-                $rv .= $over ? qq(<a\nhref="#r"\nid=t>$v</a>\n) : "$v\n";
-                $title[0] = $v;
-                for $v (@subj) { # multi-Subject message :<
-                        $v = ascii_html($v);
-                        obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx;
-                        $rv .= "Subject: $v\n";
-                }
-        } else { # dummy anchor for thread skeleton at bottom of page
-                $rv .= qq(<a\nhref="#r"\nid=t></a>) if $over;
-                $title[0] = '(no subject)';
-        }
-        for my $v ($eml->header('Date')) {
-                $v = ascii_html($v);
-                obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx; # possible :P
-                $rv .= qq{Date: $v\t<a\nhref="#r">[thread overview]</a>\n};
+        $hbuf .= "Subject: $_\n" for @subj;
+        $title[0] = $subj[0] // '(no subject)';
+        $hbuf .= "Date: $_\n" for $eml->header('Date');
+        $hbuf = ascii_html($hbuf);
+        my $t = $ts ? '?t='.ts2str($ts) : '';
+        my ($re, $addr2url) = addr2urlmap($ctx);
+        $hbuf =~ s!$re!qq(<a\nhref=").$addr2url->{lc $1}.qq($t">$1</a>)!sge;
+        $ctx->{-title_html} = ascii_html(join(' - ', @title));
+        if (my $obfs_ibx = $ctx->{-obfs_ibx}) {
+                obfuscate_addrs($obfs_ibx, $hbuf);
+                obfuscate_addrs($obfs_ibx, $ctx->{-title_html});
         }
-        if (!$nr) { # first (and only) message, common case
-                $ctx->{-title_html} = join(' - ', @title);
-                $rv = $ctx->html_top . $rv;
+
+        # [thread overview] link is typically added after Date,
+        # but added after Subject, or even nothing.
+        if ($have_over) {
+                chop $hbuf; # drop "\n", or noop if $rv eq ''
+                $hbuf .= qq{\t<a\nhref="#r">[thread overview]</a>\n};
+                $hbuf =~ s!^Subject:\x20(.*?)(\n[A-Z]|\z)
+                                !Subject: <a\nhref="#r"\nid=t>$1</a>$2!msx or
+                        $hbuf .= qq(<a\nhref="#r\nid=t></a>);
         }
         if (scalar(@$mids) == 1) { # common case
-                my $mhtml = ascii_html($mids->[0]);
-                $rv .= "Message-ID: &lt;$mhtml&gt; ";
-                $rv .= "(<a\nhref=\"raw\">raw</a>)\n";
+                my $x = ascii_html($mids->[0]);
+                $hbuf .= qq[Message-ID: &lt;$x&gt; (<a href="raw">raw</a>)\n];
+        }
+        if (!$nr) { # first (and only) message, common case
+                print { $ctx->zfh } $ctx->html_top, $hbuf;
         } else {
+                delete $ctx->{-title_html};
+                print { $ctx->zfh } $ctx->{-html_tip}, $hbuf;
+        }
+        $ctx->{-linkify} //= PublicInbox::Linkify->new;
+        $hbuf = '';
+        if (scalar(@$mids) != 1) { # unlikely, but it happens :<
                 # X-Alt-Message-ID can happen if a message is injected from
                 # public-inbox-nntpd because of multiple Message-ID headers.
-                my $lnk = PublicInbox::Linkify->new;
-                my $s = '';
                 for my $h (qw(Message-ID X-Alt-Message-ID)) {
-                        $s .= "$h: $_\n" for ($eml->header_raw($h));
+                        $hbuf .= "$h: $_\n" for ($eml->header_raw($h));
                 }
-                $lnk->linkify_mids('..', \$s, 1);
-                $rv .= $s;
+                $ctx->{-linkify}->linkify_mids('..', \$hbuf, 1); # escapes HTML
+                print { $ctx->{zfh} } $hbuf;
+                $hbuf = '';
+        }
+        my @irt = $eml->header_raw('In-Reply-To');
+        my $refs;
+        if (@irt) { # ("so-and-so's message of $DATE") added by some MUAs
+                for (grep(/=\?/, @irt)) {
+                        s/(=\?.*)\z/PublicInbox::Eml::mhdr_decode $1/se;
+                }
+        } else {
+                $refs = references($eml);
+                $irt[0] = pop(@$refs) if scalar @$refs;
         }
-        $rv .= _parent_headers($eml, $over);
-        $rv .= "\n";
-        \$rv;
+        $hbuf .= "In-Reply-To: $_\n" for @irt;
+
+        # do not display References: if search is present,
+        # we show the thread skeleton at the bottom, instead.
+        if (!$have_over) {
+                $refs //= references($eml);
+                $hbuf .= 'References: <'.join(">\n\t<", @$refs).">\n" if @$refs;
+        }
+        $ctx->{-linkify}->linkify_mids('..', \$hbuf); # escapes HTML
+        say { $ctx->{zfh} } $hbuf;
+        1;
 }
 
 sub SKEL_EXPAND () {
@@ -769,7 +799,6 @@ sub thread_skel ($$$) {
         # when multiple Subject: headers are present, so we follow suit:
         my $subj = $hdr->header('Subject') // '';
         $subj = '(no subject)' if $subj eq '';
-        $ctx->{prev_subj} = [ split(/ /, subject_normalized($subj)) ];
         $ctx->{cur} = $mid;
         $ctx->{prev_attr} = '';
         $ctx->{prev_level} = 0;
@@ -782,54 +811,47 @@ sub thread_skel ($$$) {
         $ctx->{parent_msg} = $parent;
 }
 
-sub _parent_headers {
-        my ($hdr, $over) = @_;
-        my $rv = '';
-        my @irt = $hdr->header_raw('In-Reply-To');
-        my $refs;
-        if (@irt) {
-                my $lnk = PublicInbox::Linkify->new;
-                $rv .= "In-Reply-To: $_\n" for @irt;
-                $lnk->linkify_mids('..', \$rv);
-        } else {
-                $refs = references($hdr);
-                my $irt = pop @$refs;
-                if (defined $irt) {
-                        my $html = ascii_html($irt);
-                        my $href = mid_href($irt);
-                        $rv .= "In-Reply-To: &lt;";
-                        $rv .= "<a\nhref=\"../$href/\">$html</a>&gt;\n";
-                }
-        }
-
-        # do not display References: if search is present,
-        # we show the thread skeleton at the bottom, instead.
-        return $rv if $over;
-
-        $refs //= references($hdr);
-        if (@$refs) {
-                @$refs = map { linkify_ref_no_over($_) } @$refs;
-                $rv .= 'References: '. join("\n\t", @$refs) . "\n";
-        }
-        $rv;
-}
-
-# returns a string buffer
+# writes to zbuf
 sub html_footer {
         my ($ctx, $hdr) = @_;
-        my $ibx = $ctx->{ibx};
         my $upfx = '../';
-        my $skel;
-        my $rv = '<pre>';
-        if ($ibx->over) {
+        my (@related, $skel);
+        my $foot = '<pre>';
+        my $qry = delete $ctx->{-qry};
+        if ($qry && $ctx->{ibx}->isrch) {
+                my $q = ''; # search for either ancestor or descendent patches
+                for (@{$qry->{dfpre}}, @{$qry->{dfpost}}) {
+                        chop if length > 7; # include 1 abbrev "older" patches
+                        $q .= "dfblob:$_ ";
+                }
+                chop $q; # omit trailing SP
+                local $Text::Wrap::columns = COLS;
+                local $Text::Wrap::huge = 'overflow';
+                $q = wrap('', '', $q);
+                my $rows = ($q =~ tr/\n/\n/) + 1;
+                $q = ascii_html($q);
+                $related[0] = <<EOM;
+<form id=related
+action=$upfx
+><pre>find likely ancestor, descendant, or conflicting patches for <a
+href=#t>this message</a>:
+<textarea name=q cols=${\COLS} rows=$rows>$q</textarea>
+<input type=submit value=search
+/>\t(<a href=${upfx}_/text/help/#search>help</a>)</pre></form>
+EOM
+                # TODO: related codesearch
+                # my $csrchv = $ctx->{ibx}->{-csrch} // [];
+                # push @related, '<pre>'.ascii_html(Dumper($csrchv)).'</pre>';
+        }
+        if ($ctx->{ibx}->over) {
                 my $t = ts2str($ctx->{-t_max});
                 my $t_fmt = fmt_ts($ctx->{-t_max});
-                $skel .= <<EOF;
-        other threads:[<a
+                my $fallback = @related ? "\t" : "<a id=related>\t</a>";
+                $skel = <<EOF;
+${fallback}other threads:[<a
 href="$upfx?t=$t">~$t_fmt UTC</a>|<a
 href="$upfx">newest</a>]
 EOF
-
                 thread_skel(\$skel, $ctx, $hdr);
                 my ($next, $prev);
                 my $parent = '       ';
@@ -837,43 +859,32 @@ EOF
 
                 if (my $n = $ctx->{next_msg}) {
                         $n = mid_href($n);
-                        $next = "<a\nhref=\"$upfx$n/\"\nrel=next>next</a>";
+                        $next = qq(<a\nhref="$upfx$n/"\nrel=next>next</a>);
                 }
-                my $u;
                 my $par = $ctx->{parent_msg};
-                if ($par) {
-                        $u = mid_href($par);
-                        $u = "$upfx$u/";
-                }
+                my $u = $par ? $upfx.mid_href($par).'/' : undef;
                 if (my $p = $ctx->{prev_msg}) {
                         $prev = mid_href($p);
                         if ($p && $par && $p eq $par) {
-                                $prev = "<a\nhref=\"$upfx$prev/\"\n" .
+                                $prev = qq(<a\nhref="$upfx$prev/"\n) .
                                         'rel=prev>prev parent</a>';
                                 $parent = '';
                         } else {
-                                $prev = "<a\nhref=\"$upfx$prev/\"\n" .
+                                $prev = qq(<a\nhref="$upfx$prev/"\n) .
                                         'rel=prev>prev</a>';
-                                $parent = " <a\nhref=\"$u\">parent</a>" if $u;
+                                $parent = qq( <a\nhref="$u">parent</a>) if $u;
                         }
                 } elsif ($u) { # unlikely
-                        $parent = " <a\nhref=\"$u\"\nrel=prev>parent</a>";
+                        $parent = qq( <a\nhref="$u"\nrel=prev>parent</a>);
                 }
-                $rv .= "$next $prev$parent ";
+                $foot .= "$next $prev$parent ";
         } else { # unindexed inboxes w/o over
                 $skel = qq( <a\nhref="$upfx">latest</a>);
         }
-        $rv .= qq(<a\nhref="#R">reply</a>);
-        $rv .= $skel;
-        $rv .= '</pre>';
-        $rv .= msg_reply($ctx, $hdr);
-}
-
-sub linkify_ref_no_over {
-        my ($mid) = @_;
-        my $href = mid_href($mid);
-        my $html = ascii_html($mid);
-        "&lt;<a\nhref=\"../$href/\">$html</a>&gt;";
+        # $skel may be big for big threads, don't append it to $foot
+        print { $ctx->zfh } $foot, qq(<a\nhref="#R">reply</a>),
+                                $skel, '</pre>', @related,
+                                msg_reply($ctx, $hdr);
 }
 
 sub ghost_parent {
@@ -934,8 +945,8 @@ sub thread_results {
                         my $tip = splice(@$rootset, $idx, 1);
                         @$rootset = reverse @$rootset;
                         unshift @$rootset, $tip;
-                        $ctx->{sl_note} = strict_loose_note($nr);
                 }
+                $ctx->{sl_note} = strict_loose_note($nr);
         }
         $rootset
 }
@@ -1071,6 +1082,8 @@ sub _skel_ghost {
         1;
 }
 
+# note: we favor Date: here because git-send-email increments it
+# to preserve [PATCH $N/$M] ordering in series (it can't control Received:)
 sub sort_ds {
         @{$_[0]} = sort {
                 (eval { $a->topmost->{ds} } || 0) <=>
@@ -1092,9 +1105,10 @@ sub acc_topic { # walk_thread callback
         if ($has_blob) {
                 my $subj = subject_normalized($smsg->{subject});
                 $subj = '(no subject)' if $subj eq '';
+                my $ts = $smsg->{ts};
                 my $ds = $smsg->{ds};
                 if ($level == 0) { # new, top-level topic
-                        my $topic = [ $ds, 1, { $subj => $mid }, $subj ];
+                        my $topic = [ $ts, $ds, 1, { $subj => $mid }, $subj ];
                         $ctx->{-cur_topic} = $topic;
                         push @{$ctx->{order}}, $topic;
                         return 1;
@@ -1102,10 +1116,11 @@ sub acc_topic { # walk_thread callback
 
                 # continue existing topic
                 my $topic = $ctx->{-cur_topic}; # should never be undef
-                $topic->[0] = $ds if $ds > $topic->[0];
-                $topic->[1]++; # bump N+ message counter
-                my $seen = $topic->[2];
-                if (scalar(@$topic) == 3) { # parent was a ghost
+                $topic->[0] = $ts if $ts > $topic->[0];
+                $topic->[1] = $ds if $ds > $topic->[1];
+                $topic->[2]++; # bump N+ message counter
+                my $seen = $topic->[3];
+                if (scalar(@$topic) == 4) { # parent was a ghost
                         push @$topic, $subj;
                 } elsif (!defined($seen->{$subj})) {
                         push @$topic, $level, $subj; # @extra messages
@@ -1113,7 +1128,7 @@ sub acc_topic { # walk_thread callback
                 $seen->{$subj} = $mid; # latest for subject
         } else { # ghost message
                 return 1 if $level != 0; # ignore child ghosts
-                my $topic = $ctx->{-cur_topic} = [ -666, 0, {} ];
+                my $topic = $ctx->{-cur_topic} = [ -666, -666, 0, {} ];
                 push @{$ctx->{order}}, $topic;
         }
         1;
@@ -1128,12 +1143,13 @@ sub dump_topics {
         }
 
         my @out;
-        my $ibx = $ctx->{ibx};
-        my $obfs_ibx = $ibx->{obfuscate} ? $ibx : undef;
-
+        my $obfs_ibx = $ctx->{ibx}->{obfuscate} ? $ctx->{ibx} : undef;
+        if (my $note = delete $ctx->{t_note}) {
+                push @out, $note; # "messages from ... to ..."
+        }
         # sort by recency, this allows new posts to "bump" old topics...
         foreach my $topic (sort { $b->[0] <=> $a->[0] } @$order) {
-                my ($ds, $n, $seen, $top_subj, @extra) = @$topic;
+                my ($ts, $ds, $n, $seen, $top_subj, @extra) = @$topic;
                 @$topic = ();
                 next unless defined $top_subj;  # ghost topic
                 my $mid = delete $seen->{$top_subj};
@@ -1155,9 +1171,9 @@ sub dump_topics {
 
                 my $s = "<a\nhref=\"$href/T/$anchor\">$top_subj</a>\n" .
                         " $ds UTC $n\n";
-                for (my $i = 0; $i < scalar(@extra); $i += 2) {
-                        my $level = $extra[$i];
-                        my $subj = $extra[$i + 1]; # already normalized
+                while (@extra) {
+                        my $level = shift @extra;
+                        my $subj = shift @extra; # already normalized
                         $mid = delete $seen->{$subj};
                         my @subj = split(/ /, $subj);
                         my @next_prev = @subj; # full copy
@@ -1189,7 +1205,11 @@ sub pagination_footer ($$) {
                 $next = $next ? "$next | " : '             | ';
                 $prev .= qq[ | <a\nhref="$latest">latest</a>];
         }
-        "<hr><pre>page: $next$prev</pre>";
+        my $rv = '<hr><pre id=nav>';
+        $rv .= "page: $next$prev\n" if $next || $prev;
+        $rv .= q{- recent:[<b>subjects (threaded)</b>|<a
+href="./topics_new.html">topics (new)</a>|<a
+href="./topics_active.html">topics (active)</a>]</pre>};
 }
 
 sub paginate_recent ($$) {
@@ -1204,23 +1224,30 @@ sub paginate_recent ($$) {
         $t =~ s/\A([0-9]{8,14})-// and $after = str2ts($1);
         $t =~ /\A([0-9]{8,14})\z/ and $before = str2ts($1);
 
-        my $ibx = $ctx->{ibx};
-        my $msgs = $ibx->recent($opts, $after, $before);
-        my $nr = scalar @$msgs;
-        if ($nr < $lim && defined($after)) {
+        my $msgs = $ctx->{ibx}->over->recent($opts, $after, $before);
+        if (defined($after) && scalar(@$msgs) < $lim) {
                 $after = $before = undef;
-                $msgs = $ibx->recent($opts);
-                $nr = scalar @$msgs;
+                $msgs = $ctx->{ibx}->over->recent($opts);
         }
-        my $more = $nr == $lim;
+        my $more = scalar(@$msgs) == $lim;
         my ($newest, $oldest);
-        if ($nr) {
+        if (@$msgs) {
                 $newest = $msgs->[0]->{ts};
                 $oldest = $msgs->[-1]->{ts};
                 # if we only had $after, our SQL query in ->recent ordered
                 if ($newest < $oldest) {
                         ($oldest, $newest) = ($newest, $oldest);
-                        $more = 0 if defined($after) && $after < $oldest;
+                        $more = undef if defined($after) && $after < $oldest;
+                }
+                if (defined($after // $before)) {
+                        my $n = strftime('%Y-%m-%d %H:%M:%S', gmtime($newest));
+                        my $o = strftime('%Y-%m-%d %H:%M:%S', gmtime($oldest));
+                        $ctx->{t_note} = <<EOM;
+ messages from $o to $n UTC [<a href="#nav">more...</a>]
+EOM
+                        my $s = ts2str($newest);
+                        $ctx->{prev_page} = qq[<a\nhref="?t=$s-"\nrel=prev>] .
+                                                'prev (newer)</a>';
                 }
         }
         if (defined($oldest) && $more) {
@@ -1228,11 +1255,6 @@ sub paginate_recent ($$) {
                 $ctx->{next_page} = qq[<a\nhref="?t=$s"\nrel=next>] .
                                         'next (older)</a>';
         }
-        if (defined($newest) && (defined($before) || defined($after))) {
-                my $s = ts2str($newest);
-                $ctx->{prev_page} = qq[<a\nhref="?t=$s-"\nrel=prev>] .
-                                        'prev (newer)</a>';
-        }
         $msgs;
 }
 
@@ -1240,11 +1262,8 @@ sub paginate_recent ($$) {
 sub index_topics {
         my ($ctx) = @_;
         my $msgs = paginate_recent($ctx, 200); # 200 is our window
-        if (@$msgs) {
-                walk_thread(thread_results($ctx, $msgs), $ctx, \&acc_topic);
-        }
-        html_oneshot($ctx, dump_topics($ctx), \pagination_footer($ctx, '.'));
-
+        walk_thread(thread_results($ctx, $msgs), $ctx, \&acc_topic) if @$msgs;
+        html_oneshot($ctx, dump_topics($ctx), pagination_footer($ctx, '.'));
 }
 
 sub thread_adj_level {
@@ -1278,4 +1297,30 @@ sub ghost_index_entry {
                 . '</pre>' . $end;
 }
 
+# /$INBOX/$MSGID/d/ endpoint
+sub diff_msg {
+        my ($ctx) = @_;
+        require PublicInbox::MailDiff;
+        my $ibx = $ctx->{ibx};
+        my $over = $ibx->over or return no_over_html($ctx);
+        my ($id, $prev);
+        my $md = bless { ctx => $ctx }, 'PublicInbox::MailDiff';
+        my $next_arg = $md->{next_arg} = [ $ctx->{mid}, \$id, \$prev ];
+        my $smsg = $md->{smsg} = $over->next_by_mid(@$next_arg) or
+                return; # undef == 404
+        $ctx->{-t_max} = $smsg->{ts};
+        $ctx->{-upfx} = '../../';
+        $ctx->{-apfx} = '//'; # fail on to_attr()
+        $ctx->{-linkify} = PublicInbox::Linkify->new;
+        my $mid = ascii_html($smsg->{mid});
+        $ctx->{-title_html} = "diff for duplicates of &lt;$mid&gt;";
+        PublicInbox::WwwStream::html_init($ctx);
+        print { $ctx->{zfh} } '<pre>diff for duplicates of &lt;<a href="../">',
+                                $mid, "</a>&gt;\n\n";
+        sub {
+                $ctx->attach($_[0]->([200, delete $ctx->{-res_hdr}]));
+                $md->begin_mail_diff;
+        };
+}
+
 1;
diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index fb394b7c..d078c5f9 100644
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # used by PublicInbox::View
@@ -7,15 +7,13 @@
 # (or reconstruct) blobs.
 
 package PublicInbox::ViewDiff;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(Exporter);
-our @EXPORT_OK = qw(flush_diff);
+our @EXPORT_OK = qw(flush_diff uri_escape_path);
 use URI::Escape qw(uri_escape_utf8);
-use PublicInbox::Hval qw(ascii_html to_attr);
+use PublicInbox::Hval qw(ascii_html to_attr utf8_maybe);
 use PublicInbox::Git qw(git_unquote);
 
-my $UNSAFE = "^A-Za-z0-9\-\._~/"; # '/' + $URI::Escape::Unsafe{RFC3986}
 my $OID_NULL = '0{7,}';
 my $OID_BLOB = '[a-f0-9]{7,}';
 my $LF = qr!\n!;
@@ -41,22 +39,24 @@ our $EXTRACT_DIFFS = qr/(
                 ^\+{3}\x20($FN)$LF)/msx;
 our $IS_OID = qr/\A$OID_BLOB\z/s;
 
+sub uri_escape_path {
+        # '/' + $URI::Escape::Unsafe{RFC3986}
+        uri_escape_utf8($_[0], "^A-Za-z0-9\-\._~/");
+}
+
 # link to line numbers in blobs
-sub diff_hunk ($$$$) {
-        my ($dst, $dctx, $ca, $cb) = @_;
+sub diff_hunk ($$$) {
+        my ($dctx, $ca, $cb) = @_;
         my ($oid_a, $oid_b, $spfx) = @$dctx{qw(oid_a oid_b spfx)};
 
         if (defined($spfx) && defined($oid_a) && defined($oid_b)) {
-                my ($n) = ($ca =~ /^-([0-9]+)/);
-                $n = defined($n) ? "#n$n" : '';
-
-                $$dst .= qq(@@ <a\nhref="$spfx$oid_a/s/$dctx->{Q}$n">$ca</a>);
+                my $n = ($ca =~ /^-([0-9]+)/) ? "#n$1" : '';
+                my $x = qq(@@ <a\nhref="$spfx$oid_a/s/$dctx->{Q}$n">$ca</a>);
 
-                ($n) = ($cb =~ /^\+([0-9]+)/);
-                $n = defined($n) ? "#n$n" : '';
-                $$dst .= qq( <a\nhref="$spfx$oid_b/s/$dctx->{Q}$n">$cb</a> @@);
+                $n = ($cb =~ /^\+([0-9]+)/) ? "#n$1" : '';
+                $x .= qq( <a\nhref="$spfx$oid_b/s/$dctx->{Q}$n">$cb</a> @@);
         } else {
-                $$dst .= "@@ $ca $cb @@";
+                "@@ $ca $cb @@";
         }
 }
 
@@ -66,8 +66,8 @@ sub oid ($$$) {
 }
 
 # returns true if diffstat anchor written, false otherwise
-sub anchor0 ($$$$) {
-        my ($dst, $ctx, $fn, $rest) = @_;
+sub anchor0 ($$$) {
+        my ($ctx, $fn, $rest) = @_;
 
         my $orig = $fn;
 
@@ -83,15 +83,12 @@ sub anchor0 ($$$$) {
         # long filenames will require us to check in anchor1()
         push(@{$ctx->{-long_path}}, $fn) if $fn =~ s!\A\.\.\./?!!;
 
-        if (defined(my $attr = to_attr($ctx->{-apfx}.$fn))) {
-                $ctx->{-anchors}->{$attr} = 1;
-                my $spaces = ($orig =~ s/( +)\z//) ? $1 : '';
-                $$dst .= " <a\nid=i$attr\nhref=#$attr>" .
-                        ascii_html($orig) . '</a>' . $spaces .
+        my $attr = to_attr($ctx->{-apfx}.$fn) // return;
+        $ctx->{-anchors}->{$attr} = 1;
+        my $spaces = ($orig =~ s/( +)\z//) ? $1 : '';
+        print { $ctx->{zfh} } " <a\nid=i$attr\nhref=#$attr>",
+                        ascii_html($orig), '</a>', $spaces,
                         $ctx->{-linkify}->to_html($rest);
-                return 1;
-        }
-        undef;
 }
 
 # returns "diff --git" anchor destination, undef otherwise
@@ -123,15 +120,11 @@ sub diff_header ($$$) {
         $pa = (split(m'/', git_unquote($pa), 2))[1] if $pa ne '/dev/null';
         $pb = (split(m'/', git_unquote($pb), 2))[1] if $pb ne '/dev/null';
         if ($pa eq $pb && $pb ne '/dev/null') {
-                $dctx->{Q} = "?b=".uri_escape_utf8($pb, $UNSAFE);
+                $dctx->{Q} = '?b='.uri_escape_path($pb);
         } else {
                 my @q;
-                if ($pb ne '/dev/null') {
-                        push @q, 'b='.uri_escape_utf8($pb, $UNSAFE);
-                }
-                if ($pa ne '/dev/null') {
-                        push @q, 'a='.uri_escape_utf8($pa, $UNSAFE);
-                }
+                push @q, 'b='.uri_escape_path($pb) if $pb ne '/dev/null';
+                push @q, 'a='.uri_escape_path($pa) if $pa ne '/dev/null';
                 $dctx->{Q} = '?'.join('&amp;', @q);
         }
 
@@ -141,44 +134,48 @@ sub diff_header ($$$) {
 
         # no need to capture oid_a and oid_b on add/delete,
         # we just linkify OIDs directly via s///e in conditional
-        if (($$x =~ s/$NULL_TO_BLOB/$1 . oid($dctx, $spfx, $2)/e) ||
-                ($$x =~ s/$BLOB_TO_NULL/
-                        'index ' . oid($dctx, $spfx, $1) . $2/e)) {
+        if ($$x =~ s/$NULL_TO_BLOB/$1 . oid($dctx, $spfx, $2)/e) {
+                push @{$ctx->{-qry}->{dfpost}}, $2;
+        } elsif ($$x =~ s/$BLOB_TO_NULL/'index '.oid($dctx, $spfx, $1).$2/e) {
+                push @{$ctx->{-qry}->{dfpre}}, $1;
         } elsif ($$x =~ $BLOB_TO_BLOB) {
                 # modification-only, not add/delete:
                 # linkify hunk headers later using oid_a and oid_b
                 @$dctx{qw(oid_a oid_b)} = ($1, $2);
+                push @{$ctx->{-qry}->{dfpre}}, $1;
+                push @{$ctx->{-qry}->{dfpost}}, $2;
         } else {
                 warn "BUG? <$$x> had no ^index line";
         }
         $$x =~ s!^diff --git!anchor1($ctx, $pb) // 'diff --git'!ems;
-        my $dst = $ctx->{obuf};
-        $$dst .= qq(<span\nclass="head">);
-        $$dst .= $$x;
-        $$dst .= '</span>';
+        print { $ctx->{zfh} } qq(<span\nclass="head">), $$x, '</span>';
         $dctx;
 }
 
 sub diff_before_or_after ($$) {
         my ($ctx, $x) = @_;
-        my $linkify = $ctx->{-linkify};
-        my $dst = $ctx->{obuf};
-        my $anchors = exists($ctx->{-anchors}) ? 1 : 0;
-        for my $y (split(/(^---\n)/sm, $$x)) {
-                if ($y =~ /\A---\n\z/s) {
-                        $$dst .= "---\n"; # all HTML is "\r\n" => "\n"
-                        $anchors |= 2;
-                } elsif ($anchors == 3 && $y =~ /^ [0-9]+ files? changed, /sm) {
-                        # ok, looks like a diffstat, go line-by-line:
-                        for my $l (split(/^/m, $y)) {
-                                if ($l =~ /^ (.+)( +\| .*\z)/s) {
-                                        anchor0($dst, $ctx, $1, $2) and next;
-                                }
-                                $$dst .= $linkify->to_html($l);
-                        }
-                } else { # commit message, notes, etc
-                        $$dst .= $linkify->to_html($y);
+        if (exists $ctx->{-anchors} && $$x =~ # diffstat lines:
+                        /((?:^\x20(?:[^\n]+?)(?:\x20+\|\x20[^\n]*\n))+)
+                        (\x20[0-9]+\x20files?\x20)changed,/msx) {
+                my $pre = substr($$x, 0, $-[0]); # (likely) short prefix
+                substr($$x, 0, $+[0], ''); # sv_chop on $$x ($$x may be long)
+                my @x = ($2, $1);
+                my $lnk = $ctx->{-linkify};
+                my $zfh = $ctx->{zfh};
+                # uninteresting prefix
+                print $zfh $lnk->to_html($pre);
+                for my $l (split(/^/m, pop(@x))) { # $2 per-file stat lines
+                        $l =~ /^ (.+)( +\| .*\z)/s and
+                                anchor0($ctx, $1, $2) and next;
+                         print $zfh $lnk->to_html($l);
                 }
+                my $ch = $ctx->{changed_href} // '#related';
+                print $zfh pop(@x), # $3 /^ \d+ files? /
+                        qq(<a href="$ch">changed</a>,),
+                        # insertions/deletions, notes, commit message, etc:
+                        $lnk->to_html($$x);
+        } else {
+                print { $ctx->{zfh} } $ctx->{-linkify}->to_html($$x);
         }
 }
 
@@ -189,9 +186,9 @@ sub flush_diff ($$) {
         my @top = split($EXTRACT_DIFFS, $$cur);
         undef $$cur; # free memory
 
-        my $linkify = $ctx->{-linkify};
-        my $dst = $ctx->{obuf};
+        my $lnk = $ctx->{-linkify};
         my $dctx; # {}, keys: Q, oid_a, oid_b
+        my $zfh = $ctx->zfh;
 
         while (defined(my $x = shift @top)) {
                 if (scalar(@top) >= 4 &&
@@ -199,7 +196,8 @@ sub flush_diff ($$) {
                                 $top[0] =~ $IS_OID) {
                         $dctx = diff_header(\$x, $ctx, \@top);
                 } elsif ($dctx) {
-                        my $after = '';
+                        open(my $afh, '>>:utf8', \(my $after='')) or
+                                die "open: $!";
 
                         # Quiet "Complex regular subexpression recursion limit"
                         # warning.  Perl will truncate matches upon hitting
@@ -214,29 +212,33 @@ sub flush_diff ($$) {
                         for my $s (split(/((?:(?:^\+[^\n]*\n)+)|
                                         (?:(?:^-[^\n]*\n)+)|
                                         (?:^@@ [^\n]+\n))/xsm, $x)) {
+                                undef $x;
                                 if (!defined($dctx)) {
-                                        $after .= $s;
+                                        print $afh $s;
                                 } elsif ($s =~ s/\A@@ (\S+) (\S+) @@//) {
-                                        $$dst .= qq(<span\nclass="hunk">);
-                                        diff_hunk($dst, $dctx, $1, $2);
-                                        $$dst .= $linkify->to_html($s);
-                                        $$dst .= '</span>';
-                                } elsif ($s =~ /\A\+/) {
-                                        $$dst .= qq(<span\nclass="add">);
-                                        $$dst .= $linkify->to_html($s);
-                                        $$dst .= '</span>';
+                                        print $zfh qq(<span\nclass="hunk">),
+                                                diff_hunk($dctx, $1, $2),
+                                                $lnk->to_html($s),
+                                                '</span>';
+                                } elsif ($s =~ /\A\+/) { # $s may be huge
+                                        print $zfh qq(<span\nclass="add">),
+                                                        $lnk->to_html($s),
+                                                        '</span>';
                                 } elsif ($s =~ /\A-- $/sm) { # email sig starts
                                         $dctx = undef;
-                                        $after .= $s;
-                                } elsif ($s =~ /\A-/) {
-                                        $$dst .= qq(<span\nclass="del">);
-                                        $$dst .= $linkify->to_html($s);
-                                        $$dst .= '</span>';
-                                } else {
-                                        $$dst .= $linkify->to_html($s);
+                                        print $afh $s;
+                                } elsif ($s =~ /\A-/) { # $s may be huge
+                                        print $zfh qq(<span\nclass="del">),
+                                                        $lnk->to_html($s),
+                                                        '</span>';
+                                } else { # $s may be huge
+                                        print $zfh $lnk->to_html($s);
                                 }
                         }
-                        diff_before_or_after($ctx, \$after) unless $dctx;
+                        if (!$dctx) {
+                                utf8_maybe($after);
+                                diff_before_or_after($ctx, \$after);
+                        }
                 } else {
                         diff_before_or_after($ctx, \$x);
                 }
diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 3cbc363b..790b9a2c 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -1,8 +1,7 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # show any VCS object, similar to "git show"
-# FIXME: we only show blobs for now
 #
 # This can use a "solver" to reconstruct blobs based on git
 # patches (with abbreviated OIDs in the header).  However, the
@@ -16,11 +15,21 @@
 package PublicInbox::ViewVCS;
 use strict;
 use v5.10.1;
+use File::Temp 0.19 (); # newdir
 use PublicInbox::SolverGit;
+use PublicInbox::Git;
+use PublicInbox::GitAsyncCat;
 use PublicInbox::WwwStream qw(html_oneshot);
 use PublicInbox::Linkify;
 use PublicInbox::Tmpfile;
-use PublicInbox::Hval qw(ascii_html to_filename);
+use PublicInbox::ViewDiff qw(flush_diff uri_escape_path);
+use PublicInbox::View;
+use PublicInbox::Eml;
+use Text::Wrap qw(wrap);
+use PublicInbox::Hval qw(ascii_html to_filename prurl utf8_maybe);
+use POSIX qw(strftime);
+use autodie qw(open seek truncate);
+use Fcntl qw(SEEK_SET);
 my $hl = eval {
         require PublicInbox::HlMod;
         PublicInbox::HlMod->new;
@@ -29,22 +38,52 @@ my $hl = eval {
 my %QP_MAP = ( A => 'oid_a', a => 'path_a', b => 'path_b' );
 our $MAX_SIZE = 1024 * 1024; # TODO: configurable
 my $BIN_DETECT = 8000; # same as git
+my $SHOW_FMT = '--pretty=format:'.join('%n', '%P', '%p', '%H', '%T', '%s', '%f',
+        '%an <%ae>  %ai', '%cn <%ce>  %ci', '%b%x00');
 
-sub html_page ($$$) {
-        my ($ctx, $code, $strref) = @_;
+my %GIT_MODE = (
+        '100644' => ' ', # blob
+        '100755' => 'x', # executable blob
+        '040000' => 'd', # tree
+        '120000' => 'l', # symlink
+        '160000' => 'g', # commit (gitlink)
+);
+
+# TODO: not fork safe, but we don't fork w/o exec in PublicInbox::WWW
+my (@solver_q, $solver_lim);
+my $solver_nr = 0;
+
+sub html_page ($$;@) {
+        my ($ctx, $code) = @_[0, 1];
         my $wcb = delete $ctx->{-wcb};
-        $ctx->{-upfx} = '../../'; # from "/$INBOX/$OID/s/"
-        my $res = html_oneshot($ctx, $code, $strref);
+        $ctx->{-upfx} //= '../../'; # from "/$INBOX/$OID/s/"
+        my $res = html_oneshot($ctx, $code, @_[2..$#_]);
         $wcb ? $wcb->($res) : $res;
 }
 
+sub dbg_log ($) {
+        my ($ctx) = @_;
+        my $log = delete $ctx->{lh} // die 'BUG: already captured debug log';
+        if (!CORE::seek($log, 0, SEEK_SET)) {
+                warn "seek(log): $!";
+                return '<pre>debug log seek error</pre>';
+        }
+        $log = eval { PublicInbox::IO::read_all $log } // do {
+                warn "read(log): $@";
+                return '<pre>debug log read error</pre>';
+        };
+        return '' if $log eq '';
+        $ctx->{-linkify} //= PublicInbox::Linkify->new;
+        "<hr><pre>debug log:\n\n".
+                $ctx->{-linkify}->to_html($log).'</pre>';
+}
+
 sub stream_blob_parse_hdr { # {parse_hdr} for Qspawn
         my ($r, $bref, $ctx) = @_;
-        my ($res, $logref) = delete @$ctx{qw(-res -logref)};
-        my ($git, $oid, $type, $size, $di) = @$res;
+        my ($git, $oid, $type, $size, $di) = @{$ctx->{-res}};
         my @cl = ('Content-Length', $size);
-        if (!defined $r) { # error
-                html_page($ctx, 500, $logref);
+        if (!defined $r) { # sysread error
+                html_page($ctx, 500, dbg_log($ctx));
         } elsif (index($$bref, "\0") >= 0) {
                 [200, [qw(Content-Type application/octet-stream), @cl] ];
         } else {
@@ -54,114 +93,512 @@ sub stream_blob_parse_hdr { # {parse_hdr} for Qspawn
                                 'text/plain; charset=UTF-8', @cl ] ];
                 }
                 if ($r == 0) {
-                        warn "premature EOF on $oid $$logref";
-                        return html_page($ctx, 500, $logref);
+                        my $log = dbg_log($ctx);
+                        warn "premature EOF on $oid $log";
+                        return html_page($ctx, 500, $log);
                 }
-                @$ctx{qw(-res -logref)} = ($res, $logref);
                 undef; # bref keeps growing
         }
 }
 
-sub stream_large_blob ($$$$) {
-        my ($ctx, $res, $logref, $fn) = @_;
-        $ctx->{-logref} = $logref;
+sub stream_large_blob ($$) {
+        my ($ctx, $res) = @_;
         $ctx->{-res} = $res;
         my ($git, $oid, $type, $size, $di) = @$res;
         my $cmd = ['git', "--git-dir=$git->{git_dir}", 'cat-file', $type, $oid];
         my $qsp = PublicInbox::Qspawn->new($cmd);
-        my $env = $ctx->{env};
-        $env->{'qspawn.wcb'} = delete $ctx->{-wcb};
-        $qsp->psgi_return($env, undef, \&stream_blob_parse_hdr, $ctx);
+        $ctx->{env}->{'qspawn.wcb'} = $ctx->{-wcb};
+        $qsp->psgi_yield($ctx->{env}, undef, \&stream_blob_parse_hdr, $ctx);
 }
 
-sub show_other_result ($$) {
+sub show_other_result ($$) { # future-proofing
         my ($bref, $ctx) = @_;
-        my ($qsp, $logref) = delete @$ctx{qw(-qsp -logref)};
-        if (my $err = $qsp->{err}) {
-                utf8::decode($$err);
-                $$logref .= "git show error: $err";
-                return html_page($ctx, 500, $logref);
+        if (my $qsp_err = delete $ctx->{-qsp_err}) {
+                return html_page($ctx, 500, dbg_log($ctx) .
+                                "git show error:$qsp_err");
         }
         my $l = PublicInbox::Linkify->new;
-        utf8::decode($$bref);
-        $$bref = '<pre>'. $l->to_html($$bref);
-        $$bref .= '</pre><hr>' . $$logref;
-        html_page($ctx, 200, $bref);
+        utf8_maybe($$bref);
+        html_page($ctx, 200, '<pre>', $l->to_html($$bref), '</pre><hr>',
+                dbg_log($ctx));
 }
 
-sub show_other ($$$$) {
-        my ($ctx, $res, $logref, $fn) = @_;
-        my ($git, $oid, $type, $size) = @$res;
-        if ($size > $MAX_SIZE) {
-                $$logref = "$oid is too big to show\n" . $$logref;
-                return html_page($ctx, 200, $logref);
+sub cmt_title { # git->cat_async callback
+        my ($bref, $oid, $type, $size, $ctx_cb) = @_;
+        utf8_maybe($$bref);
+        my $title = $$bref =~ /\r?\n\r?\n([^\r\n]+)\r?\n?/ ? $1 : '';
+        # $ctx_cb is [ $ctx, $cmt_fin ]
+        push @{$ctx_cb->[0]->{-cmt_pt}}, ascii_html($title);
+}
+
+sub do_cat_async {
+        my ($arg, $cb, @req) = @_;
+        # favor git(1) over Gcf2 (libgit2) for SHA-256 support
+        my $ctx = ref $arg eq 'ARRAY' ? $arg->[0] : $arg;
+        $ctx->{git}->cat_async($_, $cb, $arg) for @req;
+        if ($ctx->{env}->{'pi-httpd.async'}) {
+                $ctx->{git}->watch_async;
+        } else { # synchronous, generic PSGI
+                $ctx->{git}->cat_async_wait;
+        }
+}
+
+sub do_check_async {
+        my ($ctx, $cb, @req) = @_;
+        if ($ctx->{env}->{'pi-httpd.async'}) {
+                async_check($ctx, $_, $cb, $ctx) for @req;
+        } else { # synchronous, generic PSGI
+                $ctx->{git}->check_async($_, $cb, $ctx) for @req;
+                $ctx->{git}->check_async_wait;
+        }
+}
+
+sub cmt_hdr_prep { # psgi_qx cb
+        my ($fh, $ctx, $cmt_fin) = @_;
+        return if $ctx->{-qsp_err_h}; # let cmt_fin handle it
+        seek $fh, 0, SEEK_SET;
+        my $buf = do { local $/ = "\0"; <$fh> } // die "readline: $!";
+        chop($buf) eq "\0" or die 'no NUL in git show -z output';
+        utf8_maybe($buf); # non-UTF-8 commits exist
+        chomp $buf;
+        (my $P, my $p, @{$ctx->{cmt_info}}) = split(/\n/, $buf, 9);
+        truncate $fh, 0;
+        return unless $P;
+        seek $fh, 0, SEEK_SET;
+        my $qsp_p = PublicInbox::Qspawn->new($ctx->{git}->cmd(qw(show
+                --encoding=UTF-8 --pretty=format:%n -M --stat -p), $ctx->{oid}),
+                undef, { 1 => $fh });
+        $qsp_p->{qsp_err} = \($ctx->{-qsp_err_p} = '');
+        $qsp_p->psgi_qx($ctx->{env}, undef, \&cmt_patch_prep, $ctx, $cmt_fin);
+        @{$ctx->{-cmt_P}} = split / /, $P;
+        @{$ctx->{-cmt_p}} = split / /, $p; # abbreviated
+        do_cat_async([$ctx, $cmt_fin], \&cmt_title, @{$ctx->{-cmt_P}});
+}
+
+sub read_patchid { # psgi_qx cb
+        my ($bref, $ctx, $cmt_fin) = @_;
+        my ($patchid) = split(/ /, $$bref); # ignore commit
+        $ctx->{-q_value_html} = "patchid:$patchid" if defined $patchid;
+}
+
+sub cmt_patch_prep { # psgi_qx cb
+        my ($fh, $ctx, $cmt_fin) = @_;
+        return if $ctx->{-qsp_err_p}; # let cmt_fin handle error
+        return if -s $fh > $MAX_SIZE; # too big to show, too big to patch-id
+        seek $fh, 0, SEEK_SET;
+        my $qsp = PublicInbox::Qspawn->new(
+                                $ctx->{git}->cmd(qw(patch-id --stable)),
+                                undef, { 0 => $fh });
+        $qsp->{qsp_err} = \$ctx->{-qsp_err_p};
+        $qsp->psgi_qx($ctx->{env}, undef, \&read_patchid, $ctx, $cmt_fin);
+}
+
+sub ibx_url_for {
+        my ($ctx) = @_;
+        $ctx->{ibx} and return; # fall back to $upfx
+        $ctx->{git} or die 'BUG: no {git}';
+        if (my $ALL = $ctx->{www}->{pi_cfg}->ALL) {
+                if (defined(my $u = $ALL->base_url($ctx->{env}))) {
+                        return wantarray ? ($u) : $u;
+                }
+        }
+        my @ret;
+        if (my $ibx_names = $ctx->{git}->{ibx_names}) {
+                my $by_name = $ctx->{www}->{pi_cfg}->{-by_name};
+                for my $name (@$ibx_names) {
+                        my $ibx = $by_name->{$name} // do {
+                                warn "inbox `$name' no longer exists\n";
+                                next;
+                        };
+                        $ibx->isrch // next;
+                        my $u = defined($ibx->{url}) ?
+                                prurl($ctx->{env}, $ibx->{url}) : $name;
+                        $u .= '/' if substr($u, -1) ne '/';
+                        push @ret, $u;
+                }
         }
+        wantarray ? (@ret) : $ret[0];
+}
+
+sub cmt_fin { # OnDestroy cb
+        my ($ctx) = @_;
+        my ($eh, $ep) = delete @$ctx{qw(-qsp_err_h -qsp_err_p)};
+        if ($eh || $ep) {
+                my $e = join(' - ', grep defined, $eh, $ep);
+                return html_page($ctx, 500, dbg_log($ctx) .
+                                "git show/patch-id error:$e");
+        }
+        $ctx->{-linkify} //= PublicInbox::Linkify->new;
+        my $upfx = $ctx->{-upfx} = '../../'; # from "/$INBOX/$OID/s/"
+        my ($H, $T, $s, $f, $au, $co, $bdy) = @{delete $ctx->{cmt_info}};
+        # try to keep author and committer dates lined up
+        my $x = length($au) - length($co);
+        if ($x > 0) {
+                $x = ' ' x $x;
+                $co =~ s/>/>$x/;
+        } elsif ($x < 0) {
+                $x = ' ' x (-$x);
+                $au =~ s/>/>$x/;
+        }
+        $_ = ascii_html($_) for ($au, $co);
+        my $ibx_url = ibx_url_for($ctx) // $upfx;
+        $au =~ s!(&gt; +)([0-9]{4,}-\S+ \S+)!
+                my ($gt, $t) = ($1, $2);
+                $t =~ tr/ :-//d;
+                qq($gt<a
+href="$ibx_url?t=$t"
+title="list contemporary emails">$2</a>)
+                !e;
+
+        $ctx->{-title_html} = $s = $ctx->{-linkify}->to_html($s);
+        my ($P, $p, $pt) = delete @$ctx{qw(-cmt_P -cmt_p -cmt_pt)};
+        $_ = qq(<a href="$upfx$_/s/">).shift(@$p).'</a> '.shift(@$pt) for @$P;
+        if (@$P == 1) {
+                $x = qq{ (<a
+href="$f.patch">patch</a>)\n   <a href=#parent>parent</a> $P->[0]};
+        } elsif (@$P > 1) {
+                $x = qq(\n  <a href=#parents>parents</a> $P->[0]\n);
+                shift @$P;
+                $x .= qq(          $_\n) for @$P;
+                chop $x;
+        } else {
+                $x = ' (<a href=#root_commit>root commit</a>)';
+        }
+        PublicInbox::WwwStream::html_init($ctx);
+        my $zfh = $ctx->zfh;
+        print $zfh <<EOM;
+<pre>   <a href=#commit>commit</a> $H$x
+     <a href=#tree>tree</a> <a href="$upfx$T/s/?b=">$T</a>
+   author $au
+committer $co
+
+<b>$s</b>
+EOM
+        print $zfh "\n", $ctx->{-linkify}->to_html($bdy) if length($bdy);
+        undef $bdy; # free memory
+        my $fh = delete $ctx->{patch_fh};
+        if (-s $fh > $MAX_SIZE) {
+                print $zfh '</pre><hr><pre>patch is too large to show</pre>';
+        } else { # prepare flush_diff:
+                seek $fh, 0, SEEK_SET;
+                PublicInbox::IO::read_all $fh, -s _, \$x;
+                utf8_maybe($x);
+                $ctx->{-apfx} = $ctx->{-spfx} = $upfx;
+                $x =~ s/\r?\n/\n/gs;
+                $ctx->{-anchors} = {} if $x =~ /^diff --git /sm;
+                flush_diff($ctx, \$x); # undefs $x
+                # TODO: should there be another textarea which attempts to
+                # search for the exact email which was applied to make this
+                # commit?
+                if (my $qry = delete $ctx->{-qry}) {
+                        my $q = '';
+                        for (@{$qry->{dfpost}}, @{$qry->{dfpre}}) {
+                                # keep blobs as short as reasonable, emails
+                                # are going to be older than what's in git
+                                substr($_, 7, 64, '');
+                                $q .= "dfblob:$_ ";
+                        }
+                        chop $q; # no trailing SP
+                        local $Text::Wrap::columns = PublicInbox::View::COLS;
+                        local $Text::Wrap::huge = 'overflow';
+                        $q = wrap('', '', $q);
+                        my $rows = ($q =~ tr/\n/\n/) + 1;
+                        $q = ascii_html($q);
+                        my $ibx_url = ibx_url_for($ctx);
+                        my $alt;
+                        if (defined $ibx_url) {
+                                $alt = " `$ibx_url'";
+                                $ibx_url =~ m!://! or
+                                        substr($ibx_url, 0, 0, '../../../');
+                                $ibx_url = ascii_html($ibx_url);
+                        } else {
+                                $ibx_url = $upfx;
+                                $alt = '';
+                        }
+                        print $zfh <<EOM;
+</pre><hr><form action="$ibx_url"
+id=related><pre>find related emails, including ancestors/descendants/conflicts
+<textarea name=q cols=${\PublicInbox::View::COLS} rows=$rows>$q</textarea>
+<input type=submit value="search$alt"
+/>\t(<a href="${ibx_url}_/text/help/">help</a>)</pre></form>
+EOM
+                }
+        }
+        chop($x = <<EOM);
+<hr><pre>glossary
+--------
+<dfn
+id=commit>Commit</dfn> objects reference one tree, and zero or more parents.
+
+Single <dfn
+id=parent>parent</dfn> commits can typically generate a patch in
+unified diff format via `git format-patch'.
+
+Multiple <dfn id=parents>parents</dfn> means the commit is a merge.
+
+<dfn id=root_commit>Root commits</dfn> have no ancestor.  Note that it is
+possible to have multiple root commits when merging independent histories.
+
+Every commit references one top-level <dfn id=tree>tree</dfn> object.</pre>
+EOM
+        delete($ctx->{-wcb})->($ctx->html_done($x));
+}
+
+sub stream_patch_parse_hdr { # {parse_hdr} for Qspawn
+        my ($r, $bref, $ctx) = @_;
+        if (!defined $r) { # sysread error
+                html_page($ctx, 500, dbg_log($ctx));
+        } elsif (index($$bref, "\n\n") >= 0) {
+                my $eml = bless { hdr => $bref }, 'PublicInbox::Eml';
+                my $fn = to_filename($eml->header('Subject') // '');
+                $fn = substr($fn // 'PATCH-no-subject', 6); # drop "PATCH-"
+                return [ 200, [ 'Content-Type', 'text/plain; charset=UTF-8',
+                                'Content-Disposition',
+                                qq(inline; filename=$fn.patch) ] ];
+        } elsif ($r == 0) {
+                my $log = dbg_log($ctx);
+                warn "premature EOF on $ctx->{patch_oid} $log";
+                return html_page($ctx, 500, $log);
+        } else {
+                undef; # bref keeps growing until "\n\n"
+        }
+}
+
+sub show_patch ($$) {
+        my ($ctx, $res) = @_;
+        my ($git, $oid) = @$res;
+        my @cmd = ('git', "--git-dir=$git->{git_dir}",
+                qw(format-patch -1 --stdout -C),
+                "--signature=git format-patch -1 --stdout -C $oid", $oid);
+        my $qsp = PublicInbox::Qspawn->new(\@cmd);
+        $ctx->{env}->{'qspawn.wcb'} = $ctx->{-wcb};
+        $ctx->{patch_oid} = $oid;
+        $qsp->psgi_yield($ctx->{env}, undef, \&stream_patch_parse_hdr, $ctx);
+}
+
+sub show_commit ($$) {
+        my ($ctx, $res) = @_;
+        return show_patch($ctx, $res) if ($ctx->{fn} // '') =~ /\.patch\z/;
+        my ($git, $oid) = @$res;
+        # patch-id needs two passes, and we use the initial show to ensure
+        # a patch embedded inside the commit message body doesn't get fed
+        # to patch-id:
+        open $ctx->{patch_fh}, '+>', "$ctx->{-tmp}/show";
+        my $qsp_h = PublicInbox::Qspawn->new($git->cmd('show', $SHOW_FMT,
+                        qw(--encoding=UTF-8 -z --no-notes --no-patch), $oid),
+                        undef, { 1 => $ctx->{patch_fh} });
+        $qsp_h->{qsp_err} = \($ctx->{-qsp_err_h} = '');
+        my $cmt_fin = PublicInbox::OnDestroy->new($$, \&cmt_fin, $ctx);
+        $ctx->{git} = $git;
+        $ctx->{oid} = $oid;
+        $qsp_h->psgi_qx($ctx->{env}, undef, \&cmt_hdr_prep, $ctx, $cmt_fin);
+}
+
+sub show_other ($$) { # just in case...
+        my ($ctx, $res) = @_;
+        my ($git, $oid, $type, $size) = @$res;
+        $size > $MAX_SIZE and return html_page($ctx, 200,
+                ascii_html($type)." $oid is too big to show\n". dbg_log($ctx));
         my $cmd = ['git', "--git-dir=$git->{git_dir}",
                 qw(show --encoding=UTF-8 --no-color --no-abbrev), $oid ];
         my $qsp = PublicInbox::Qspawn->new($cmd);
-        my $env = $ctx->{env};
-        $ctx->{-qsp} = $qsp;
-        $ctx->{-logref} = $logref;
-        $qsp->psgi_qx($env, undef, \&show_other_result, $ctx);
+        $qsp->{qsp_err} = \($ctx->{-qsp_err} = '');
+        $qsp->psgi_qx($ctx->{env}, undef, \&show_other_result, $ctx);
 }
 
-# user_cb for SolverGit, called as: user_cb->($result_or_error, $uarg)
-sub solve_result {
-        my ($res, $ctx) = @_;
-        my ($log, $hints, $fn) = delete @$ctx{qw(log hints fn)};
-
-        unless (seek($log, 0, 0)) {
-                warn "seek(log): $!";
-                return html_page($ctx, 500, \'seek error');
+sub show_tree_result ($$) {
+        my ($bref, $ctx) = @_;
+        if (my $qsp_err = delete $ctx->{-qsp_err}) {
+                return html_page($ctx, 500, dbg_log($ctx) .
+                                "git ls-tree -z error:$qsp_err");
         }
-        $log = do { local $/; <$log> };
+        my @ent = split(/\0/, $$bref);
+        my $qp = delete $ctx->{qp};
+        my $l = $ctx->{-linkify} //= PublicInbox::Linkify->new;
+        my $pfx = $ctx->{-path} // $qp->{b}; # {-path} is from RepoTree
+        $$bref = "<pre><a href=#tree>tree</a> $ctx->{tree_oid}";
+        # $REPO/tree/$path already sets {-upfx}
+        my $upfx = $ctx->{-upfx} //= '../../';
+        if (defined $pfx) {
+                $pfx =~ s!/+\z!!s;
+                if (my $t = $ctx->{-obj}) {
+                        my $t = ascii_html($t);
+                        $$bref .= <<EOM
+\n\$ git ls-tree -l $t        # shows similar output on the CLI
+EOM
+                } elsif ($pfx eq '') {
+                        $$bref .= "  (root)\n";
+                } else {
+                        my $x = ascii_html($pfx);
+                        $pfx .= '/';
+                        $$bref .= qq(  <a href=#path>path</a>: $x</a>\n);
+                }
+        } else {
+                $pfx = '';
+                $$bref .= qq[  (<a href=#path>path</a> unknown)\n];
+        }
+        my ($x, $m, $t, $oid, $sz, $f, $n, $gitlink);
+        $$bref .= "\n        size        name";
+        for (@ent) {
+                ($x, $f) = split(/\t/, $_, 2);
+                undef $_;
+                ($m, $t, $oid, $sz) = split(/ +/, $x, 4);
+                $m = $GIT_MODE{$m} // '?';
+                utf8_maybe($f);
+                $n = ascii_html($f);
+                if ($m eq 'g') { # gitlink submodule commit
+                        $$bref .= "\ng\t\t$n @ <a\nhref=#g>commit</a>$oid";
+                        $gitlink = 1;
+                        next;
+                }
+                my $q = 'b='.ascii_html(uri_escape_path($pfx.$f));
+                if ($m eq 'd') { $n .= '/' }
+                elsif ($m eq 'x') { $n = "<b>$n</b>" }
+                elsif ($m eq 'l') { $n = "<i>$n</i>" }
+                $$bref .= qq(\n$m\t$sz\t<a\nhref="$upfx$oid/s/?$q">$n</a>);
+        }
+        $$bref .= dbg_log($ctx);
+        $$bref .= <<EOM;
+<hr><pre>glossary
+--------
+<dfn
+id=tree>Tree</dfn> objects belong to commits or other tree objects.  Trees may
+reference blobs, sub-trees, or (rarely) commits of submodules.
+
+<dfn
+id=path>Path</dfn> names are stored in tree objects, but trees do not know
+their own path name.  A tree's path name comes from their parent tree,
+or it is the root tree referenced by a commit object.  Thus, this web UI
+relies on the `b=' URI parameter as a hint to display the path name.
+EOM
+
+        $$bref .= <<EOM if $gitlink;
 
-        my $ref = ref($res);
+<dfn title="submodule commit"
+id=g>Commit</dfn> objects may be stored in trees to reference submodules.</pre>
+EOM
+        chop $$bref;
+        html_page($ctx, 200, $$bref);
+}
+
+sub show_tree ($$) { # also used by RepoTree
+        my ($ctx, $res) = @_;
+        my ($git, $oid, undef, $size) = @$res;
+        $size > $MAX_SIZE and return html_page($ctx, 200,
+                        "tree $oid is too big to show\n". dbg_log($ctx));
+        my $cmd = [ 'git', "--git-dir=$git->{git_dir}",
+                qw(ls-tree -z -l --no-abbrev), $oid ];
+        my $qsp = PublicInbox::Qspawn->new($cmd);
+        $ctx->{tree_oid} = $oid;
+        $qsp->{qsp_err} = \($ctx->{-qsp_err} = '');
+        $qsp->psgi_qx($ctx->{env}, undef, \&show_tree_result, $ctx);
+}
+
+# returns seconds offset from git TZ offset
+sub tz_adj ($) {
+        my ($tz) = @_; # e.g "-0700"
+        $tz = int($tz);
+        my $mm = $tz < 0 ? -$tz : $tz;
+        $mm = int($mm / 100) * 60 + ($mm % 100);
+        $mm = $tz < 0 ? -$mm : $mm;
+        ($mm * 60);
+}
+
+sub show_tag_result { # git->cat_async callback
+        my ($bref, $oid, $type, $size, $ctx) = @_;
+        utf8_maybe($$bref);
         my $l = PublicInbox::Linkify->new;
-        $log = '<pre>debug log:</pre><hr /><pre>' .
-                $l->to_html($log) . '</pre>';
+        $$bref = $l->to_html($$bref);
+        $$bref =~ s!^object ([a-f0-9]+)!object <a
+href=../../$1/s/>$1</a>!;
 
-        $res or return html_page($ctx, 404, \$log);
-        $ref eq 'ARRAY' or return html_page($ctx, 500, \$log);
+        $$bref =~ s/^(tagger .*&gt; )([0-9]+) ([\-+]?[0-9]+)/$1.strftime(
+                '%Y-%m-%d %H:%M:%S', gmtime($2 + tz_adj($3)))." $3"/sme;
+        # TODO: download link
+        html_page($ctx, 200, '<pre>', $$bref, '</pre>', dbg_log($ctx));
+}
+
+sub show_tag ($$) {
+        my ($ctx, $res) = @_;
+        my ($git, $oid) = @$res;
+        $ctx->{git} = $git;
+        do_cat_async($ctx, \&show_tag_result, $oid);
+}
+
+# user_cb for SolverGit, called as: user_cb->($result_or_error, $uarg)
+sub solve_result {
+        my ($res, $ctx) = @_;
+        my $hints = delete $ctx->{hints};
+        $res or return html_page($ctx, 404, 'Not found', dbg_log($ctx));
+        ref($res) eq 'ARRAY' or
+                return html_page($ctx, 500, 'Internal error', dbg_log($ctx));
 
         my ($git, $oid, $type, $size, $di) = @$res;
-        return show_other($ctx, $res, \$log, $fn) if $type ne 'blob';
-        my $path = to_filename($di->{path_b} // $hints->{path_b} // 'blob');
-        my $raw_link = "(<a\nhref=$path>raw</a>)";
+        return show_commit($ctx, $res) if $type eq 'commit';
+        return show_tree($ctx, $res) if $type eq 'tree';
+        return show_tag($ctx, $res) if $type eq 'tag';
+        return show_other($ctx, $res) if $type ne 'blob';
+        my $fn = $di->{path_b} // $hints->{path_b};
+        my $paths = $ctx->{-paths} //= do {
+                my $path = to_filename($fn // 'blob') // 'blob';
+                my $raw_more = qq[(<a\nhref="$path">raw</a>)];
+                my @def;
+
+                # XXX not sure if this is the correct wording
+                if (defined($fn)) {
+                        $raw_more .= qq(
+name: ${\ascii_html($fn)} \t # note: path name is non-authoritative<a
+href="#pathdef" id=top>(*)</a>);
+                        $def[0] = "<hr><pre\nid=pathdef>" .
+'(*) Git path names are given by the tree(s) the blob belongs to.
+    Blobs themselves have no identifier aside from the hash of its contents.'.
+qq(<a\nhref="#top">^</a></pre>);
+                }
+                [ $path, $raw_more, @def ];
+        };
+        $ctx->{-q_value_html} //= do {
+                my $s = defined($fn) ? 'dfn:'.ascii_html($fn).' ' : '';
+                $s.'dfpost:'.substr($oid, 0, 7);
+        };
+
         if ($size > $MAX_SIZE) {
-                return stream_large_blob($ctx, $res, \$log, $fn) if defined $fn;
-                $log = "<pre><b>Too big to show, download available</b>\n" .
-                        "$oid $type $size bytes $raw_link</pre>" . $log;
-                return html_page($ctx, 200, \$log);
+                return stream_large_blob($ctx, $res) if defined $ctx->{fn};
+                return html_page($ctx, 200, <<EOM . dbg_log($ctx));
+<pre><b>Too big to show, download available</b>
+blob $oid $size bytes $paths->[1]</pre>
+EOM
         }
+        bless $ctx, 'PublicInbox::WwwStream'; # for DESTROY
+        $ctx->{git} = $git;
+        do_cat_async($ctx, \&show_blob, $oid);
+}
 
-        my $blob = $git->cat_file($oid);
-        if (!$blob) { # WTF?
+sub show_blob { # git->cat_async callback
+        my ($blob, $oid, $type, $size, $ctx) = @_;
+        if (!$blob) {
                 my $e = "Failed to retrieve generated blob ($oid)";
-                warn "$e ($git->{git_dir})";
-                $log = "<pre><b>$e</b></pre>" . $log;
-                return html_page($ctx, 500, \$log);
+                warn "$e ($ctx->{git}->{git_dir}) type=$type";
+                return html_page($ctx, 500, "<pre><b>$e</b></pre>".dbg_log($ctx))
         }
 
         my $bin = index(substr($$blob, 0, $BIN_DETECT), "\0") >= 0;
-        if (defined $fn) {
+        if (defined $ctx->{fn}) {
                 my $h = [ 'Content-Length', $size, 'Content-Type' ];
                 push(@$h, ($bin ? 'application/octet-stream' : 'text/plain'));
                 return delete($ctx->{-wcb})->([200, $h, [ $$blob ]]);
         }
 
-        if ($bin) {
-                $log = "<pre>$oid $type $size bytes (binary)" .
-                        " $raw_link</pre>" . $log;
-                return html_page($ctx, 200, \$log);
-        }
+        my ($path, $raw_more, @def) = @{delete $ctx->{-paths}};
+        $bin and return html_page($ctx, 200,
+                                "<pre>blob $oid $size bytes (binary)" .
+                                " $raw_more</pre>".dbg_log($ctx));
 
         # TODO: detect + convert to ensure validity
-        utf8::decode($$blob);
+        utf8_maybe($$blob);
         my $nl = ($$blob =~ s/\r?\n/\n/sg);
         my $pad = length($nl);
 
-        $l->linkify_1($$blob);
+        ($ctx->{-linkify} //= PublicInbox::Linkify->new)->linkify_1($$blob);
         my $ok = $hl->do_hl($blob, $path) if $hl;
         if ($ok) {
                 $blob = $ok;
@@ -170,38 +607,63 @@ sub solve_result {
         }
 
         # using some of the same CSS class names and ids as cgit
-        $log = "<pre>$oid $type $size bytes $raw_link</pre>" .
+        my $x = "<pre>blob $oid $size bytes $raw_more</pre>" .
                 "<hr /><table\nclass=blob>".
-                "<tr><td\nclass=linenumbers><pre>" . join('', map {
-                        sprintf("<a id=n$_ href=#n$_>% ${pad}u</a>\n", $_)
-                } (1..$nl)) . '</pre></td>' .
-                '<td><pre> </pre></td>'. # pad for non-CSS users
-                "<td\nclass=lines><pre\nstyle='white-space:pre'><code>" .
-                $l->linkify_2($$blob) .
-                '</code></pre></td></tr></table>' . $log;
-
-        html_page($ctx, 200, \$log);
+                "<tr><td\nclass=linenumbers><pre>";
+        # scratchpad in this loop is faster here than `printf $zfh':
+        $x .= sprintf("<a id=n$_ href=#n$_>% ${pad}u</a>\n", $_) for (1..$nl);
+        $x .= '</pre></td><td><pre> </pre></td>'. # pad for non-CSS users
+                "<td\nclass=lines><pre\nstyle='white-space:pre'><code>";
+        html_page($ctx, 200, $x, $ctx->{-linkify}->linkify_2($$blob),
+                '</code></pre></td></tr></table>'.dbg_log($ctx), @def);
 }
 
-# GET /$INBOX/$GIT_OBJECT_ID/s/
-# GET /$INBOX/$GIT_OBJECT_ID/s/$FILENAME
-sub show ($$;$) {
-        my ($ctx, $oid_b, $fn) = @_;
-        my $qp = $ctx->{qp};
-        my $hints = $ctx->{hints} = {};
+sub start_solver ($) {
+        my ($ctx) = @_;
         while (my ($from, $to) = each %QP_MAP) {
-                defined(my $v = $qp->{$from}) or next;
-                $hints->{$to} = $v if $v ne '';
+                my $v = $ctx->{qp}->{$from} // next;
+                $ctx->{hints}->{$to} = $v if $v ne '';
         }
-
-        $ctx->{'log'} = tmpfile("solve.$oid_b") // die "tmpfile: $!";
-        $ctx->{fn} = $fn;
+        $ctx->{-next_solver} = PublicInbox::OnDestroy->new($$, \&next_solver);
+        ++$solver_nr;
+        $ctx->{-tmp} = File::Temp->newdir("solver.$ctx->{oid_b}-XXXX",
+                                                TMPDIR => 1);
+        $ctx->{lh} or open $ctx->{lh}, '+>>', "$ctx->{-tmp}/solve.log";
         my $solver = PublicInbox::SolverGit->new($ctx->{ibx},
                                                 \&solve_result, $ctx);
+        $solver->{limiter} = $solver_lim;
+        $solver->{gits} //= [ $ctx->{git} ];
+        $solver->{tmp} = $ctx->{-tmp}; # share tmpdir
         # PSGI server will call this immediately and give us a callback (-wcb)
+        $solver->solve(@$ctx{qw(env lh oid_b hints)});
+}
+
+# run the next solver job when done and DESTROY-ed
+sub next_solver {
+        --$solver_nr;
+        # XXX FIXME: client may've disconnected if it waited a long while
+        start_solver(shift(@solver_q) // return);
+}
+
+sub may_start_solver ($) {
+        my ($ctx) = @_;
+        $solver_lim //= $ctx->{www}->{pi_cfg}->limiter('codeblob');
+        if ($solver_nr >= $solver_lim->{max}) {
+                @solver_q > 128 ? html_page($ctx, 503, 'too busy')
+                                : push(@solver_q, $ctx);
+        } else {
+                start_solver($ctx);
+        }
+}
+
+# GET /$INBOX/$GIT_OBJECT_ID/s/
+# GET /$INBOX/$GIT_OBJECT_ID/s/$FILENAME
+sub show ($$;$) {
+        my ($ctx, $oid_b, $fn) = @_;
+        @$ctx{qw(oid_b fn)} = ($oid_b, $fn);
         sub {
                 $ctx->{-wcb} = $_[0]; # HTTP write callback
-                $solver->solve($ctx->{env}, $ctx->{log}, $oid_b, $hints);
+                may_start_solver $ctx;
         };
 }
 
diff --git a/lib/PublicInbox/WQBlocked.pm b/lib/PublicInbox/WQBlocked.pm
new file mode 100644
index 00000000..8d931fa9
--- /dev/null
+++ b/lib/PublicInbox/WQBlocked.pm
@@ -0,0 +1,48 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# non-blocking workqueues, currently used by LeiNoteEvent to track renames
+package PublicInbox::WQBlocked;
+use v5.12;
+use parent qw(PublicInbox::DS);
+use PublicInbox::Syscall qw(EPOLLOUT EPOLLONESHOT);
+use PublicInbox::IPC;
+use Carp ();
+
+sub new {
+        my ($cls, $wq, $buf) = @_;
+        my $self = bless { msgq => [$buf], }, $cls;
+        $wq->{wqb} = $self->SUPER::new($wq->{-wq_s1}, EPOLLOUT|EPOLLONESHOT);
+}
+
+sub flush_send {
+        my ($self) = @_;
+        push(@{$self->{msgq}}, $_[1]) if defined($_[1]);
+        while (defined(my $buf = shift @{$self->{msgq}})) {
+                if (ref($buf) eq 'CODE') {
+                        $buf->($self); # could be \&PublicInbox::DS::close
+                } else {
+                        my $wq_s1 = $self->{sock};
+                        my $n = $PublicInbox::IPC::send_cmd->($wq_s1, [], $buf,
+                                                                0);
+                        next if defined($n);
+                        Carp::croak("sendmsg: $!") unless $!{EAGAIN};
+                        PublicInbox::DS::epwait($wq_s1, EPOLLOUT|EPOLLONESHOT);
+                        unshift @{$self->{msgq}}, $buf;
+                        last; # wait for ->event_step
+                }
+        }
+}
+
+sub enq_close { flush_send($_[0], $_[0]->can('close')) }
+
+sub event_step { # called on EPOLLOUT wakeup
+        my ($self) = @_;
+        eval { flush_send($self) } if $self->{sock};
+        if ($@) {
+                warn $@;
+                $self->close;
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 755d7558..289599b8 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -14,6 +14,7 @@ package PublicInbox::WWW;
 use strict;
 use v5.10.1;
 use PublicInbox::Config;
+use PublicInbox::Git;
 use PublicInbox::Hval;
 use URI::Escape qw(uri_unescape);
 use PublicInbox::MID qw(mid_escape);
@@ -23,9 +24,9 @@ use PublicInbox::WwwStatic qw(r path_info_raw);
 use PublicInbox::Eml;
 
 # TODO: consider a routing tree now that we have more endpoints:
-our $INBOX_RE = qr!\A/([\w\-][\w\.\-]*)!;
+our $INBOX_RE = qr!\A/([\w\-][\w\.\-\+]*)!;
 our $MID_RE = qr!([^/]+)!;
-our $END_RE = qr!(T/|t/|t\.mbox(?:\.gz)?|t\.atom|raw|)!;
+our $END_RE = qr!(T/|t/|d/|t\.mbox(?:\.gz)?|t\.atom|raw|)!;
 our $ATTACH_RE = qr!([0-9][0-9\.]*)-($PublicInbox::Hval::FN)!;
 our $OID_RE = qr![a-f0-9]{7,}!;
 
@@ -45,14 +46,21 @@ sub call {
         my $ctx = { env => $env, www => $self };
 
         # we don't care about multi-value
-        %{$ctx->{qp}} = map {
-                utf8::decode($_);
-                tr/+/ /;
-                my ($k, $v) = split(/=/, $_, 2);
-                # none of the keys we care about will need escaping
-                ($k // '', uri_unescape($v // ''))
-        } split(/[&;]+/, $env->{QUERY_STRING});
-
+        # '0' isn't a QUERY_STRING we care about
+        if (my $qs = $env->{QUERY_STRING}) {
+                utf8::decode($qs);
+                $qs =~ tr/+/ /;
+                %{$ctx->{qp}} = map {
+                        # we only use single-char query param keys
+                        if (s/\A([A-Za-z])=//) {
+                                $1 => uri_unescape($_)
+                        } elsif (/\A[a-z]\z/) { # some boolean options
+                                $_ => ''
+                        } else {
+                                () # ignored
+                        }
+                } split(/[&;]+/, $qs);
+        }
         my $path_info = path_info_raw($env);
         my $method = $env->{REQUEST_METHOD};
 
@@ -68,7 +76,9 @@ sub call {
                         my ($idx, $fn) = ($3, $4);
                         return invalid_inbox_mid($ctx, $1, $2) ||
                                 get_attach($ctx, $idx, $fn);
-                } elsif ($path_info =~ m!$INBOX_RE/!o) {
+                } elsif ($path_info =~ m!$INBOX_RE/$MID_RE/\z!o) {
+                        return invalid_inbox_mid($ctx, $1, $2) || mbox_results($ctx);
+                } elsif ($path_info =~ m!$INBOX_RE/\z!o) {
                         return invalid_inbox($ctx, $1) || mbox_results($ctx);
                 }
         }
@@ -91,6 +101,9 @@ sub call {
                 invalid_inbox($ctx, $1) || get_atom($ctx);
         } elsif ($path_info =~ m!$INBOX_RE/new\.html\z!o) {
                 invalid_inbox($ctx, $1) || get_new($ctx);
+        } elsif ($path_info =~
+                        m!$INBOX_RE/topics_(new|active)\.(atom|html)\z!o) {
+                get_topics($ctx, $1, $2, $3);
         } elsif ($path_info =~ m!$INBOX_RE/description\z!o) {
                 get_description($ctx, $1);
         } elsif ($path_info =~ m!$INBOX_RE/(?:(?:git/)?([0-9]+)(?:\.git)?/)?
@@ -176,6 +189,7 @@ sub preload {
                 }
                 $pi_cfg->ALL and require PublicInbox::Isearch;
                 $self->cgit;
+                $self->coderepo;
                 $self->stylesheets_prepare($_) for ('', '../', '../../');
                 $self->news_www;
         }
@@ -194,10 +208,20 @@ sub r404 {
 
 sub news_cgit_fallback ($) {
         my ($ctx) = @_;
-        my $www = $ctx->{www};
-        my $env = $ctx->{env};
-        my $res = $www->news_www->call($env);
-        $res->[0] == 404 ? $www->cgit->call($env) : $res;
+        my $res = $ctx->{www}->news_www->call($ctx->{env});
+
+        $res->[0] == 404 and ($ctx->{www}->{cgit_fallback} //= do {
+                my $c = $ctx->{www}->{pi_cfg}->{'publicinbox.cgit'} // 'first';
+                $c ne 'first' # `fallback' and `rewrite' => true
+        } // 0) and $res = $ctx->{www}->coderepo->srv($ctx);
+
+        ref($res) eq 'ARRAY' && $res->[0] == 404 and
+                $res = $ctx->{www}->cgit->call($ctx->{env}, $ctx);
+
+        ref($res) eq 'ARRAY' && $res->[0] == 404 &&
+                        !$ctx->{www}->{cgit_fallback} and
+                $res = $ctx->{www}->coderepo->srv($ctx);
+        $res;
 }
 
 # returns undef if valid, array ref response if invalid
@@ -250,6 +274,13 @@ sub get_new {
         PublicInbox::Feed::new_html($ctx);
 }
 
+# /$INBOX/topics_(new|active).(html|atom)
+sub get_topics {
+        my ($ctx, $ibx_name, $category, $type) = @_;
+        require PublicInbox::WwwTopics;
+        PublicInbox::WwwTopics::response($ctx, $ibx_name, $category, $type);
+}
+
 # /$INBOX/?r=$GIT_COMMIT                 -> HTML only
 sub get_index {
         my ($ctx) = @_;
@@ -303,7 +334,8 @@ sub get_text {
 sub get_vcs_object ($$$;$) {
         my ($ctx, $inbox, $oid, $filename) = @_;
         my $r404 = invalid_inbox($ctx, $inbox);
-        return $r404 if $r404 || !$ctx->{www}->{pi_cfg}->repo_objs($ctx->{ibx});
+        return $r404 if $r404;
+        return r(404) if !$ctx->{www}->{pi_cfg}->repo_objs($ctx->{ibx});
         require PublicInbox::ViewVCS;
         PublicInbox::ViewVCS::show($ctx, $oid, $filename);
 }
@@ -317,11 +349,12 @@ sub get_altid_dump {
 }
 
 sub need {
-        my ($ctx, $extra) = @_;
+        my ($ctx, $extra, $upref) = @_;
         require PublicInbox::WwwStream;
-        PublicInbox::WwwStream::html_oneshot($ctx, 501, \<<EOF);
+        $upref //= '../';
+        PublicInbox::WwwStream::html_oneshot($ctx, 501, <<EOF);
 <pre>$extra is not available for this public-inbox
-<a\nhref="../">Return to index</a></pre>
+<a\nhref="$upref">Return to index</a></pre>
 EOF
 }
 
@@ -441,6 +474,10 @@ sub msg_page {
 
         # legacy, but no redirect for compatibility:
         'f/' eq $e and return get_mid_html($ctx);
+        if ($e eq 'd/') {
+                require PublicInbox::View;
+                return PublicInbox::View::diff_msg($ctx);
+        }
         r404($ctx);
 }
 
@@ -480,16 +517,21 @@ sub news_www {
 
 sub cgit {
         my ($self) = @_;
-        $self->{cgit} //= do {
-                my $pi_cfg = $self->{pi_cfg};
-
-                if (defined($pi_cfg->{'publicinbox.cgitrc'})) {
+        $self->{cgit} //=
+                (defined($self->{pi_cfg}->{'publicinbox.cgitrc'}) ? do {
                         require PublicInbox::Cgit;
-                        PublicInbox::Cgit->new($pi_cfg);
-                } else {
+                        PublicInbox::Cgit->new($self->{pi_cfg});
+                } : undef) // do {
                         require Plack::Util;
                         Plack::Util::inline_object(call => sub { r404() });
-                }
+                };
+}
+
+sub coderepo {
+        my ($self) = @_;
+        $self->{coderepo} //= do {
+                require PublicInbox::WwwCoderepo;
+                PublicInbox::WwwCoderepo->new($self->{pi_cfg});
         }
 }
 
@@ -558,9 +600,9 @@ sub stylesheets_prepare ($$) {
                                 next;
                         };
                         my $ctime = 0;
-                        my $local = do { local $/; <$fh> };
+                        my $local = PublicInbox::IO::read_all $fh; # sets _
                         if ($local =~ /\S/) {
-                                $ctime = sprintf('%x',(stat($fh))[10]);
+                                $ctime = sprintf('%x',(stat(_))[10]);
                                 $local = $mini->($local);
                         }
 
diff --git a/lib/PublicInbox/WWW.pod b/lib/PublicInbox/WWW.pod
index 9f6ba466..b55f010e 100644
--- a/lib/PublicInbox/WWW.pod
+++ b/lib/PublicInbox/WWW.pod
@@ -20,13 +20,14 @@ The PSGI web interface for public-inbox.
 
 Using this directly is not needed unless you wish to customize
 your public-inbox PSGI deployment or are using a PSGI server
-other than L<public-inbox-httpd(1)>.
+other than L<public-inbox-netd(1)> (C<-netd>) /
+L<public-inbox-httpd(1)> (C<-httpd>)
 
-While this PSGI application works with all PSGI/Plack web
+While this PSGI application should work with all PSGI/Plack web
 servers such as L<starman(1)>, L<starlet(1)> or L<twiggy(1)>;
-PublicInbox::WWW takes advantage of currently-undocumented APIs
-of L<public-inbox-httpd(1)> to improve fairness when serving
-large responses for thread views and git clones.
+PublicInbox::WWW takes advantage of internal APIs of C<-netd>
+and C<-httpd> to improve fairness when serving large responses
+for thread views, mbox downloads, and git clones.
 
 =head1 ENVIRONMENT
 
@@ -47,10 +48,11 @@ and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta
 
 =head1 COPYRIGHT
 
-Copyright (C) 2016-2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright (C) all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<http://www.gnu.org/licenses/agpl-3.0.txt>
 
 =head1 SEE ALSO
 
-L<http://plackperl.org/>, L<Plack>, L<public-inbox-httpd(1)>
+L<http://plackperl.org/>, L<Plack>, L<public-inbox-netd(1)>,
+L<public-inbox-httpd(1)>
diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index 3f6fe21b..1ec574ea 100644
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # ref: https://cr.yp.to/proto/maildir.html
@@ -12,11 +12,11 @@ use PublicInbox::MdirReader;
 use PublicInbox::NetReader;
 use PublicInbox::Filter::Base qw(REJECT);
 use PublicInbox::Spamcheck;
-use PublicInbox::DS qw(now add_timer);
+use PublicInbox::DS qw(now add_timer awaitpid);
 use PublicInbox::MID qw(mids);
 use PublicInbox::ContentHash qw(content_hash);
-use PublicInbox::EOFpipe;
 use POSIX qw(_exit WNOHANG);
+use constant { D_MAILDIR => 1, D_MH => 2 };
 
 sub compile_watchheaders ($) {
         my ($ibx) = @_;
@@ -41,11 +41,25 @@ sub compile_watchheaders ($) {
         $ibx->{-watchheaders} = $watch_hdrs if scalar @$watch_hdrs;
 }
 
+sub d_type_set ($$$) {
+        my ($d_type, $dir, $is) = @_;
+        my $isnt = D_MAILDIR;
+        if ($is == D_MAILDIR) {
+                $isnt = D_MH;
+                $d_type->{"$dir/cur"} |= $is;
+                $d_type->{"$dir/new"} |= $is;
+        }
+        warn <<EOM if ($d_type->{$dir} |= $is) & $isnt;
+W: `$dir' is both Maildir and MH (non-fatal)
+EOM
+}
+
 sub new {
         my ($class, $cfg) = @_;
-        my (%mdmap, $spamc);
+        my (%d_map, %d_type);
         my (%imap, %nntp); # url => [inbox objects] or 'watchspam'
         my (@imap, @nntp);
+        PublicInbox::Import::load_config($cfg);
 
         # "publicinboxwatch" is the documented namespace
         # "publicinboxlearn" is legacy but may be supported
@@ -57,7 +71,11 @@ sub new {
                         my $uri;
                         if (is_maildir($dir)) {
                                 # skip "new", no MUA has seen it, yet.
-                                $mdmap{"$dir/cur"} = 'watchspam';
+                                $d_map{"$dir/cur"} = 'watchspam';
+                                d_type_set \%d_type, $dir, D_MAILDIR;
+                        } elsif (is_mh($dir)) {
+                                $d_map{$dir} = 'watchspam';
+                                d_type_set \%d_type, $dir, D_MH;
                         } elsif ($uri = imap_uri($dir)) {
                                 $imap{$$uri} = 'watchspam';
                                 push @imap, $uri;
@@ -69,7 +87,6 @@ sub new {
                         }
                 }
         }
-
         my $k = 'publicinboxwatch.spamcheck';
         my $default = undef;
         my $spamcheck = PublicInbox::Spamcheck::get($cfg, $k, $default);
@@ -80,16 +97,28 @@ sub new {
                 my $ibx = $_[0] = PublicInbox::InboxWritable->new($_[0]);
 
                 my $watches = $ibx->{watch} or return;
+
+                $ibx->{indexlevel} //= $ibx->detect_indexlevel;
                 $watches = PublicInbox::Config::_array($watches);
                 for my $watch (@$watches) {
                         my $uri;
-                        if (is_maildir($watch)) {
+                        my $bool = $cfg->git_bool($watch);
+                        if (defined $bool && !$bool) {
+                                $ibx->{-watch_disabled} = 1;
+                        } elsif (is_maildir($watch)) {
                                 compile_watchheaders($ibx);
                                 my ($new, $cur) = ("$watch/new", "$watch/cur");
-                                my $cur_dst = $mdmap{$cur} //= [];
+                                my $cur_dst = $d_map{$cur} //= [];
                                 return if is_watchspam($cur, $cur_dst, $ibx);
-                                push @{$mdmap{$new} //= []}, $ibx;
+                                push @{$d_map{$new} //= []}, $ibx;
                                 push @$cur_dst, $ibx;
+                                d_type_set \%d_type, $watch, D_MAILDIR;
+                        } elsif (is_mh($watch)) {
+                                my $cur_dst = $d_map{$watch} //= [];
+                                return if is_watchspam($watch, $cur_dst, $ibx);
+                                compile_watchheaders($ibx);
+                                push(@$cur_dst, $ibx);
+                                d_type_set \%d_type, $watch, D_MH;
                         } elsif ($uri = imap_uri($watch)) {
                                 my $cur_dst = $imap{$$uri} //= [];
                                 return if is_watchspam($uri, $cur_dst, $ibx);
@@ -106,18 +135,19 @@ sub new {
                 }
         });
 
-        my $mdre;
-        if (scalar keys %mdmap) {
-                $mdre = join('|', map { quotemeta($_) } keys %mdmap);
-                $mdre = qr!\A($mdre)/!;
+        my $d_re;
+        if (scalar keys %d_map) {
+                $d_re = join('|', map quotemeta, keys %d_map);
+                $d_re = qr!\A($d_re)/!;
         }
-        return unless $mdre || scalar(keys %imap) || scalar(keys %nntp);
+        return unless $d_re || scalar(keys %imap) || scalar(keys %nntp);
 
         bless {
                 max_batch => 10, # avoid hogging locks for too long
                 spamcheck => $spamcheck,
-                mdmap => \%mdmap,
-                mdre => $mdre,
+                d_map => \%d_map,
+                d_re => $d_re,
+                d_type => \%d_type,
                 pi_cfg => $cfg,
                 imap => scalar keys %imap ? \%imap : undef,
                 nntp => scalar keys %nntp? \%nntp : undef,
@@ -141,6 +171,7 @@ sub _done_for_now {
 
 sub remove_eml_i { # each_inbox callback
         my ($ibx, $self, $eml, $loc) = @_;
+        return if $ibx->{-watch_disabled};
 
         eval {
                 # try to avoid taking a lock or unnecessary spawning
@@ -214,17 +245,23 @@ sub import_eml ($$$) {
 
 sub _try_path {
         my ($self, $path) = @_;
-        my $fl = PublicInbox::MdirReader::maildir_path_flags($path) // return;
-        return if $fl =~ /[DT]/; # no Drafts or Trash
-        if ($path !~ $self->{mdre}) {
-                warn "unrecognized path: $path\n";
-                return;
-        }
-        my $inboxes = $self->{mdmap}->{$1};
-        unless ($inboxes) {
-                warn "unmappable dir: $1\n";
-                return;
-        }
+        $path =~ $self->{d_re} or
+                return warn("BUG? unrecognized path: $path\n");
+        my $dir = $1;
+        my $inboxes = $self->{d_map}->{$dir} //
+                return warn("W: unmappable dir: $dir\n");
+        my ($md_fl, $mh_seq);
+        if ($self->{d_type}->{$dir} & D_MH) {
+                $path =~ m!/([0-9]+)\z! ? ($mh_seq = $1) : return;
+        }
+        $self->{d_type}->{$dir} & D_MAILDIR and
+                $md_fl = PublicInbox::MdirReader::maildir_path_flags($path);
+        $md_fl // $mh_seq // return;
+        return if ($md_fl // '') =~ /[DT]/; # no Drafts or Trash
+        # n.b. none of the MH keywords are relevant for public mail,
+        # mh_seq is only used to validate we're reading an email
+        # and not treating .mh_sequences as an email
+
         my $warn_cb = $SIG{__WARN__} || \&CORE::warn;
         local $SIG{__WARN__} = sub {
                 my $pfx = ($_[0] // '') =~ /^([A-Z]: )/g ? $1 : '';
@@ -244,25 +281,24 @@ sub quit_done ($) {
         return unless $self->{quit};
 
         # don't have reliable wakeups, keep signalling
-        my $done = 1;
-        for (qw(idle_pids poll_pids)) {
-                my $pids = $self->{$_} or next;
-                for (keys %$pids) {
-                        $done = undef if kill('QUIT', $_);
-                }
-        }
-        $done;
+        my $live = grep { kill('QUIT', $_) } keys %{$self->{pids}};
+        add_timer(0.01, \&quit_done, $self) if $live;
+        $live == 0;
 }
 
-sub quit {
+sub quit { # may be called in IMAP/NNTP children
         my ($self) = @_;
         $self->{quit} = 1;
         %{$self->{opendirs}} = ();
         _done_for_now($self);
         quit_done($self);
-        if (my $idle_mic = $self->{idle_mic}) {
+        if (my $dir_idle = delete $self->{dir_idle}) {
+                $dir_idle->close if $dir_idle;
+        }
+        if (my $idle_mic = delete $self->{idle_mic}) { # IMAP child
+                return unless $idle_mic->IsConnected && $idle_mic->Socket;
                 eval { $idle_mic->done };
-                if ($@) {
+                if ($@ && $idle_mic->IsConnected && $idle_mic->Socket) {
                         warn "IDLE DONE error: $@\n";
                         eval { $idle_mic->disconnect };
                         warn "IDLE LOGOUT error: $@\n" if $@;
@@ -282,8 +318,8 @@ sub watch_fs_init ($) {
         };
         require PublicInbox::DirIdle;
         # inotify_create + EPOLL_CTL_ADD
-        my $dir_idle = PublicInbox::DirIdle->new($cb);
-        $dir_idle->add_watches([keys %{$self->{mdmap}}]);
+        my $dir_idle = $self->{dir_idle} = PublicInbox::DirIdle->new($cb);
+        $dir_idle->add_watches([keys %{$self->{d_map}}]);
 }
 
 sub net_cb { # NetReader::(nntp|imap)_each callback
@@ -318,7 +354,7 @@ sub imap_fetch_all ($$) {
         local $SIG{__WARN__} = sub {
                 my $pfx = ($_[0] // '') =~ /^([A-Z]: |# )/g ? $1 : '';
                 my $uid = $self->{cur_uid};
-                $warn_cb->("$pfx$uri", $uid ? ("UID:$uid") : (), "\n", @_);
+                $warn_cb->("$pfx$uri", $uid ? (" UID:$uid") : (), "\n", @_);
         };
         PublicInbox::NetReader::imap_each($self, $uri, \&net_cb, $self,
                                         $self->{imap}->{$$uri});
@@ -328,7 +364,7 @@ sub imap_idle_once ($$$$) {
         my ($self, $mic, $intvl, $uri) = @_;
         my $i = $intvl //= (29 * 60);
         my $end = now() + $intvl;
-        warn "I: $uri idling for ${intvl}s\n";
+        warn "# $uri idling for ${intvl}s\n";
         local $0 = "IDLE $0";
         return if $self->{quit};
         unless ($mic->idle) {
@@ -381,63 +417,42 @@ sub watch_imap_idle_1 ($$$) {
 
 sub watch_atfork_child ($) {
         my ($self) = @_;
-        delete $self->{idle_pids};
-        delete $self->{poll_pids};
-        delete $self->{opendirs};
-        PublicInbox::DS->Reset;
+        delete @$self{qw(dir_idle pids opendirs)};
         my $sig = delete $self->{sig};
-        $sig->{CHLD} = 'DEFAULT';
+        $sig->{CHLD} = $sig->{HUP} = $sig->{USR1} = 'DEFAULT';
+        # TERM/QUIT/INT call ->quit, which works in both parent+child
         @SIG{keys %$sig} = values %$sig;
-        PublicInbox::DS::sig_setmask($self->{oldset});
+        PublicInbox::DS::sig_setmask(PublicInbox::DS::allowset($sig));
 }
 
 sub watch_atfork_parent ($) { _done_for_now($_[0]) }
 
 sub imap_idle_requeue { # DS::add_timer callback
-        my ($self, $uri_intvl) = @_;
+        my ($self, $uri, $intvl) = @_;
         return if $self->{quit};
-        push @{$self->{idle_todo}}, $uri_intvl;
+        push @{$self->{idle_todo}}, $uri, $intvl;
         event_step($self);
 }
 
-sub imap_idle_reap { # PublicInbox::DS::dwaitpid callback
-        my ($self, $pid) = @_;
-        my $uri_intvl = delete $self->{idle_pids}->{$pid} or
-                die "BUG: PID=$pid (unknown) reaped: \$?=$?\n";
-
-        my ($uri, $intvl) = @$uri_intvl;
+sub imap_idle_reap { # awaitpid callback
+        my ($pid, $self, $uri, $intvl) = @_;
+        delete $self->{pids}->{$pid};
         return if $self->{quit};
         warn "W: PID=$pid on $uri died: \$?=$?\n" if $?;
-        add_timer(60, \&imap_idle_requeue, $self, $uri_intvl);
+        add_timer(60, \&imap_idle_requeue, $self, $uri, $intvl);
 }
 
-sub reap { # callback for EOFpipe
-        my ($pid, $cb, $self) = @{$_[0]};
-        my $ret = waitpid($pid, 0);
-        if ($ret == $pid) {
-                $cb->($self, $pid); # poll_fetch_reap || imap_idle_reap
-        } else {
-                warn "W: waitpid($pid) => ", $ret // "($!)", "\n";
-        }
-}
-
-sub imap_idle_fork ($$) {
-        my ($self, $uri_intvl) = @_;
-        my ($uri, $intvl) = @$uri_intvl;
-        pipe(my ($r, $w)) or die "pipe: $!";
-        my $seed = rand(0xffffffff);
-        my $pid = fork // die "fork: $!";
+sub imap_idle_fork {
+        my ($self, $uri, $intvl) = @_;
+        return if $self->{quit};
+        my $pid = PublicInbox::DS::do_fork;
         if ($pid == 0) {
-                srand($seed);
-                eval { Net::SSLeay::randomize() };
-                close $r;
                 watch_atfork_child($self);
                 watch_imap_idle_1($self, $uri, $intvl);
-                close $w;
                 _exit(0);
         }
-        $self->{idle_pids}->{$pid} = $uri_intvl;
-        PublicInbox::EOFpipe->new($r, \&reap, [$pid, \&imap_idle_reap, $self]);
+        $self->{pids}->{$pid} = undef;
+        awaitpid($pid, \&imap_idle_reap, $self, $uri, $intvl);
 }
 
 sub event_step {
@@ -447,13 +462,13 @@ sub event_step {
         if ($idle_todo && @$idle_todo) {
                 watch_atfork_parent($self);
                 eval {
-                        while (my $uri_intvl = shift(@$idle_todo)) {
-                                imap_idle_fork($self, $uri_intvl);
+                        while (my ($uri, $intvl) = splice(@$idle_todo, 0, 2)) {
+                                imap_idle_fork($self, $uri, $intvl);
                         }
                 };
                 die $@ if $@;
         }
-        fs_scan_step($self) if $self->{mdre};
+        fs_scan_step($self) if $self->{d_re};
 }
 
 sub watch_imap_fetch_all ($$) {
@@ -474,7 +489,7 @@ sub watch_nntp_fetch_all ($$) {
         local $SIG{__WARN__} = sub {
                 my $pfx = ($_[0] // '') =~ /^([A-Z]: |# )/g ? $1 : '';
                 my $art = $self->{cur_uid};
-                $warn_cb->("$pfx$uri", $art ? ("ARTICLE $art") : (), "\n", @_);
+                $warn_cb->("$pfx$uri", $art ? (" ARTICLE $art") : (), "\n", @_);
         };
         for $uri (@$uris) {
                 PublicInbox::NetReader::nntp_each($self, $uri, \&net_cb, $self,
@@ -486,52 +501,44 @@ sub watch_nntp_fetch_all ($$) {
 sub poll_fetch_fork { # DS::add_timer callback
         my ($self, $intvl, $uris) = @_;
         return if $self->{quit};
-        pipe(my ($r, $w)) or die "pipe: $!";
         watch_atfork_parent($self);
-        my $seed = rand(0xffffffff);
-        my $pid = fork;
-        if (defined($pid) && $pid == 0) {
-                srand($seed);
-                eval { Net::SSLeay::randomize() };
-                close $r;
+        my @nntp;
+        my @imap = grep { # push() always returns > 0
+                $_->scheme =~ m!\Aimaps?!i ? 1 : (push(@nntp, $_) < 0)
+        } @$uris;
+        my $pid = PublicInbox::DS::do_fork;
+        if ($pid == 0) {
                 watch_atfork_child($self);
-                if ($uris->[0]->scheme =~ m!\Aimaps?!i) {
-                        watch_imap_fetch_all($self, $uris);
-                } else {
-                        watch_nntp_fetch_all($self, $uris);
-                }
-                close $w;
+                watch_imap_fetch_all($self, \@imap) if @imap;
+                watch_nntp_fetch_all($self, \@nntp) if @nntp;
                 _exit(0);
         }
-        die "fork: $!"  unless defined $pid;
-        $self->{poll_pids}->{$pid} = [ $intvl, $uris ];
-        PublicInbox::EOFpipe->new($r, \&reap, [$pid, \&poll_fetch_reap, $self]);
+        $self->{pids}->{$pid} = undef;
+        awaitpid($pid, \&poll_fetch_reap, $self, $intvl, $uris);
 }
 
-sub poll_fetch_reap {
-        my ($self, $pid) = @_;
-        my $intvl_uris = delete $self->{poll_pids}->{$pid} or
-                die "BUG: PID=$pid (unknown) reaped: \$?=$?\n";
+sub poll_fetch_reap { # awaitpid callback
+        my ($pid, $self, $intvl, $uris) = @_;
+        delete $self->{pids}->{$pid};
         return if $self->{quit};
-        my ($intvl, $uris) = @$intvl_uris;
         if ($?) {
                 warn "W: PID=$pid died: \$?=$?\n", map { "$_\n" } @$uris;
         }
-        warn("I: will check $_ in ${intvl}s\n") for @$uris;
+        warn("# will check $_ in ${intvl}s\n") for @$uris;
         add_timer($intvl, \&poll_fetch_fork, $self, $intvl, $uris);
 }
 
 sub watch_imap_init ($$) {
         my ($self, $poll) = @_;
-        my $mics = PublicInbox::NetReader::imap_common_init($self);
-        my $idle = []; # [ [ uri1, intvl1 ], [uri2, intvl2] ]
+        my $mics = PublicInbox::NetReader::imap_common_init($self) or return;
+        my $idle = []; # [ uri1, intvl1, uri2, intvl2 ]
         for my $uri (@{$self->{imap_order}}) {
                 my $sec = uri_section($uri);
                 my $mic = $mics->{$sec};
                 my $intvl = $self->{cfg_opt}->{$sec}->{pollInterval};
                 if ($mic->has_capability('IDLE') && !$intvl) {
                         $intvl = $self->{cfg_opt}->{$sec}->{idleInterval};
-                        push @$idle, [ $uri, $intvl // () ];
+                        push @$idle, $uri, $intvl;
                 } else {
                         push @{$poll->{$intvl || 120}}, $uri;
                 }
@@ -552,10 +559,12 @@ sub watch_nntp_init ($$) {
         }
 }
 
+sub quit_inprogress { !$_[0]->quit_done } # post_loop_do CB
+
 sub watch { # main entry point
-        my ($self, $sig, $oldset) = @_;
-        $self->{oldset} = $oldset;
-        $self->{sig} = $sig;
+        my ($self, $sig) = @_;
+        my $first_sig;
+        $self->{sig} //= ($first_sig = $sig);
         my $poll = {}; # intvl_seconds => [ uri1, uri2 ]
         watch_imap_init($self, $poll) if $self->{imap};
         watch_nntp_init($self, $poll) if $self->{nntp};
@@ -563,9 +572,9 @@ sub watch { # main entry point
                 # poll all URIs for a given interval sequentially
                 add_timer(0, \&poll_fetch_fork, $self, $intvl, $uris);
         }
-        watch_fs_init($self) if $self->{mdre};
-        PublicInbox::DS->SetPostLoopCallback(sub { !$self->quit_done });
-        PublicInbox::DS::event_loop($sig, $oldset); # calls ->event_step
+        watch_fs_init($self) if $self->{d_re};
+        local @PublicInbox::DS::post_loop_do = (\&quit_inprogress, $self);
+        PublicInbox::DS::event_loop($first_sig); # calls ->event_step
         _done_for_now($self);
 }
 
@@ -594,7 +603,7 @@ sub fs_scan_step {
                 $opendirs->{$dir} = $dh if $n < 0;
         }
         if ($op && $op eq 'full') {
-                foreach my $dir (keys %{$self->{mdmap}}) {
+                foreach my $dir (keys %{$self->{d_map}}) {
                         next if $opendirs->{$dir}; # already in progress
                         my $ok = opendir(my $dh, $dir);
                         unless ($ok) {
@@ -669,6 +678,13 @@ sub is_maildir {
         $_[0];
 }
 
+sub is_mh {
+        $_[0] =~ s!\Amh:!!i or return;
+        $_[0] =~ tr!/!/!s;
+        $_[0] =~ s!/\z!!;
+        $_[0];
+}
+
 sub is_watchspam {
         my ($cur, $ws, $ibx) = @_;
         if ($ws && !ref($ws) && $ws eq 'watchspam') {
diff --git a/lib/PublicInbox/WwwAltId.pm b/lib/PublicInbox/WwwAltId.pm
index e107dfe0..31d9b607 100644
--- a/lib/PublicInbox/WwwAltId.pm
+++ b/lib/PublicInbox/WwwAltId.pm
@@ -1,9 +1,9 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # dumps using the ".dump" command of sqlite3(1)
 package PublicInbox::WwwAltId;
-use strict;
+use v5.12;
 use PublicInbox::Qspawn;
 use PublicInbox::WwwStream qw(html_oneshot);
 use PublicInbox::AltId;
@@ -33,14 +33,14 @@ sub sqldump ($$) {
         my $altid_map = $ibx->altid_map;
         my $fn = $altid_map->{$altid_pfx};
         unless (defined $fn) {
-                return html_oneshot($ctx, 404, \<<EOF);
+                return html_oneshot($ctx, 404, <<EOF);
 <pre>`$altid_pfx' is not a valid altid for this inbox</pre>
 EOF
         }
 
         if ($env->{REQUEST_METHOD} ne 'POST') {
                 my $url = $ibx->base_url($ctx->{env}) . "$altid_pfx.sql.gz";
-                return html_oneshot($ctx, 405, \<<EOF);
+                return html_oneshot($ctx, 405, <<EOF);
 <pre>A POST request is required to retrieve $altid_pfx.sql.gz
 
         curl -d '' -O $url
@@ -54,24 +54,19 @@ or
 EOF
         }
 
-        $sqlite3 //= which('sqlite3') // return html_oneshot($ctx, 501, \<<EOF);
+        $sqlite3 //= which('sqlite3') // return html_oneshot($ctx, 501, <<EOF);
 <pre>sqlite3 not available
 
 The administrator needs to install the sqlite3(1) binary
 to support gzipped sqlite3 dumps.</pre>
 EOF
 
-        # setup stdin, POSIX requires writes <= 512 bytes to succeed so
-        # we can close the pipe right away.
-        pipe(my ($r, $w)) or die "pipe: $!";
-        syswrite($w, ".dump\n") == 6 or die "write: $!";
-        close($w) or die "close: $!";
-
         # TODO: use -readonly if available with newer sqlite3(1)
-        my $qsp = PublicInbox::Qspawn->new([$sqlite3, $fn], undef, { 0 => $r });
+        my $qsp = PublicInbox::Qspawn->new([$sqlite3, $fn], undef,
+                                                        { 0 => \".dump\n" });
         $ctx->{altid_pfx} = $altid_pfx;
         $env->{'qspawn.filter'} = PublicInbox::GzipFilter->new;
-        $qsp->psgi_return($env, undef, \&check_output, $ctx);
+        $qsp->psgi_yield($env, undef, \&check_output, $ctx);
 }
 
 1;
diff --git a/lib/PublicInbox/WwwAtomStream.pm b/lib/PublicInbox/WwwAtomStream.pm
index 82895db6..26b366f5 100644
--- a/lib/PublicInbox/WwwAtomStream.pm
+++ b/lib/PublicInbox/WwwAtomStream.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Atom body stream for HTTP responses
@@ -8,7 +8,7 @@ use strict;
 use parent 'PublicInbox::GzipFilter';
 
 use POSIX qw(strftime);
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
 use PublicInbox::Address;
 use PublicInbox::Hval qw(ascii_html mid_href);
 use PublicInbox::MsgTime qw(msg_timestamp);
@@ -16,6 +16,7 @@ use PublicInbox::MsgTime qw(msg_timestamp);
 sub new {
         my ($class, $ctx, $cb) = @_;
         $ctx->{feed_base_url} = $ctx->{ibx}->base_url($ctx->{env});
+        $ctx->{-spfx} = $ctx->{feed_base_url} if $ctx->{ibx}->{coderepo};
         $ctx->{cb} = $cb || \&PublicInbox::GzipFilter::close;
         $ctx->{emit_header} = 1;
         bless $ctx, $class;
@@ -38,14 +39,15 @@ sub async_next ($) {
 sub async_eml { # for async_blob_cb
         my ($ctx, $eml) = @_;
         my $smsg = delete $ctx->{smsg};
+        $smsg->{mid} // $smsg->populate($eml);
         $ctx->write(feed_entry($ctx, $smsg, $eml));
 }
 
 sub response {
-        my ($class, $ctx, $code, $cb) = @_;
+        my ($class, $ctx, $cb) = @_;
         my $res_hdr = [ 'Content-Type' => 'application/atom+xml' ];
         $class->new($ctx, $cb);
-        $ctx->psgi_response($code, $res_hdr);
+        $ctx->psgi_response(200, $res_hdr);
 }
 
 # called once for each message by PSGI server
@@ -97,15 +99,16 @@ sub atom_header {
                 $base_url .= '?' . $search_q->qs_html(x => undef);
                 $self_url .= '?' . $search_q->qs_html;
                 $page_id = to_uuid("q\n".$query);
+        } elsif (defined(my $cat = $ctx->{topic_category})) {
+                $title = title_tag("$cat topics - ".$ibx->description);
+                $self_url .= "topics_$cat.atom";
         } else {
                 $title = title_tag($ibx->description);
                 $self_url .= 'new.atom';
-                if (defined(my $addr = $ibx->{-primary_address})) {
-                        $page_id = "mailto:$addr";
-                } else {
-                        $page_id = to_uuid($self_url);
-                }
+                my $addr = $ibx->{-primary_address};
+                $page_id = "mailto:$addr" if defined $addr;
         }
+        $page_id //= to_uuid($self_url);
         qq(<?xml version="1.0" encoding="us-ascii"?>\n) .
         qq(<feed\nxmlns="http://www.w3.org/2005/Atom"\n) .
         qq(xmlns:thr="http://purl.org/syndication/thread/1.0">) .
@@ -145,19 +148,19 @@ sub feed_entry {
         my $name = ascii_html(join(', ', PublicInbox::Address::names($from)));
         $email = ascii_html($email // $ctx->{ibx}->{-primary_address});
 
-        my $s = delete($ctx->{emit_header}) ? atom_header($ctx, $title) : '';
-        $s .= "<entry><author><name>$name</name><email>$email</email>" .
+        print { $ctx->zfh }
+                (delete($ctx->{emit_header}) ? atom_header($ctx, $title) : ''),
+                "<entry><author><name>$name</name><email>$email</email>" .
                 "</author>$title$updated" .
-                qq(<link\nhref="$href"/>).
+                qq(<link\nhref="$href"/>) .
                 "<id>$uuid</id>$irt" .
                 qq{<content\ntype="xhtml">} .
                 qq{<div\nxmlns="http://www.w3.org/1999/xhtml">} .
                 qq(<pre\nstyle="white-space:pre-wrap">);
-        $ctx->{obuf} = \$s;
         $ctx->{mhref} = $href;
-        PublicInbox::View::multipart_text_as_html($eml, $ctx);
-        delete $ctx->{obuf};
-        $s .= '</pre></div></content></entry>';
+        $ctx->{changed_href} = "${href}#related";
+        $eml->each_part(\&PublicInbox::View::add_text_body, $ctx, 1);
+        '</pre></div></content></entry>';
 }
 
 sub feed_updated {
diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
new file mode 100644
index 00000000..61aa7862
--- /dev/null
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -0,0 +1,377 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# Standalone code repository viewer for users w/o cgit.
+# This isn't intended to replicate all of cgit, but merely to be a
+# "good enough" viewer with search support and some UI hints to encourage
+# cloning + command-line usage.
+package PublicInbox::WwwCoderepo;
+use v5.12;
+use parent qw(PublicInbox::WwwStream);
+use File::Temp 0.19 (); # newdir
+use POSIX qw(O_RDWR F_GETFL);
+use PublicInbox::ViewVCS;
+use PublicInbox::WwwStatic qw(r);
+use PublicInbox::GitHTTPBackend;
+use PublicInbox::WwwStream;
+use PublicInbox::Hval qw(prurl ascii_html utf8_maybe);
+use PublicInbox::ViewDiff qw(uri_escape_path);
+use PublicInbox::RepoSnapshot;
+use PublicInbox::RepoAtom;
+use PublicInbox::RepoTree;
+use PublicInbox::RepoList;
+use PublicInbox::OnDestroy;
+use URI::Escape qw(uri_escape_utf8);
+use File::Spec;
+use autodie qw(fcntl open);
+
+my @EACH_REF = (qw(git for-each-ref --sort=-creatordate),
+                "--format=%(HEAD)%00".join('%00', map { "%($_)" }
+                qw(objectname refname:short subject creatordate:short)));
+my $HEADS_CMD = <<'';
+# heads (aka `branches'):
+$ git for-each-ref --sort=-creatordate refs/heads \
+        --format='%(HEAD) %(refname:short) %(subject) (%(creatordate:short))'
+
+my $TAGS_CMD = <<'';
+# tags:
+$ git for-each-ref --sort=-creatordate refs/tags \
+        --format='%(refname:short) %(subject) (%(creatordate:short))'
+
+my $NO_HEADS = "# no heads (branches), yet...\n";
+my $NO_TAGS = "# no tags, yet...\n";
+
+# shared with PublicInbox::Cgit
+sub prepare_coderepos {
+        my ($self) = @_;
+        my $pi_cfg = $self->{pi_cfg};
+
+        # TODO: support gitweb and other repository viewers?
+        $pi_cfg->parse_cgitrc(undef, 0);
+
+        my $coderepos = $pi_cfg->{-coderepos};
+        for my $k (grep(/\Acoderepo\.(?:.+)\.dir\z/, keys %$pi_cfg)) {
+                $k = substr($k, length('coderepo.'), -length('.dir'));
+                $coderepos->{$k} //= $pi_cfg->fill_coderepo($k);
+        }
+
+        # associate inboxes and extindices with coderepos for search:
+        for my $k (grep(/\Apublicinbox\.(?:.+)\.coderepo\z/, keys %$pi_cfg)) {
+                $k = substr($k, length('publicinbox.'), -length('.coderepo'));
+                my $ibx = $pi_cfg->lookup_name($k) // next;
+                $pi_cfg->repo_objs($ibx);
+        }
+        for my $k (grep(/\Aextindex\.(?:.+)\.coderepo\z/, keys %$pi_cfg)) {
+                $k = substr($k, length('extindex.'), -length('.coderepo'));
+                my $eidx = $pi_cfg->lookup_ei($k) // next;
+                $pi_cfg->repo_objs($eidx);
+        }
+        $pi_cfg->each_cindex('load_coderepos', $pi_cfg);
+}
+
+sub new {
+        my ($cls, $pi_cfg) = @_;
+        my $self = bless { pi_cfg => $pi_cfg }, $cls;
+        prepare_coderepos($self);
+        $self->{snapshots} = do {
+                my $s = $pi_cfg->{'coderepo.snapshots'} // '';
+                $s eq 'all' ? \%PublicInbox::RepoSnapshot::FMT_TYPES :
+                        +{ map { $_ => 1 } split(/\s+/, $s) };
+        };
+        $self->{$_} = 10 for qw(summary_branches summary_tags);
+        $self->{$_} = 10 for qw(summary_log);
+
+        # try reuse STDIN if it's already /dev/null
+        open $self->{log_fh}, '+>', '/dev/null';
+        my @l = stat($self->{log_fh}) or die "stat: $!";
+        my @s = stat(STDIN) or die "stat(STDIN): $!";
+        if ("@l[0, 1]" eq "@s[0, 1]") {
+                my $f = fcntl(STDIN, F_GETFL, 0);
+                $self->{log_fh} = *STDIN{IO} if $f & O_RDWR;
+        }
+        $self;
+}
+
+sub _snapshot_link_prep {
+        my ($ctx) = @_;
+        my @s = sort keys %{$ctx->{wcr}->{snapshots}} or return ();
+        my $n = $ctx->{git}->local_nick // die "BUG: $ctx->{git_dir} nick";
+        $n =~ s!\.git/*\z!!;
+        ($n) = ($n =~ m!([^/]+)/*\z!);
+        (ascii_html($n).'-', @s);
+}
+
+sub _refs_heads_link {
+        my ($line, $upfx) = @_;
+        my ($pfx, $oid, $ref, $s, $cd) = split(/\0/, $line);
+        my $align = length($ref) < 12 ? ' ' x (12 - length($ref)) : '';
+        ("$pfx <a\nhref=$upfx$oid/s/>", ascii_html($ref),
+                "</a>$align ", ascii_html($s), " ($cd)\n")
+}
+
+sub _refs_tags_link {
+        my ($line, $upfx, $snap_pfx, @snap_fmt) = @_;
+        my (undef, $oid, $ref, $s, $cd) = split(/\0/, $line);
+        my $align = length($ref) < 12 ? ' ' x (12 - length($ref)) : '';
+        if (@snap_fmt) {
+                my $v = $ref;
+                $v =~ s/\A[vV]//;
+                @snap_fmt = map {
+                        qq{ <a href="${upfx}snapshot/$snap_pfx$v.$_">$_</a>}
+                } @snap_fmt;
+        }
+        ("<a\nhref=$upfx$oid/s/>", ascii_html($ref),
+                "</a>$align ", ascii_html($s), " ($cd)", @snap_fmt, "\n");
+}
+
+sub emit_joined_inboxes ($) {
+        my ($ctx) = @_;
+        my $names = $ctx->{git}->{ibx_names}; # coderepo directives in config
+        my $score = $ctx->{git}->{ibx_score}; # generated w/ cindex --join
+        ($names || $score) or return;
+        my $pi_cfg = $ctx->{wcr}->{pi_cfg};
+        my ($u, $h);
+        my $zfh = $ctx->zfh;
+        print $zfh "\n# associated public inboxes:",
+                "\n# (number on the left is used for dev purposes)";
+        my @ns = map { [ 0, $_ ] } @$names;
+        my $env = $ctx->{env};
+        for (@ns, @$score) {
+                my ($nr, $name) = @$_;
+                my $ibx = $pi_cfg->lookup_name($name) // do {
+                        warn "W: inbox `$name' gone for $ctx->{git}->{git_dir}";
+                        say $zfh '# ', ascii_html($name), ' (missing inbox?)';
+                        next;
+                };
+                if (scalar(@{$ibx->{url} // []})) {
+                        $u = $h = ascii_html(prurl($env, $ibx->{url}));
+                } else {
+                        $h = ascii_html(prurl($env, uri_escape_utf8($name)));
+                        $h .= '/';
+                        $u = ascii_html($name);
+                }
+                if ($nr) {
+                        printf $zfh "\n% 11u", $nr;
+                } else {
+                        print $zfh "\n", ' 'x11;
+                }
+                print $zfh qq{ <a\nhref="$h">$u</a>};
+        }
+}
+
+sub summary_END { # called via OnDestroy
+        my ($ctx) = @_;
+        my $wcb = delete($ctx->{-wcb}) or return; # already done
+        PublicInbox::WwwStream::html_init($ctx);
+        my $zfh = $ctx->zfh;
+
+        my @r = split(/\n/s, delete($ctx->{qx_res}->{'log'}) // '');
+        my $last = scalar(@r) > $ctx->{wcr}->{summary_log} ? pop(@r) : undef;
+        my $tip_html = '';
+        my $tip = $ctx->{qp}->{h};
+        $tip_html .= ' '.ascii_html($tip).' --' if defined $tip;
+        print $zfh <<EOM;
+<pre><a id=log>\$</a> git log --pretty=format:'%h %s (%cs)%d'$tip_html
+EOM
+        for (@r) {
+                my $d; # decorations
+                s/^ \(([^\)]+)\)// and $d = $1;
+                substr($_, 0, 1, '');
+                my ($H, $h, $cs, $s) = split(/ /, $_, 4);
+                print $zfh "<a\nhref=./$H/s/>$h</a> ", ascii_html($s),
+                        " (", $cs, ")\n";
+                print $zfh "\t(", ascii_html($d), ")\n" if $d;
+        }
+        print $zfh '# no commits in `', ($tip//'HEAD'),"', yet\n\n" if !@r;
+        print $zfh "...\n" if $last;
+
+        # README
+        my ($bref, $oid, $ref_path) = @{delete $ctx->{qx_res}->{readme}};
+        if ($bref) {
+                my $l = PublicInbox::Linkify->new;
+                $$bref =~ s/\s*\z//sm;
+                my (undef, $path) = split(/:/, $ref_path, 2); # HEAD:README
+                print $zfh "\n<a id=readme>\$</a> " .
+                        qq(git cat-file blob <a href="./$oid/s/?b=) .
+                        ascii_html(uri_escape_path($path)) . q(">).
+                        ascii_html($ref_path), "</a>\n",
+                        $l->to_html($$bref), '</pre><hr><pre>';
+        }
+
+        # refs/heads
+        print $zfh '<a id=heads>', $HEADS_CMD , '</a>';
+        @r = split(/^/sm, delete($ctx->{qx_res}->{heads}) // '');
+        $last = scalar(@r) > $ctx->{wcr}->{summary_branches} ? pop(@r) : undef;
+        chomp(@r);
+        for (@r) { print $zfh _refs_heads_link($_, './') }
+        print $zfh $NO_HEADS if !@r;
+        print $zfh qq(<a href="refs/heads/">...</a>\n) if $last;
+        print $zfh "\n<a id=tags>", $TAGS_CMD, '</a>';
+        @r = split(/^/sm, delete($ctx->{qx_res}->{tags}) // '');
+        $last = scalar(@r) > $ctx->{wcr}->{summary_tags} ? pop(@r) : undef;
+        my ($snap_pfx, @snap_fmt) = _snapshot_link_prep($ctx);
+        chomp @r;
+        for (@r) { print $zfh _refs_tags_link($_, './', $snap_pfx, @snap_fmt) }
+        print $zfh $NO_TAGS if !@r;
+        print $zfh qq(<a href="refs/tags/">...</a>\n) if $last;
+        emit_joined_inboxes $ctx;
+        $wcb->($ctx->html_done('</pre>'));
+}
+
+sub capture { # psgi_qx callback to capture git-for-each-ref
+        my ($bref, $ctx, $key) = @_; #  $_[3] = OnDestroy(summary_END)
+        $ctx->{qx_res}->{$key} = $$bref;
+        # summary_END may be called via OnDestroy $arg->[2]
+}
+
+sub set_readme { # git->cat_async callback
+        my ($bref, $oid, $type, $size, $ctx) = @_;
+        my $ref_path = shift @{$ctx->{-readme_tries}}; # e.g. HEAD:README
+        if ($type eq 'blob' && !$ctx->{qx_res}->{readme}) {
+                $ctx->{qx_res}->{readme} = [ $bref, $oid, $ref_path ];
+        } elsif (scalar @{$ctx->{-readme_tries}} == 0) {
+                $ctx->{qx_res}->{readme} //= []; # nothing left to try
+        } # or try another README...
+        # summary_END may be called via OnDestroy ($ctx->{-END})
+}
+
+sub summary ($$) {
+        my ($ctx, $wcb) = @_;
+        $ctx->{-wcb} = $wcb; # PublicInbox::HTTP::{Identity,Chunked}
+        my $tip = $ctx->{qp}->{h}; # same as cgit
+        if (defined $tip && $tip eq '') {
+                delete $ctx->{qp}->{h};
+                undef($tip);
+        }
+        my ($nb, $nt, $nl) = map { $_ + 1 } @{$ctx->{wcr}}{qw(
+                summary_branches summary_tags summary_log)};
+        $ctx->{qx_res} = {};
+        my $qsp_err = \($ctx->{-qsp_err} = '');
+        my %opt = (quiet => 1, 2 => $ctx->{wcr}->{log_fh});
+        my %env = (GIT_DIR => $ctx->{git}->{git_dir});
+        my @log = (qw(git log), "-$nl", '--pretty=format:%d %H %h %cs %s');
+        push(@log, $tip) if defined $tip;
+
+        # limit scope for MockHTTP test (t/solver_git.t)
+        my $END = PublicInbox::OnDestroy->new($$, \&summary_END, $ctx);
+        for (['log', \@log],
+                 [ 'heads', [@EACH_REF, "--count=$nb", 'refs/heads'] ],
+                 [ 'tags', [@EACH_REF, "--count=$nt", 'refs/tags'] ]) {
+                my ($k, $cmd) = @$_;
+                my $qsp = PublicInbox::Qspawn->new($cmd, \%env, \%opt);
+                $qsp->{qsp_err} = $qsp_err;
+                $qsp->psgi_qx($ctx->{env}, undef, \&capture, $ctx, $k, $END);
+        }
+        $tip //= 'HEAD';
+        my @try = ("$tip:README", "$tip:README.md"); # TODO: configurable
+        my %ctx = (%$ctx, -END => $END, -readme_tries => [ @try ]);
+        PublicInbox::ViewVCS::do_cat_async(\%ctx, \&set_readme, @try);
+}
+
+# called by GzipFilter->close after translate
+sub zflush { $_[0]->SUPER::zflush('</pre>', $_[0]->_html_end) }
+
+# called by GzipFilter->write or GetlineResponse->getline
+sub translate {
+        my $ctx = shift;
+        $_[0] // return zflush($ctx); # getline caller
+        my @out;
+        my $fbuf = delete($ctx->{fbuf}) // shift;
+        $fbuf .= shift while @_;
+        if ($ctx->{-heads}) {
+                while ($fbuf =~ s/\A([^\n]+)\n//s) {
+                        utf8_maybe(my $x = $1);
+                        push @out, _refs_heads_link($x, '../../');
+                }
+        } else {
+                my ($snap_pfx, @snap_fmt) = _snapshot_link_prep($ctx);
+                while ($fbuf =~ s/\A([^\n]+)\n//s) {
+                        utf8_maybe(my $x = $1);
+                        push @out, _refs_tags_link($x, '../../',
+                                                $snap_pfx, @snap_fmt);
+                }
+        }
+        $ctx->{fbuf} = $fbuf; # may be incomplete
+        @out ? $ctx->SUPER::translate(@out) : ''; # not EOF, yet
+}
+
+sub _refs_parse_hdr { # {parse_hdr} for Qspawn
+        my ($r, $bref, $ctx) = @_;
+        my ($code, $top);
+        if ($r == 0) {
+                $code = 404;
+                $top = $ctx->{-heads} ? $NO_HEADS : $NO_TAGS;
+        } else {
+                $code = 200;
+                $top = $ctx->{-heads} ? $HEADS_CMD : $TAGS_CMD;
+        }
+        PublicInbox::WwwStream::html_init($ctx);
+        bless $ctx, __PACKAGE__; # re-bless for ->translate
+        print { $ctx->{zfh} } '<pre>', $top;
+        [ $code, delete($ctx->{-res_hdr}), $ctx ]; # [2] is qspawn.filter
+}
+
+sub refs_foo { # /$REPO/refs/{heads,tags} endpoints
+        my ($self, $ctx, $pfx) = @_;
+        $ctx->{wcr} = $self;
+        $ctx->{-upfx} = '../../';
+        $ctx->{-heads} = 1 if $pfx eq 'refs/heads';
+        my $qsp = PublicInbox::Qspawn->new([@EACH_REF, $pfx ],
+                                        { GIT_DIR => $ctx->{git}->{git_dir} });
+        $qsp->psgi_yield($ctx->{env}, undef, \&_refs_parse_hdr, $ctx);
+}
+
+sub srv { # endpoint called by PublicInbox::WWW
+        my ($self, $ctx) = @_;
+        my $path_info = $ctx->{env}->{PATH_INFO};
+        my $git;
+        # handle clone requests
+        my $pi_cfg = $self->{pi_cfg};
+        if ($path_info =~ m!\A/(.+?)/($PublicInbox::GitHTTPBackend::ANY)\z!x and
+                ($git = $pi_cfg->get_coderepo($1))) {
+                        PublicInbox::GitHTTPBackend::serve($ctx->{env},$git,$2);
+        } elsif ($path_info =~ m!\A/(.+?)/\z! and
+                        ($ctx->{git} = $pi_cfg->get_coderepo($1))) {
+                $ctx->{wcr} = $self;
+                sub { summary($ctx, $_[0]) }; # $_[0] = wcb
+        } elsif ($path_info =~ m!\A/(.+?)/([a-f0-9]+)/s/([^/]+)?\z! and
+                        ($ctx->{git} = $pi_cfg->get_coderepo($1))) {
+                $ctx->{lh} = $self->{log_fh};
+                PublicInbox::ViewVCS::show($ctx, $2, $3);
+        } elsif ($path_info =~ m!\A/(.+?)/tree/(.*)\z! and
+                        ($ctx->{git} = $pi_cfg->get_coderepo($1))) {
+                $ctx->{lh} = $self->{log_fh};
+                PublicInbox::RepoTree::srv_tree($ctx, $2) // r(404);
+        } elsif ($path_info =~ m!\A/(.+?)/snapshot/([^/]+)\z! and
+                        ($ctx->{git} = $pi_cfg->get_coderepo($1))) {
+                $ctx->{wcr} = $self;
+                PublicInbox::RepoSnapshot::srv($ctx, $2) // r(404);
+        } elsif ($path_info =~ m!\A/(.+?)/atom/(.*)\z! and
+                        ($ctx->{git} = $pi_cfg->get_coderepo($1))) {
+                $ctx->{lh} = $self->{log_fh};
+                PublicInbox::RepoAtom::srv_atom($ctx, $2) // r(404);
+        } elsif ($path_info =~ m!\A/(.+?)/tags\.atom\z! and
+                        ($ctx->{git} = $pi_cfg->get_coderepo($1))) {
+                PublicInbox::RepoAtom::srv_tags_atom($ctx);
+        } elsif ($path_info =~ m!\A/(.+?)/(refs/(?:heads|tags))/\z! and
+                        ($ctx->{git} = $pi_cfg->get_coderepo($1))) {
+                refs_foo($self, $ctx, $2);
+        } elsif ($path_info =~ m!\A/(.*?\*.*?)/*\z!) {
+                my $re = PublicInbox::Config::glob2re($1);
+                PublicInbox::RepoList::html($self, $ctx, qr!$re\z!) // r(404);
+        } elsif ($path_info =~ m!\A/(.+?)/\z!) {
+                my $re = qr!\A\Q$1\E/!;
+                PublicInbox::RepoList::html($self, $ctx, $re) // r(404);
+        } elsif ($path_info =~ m!\A/(.+?)\z! and
+                        ($git = $pi_cfg->get_coderepo($1))) {
+                my $qs = $ctx->{env}->{QUERY_STRING};
+                my $url = $git->base_url($ctx->{env});
+                $url .= "?$qs" if $qs ne '';
+                [ 301, [ Location => $url, 'Content-Type' => 'text/plain' ],
+                        [ "Redirecting to $url\n" ] ];
+        } else {
+                r(404);
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm
index 79c0a8ec..2d6c74da 100644
--- a/lib/PublicInbox/WwwListing.pm
+++ b/lib/PublicInbox/WwwListing.pm
@@ -41,10 +41,7 @@ sub list_match_i { # ConfigIter callback
         if (defined($section)) {
                 return if $section !~ m!\Apublicinbox\.([^/]+)\z!;
                 my $ibx = $cfg->lookup_name($1) or return;
-                if (!$ibx->{-hide}->{$ctx->hide_key} &&
-                                        grep(/$re/, @{$ibx->{url} // []})) {
-                        $ctx->ibx_entry($ibx);
-                }
+                $ctx->ibx_entry($ibx) unless $ctx->hide_inbox($ibx, $re);
         } else { # undef == "EOF"
                 $ctx->{-wcb}->($ctx->psgi_triple);
         }
@@ -54,13 +51,17 @@ sub url_filter {
         my ($ctx, $key, $default) = @_;
         $key //= 'publicInbox.wwwListing';
         $default //= '404';
-        my $v = $ctx->{www}->{pi_cfg}->{lc $key} // $default;
+        my $cfg = $ctx->{www}->{pi_cfg};
+        my $v = $cfg->{lc $key} // $default;
 again:
         if ($v eq 'match=domain') {
                 my $h = $ctx->{env}->{HTTP_HOST} // $ctx->{env}->{SERVER_NAME};
                 $h =~ s/:[0-9]+\z//;
                 (qr!\A(?:https?:)?//\Q$h\E(?::[0-9]+)?/!i, "url:$h");
         } elsif ($v eq 'all') {
+                my $niu = $cfg->{lc 'publicinbox.nameIsUrl'};
+                defined($niu) && $cfg->git_bool($niu) and
+                        $ctx->{-name_is_url} = [ '.' ];
                 (qr/./, undef);
         } elsif ($v eq '404') {
                 (undef, undef);
@@ -76,6 +77,12 @@ EOF
 
 sub hide_key { 'www' }
 
+sub hide_inbox {
+        my ($ctx, $ibx, $re) = @_;
+        $ibx->{'-hide_'.$ctx->hide_key} ||
+                !grep(/$re/, @{$ibx->{url} // $ctx->{-name_is_url} // []})
+}
+
 sub add_misc_ibx { # MiscSearch->retry_reopen callback
         my ($misc, $ctx, $re, $qs) = @_;
         require PublicInbox::SearchQuery;
@@ -104,15 +111,13 @@ sub add_misc_ibx { # MiscSearch->retry_reopen callback
                 $ctx->ibx_entry($pi_cfg->ALL // die('BUG: ->ALL expected'), {});
         }
         my $mset = $misc->mset($qs, $opt); # sorts by $MODIFIED (mtime)
-        my $hide_key = $ctx->hide_key;
 
         for my $mi ($mset->items) {
                 my $doc = $mi->get_document;
                 my ($eidx_key) = PublicInbox::Search::xap_terms('Q', $doc);
                 $eidx_key // next;
                 my $ibx = $pi_cfg->lookup_eidx_key($eidx_key) // next;
-                next if $ibx->{-hide}->{$hide_key};
-                grep(/$re/, @{$ibx->{url} // []}) or next;
+                next if $ctx->hide_inbox($ibx, $re);
                 $ctx->ibx_entry($ibx, $misc->doc2ibx_cache_ent($doc));
                 if ($r) { # for descriptions in search_nav_bot
                         my $pct = PublicInbox::Search::get_pct($mi);
@@ -162,24 +167,22 @@ sub mset_footer ($$) {
         # no footer if too few matches
         return '' if $mset->get_matches_estimated == $mset->size;
         require PublicInbox::SearchView;
-        PublicInbox::SearchView::search_nav_bot($mset, $ctx->{-sq});
+        PublicInbox::SearchView::search_nav_bot($ctx, $mset, $ctx->{-sq});
 }
 
 sub mset_nav_top {
         my ($ctx, $mset) = @_;
         my $q = $ctx->{-sq};
         my $qh = $q->{'q'} // '';
-        utf8::decode($qh);
-        $qh = ascii_html($qh);
-        $qh = qq[\nvalue="$qh"] if $qh ne '';
-        my $rv = <<EOM;
-<form
-action="./"><pre><input name=q type=text$qh
-/><input type=submit value="locate inbox"
-/><input type=submit name=a value="search all inboxes"
-/></pre></form><pre>
+        if ($qh ne '') {
+                utf8::decode($qh);
+                $qh = qq[\nvalue="].ascii_html($qh).'"';
+        }
+        chop(my $rv = <<EOM);
+<form action="./"><pre><input name=q type=text$qh/><input
+type=submit value="locate inbox"/><input type=submit name=a
+value="search all inboxes"/></pre></form><pre>
 EOM
-        chomp $rv;
         if (defined($q->{'q'})) {
                 my $initial_q = $ctx->{-uxs_retried};
                 if (defined $initial_q) {
@@ -210,28 +213,28 @@ sub psgi_triple {
         my $h = [ 'Content-Type', 'text/html; charset=UTF-8',
                         'Content-Length', undef ];
         my $gzf = gzf_maybe($h, $ctx->{env});
-        $gzf->zmore('<html><head><title>public-inbox listing</title>' .
-                        $ctx->{www}->style('+/') .
-                        '</head><body>');
+        my $zfh = $gzf->zfh;
+        print $zfh '<html><head><title>public-inbox listing</title>',
+                        $ctx->{www}->style('+/'),
+                        '</head><body>';
         my $code = 404;
         if (my $list = delete $ctx->{-list}) {
                 my $mset = delete $ctx->{-mset};
                 $code = 200;
                 if ($mset) { # already sorted, so search bar:
-                        $gzf->zmore(mset_nav_top($ctx, $mset));
+                        print $zfh mset_nav_top($ctx, $mset);
                 } else { # sort config dump by ->modified
                         @$list = map { $_->[1] }
                                 sort { $b->[0] <=> $a->[0] } @$list;
                 }
-                $gzf->zmore('<pre>');
-                $gzf->zmore(join("\n", @$list));
-                $gzf->zmore(mset_footer($ctx, $mset)) if $mset;
+                print $zfh '<pre>', join("\n", @$list); # big
+                print $zfh mset_footer($ctx, $mset) if $mset;
         } elsif (my $mset = delete $ctx->{-mset}) {
-                $gzf->zmore(mset_nav_top($ctx, $mset));
-                $gzf->zmore('<pre>no matching inboxes');
-                $gzf->zmore(mset_footer($ctx, $mset));
+                print $zfh mset_nav_top($ctx, $mset),
+                                '<pre>no matching inboxes',
+                                mset_footer($ctx, $mset);
         } else {
-                $gzf->zmore('<pre>no inboxes, yet');
+                print $zfh '<pre>no inboxes, yet';
         }
         my $out = $gzf->zflush('</pre><hr><pre>'.
 qq(This is a listing of public inboxes, see the `mirror' link of each inbox
diff --git a/lib/PublicInbox/WwwStatic.pm b/lib/PublicInbox/WwwStatic.pm
index eeb5e565..d8902193 100644
--- a/lib/PublicInbox/WwwStatic.pm
+++ b/lib/PublicInbox/WwwStatic.pm
@@ -12,13 +12,12 @@ use strict;
 use v5.10.1;
 use parent qw(Exporter);
 use Fcntl qw(SEEK_SET O_RDONLY O_NONBLOCK);
-use POSIX qw(strftime);
 use HTTP::Date qw(time2str);
 use HTTP::Status qw(status_message);
 use Errno qw(EACCES ENOTDIR ENOENT);
 use URI::Escape qw(uri_escape_utf8);
 use PublicInbox::GzipFilter qw(gzf_maybe);
-use PublicInbox::Hval qw(ascii_html);
+use PublicInbox::Hval qw(ascii_html fmt_ts);
 use Plack::MIME;
 our @EXPORT_OK = qw(@NO_CACHE r path_info_raw);
 
@@ -275,12 +274,11 @@ sub dir_response ($$$) {
         my $path_info = $env->{PATH_INFO};
         push @entries, '..' if $path_info ne '/';
         for my $base (@entries) {
+                my @st = stat($fs_path . $base) or next; # unlikely
                 my $href = ascii_html(uri_escape_utf8($base));
                 my $name = ascii_html($base);
-                my @st = stat($fs_path . $base) or next; # unlikely
-                my ($gzipped, $uncompressed, $hsize);
-                my $entry = '';
                 my $mtime = $st[9];
+                my ($entry, $hsize);
                 if (-d _) {
                         $href .= '/';
                         $name .= '/';
@@ -296,12 +294,12 @@ sub dir_response ($$$) {
                         next;
                 }
                 # 54 = 80 - (SP length(strftime(%Y-%m-%d %k:%M)) SP human_size)
-                $hsize = sprintf('% 8s', $hsize);
                 my $pad = 54 - length($name);
                 $pad = 1 if $pad <= 0;
-                $entry .= qq(<a\nhref="$href">$name</a>) . (' ' x $pad);
-                $mtime = strftime('%Y-%m-%d %k:%M', gmtime($mtime));
-                $entry .= $mtime . $hsize;
+                $entry = qq(\n<a\nhref="$href">$name</a>) .
+                                (' ' x $pad) .
+                                fmt_ts($mtime) .
+                                sprintf('% 8s', $hsize);
         }
 
         # filter out '.gz' files as long as the mtime matches the
@@ -309,17 +307,16 @@ sub dir_response ($$$) {
         delete(@other{keys %want_gz});
         @entries = ((map { ${$dirs{$_}} } sort keys %dirs),
                         (map { ${$other{$_}} } sort keys %other));
-
         my $path_info_html = ascii_html($path_info);
-        my $h = [qw(Content-Type text/html Content-Length), undef];
-        my $gzf = gzf_maybe($h, $env);
-        $gzf->zmore("<html><head><title>Index of $path_info_html</title>" .
-                ${$self->{style}} .
-                "</head><body><pre>Index of $path_info_html</pre><hr><pre>\n");
-        $gzf->zmore(join("\n", @entries));
-        my $out = $gzf->zflush("</pre><hr></body></html>\n");
-        $h->[3] = length($out);
-        [ 200, $h, [ $out ] ]
+        my @h = qw(Content-Type text/html);
+        my $gzf = gzf_maybe(\@h, $env);
+        print { $gzf->zfh } '<html><head><title>Index of ', $path_info_html,
+                '</title>', ${$self->{style}}, '</head><body><pre>Index of ',
+                $path_info_html, '</pre><hr><pre>', @entries,
+                '</pre><hr></body></html>';
+        my $out = $gzf->zflush;
+        push @h, 'Content-Length', length($out);
+        [ 200, \@h, [ $out ] ]
 }
 
 sub call { # PSGI app endpoint
diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index aee78170..8d32074f 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # HTML body stream for which yields getline+close methods for
@@ -17,8 +17,9 @@ http://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/public-inb
 https://public-inbox.org/public-inbox.git) ];
 
 sub base_url ($) {
-        my $ctx = shift;
-        my $base_url = $ctx->{ibx}->base_url($ctx->{env});
+        my ($ctx) = @_;
+        my $thing = $ctx->{ibx} // $ctx->{git} // return;
+        my $base_url = $thing->base_url($ctx->{env});
         chop $base_url; # no trailing slash for clone
         $base_url;
 }
@@ -27,6 +28,9 @@ sub init {
         my ($ctx, $cb) = @_;
         $ctx->{cb} = $cb;
         $ctx->{base_url} = base_url($ctx);
+        $ctx->{-res_hdr} = [ 'Content-Type' => 'text/html; charset=UTF-8' ];
+        $ctx->{gz} = PublicInbox::GzipFilter::gz_or_noop($ctx->{-res_hdr},
+                                                        $ctx->{env});
         bless $ctx, __PACKAGE__;
 }
 
@@ -35,9 +39,60 @@ sub async_eml { # for async_blob_cb
         $ctx->write($ctx->{cb}->($ctx, $eml));
 }
 
+sub html_repo_top ($) {
+        my ($ctx) = @_;
+        my $git = $ctx->{git} // return $ctx->html_top_fallback;
+        my $desc = ascii_html($git->description);
+        my $title = delete($ctx->{-title_html}) // $desc;
+        my $upfx = $ctx->{-upfx} // '';
+        my $atom = $ctx->{-atom} // (substr($upfx, -1) eq '/' ?
+                                        "${upfx}atom/" : "$upfx/atom/");
+        my $top = ascii_html($git->{nick});
+        $top = qq(<a\nhref="$upfx">$top</a>) if length($upfx);
+        $top .= <<EOM;
+  <a href='$upfx#readme'>about</a> / <a
+href='$upfx#heads'>heads</a> / <a
+href='$upfx#tags'>tags</a>
+<b>$desc</b>
+EOM
+        my @url = PublicInbox::ViewVCS::ibx_url_for($ctx);
+        if (@url) {
+                $ctx->{-has_srch} = 1;
+                my $base_url = base_url($ctx);
+                my ($pfx, $sfx) = ($base_url =~ m!\A(https?://[^/]+/)(.*)\z!i);
+                my $iupfx = '../' x (($sfx =~ tr!/!/!) + 1);
+                $pfx = ascii_html($pfx);
+                $pfx = qr/\A\Q$pfx\E/i;
+                my $tmp = $top;
+                $top = '';
+                my ($s, $u);
+                my $q_val = delete($ctx->{-q_value_html}) // '';
+                $q_val = qq(\nvalue="$q_val") if $q_val ne '';
+                for (@url) {
+                        $u = $s = ascii_html($_);
+                        substr($u, 0, 0, $iupfx) if $u !~ m!://!;
+                        $s =~ s!$pfx!!;
+                        $s =~ s!/\z!!;
+                        $top .= qq{<form\naction="$u"><pre>$tmp} .
+                                qq{<input\nname=q type=text$q_val />} .
+                                qq{<input type=submit\n} .
+                                qq{value="search mail in `$s&#39;"/>} .
+                                q{</pre></form>};
+                        $tmp = '';
+                }
+        } else {
+                $top = "<pre>$top</pre>";
+        }
+        "<html><head><title>$title</title>" .
+                qq(<link\nrel=alternate\ntitle="Atom feed"\n).
+                qq(href="$atom"\ntype="application/atom+xml"/>) .
+                $ctx->{www}->style($upfx) .
+                '</head><body>'.$top;
+}
+
 sub html_top ($) {
         my ($ctx) = @_;
-        my $ibx = $ctx->{ibx};
+        my $ibx = $ctx->{ibx} // return html_repo_top($ctx);
         my $desc = ascii_html($ibx->description);
         my $title = delete($ctx->{-title_html}) // $desc;
         my $upfx = $ctx->{-upfx} || '';
@@ -59,6 +114,7 @@ sub html_top ($) {
                         qq(<a\nid=mirror) .
                         qq(\nhref="${upfx}_/text/mirror/">mirror</a>$code / ).
                         qq(<a\nhref="$atom">Atom feed</a>);
+        $links .= delete($ctx->{-html_more_links}) if $ctx->{-html_more_links};
         if ($ibx->isrch) {
                 my $q_val = delete($ctx->{-q_value_html}) // '';
                 $q_val = qq(\nvalue="$q_val") if $q_val ne '';
@@ -81,38 +137,36 @@ sub html_top ($) {
                 '</head><body>'. $top . (delete($ctx->{-html_tip}) // '');
 }
 
+sub inboxes { () } # TODO
+
 sub coderepos ($) {
         my ($ctx) = @_;
+        $ctx->{ibx} // return inboxes($ctx);
         my $cr = $ctx->{ibx}->{coderepo} // return ();
-        my $cfg = $ctx->{www}->{pi_cfg};
         my $upfx = ($ctx->{-upfx} // ''). '../';
-        my @ret;
-        for my $cr_name (@$cr) {
-                $ret[0] //= <<EOF;
-<a id=code>Code repositories for project(s) associated with this inbox:
-EOF
-                my $urls = $cfg->get_all("coderepo.$cr_name.cgiturl");
-                if ($urls) {
-                        for (@$urls) {
-                                # relative or absolute URL?, prefix relative
-                                # "foo.git" with appropriate number of "../"
-                                my $u = m!\A(?:[a-z\+]+:)?//! ? $_ : $upfx.$_;
-                                $u = ascii_html(prurl($ctx->{env}, $u));
-                                $ret[0] .= qq(\n\t<a\nhref="$u">$u</a>);
-                        }
-                } else {
-                        $ret[0] .= qq[\n\t$cr_name.git (no URL configured)];
+        my $pfx = $ctx->{base_url} //= $ctx->base_url;
+        my $up = $upfx =~ tr!/!/!;
+        $pfx =~ s!/[^/]+\z!/! for (1..$up);
+        $pfx .= '/' if substr($pfx, -1, 1) ne '/';
+        my $buf = '<a id=code>' .
+                'Code repositories for project(s) associated with this '.
+                $ctx->{ibx}->thing_type . "\n";
+        for my $git (@{$ctx->{www}->{pi_cfg}->repo_objs($ctx->{ibx})}) {
+                for ($git->pub_urls($ctx->{env})) {
+                        my $u = m!\A(?:[a-z\+]+:)?//!i ? $_ : $pfx.$_;
+                        $u = ascii_html(prurl($ctx->{env}, $u));
+                        $buf .= qq(\n\t<a\nhref="$u">$u</a>);
                 }
         }
-        @ret; # may be empty, this sub is called as an arg for join()
+        ($buf);
 }
 
 sub _html_end {
         my ($ctx) = @_;
         my $upfx = $ctx->{-upfx} || '';
         my $m = "${upfx}_/text/mirror/";
-        my $x;
-        if ($ctx->{ibx}->can('cloneurl')) {
+        my $x = '';
+        if ($ctx->{ibx} && $ctx->{ibx}->can('cloneurl')) {
                 $x = <<EOF;
 This is a public inbox, see <a
 href="$m">mirroring instructions</a>
@@ -136,12 +190,15 @@ as well as URLs for IMAP folder(s).
 EOM
                         }
                 }
-        } else {
+        } elsif ($ctx->{ibx}) { # extindex
                 $x = <<EOF;
 This is an external index of several public inboxes,
 see <a href="$m">mirroring instructions</a> on how to clone and mirror
 all data and code used by this external index.
 EOF
+        } elsif ($ctx->{git}) { # coderepo
+                $x = join('', map { "git clone $_\n" }
+                        @{$ctx->{git}->cloneurl($ctx->{env})});
         }
         chomp $x;
         '<hr><pre>'.join("\n\n", coderepos($ctx), $x).'</pre></body></html>'
@@ -164,18 +221,26 @@ sub getline {
         $ctx->zflush(_html_end($ctx));
 }
 
-sub html_oneshot ($$;$) {
-        my ($ctx, $code, $sref) = @_;
+sub html_done ($;@) {
+        my $ctx = $_[0];
+        my $bdy = $ctx->zflush(@_[1..$#_], _html_end($ctx));
+        my $res_hdr = delete $ctx->{-res_hdr};
+        push @$res_hdr, 'Content-Length', length($bdy);
+        [ 200, $res_hdr, [ $bdy ] ]
+}
+
+sub html_oneshot ($$;@) {
+        my ($ctx, $code) = @_[0, 1];
         my $res_hdr = [ 'Content-Type' => 'text/html; charset=UTF-8',
                 'Content-Length' => undef ];
         bless $ctx, __PACKAGE__;
         $ctx->{gz} = PublicInbox::GzipFilter::gz_or_noop($res_hdr, $ctx->{env});
+        my @top;
         $ctx->{base_url} // do {
-                $ctx->zmore(html_top($ctx));
+                @top = html_top($ctx);
                 $ctx->{base_url} = base_url($ctx);
         };
-        $ctx->zmore($$sref) if $sref;
-        my $bdy = $ctx->zflush(_html_end($ctx));
+        my $bdy = $ctx->zflush(@top, @_[2..$#_], _html_end($ctx));
         $res_hdr->[3] = length($bdy);
         [ $code, $res_hdr, [ $bdy ] ]
 }
@@ -195,10 +260,23 @@ sub async_next ($) {
 }
 
 sub aresponse {
-        my ($ctx, $code, $cb) = @_;
-        my $res_hdr = [ 'Content-Type' => 'text/html; charset=UTF-8' ];
+        my ($ctx, $cb) = @_;
         init($ctx, $cb);
-        $ctx->psgi_response($code, $res_hdr);
+        $ctx->psgi_response(200, delete $ctx->{-res_hdr});
+}
+
+sub html_init {
+        my $ctx = $_[-1];
+        $ctx->{base_url} = base_url($ctx);
+        my $h = $ctx->{-res_hdr} = ['Content-Type', 'text/html; charset=UTF-8'];
+        $ctx->{gz} = PublicInbox::GzipFilter::gz_or_noop($h, $ctx->{env});
+        bless $ctx, @_ > 1 ? $_[0] : __PACKAGE__;
+        print { $ctx->zfh } html_top($ctx);
+}
+
+sub DESTROY {
+        my ($ctx) = @_;
+        $ctx->{git}->cleanup if $ctx->{git} && $ctx->{git}->{-tmp};
 }
 
 1;
diff --git a/lib/PublicInbox/WwwText.pm b/lib/PublicInbox/WwwText.pm
index 2b4e69fe..5e23005e 100644
--- a/lib/PublicInbox/WwwText.pm
+++ b/lib/PublicInbox/WwwText.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # used for displaying help texts and other non-mail content
@@ -7,7 +7,7 @@ use strict;
 use v5.10.1;
 use PublicInbox::Linkify;
 use PublicInbox::WwwStream;
-use PublicInbox::Hval qw(ascii_html prurl);
+use PublicInbox::Hval qw(ascii_html prurl fmt_ts);
 use HTTP::Date qw(time2str);
 use URI::Escape qw(uri_escape_utf8);
 use PublicInbox::GzipFilter qw(gzf_maybe);
@@ -31,20 +31,17 @@ sub get_text {
         my $have_tslash = ($key =~ s!/\z!!) if !$raw;
 
         my $txt = '';
-        my $hdr = [ 'Content-Type', 'text/plain', 'Content-Length', undef ];
-        if (!_default_text($ctx, $key, $hdr, \$txt)) {
+        if (!_default_text($ctx, $key, \$txt)) {
                 $code = 404;
                 $txt = "404 Not Found ($key)\n";
         }
         my $env = $ctx->{env};
         if ($raw) {
-                if ($code == 200) {
-                        my $gzf = gzf_maybe($hdr, $env);
-                        $txt = $gzf->translate($txt);
-                        $txt .= $gzf->zflush;
-                }
-                $hdr->[3] = length($txt);
-                return [ $code, $hdr, [ $txt ] ]
+                my $h = delete $ctx->{-res_hdr};
+                $txt = gzf_maybe($h, $env)->zflush($txt) if $code == 200;
+                push @$h, 'Content-Type', 'text/plain',
+                        'Content-Length', length($txt);
+                return [ $code, $h, [ $txt ] ]
         }
 
         # enforce trailing slash for "wget -r" compatibility
@@ -71,7 +68,11 @@ sub get_text {
                 $txt = ascii_html($txt);
         }
         $txt = '<pre>' . $l->linkify_2($txt) . '</pre>';
-        PublicInbox::WwwStream::html_oneshot($ctx, $code, \$txt);
+        $txt =~ s!^search$!<a\nid=search>search</a>!sm;
+        $txt =~ s!\bPOP3\b!<a\nid=pop3>POP3</a>!;
+        $txt =~ s!\b(Newsgroups?)\b!<a\nid=nntp>$1</a>!;
+        $txt =~ s!\bIMAP\b!<a\nid=imap>IMAP</a>!;
+        PublicInbox::WwwStream::html_oneshot($ctx, $code, $txt);
 }
 
 sub _srch_prefix ($$) {
@@ -167,12 +168,13 @@ EOF
 }
 
 # n.b. this is a perfect candidate for memoization
-sub inbox_config ($$$) {
-        my ($ctx, $hdr, $txt) = @_;
+sub inbox_config ($$) {
+        my ($ctx, $txt) = @_;
         my $ibx = $ctx->{ibx};
-        push @$hdr, 'Content-Disposition', 'inline; filename=inbox.config';
+        push @{$ctx->{-res_hdr}},
+                'Content-Disposition', 'inline; filename=inbox.config';
         my $t = eval { $ibx->mm->created_at };
-        push(@$hdr, 'Last-Modified', time2str($t)) if $t;
+        push(@{$ctx->{-res_hdr}}, 'Last-Modified', time2str($t)) if $t;
         my $name = dq_escape($ibx->{name});
         my $inboxdir = '/path/to/top-level-inbox';
         my $base_url = $ibx->base_url($ctx->{env});
@@ -219,10 +221,11 @@ EOF
 }
 
 # n.b. this is a perfect candidate for memoization
-sub extindex_config ($$$) {
-        my ($ctx, $hdr, $txt) = @_;
+sub extindex_config ($$) {
+        my ($ctx, $txt) = @_;
         my $ibx = $ctx->{ibx};
-        push @$hdr, 'Content-Disposition', 'inline; filename=extindex.config';
+        push @{$ctx->{-res_hdr}},
+                'Content-Disposition', 'inline; filename=extindex.config';
         my $name = dq_escape($ibx->{name});
         my $base_url = $ibx->base_url($ctx->{env});
         $$txt .= <<EOS;
@@ -245,52 +248,64 @@ EOS
 
 sub coderepos_raw ($$) {
         my ($ctx, $top_url) = @_;
-        my $cr = $ctx->{ibx}->{coderepo} // return ();
         my $cfg = $ctx->{www}->{pi_cfg};
-        my @ret;
-        for my $cr_name (@$cr) {
-                $ret[0] //= do {
-                        my $thing = $ctx->{ibx}->can('cloneurl') ?
-                                'public inbox' : 'external index';
-                        <<EOF;
-Code repositories for project(s) associated with this $thing
-EOF
-                };
-                my $urls = $cfg->get_all("coderepo.$cr_name.cgiturl");
-                if ($urls) {
-                        for (@$urls) {
-                                # relative or absolute URL?, prefix relative
-                                # "foo.git" with appropriate number of "../"
-                                my $u = m!\A(?:[a-z\+]+:)?//!i ? $_ :
-                                        $top_url.$_;
-                                $ret[0] .= "\n\t" . prurl($ctx->{env}, $u);
-                        }
-                } else {
-                        $ret[0] .= qq[\n\t$cr_name.git (no URL configured)];
+        my $cr = $cfg->repo_objs($ctx->{ibx}) or return ();
+        my $buf = 'Code repositories for project(s) associated with this '.
+                $ctx->{ibx}->thing_type . ":\n";
+        my @recs = PublicInbox::CodeSearch::repos_sorted($cfg, @$cr);
+        my $cr_score = $ctx->{ibx}->{-cr_score};
+        my $env = $ctx->{env};
+        for (@recs) {
+                my ($t, $git) = @$_;
+                for ($git->pub_urls($env)) {
+                        my $u = m!\A(?:[a-z\+]+:)?//!i ? $_ : $top_url.$_;
+                        my $nr = $cr_score->{$git->{nick}};
+                        $buf .= "\n";
+                        $buf .= $nr ? sprintf('% 9u', $nr) : (' 'x9);
+                        $buf .= ' '.fmt_ts($t).' '.prurl($env, $u);
                 }
         }
-        @ret; # may be empty, this sub is called as an arg for join()
+        ($buf);
 }
 
-sub _add_imap_nntp_urls ($$) {
+sub _add_non_http_urls ($$) {
         my ($ctx, $txt) = @_;
         $ctx->{ibx}->can('nntp_url') or return; # TODO extindex can have IMAP
         my $urls = $ctx->{ibx}->imap_url($ctx);
         if (@$urls) {
-                $$txt .= "\nIMAP subfolder(s) are available under:";
-                $$txt .= "\n  " . join("\n  ", @$urls);
+                $urls = join("\n  ", @$urls);
+                $urls =~ s!://([^/@]+)/!://;AUTH=ANONYMOUS\@$1/!sg;
                 $$txt .= <<EOM
 
+IMAP subfolder(s) are available under:
+  $urls
   # each subfolder (starting with `0') holds 50K messages at most
 EOM
         }
         $urls = $ctx->{ibx}->nntp_url($ctx);
         if (@$urls) {
-                $$txt .= "\n";
-                $$txt .= @$urls == 1 ? 'Newsgroup' : 'Newsgroups are';
+                $$txt .= @$urls == 1 ? "\nNewsgroup" : "\nNewsgroups are";
                 $$txt .= ' available over NNTP:';
                 $$txt .= "\n  " . join("\n  ", @$urls) . "\n";
         }
+        $urls = $ctx->{ibx}->pop3_url($ctx);
+        if (@$urls) {
+                $urls = join("\n  ", @$urls);
+                $$txt .= <<EOM;
+
+POP3 access is available:
+  $urls
+
+The POP3 password is: anonymous
+The POP3 username is: \$(uuidgen)\@$ctx->{ibx}->{newsgroup}
+where \$(uuidgen) in the output of the `uuidgen' command on your system.
+The UUID in the username functions as a private cookie (don't share it).
+By default, only 1000 messages are retrieved.  You may download more
+by appending `?limit=NUM' (without quotes) to the username, where
+`NUM' is an integer between 1 and 50000.
+Idle accounts will expire periodically.
+EOM
+        }
 }
 
 sub _add_onion_note ($) {
@@ -379,7 +394,7 @@ EOM
 
 Example config snippet for mirrors: $cfg_link
 EOF
-        _add_imap_nntp_urls($ctx, $txt);
+        _add_non_http_urls($ctx, $txt);
         _add_onion_note($txt);
 
         my $code_url = prurl($ctx->{env}, $PublicInbox::WwwStream::CODE_URL);
@@ -389,16 +404,16 @@ EOF
         1;
 }
 
-sub _default_text ($$$$) {
-        my ($ctx, $key, $hdr, $txt) = @_;
+sub _default_text ($$$) {
+        my ($ctx, $key, $txt) = @_;
         if ($key eq 'mirror') {
                 return _mirror_help($ctx, $txt);
         } elsif ($key eq 'color') {
                 return _colors_help($ctx, $txt);
         } elsif ($key eq 'config') {
                 return $ctx->{ibx}->can('cloneurl') ?
-                        inbox_config($ctx, $hdr, $txt) :
-                        extindex_config($ctx, $hdr, $txt);
+                        inbox_config($ctx, $txt) :
+                        extindex_config($ctx, $txt);
         }
         return if $key ne 'help'; # TODO more keys?
 
@@ -508,7 +523,7 @@ message threading
 EOF
         } # $over
 
-        _add_imap_nntp_urls($ctx, \(my $note = ''));
+        _add_non_http_urls($ctx, \(my $note = ''));
         $note and $note =~ s/^/  /gms and $$txt .= <<EOF;
 additional protocols
 --------------------
diff --git a/lib/PublicInbox/WwwTopics.pm b/lib/PublicInbox/WwwTopics.pm
new file mode 100644
index 00000000..9d270732
--- /dev/null
+++ b/lib/PublicInbox/WwwTopics.pm
@@ -0,0 +1,85 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+package PublicInbox::WwwTopics;
+use v5.12;
+use PublicInbox::Hval qw(ascii_html mid_href fmt_ts);
+
+sub add_topic_html ($$) {
+        my (undef, $smsg) = @_;
+        my $s = ascii_html($smsg->{subject});
+        $s = '(no subject)' if $s eq '';
+        $_[0] .= "\n".fmt_ts($smsg->{ds}) .
+                qq{ <a\nhref="}.mid_href($smsg->{mid}).qq{/#r">$s</a>};
+        my $nr = $smsg->{'COUNT(num)'};
+        $_[0] .= " $nr+ messages" if $nr > 1;
+}
+
+# n.b. the `SELECT DISTINCT(tid)' subquery is critical for performance
+# with giant inboxes and extindices
+sub topics_new ($) {
+        $_[0]->do_get(<<EOS);
+SELECT ds,ddd,COUNT(num) FROM over WHERE tid IN
+(SELECT DISTINCT(tid) FROM over WHERE tid > 0 ORDER BY ts DESC LIMIT 200)
+AND +num > 0
+GROUP BY tid
+ORDER BY ds ASC
+EOS
+}
+
+sub topics_active ($) {
+        $_[0]->do_get(<<EOS);
+SELECT ddd,MAX(ds) as ds,COUNT(num) FROM over WHERE tid IN
+(SELECT DISTINCT(tid) FROM over WHERE tid > 0 ORDER BY ts DESC LIMIT 200)
+AND +num > 0
+GROUP BY tid
+ORDER BY ds ASC
+EOS
+}
+
+sub topics_i { pop @{$_[0]->{msgs}} }
+
+sub topics_atom { # GET /$INBOX_NAME/topics_(new|active).atom
+        my ($ctx) = @_;
+        require PublicInbox::WwwAtomStream;
+        my ($hdr, $smsg, $val);
+        PublicInbox::WwwAtomStream->response($ctx, \&topics_i);
+}
+
+sub topics_html { # GET /$INBOX_NAME/topics_(new|active).html
+        my ($ctx) = @_;
+        require PublicInbox::WwwStream;
+        my $buf = '<pre>';
+        $ctx->{-html_more_links} = qq{\n- recent:[<a
+href="./">subjects (threaded)</a>|};
+
+        if ($ctx->{topic_category} eq 'new') {
+                $ctx->{-html_more_links} .= qq{<b>topics (new)</b>|<a
+href="./topics_active.html">topics (active)</a>]};
+        } else { # topic_category eq "active" - topics with recent replies
+                $ctx->{-html_more_links} .= qq{<a
+href="./topics_new.html">topics (new)</a>|<b>topics (active)</b>]};
+        }
+        # can't use SQL to filter references since our schema wasn't designed
+        # for it, but our SQL sorts by ascending time to favor top-level
+        # messages while our final result (post-references filter) favors
+        # recent messages
+        my $msgs = delete $ctx->{msgs};
+        add_topic_html($buf, pop @$msgs) while scalar(@$msgs);
+        $buf .= '</pre>';
+        PublicInbox::WwwStream::html_oneshot($ctx, 200, $buf);
+}
+
+sub response {
+        my ($ctx, $ibx_name, $category, $type) = @_;
+        my ($ret, $over);
+        $ret = PublicInbox::WWW::invalid_inbox($ctx, $ibx_name) and return $ret;
+        $over = $ctx->{ibx}->over or
+                return PublicInbox::WWW::need($ctx, 'Overview', './');
+        $ctx->{msgs} = $category eq 'new' ? topics_new($over) :
+                        topics_active($over);
+        $ctx->{topic_category} = $category;
+        $type eq 'atom' ? topics_atom($ctx) : topics_html($ctx);
+}
+
+1;
diff --git a/lib/PublicInbox/XapClient.pm b/lib/PublicInbox/XapClient.pm
new file mode 100644
index 00000000..4dcbbe5d
--- /dev/null
+++ b/lib/PublicInbox/XapClient.pm
@@ -0,0 +1,48 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# This talks to (XapHelperCxx.pm + xap_helper.h) or XapHelper.pm
+# and will eventually allow users with neither XS nor SWIG Perl
+# bindings to use Xapian as long as they have Xapian development
+# headers/libs and a C++ compiler
+package PublicInbox::XapClient;
+use v5.12;
+use PublicInbox::Spawn qw(spawn);
+use Socket qw(AF_UNIX SOCK_SEQPACKET);
+use PublicInbox::IPC;
+use autodie qw(fork pipe socketpair);
+
+sub mkreq {
+        my ($self, $ios, @arg) = @_;
+        my ($r, $n);
+        pipe($r, $ios->[0]) if !defined($ios->[0]);
+        my @fds = map fileno($_), @$ios;
+        my $buf = join("\0", @arg, '');
+        $n = $PublicInbox::IPC::send_cmd->($self->{io}, \@fds, $buf, 0) //
+                die "send_cmd: $!";
+        $n == length($buf) or die "send_cmd: $n != ".length($buf);
+        $r;
+}
+
+sub start_helper {
+        my @argv = @_;
+        socketpair(my $sock, my $in, AF_UNIX, SOCK_SEQPACKET, 0);
+        my $cls = 'PublicInbox::XapHelperCxx';
+        my $env;
+        my $cmd = eval "require $cls; ${cls}::cmd()";
+        if ($@) { # fall back to Perl + XS|SWIG
+                $cls = 'PublicInbox::XapHelper';
+                # ensure the child process has the same @INC we do:
+                $env = { PERL5LIB => join(':', @INC) };
+                $cmd = [$^X, ($^W ? ('-w') : ()), "-M$cls", '-e',
+                        $cls.'::start(@ARGV)', '--' ];
+        }
+        push @$cmd, @argv;
+        my $pid = spawn($cmd, $env, { 0 => $in });
+        my $self = bless { io => $sock, impl => $cls }, __PACKAGE__;
+        PublicInbox::IO::attach_pid($sock, $pid);
+        $self;
+}
+
+1;
diff --git a/lib/PublicInbox/XapHelper.pm b/lib/PublicInbox/XapHelper.pm
new file mode 100644
index 00000000..ed11a2f8
--- /dev/null
+++ b/lib/PublicInbox/XapHelper.pm
@@ -0,0 +1,313 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Perl + SWIG||XS implementation if XapHelperCxx / xap_helper.h isn't usable.
+package PublicInbox::XapHelper;
+use v5.12;
+use Getopt::Long (); # good API even if we only use short options
+our $GLP = Getopt::Long::Parser->new;
+$GLP->configure(qw(require_order bundling no_ignore_case no_auto_abbrev));
+use PublicInbox::Search qw(xap_terms);
+use PublicInbox::CodeSearch;
+use PublicInbox::IPC;
+use PublicInbox::IO qw(read_all);
+use Socket qw(SOL_SOCKET SO_TYPE SOCK_SEQPACKET AF_UNIX);
+use PublicInbox::DS qw(awaitpid);
+use autodie qw(open getsockopt);
+use POSIX qw(:signal_h);
+use Fcntl qw(LOCK_UN LOCK_EX);
+use Carp qw(croak);
+my $X = \%PublicInbox::Search::X;
+our (%SRCH, %WORKERS, $nworker, $workerset, $in);
+our $stderr = \*STDERR;
+
+sub cmd_test_inspect {
+        my ($req) = @_;
+        print { $req->{0} } "pid=$$ has_threadid=",
+                ($req->{srch}->has_threadid ? 1 : 0)
+}
+
+sub iter_retry_check ($) {
+        if (ref($@) =~ /\bDatabaseModifiedError\b/) {
+                $_[0]->{srch}->reopen;
+                undef; # retries
+        } elsif (ref($@) =~ /\bDocNotFoundError\b/) {
+                warn "doc not found: $@";
+                0; # continue to next doc
+        } else {
+                die;
+        }
+}
+
+sub term_length_extract ($) {
+        my ($req) = @_;
+        @{$req->{A_len}} = map {
+                my $len = s/([0-9]+)\z// ? ($1 + 0) : undef;
+                [ $_, $len ];
+        } @{$req->{A}};
+}
+
+sub dump_ibx_iter ($$$) {
+        my ($req, $ibx_id, $it) = @_;
+        my $out = $req->{0};
+        eval {
+                my $doc = $it->get_document;
+                for my $pair (@{$req->{A_len}}) {
+                        my ($pfx, $len) = @$pair;
+                        my @t = xap_terms($pfx, $doc);
+                        @t = grep { length == $len } @t if defined($len);
+                        for (@t) {
+                                print $out "$_ $ibx_id\n" or die "print: $!";
+                                ++$req->{nr_out};
+                        }
+                }
+        };
+        $@ ? iter_retry_check($req) : 0;
+}
+
+sub emit_mset_stats ($$) {
+        my ($req, $mset) = @_;
+        my $err = $req->{1} or croak "BUG: caller only passed 1 FD";
+        say $err 'mset.size='.$mset->size.' nr_out='.$req->{nr_out}
+}
+
+sub cmd_dump_ibx {
+        my ($req, $ibx_id, $qry_str) = @_;
+        $qry_str // die 'usage: dump_ibx [OPTIONS] IBX_ID QRY_STR';
+        $req->{A} or die 'dump_ibx requires -A PREFIX';
+        term_length_extract $req;
+        my $max = $req->{'m'} // $req->{srch}->{xdb}->get_doccount;
+        my $opt = { relevance => -1, limit => $max, offset => $req->{o} // 0 };
+        $opt->{eidx_key} = $req->{O} if defined $req->{O};
+        my $mset = $req->{srch}->mset($qry_str, $opt);
+        $req->{0}->autoflush(1);
+        for my $it ($mset->items) {
+                for (my $t = 10; $t > 0; --$t) {
+                        $t = dump_ibx_iter($req, $ibx_id, $it) // $t;
+                }
+        }
+        emit_mset_stats($req, $mset);
+}
+
+sub dump_roots_iter ($$$) {
+        my ($req, $root2off, $it) = @_;
+        eval {
+                my $doc = $it->get_document;
+                my $G = join(' ', map { $root2off->{$_} } xap_terms('G', $doc));
+                for my $pair (@{$req->{A_len}}) {
+                        my ($pfx, $len) = @$pair;
+                        my @t = xap_terms($pfx, $doc);
+                        @t = grep { length == $len } @t if defined($len);
+                        for (@t) {
+                                $req->{wbuf} .= "$_ $G\n";
+                                ++$req->{nr_out};
+                        }
+                }
+        };
+        $@ ? iter_retry_check($req) : 0;
+}
+
+sub dump_roots_flush ($$) {
+        my ($req, $fh) = @_;
+        if ($req->{wbuf} ne '') {
+                until (flock($fh, LOCK_EX)) { die "LOCK_EX: $!" if !$!{EINTR} }
+                print { $req->{0} } $req->{wbuf} or die "print: $!";
+                until (flock($fh, LOCK_UN)) { die "LOCK_UN: $!" if !$!{EINTR} }
+                $req->{wbuf} = '';
+        }
+}
+
+sub cmd_dump_roots {
+        my ($req, $root2off_file, $qry_str) = @_;
+        $qry_str // die 'usage: dump_roots [OPTIONS] ROOT2ID_FILE QRY_STR';
+        $req->{A} or die 'dump_roots requires -A PREFIX';
+        term_length_extract $req;
+        open my $fh, '<', $root2off_file;
+        my $root2off; # record format: $OIDHEX "\0" uint32_t
+        my @x = split(/\0/, read_all $fh);
+        while (defined(my $oidhex = shift @x)) {
+                $root2off->{$oidhex} = shift @x;
+        }
+        my $opt = { relevance => -1, limit => $req->{'m'},
+                        offset => $req->{o} // 0 };
+        my $mset = $req->{srch}->mset($qry_str, $opt);
+        $req->{0}->autoflush(1);
+        $req->{wbuf} = '';
+        for my $it ($mset->items) {
+                for (my $t = 10; $t > 0; --$t) {
+                        $t = dump_roots_iter($req, $root2off, $it) // $t;
+                }
+                if (!($req->{nr_out} & 0x3fff)) {
+                        dump_roots_flush($req, $fh);
+                }
+        }
+        dump_roots_flush($req, $fh);
+        emit_mset_stats($req, $mset);
+}
+
+sub mset_iter ($$) {
+        my ($req, $it) = @_;
+        eval {
+                my $buf = $it->get_docid;
+                $buf .= "\0".$it->get_percent if $req->{p};
+                my $doc = ($req->{A} || $req->{D}) ? $it->get_document : undef;
+                for my $p (@{$req->{A}}) {
+                        $buf .= "\0".$p.$_ for xap_terms($p, $doc);
+                }
+                $buf .= "\0".$doc->get_data if $req->{D};
+                say { $req->{0} } $buf;
+        };
+        $@ ? iter_retry_check($req) : 0;
+}
+
+sub cmd_mset { # to be used by WWW + IMAP
+        my ($req, $qry_str) = @_;
+        $qry_str // die 'usage: mset [OPTIONS] QRY_STR';
+        my $opt = { limit => $req->{'m'}, offset => $req->{o} // 0 };
+        $opt->{relevance} = 1 if $req->{r};
+        $opt->{threads} = 1 if defined $req->{t};
+        $opt->{git_dir} = $req->{g} if defined $req->{g};
+        $opt->{eidx_key} = $req->{O} if defined $req->{O};
+        $opt->{threadid} = $req->{T} if defined $req->{T};
+        my $mset = $req->{srch}->mset($qry_str, $opt);
+        say { $req->{0} } 'mset.size=', $mset->size;
+        for my $it ($mset->items) {
+                for (my $t = 10; $t > 0; --$t) {
+                        $t = mset_iter($req, $it) // $t;
+                }
+        }
+}
+
+sub dispatch {
+        my ($req, $cmd, @argv) = @_;
+        my $fn = $req->can("cmd_$cmd") or return;
+        $GLP->getoptionsfromarray(\@argv, $req, @PublicInbox::Search::XH_SPEC)
+                or return;
+        my $dirs = delete $req->{d} or die 'no -d args';
+        my $key = join("\0", @$dirs);
+        $req->{srch} = $SRCH{$key} //= do {
+                my $new = { qp_flags => $PublicInbox::Search::QP_FLAGS };
+                my $first = shift @$dirs;
+                my $slow_phrase = -f "$first/iamchert";
+                $new->{xdb} = $X->{Database}->new($first);
+                for (@$dirs) {
+                        $slow_phrase ||= -f "$_/iamchert";
+                        $new->{xdb}->add_database($X->{Database}->new($_));
+                }
+                $slow_phrase or
+                        $new->{qp_flags} |= PublicInbox::Search::FLAG_PHRASE();
+                bless $new, $req->{c} ? 'PublicInbox::CodeSearch' :
+                                        'PublicInbox::Search';
+                $new->{qp} = $new->qparse_new;
+                $new;
+        };
+        $fn->($req, @argv);
+}
+
+sub recv_loop {
+        local $SIG{__WARN__} = sub { print $stderr @_ };
+        my $rbuf;
+        local $SIG{TERM} = sub { undef $in };
+        while (defined($in)) {
+                PublicInbox::DS::sig_setmask($workerset);
+                my @fds = eval { # we undef $in in SIG{TERM}
+                        $PublicInbox::IPC::recv_cmd->($in, $rbuf, 4096*33)
+                };
+                if ($@) {
+                        exit if !$in; # hit by SIGTERM
+                        die;
+                }
+                scalar(@fds) or exit(66); # EX_NOINPUT
+                die "recvmsg: $!" if !defined($fds[0]);
+                PublicInbox::DS::block_signals();
+                my $req = bless {}, __PACKAGE__;
+                my $i = 0;
+                open($req->{$i++}, '+<&=', $_) for @fds;
+                local $stderr = $req->{1} // \*STDERR;
+                die "not NUL-terminated" if chop($rbuf) ne "\0";
+                my @argv = split(/\0/, $rbuf);
+                $req->{nr_out} = 0;
+                $req->dispatch(@argv) if @argv;
+        }
+}
+
+sub reap_worker { # awaitpid CB
+        my ($pid, $nr) = @_;
+        delete $WORKERS{$nr};
+        if (($? >> 8) == 66) { # EX_NOINPUT
+                undef $in;
+        } elsif ($?) {
+                warn "worker[$nr] died \$?=$?\n";
+        }
+        PublicInbox::DS::requeue(\&start_workers) if $in;
+}
+
+sub start_worker ($) {
+        my ($nr) = @_;
+        my $pid = eval { PublicInbox::DS::do_fork } // return(warn($@));
+        if ($pid == 0) {
+                undef %WORKERS;
+                $SIG{TTIN} = $SIG{TTOU} = 'IGNORE';
+                $SIG{CHLD} = 'DEFAULT'; # Xapian may use this
+                recv_loop();
+                exit(0);
+        } else {
+                $WORKERS{$nr} = $pid;
+                awaitpid($pid, \&reap_worker, $nr);
+        }
+}
+
+sub start_workers {
+        for my $nr (grep { !defined($WORKERS{$_}) } (0..($nworker - 1))) {
+                start_worker($nr) if $in;
+        }
+}
+
+sub do_sigttou {
+        if ($in && $nworker > 1) {
+                --$nworker;
+                my @nr = grep { $_ >= $nworker } keys %WORKERS;
+                kill('TERM', @WORKERS{@nr});
+        }
+}
+
+sub xh_alive { $in || scalar(keys %WORKERS) }
+
+sub start (@) {
+        my (@argv) = @_;
+        my $c = getsockopt(local $in = \*STDIN, SOL_SOCKET, SO_TYPE);
+        unpack('i', $c) == SOCK_SEQPACKET or die 'stdin is not SOCK_SEQPACKET';
+
+        local (%SRCH, %WORKERS);
+        PublicInbox::Search::load_xapian();
+        $GLP->getoptionsfromarray(\@argv, my $opt = { j => 1 }, 'j=i') or
+                die 'bad args';
+        local $workerset = POSIX::SigSet->new;
+        $workerset->fillset or die "fillset: $!";
+        for (@PublicInbox::DS::UNBLOCKABLE) {
+                $workerset->delset($_) or die "delset($_): $!";
+        }
+
+        local $nworker = $opt->{j};
+        return recv_loop() if $nworker == 0;
+        die '-j must be >= 0' if $nworker < 0;
+        for (POSIX::SIGTERM, POSIX::SIGCHLD) {
+                $workerset->delset($_) or die "delset($_): $!";
+        }
+        my $sig = {
+                TTIN => sub {
+                        if ($in) {
+                                ++$nworker;
+                                PublicInbox::DS::requeue(\&start_workers)
+                        }
+                },
+                TTOU => \&do_sigttou,
+                CHLD => \&PublicInbox::DS::enqueue_reap,
+        };
+        PublicInbox::DS::block_signals();
+        start_workers();
+        @PublicInbox::DS::post_loop_do = \&xh_alive;
+        PublicInbox::DS::event_loop($sig);
+}
+
+1;
diff --git a/lib/PublicInbox/XapHelperCxx.pm b/lib/PublicInbox/XapHelperCxx.pm
new file mode 100644
index 00000000..eafe61a8
--- /dev/null
+++ b/lib/PublicInbox/XapHelperCxx.pm
@@ -0,0 +1,130 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Just-ahead-of-time builder for the lib/PublicInbox/xap_helper.h shim.
+# I never want users to be without source code for repairs, so this
+# aims to replicate the feel of a scripting language using C++.
+# The resulting executable is not linked to Perl in any way.
+package PublicInbox::XapHelperCxx;
+use v5.12;
+use PublicInbox::Spawn qw(run_die run_qx run_wait which);
+use PublicInbox::IO qw(try_cat write_file);
+use PublicInbox::Search;
+use Fcntl qw(SEEK_SET);
+use Config;
+use autodie;
+my $cxx = which($ENV{CXX} // 'c++') // which('clang') // die 'no C++ compiler';
+my $dir = substr("$cxx-$Config{archname}", 1); # drop leading '/'
+$dir =~ tr!/!-!;
+my $idir = ($ENV{XDG_CACHE_HOME} //
+        (($ENV{HOME} // die('HOME unset')).'/.cache')).'/public-inbox/jaot';
+substr($dir, 0, 0) = "$idir/";
+my $bin = "$dir/xap_helper";
+my ($srcpfx) = (__FILE__ =~ m!\A(.+/)[^/]+\z!);
+my @srcs = map { $srcpfx.$_ } qw(xh_mset.h xh_cidx.h xap_helper.h);
+my @pm_dep = map { $srcpfx.$_ } qw(Search.pm CodeSearch.pm);
+my $ldflags = '-Wl,-O1';
+$ldflags .= ' -Wl,--compress-debug-sections=zlib' if $^O ne 'openbsd';
+my $xflags = ($ENV{CXXFLAGS} // '-Wall -ggdb3 -pipe') . ' ' .
+        ' -DTHREADID=' . PublicInbox::Search::THREADID .
+        ' -DXH_SPEC="'.join('',
+                map { s/=.*/:/; $_ } @PublicInbox::Search::XH_SPEC) . '" ' .
+        ($ENV{LDFLAGS} // $ldflags);
+substr($xflags, 0, 0, '-O2 ') if !defined($ENV{CXXFLAGS}) && !-w __FILE__;
+my $xap_modversion;
+
+sub xap_cfg (@) {
+        my $cmd = [ $ENV{PKG_CONFIG} // 'pkg-config', @_, 'xapian-core' ];
+        chomp(my $ret = run_qx($cmd, undef, { 2 => \(my $err) }));
+        return $ret if !$?;
+        die <<EOM;
+@$cmd failed: Xapian development files missing? (\$?=$?)
+$err
+EOM
+}
+
+sub needs_rebuild () {
+        my $prev = try_cat("$dir/XFLAGS") or return 1;
+        chomp $prev;
+        return 1 if $prev ne $xflags;
+
+        $prev = try_cat("$dir/xap_modversion") or return 1;
+        chomp $prev;
+
+        $xap_modversion = xap_cfg('--modversion');
+        $xap_modversion ne $prev;
+}
+
+sub build () {
+        if (!-d $dir) {
+                require File::Path;
+                File::Path::make_path($dir);
+        }
+        require PublicInbox::CodeSearch;
+        require PublicInbox::Lock;
+        my ($prog) = ($bin =~ m!/([^/]+)\z!);
+        my $lk = PublicInbox::Lock->new("$dir/$prog.lock")->lock_for_scope;
+        write_file '>', "$dir/$prog.cpp", qq{#include "xap_helper.h"\n},
+                        PublicInbox::Search::generate_cxx(),
+                        PublicInbox::CodeSearch::generate_cxx();
+
+        # xap_modversion may be set by needs_rebuild
+        $xap_modversion //= xap_cfg('--modversion');
+        my $fl = xap_cfg(qw(--libs --cflags));
+
+        # Using rpath seems acceptable/encouraged in the NetBSD packaging world
+        # since /usr/pkg/lib isn't searched by the dynamic loader by default.
+        # Not sure if other OSes need this, but rpath seems fine for JAOT
+        # binaries (like this one) even if other distros discourage it for
+        # distributed packages.
+        $^O eq 'netbsd' and $fl =~ s/(\A|[ \t])\-L([^ \t]+)([ \t]|\z)/
+                                "$1-L$2 -Wl,-rpath=$2$3"/egsx;
+        my @xflags = split(' ', "$fl $xflags"); # ' ' awk-mode eats leading WS
+        my @cflags = ('-I', $srcpfx, grep(!/\A-(?:Wl|l|L)/, @xflags));
+        run_die([$cxx, '-o', "$dir/$prog.o", '-c', "$dir/$prog.cpp", @cflags]);
+
+        # xapian on Alpine Linux (tested 3.19.0) is linked against libstdc++,
+        # and clang needs to be told to use it (rather than libc++):
+        my @try = rindex($cxx, 'clang') >= 0 ? qw(-lstdc++) : ();
+        my @cmd = ($cxx, '-o', "$dir/$prog.tmp", "$dir/$prog.o", @xflags);
+        while (run_wait(\@cmd) and @try) {
+                warn("# attempting to link again with $try[0]...\n");
+                push(@cmd, shift(@try));
+        }
+        die "# @cmd failed: \$?=$?" if $?;
+        unlink "$dir/$prog.cpp", "$dir/$prog.o";
+        write_file '>', "$dir/XFLAGS.tmp", $xflags, "\n";
+        write_file '>', "$dir/xap_modversion.tmp", $xap_modversion, "\n";
+        undef $xap_modversion; # do we ever build() twice?
+        # not quite atomic, but close enough :P
+        rename("$dir/$_.tmp", "$dir/$_") for ($prog, qw(XFLAGS xap_modversion));
+}
+
+sub check_build () {
+        use Time::HiRes qw(stat);
+        my $ctime = 0;
+        my @bin = stat($bin) or return build();
+        for (@srcs, @pm_dep) {
+                my @st = stat($_) or die "stat $_: $!";
+                if ($st[10] > $ctime) {
+                        $ctime = $st[10];
+                        return build() if $ctime > $bin[10];
+                }
+        }
+        needs_rebuild() ? build() : 0;
+}
+
+# returns spawn arg
+sub cmd {
+        die 'PI_NO_CXX set' if $ENV{PI_NO_CXX};
+        check_build();
+        my @cmd;
+        if (my $v = $ENV{VALGRIND}) {
+                $v = 'valgrind -v' if $v eq '1';
+                @cmd = split(/\s+/, $v);
+        }
+        push @cmd, $bin;
+        \@cmd;
+}
+
+1;
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 10685636..69f0af43 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -1,15 +1,17 @@
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 package PublicInbox::Xapcmd;
-use strict;
+use v5.12;
 use PublicInbox::Spawn qw(which popen_rd);
 use PublicInbox::Syscall;
 use PublicInbox::Admin qw(setup_signals);
 use PublicInbox::Over;
+use PublicInbox::Search qw(xap_terms);
 use PublicInbox::SearchIdx;
 use File::Temp 0.19 (); # ->newdir
 use File::Path qw(remove_tree);
 use POSIX qw(WNOHANG _exit);
+use PublicInbox::DS;
 
 # support testing with dev versions of Xapian which installs
 # commands with a version number suffix (e.g. "xapian-compact-1.5")
@@ -75,15 +77,15 @@ sub commit_changes ($$$$) {
         $tmp = undef;
         if (!$opt->{-coarse_lock}) {
                 $opt->{-skip_lock} = 1;
-                $im //= $ibx if $ibx->can('eidx_sync');
-                if ($im->can('count_shards')) { # v2w or eidx
+                $im //= $ibx if $ibx->can('eidx_sync') || $ibx->can('cidx_run');
+                if ($im->can('count_shards')) { # v2w, eidx, cidx
                         my $pr = $opt->{-progress};
                         my $n = $im->count_shards;
                         if (defined $reshard && $n != $reshard) {
                                 die
 "BUG: counted $n shards after resharding to $reshard";
                         }
-                        my $prev = $im->{shards};
+                        my $prev = $im->{shards} // $ibx->{nshard};
                         if ($pr && $prev != $n) {
                                 $pr->("shard count changed: $prev => $n\n");
                                 $im->{shards} = $n;
@@ -93,7 +95,7 @@ sub commit_changes ($$$$) {
                 local %ENV = (%ENV, %$env) if $env;
                 if ($ibx->can('eidx_sync')) {
                         $ibx->eidx_sync($opt);
-                } else {
+                } elsif (!$ibx->can('cidx_run')) {
                         PublicInbox::Admin::index_inbox($ibx, $im, $opt);
                 }
         }
@@ -101,10 +103,8 @@ sub commit_changes ($$$$) {
 
 sub cb_spawn {
         my ($cb, $args, $opt) = @_; # $cb = cpdb() or compact()
-        my $seed = rand(0xffffffff);
-        my $pid = fork // die "fork: $!";
+        my $pid = PublicInbox::DS::do_fork;
         return $pid if $pid > 0;
-        srand($seed);
         $SIG{__DIE__} = sub { warn @_; _exit(1) }; # don't jump up stack
         $cb->($args, $opt);
         _exit(0);
@@ -117,7 +117,8 @@ sub runnable_or_die ($) {
 
 sub prepare_reindex ($$) {
         my ($ibx, $opt) = @_;
-        if ($ibx->can('eidx_sync')) { # no prep needed for ExtSearchIdx
+        if ($ibx->can('eidx_sync') || $ibx->can('cidx_run')) {
+                # no prep needed for ExtSearchIdx nor CodeSearchIdx
         } elsif ($ibx->version == 1) {
                 my $dir = $ibx->search->xdir(1);
                 my $xdb = $PublicInbox::Search::X{Database}->new($dir);
@@ -147,8 +148,9 @@ sub kill_pids {
 }
 
 sub process_queue {
-        my ($queue, $cb, $opt) = @_;
+        my ($queue, $task, $opt) = @_;
         my $max = $opt->{jobs} // scalar(@$queue);
+        my $cb = \&$task;
         if ($max <= 1) {
                 while (defined(my $args = shift @$queue)) {
                         $cb->($args, $opt);
@@ -186,7 +188,9 @@ sub prepare_run {
         my $tmp = {}; # old shard dir => File::Temp->newdir object or undef
         my @queue; # ([old//src,newdir]) - list of args for cpdb() or compact()
         my ($old, $misc_ok);
-        if ($ibx->can('eidx_sync')) {
+        if ($ibx->can('cidx_run')) {
+                $old = $ibx->xdir(1);
+        } elsif ($ibx->can('eidx_sync')) {
                 $misc_ok = 1;
                 $old = $ibx->xdir(1);
         } elsif (my $srch = $ibx->search) {
@@ -219,7 +223,7 @@ sub prepare_run {
                 my @old_shards;
                 while (defined(my $dn = readdir($dh))) {
                         if ($dn =~ /\A[0-9]+\z/) {
-                                push @old_shards, $dn;
+                                push(@old_shards, $dn + 0);
                         } elsif ($dn eq '.' || $dn eq '..') {
                         } elsif ($dn =~ /\Aover\.sqlite3/) {
                         } elsif ($dn eq 'misc' && $misc_ok) {
@@ -228,7 +232,7 @@ sub prepare_run {
                         }
                 }
                 die "No Xapian shards found in $old\n" unless @old_shards;
-
+                @old_shards = sort { $a <=> $b } @old_shards;
                 my ($src, $max_shard);
                 if (!defined($reshard) || $reshard == scalar(@old_shards)) {
                         # 1:1 copy
@@ -256,38 +260,21 @@ sub prepare_run {
 
 sub check_compact () { runnable_or_die($XAPIAN_COMPACT) }
 
-sub _run { # with_umask callback
-        my ($ibx, $cb, $opt) = @_;
-        my $im = $ibx->can('importer') ? $ibx->importer(0) : undef;
-        ($im // $ibx)->lock_acquire;
-        my ($tmp, $queue) = prepare_run($ibx, $opt);
-
-        # fine-grained locking if we prepare for reindex
-        if (!$opt->{-coarse_lock}) {
-                prepare_reindex($ibx, $opt);
-                ($im // $ibx)->lock_release;
-        }
-
-        $ibx->cleanup if $ibx->can('cleanup');
-        process_queue($queue, $cb, $opt);
-        ($im // $ibx)->lock_acquire if !$opt->{-coarse_lock};
-        commit_changes($ibx, $im, $tmp, $opt);
-}
-
 sub run {
         my ($ibx, $task, $opt) = @_; # task = 'cpdb' or 'compact'
-        my $cb = \&$task;
         PublicInbox::Admin::progress_prepare($opt ||= {});
         my $dir;
-        for my $fld (qw(inboxdir topdir)) {
+        for my $fld (qw(inboxdir topdir cidx_dir)) {
                 my $d = $ibx->{$fld} // next;
                 -d $d or die "$fld=$d does not exist\n";
                 $dir = $d;
                 last;
         }
-        check_compact() if $opt->{compact} && $ibx->search;
+        check_compact() if $opt->{compact} &&
+                                ($ibx->can('cidx_run') || $ibx->search);
 
-        if (!$ibx->can('eidx_sync') && !$opt->{-coarse_lock}) {
+        if (!$ibx->can('eidx_sync') && $ibx->can('version') &&
+                                        !$opt->{-coarse_lock}) {
                 # per-epoch ranges for v2
                 # v1:{ from => $OID }, v2:{ from => [ $OID, $OID, $OID ] } }
                 $opt->{reindex} = { from => $ibx->version == 1 ? '' : [] };
@@ -296,7 +283,26 @@ sub run {
 
         local @SIG{keys %SIG} = values %SIG;
         setup_signals();
-        $ibx->with_umask(\&_run, $ibx, $cb, $opt);
+        my $restore = $ibx->with_umask;
+
+        my $im = $ibx->can('importer') ? $ibx->importer(0) : undef;
+        ($im // $ibx)->lock_acquire;
+        my ($tmp, $queue) = prepare_run($ibx, $opt);
+
+        # fine-grained locking if we prepare for reindex
+        if (!$opt->{-coarse_lock}) {
+                prepare_reindex($ibx, $opt);
+                ($im // $ibx)->lock_release;
+        }
+
+        $ibx->cleanup if $ibx->can('cleanup');
+        if ($task eq 'cpdb' && $opt->{reshard} && $ibx->can('cidx_run')) {
+                cidx_reshard($ibx, $queue, $opt);
+        } else {
+                process_queue($queue, $task, $opt);
+        }
+        ($im // $ibx)->lock_acquire if !$opt->{-coarse_lock};
+        commit_changes($ibx, $im, $tmp, $opt);
 }
 
 sub cpdb_retryable ($$) {
@@ -315,15 +321,16 @@ sub cpdb_retryable ($$) {
 
 sub progress_pfx ($) {
         my ($wip) = @_; # tempdir v2: ([0-9])+-XXXX
-        my @p = split('/', $wip);
+        my @p = split(m'/', $wip);
 
-        # return "xap15/0" for v2, or "xapian15" for v1:
-        ($p[-1] =~ /\A([0-9]+)/) ? "$p[-2]/$1" : $p[-1];
+        # "basename(inboxdir)/xap15/0" for v2,
+        # "basename(inboxdir)/xapian15" for v1:
+        ($p[-1] =~ /\A([0-9]+)/) ? "$p[-3]/$p[-2]/$1" : "$p[-2]/$p[-1]";
 }
 
 sub kill_compact { # setup_signals callback
-        my ($sig, $pidref) = @_;
-        kill($sig, $$pidref) if defined($$pidref);
+        my ($sig, $ioref) = @_;
+        kill($sig, $$ioref->attached_pid // return) if defined($$ioref);
 }
 
 # xapian-compact wrapper
@@ -351,18 +358,16 @@ sub compact ($$) { # cb_spawn callback
         }
         $pr->("$pfx `".join(' ', @$cmd)."'\n") if $pr;
         push @$cmd, $src, $dst;
-        my ($rd, $pid);
         local @SIG{keys %SIG} = values %SIG;
-        setup_signals(\&kill_compact, \$pid);
-        ($rd, $pid) = popen_rd($cmd, undef, $rdr);
+        setup_signals(\&kill_compact, \my $rd);
+        $rd = popen_rd($cmd, undef, $rdr);
         while (<$rd>) {
                 if ($pr) {
                         s/\r/\r$pfx /g;
                         $pr->("$pfx $_");
                 }
         }
-        waitpid($pid, 0);
-        die "@$cmd failed: \$?=$?\n" if $?;
+        $rd->close or die "@$cmd failed: \$?=$?\n";
 }
 
 sub cpdb_loop ($$$;$$) {
@@ -406,17 +411,95 @@ sub cpdb_loop ($$$;$$) {
         } while (cpdb_retryable($src, $pfx));
 }
 
+sub xapian_write_prep ($) {
+        my ($opt) = @_;
+        PublicInbox::SearchIdx::load_xapian_writable();
+        my $flag = eval($PublicInbox::Search::Xap.'::DB_CREATE()');
+        die if $@;
+        $flag |= $PublicInbox::SearchIdx::DB_NO_SYNC if !$opt->{fsync};
+        (\%PublicInbox::Search::X, $flag);
+}
+
+sub compact_tmp_shard ($) {
+        my ($wip) = @_;
+        my $new = $wip->dirname;
+        my ($dir) = ($new =~ m!(.*?/)[^/]+/*\z!);
+        same_fs_or_die($dir, $new);
+        my $ft = File::Temp->newdir("$new.compact-XXXX", DIR => $dir);
+        PublicInbox::Syscall::nodatacow_dir($ft->dirname);
+        $ft;
+}
+
+sub cidx_reshard { # not docid based
+        my ($cidx, $queue, $opt) = @_;
+        my ($X, $flag) = xapian_write_prep($opt);
+        my $src = $cidx->xdb;
+        delete($cidx->{xdb}) == $src or die "BUG: xdb != $src";
+        my $pfx = $opt->{-progress_pfx} = progress_pfx($cidx->xdir.'/0');
+        my $pr = $opt->{-progress};
+        my $pr_data = { pr => $pr, pfx => $pfx, nr => 0 } if $pr;
+        local @SIG{keys %SIG} = values %SIG;
+
+        # like copydatabase(1), be sure we don't overwrite anything in case
+        # of other bugs:
+        setup_signals() if $opt->{compact};
+        my @tmp;
+        my @dst = map {
+                my $wip = $_->[1];
+                my $tmp = $opt->{compact} ? compact_tmp_shard($wip) : $wip;
+                push @tmp, $tmp;
+                $X->{WritableDatabase}->new($tmp->dirname, $flag);
+        } @$queue;
+        my $l = $src->get_metadata('indexlevel');
+        $dst[0]->set_metadata('indexlevel', $l) if $l eq 'medium';
+        my $fmt;
+        if ($pr_data) {
+                my $tot = $src->get_doccount;
+                $fmt = "$pfx % ".length($tot)."u/$tot\n";
+                $pr->("$pfx copying $tot documents\n");
+        }
+        my $cur = $src->postlist_begin('');
+        my $end = $src->postlist_end('');
+        my $git_dir_hash = $cidx->can('git_dir_hash');
+        my ($n, $nr);
+        for (; $cur != $end; $cur++) {
+                my $doc = $src->get_document($cur->get_docid);
+                if (my @cmt = xap_terms('Q', $doc)) {
+                        $n = hex(substr($cmt[0], 0, 8)) % scalar(@dst);
+                        warn "W: multi-commit: @cmt" if scalar(@cmt) != 1;
+                } elsif (my @P = xap_terms('P', $doc)) {
+                        $n = $git_dir_hash->($P[0]) % scalar(@dst);
+                        warn "W: multi-path @P " if scalar(@P) != 1;
+                } else {
+                        warn "W: skipped, no terms in ".$cur->get_docid;
+                        next;
+                }
+                $dst[$n]->add_document($doc);
+                $pr->(sprintf($fmt, $nr)) if $pr_data && !(++$nr & 1023);
+        }
+        return if !$opt->{compact};
+        $src = undef;
+        @dst = (); # flushes and closes
+        my @q;
+        for my $tmp (@tmp) {
+                my $arg = shift @$queue // die 'BUG: $queue empty';
+                my $wip = $arg->[1] // die 'BUG: no $wip';
+                push @q, [ "$tmp", $wip ];
+        }
+        delete $opt->{-progress_pfx};
+        process_queue(\@q, 'compact', $opt);
+}
+
 # Like copydatabase(1), this is horribly slow; and it doesn't seem due
 # to the overhead of Perl.
 sub cpdb ($$) { # cb_spawn callback
         my ($args, $opt) = @_;
-        my ($old, $newdir) = @$args;
-        my $new = $newdir->dirname;
+        my ($old, $wip) = @$args;
         my ($src, $cur_shard);
         my $reshard;
-        PublicInbox::SearchIdx::load_xapian_writable();
-        my $XapianDatabase = $PublicInbox::Search::X{Database};
+        my ($X, $flag) = xapian_write_prep($opt);
         if (ref($old) eq 'ARRAY') {
+                my $new = $wip->dirname;
                 ($cur_shard) = ($new =~ m!(?:xap|ei)[0-9]+/([0-9]+)\b!);
                 defined $cur_shard or
                         die "BUG: could not extract shard # from $new";
@@ -426,36 +509,27 @@ sub cpdb ($$) { # cb_spawn callback
                 # resharding, M:N copy means have full read access
                 foreach (@$old) {
                         if ($src) {
-                                my $sub = $XapianDatabase->new($_);
+                                my $sub = $X->{Database}->new($_);
                                 $src->add_database($sub);
                         } else {
-                                $src = $XapianDatabase->new($_);
+                                $src = $X->{Database}->new($_);
                         }
                 }
         } else {
-                $src = $XapianDatabase->new($old);
+                $src = $X->{Database}->new($old);
         }
 
-        my ($tmp, $ft);
+        my $tmp = $wip;
         local @SIG{keys %SIG} = values %SIG;
         if ($opt->{compact}) {
-                my ($dir) = ($new =~ m!(.*?/)[^/]+/*\z!);
-                same_fs_or_die($dir, $new);
-                $ft = File::Temp->newdir("$new.compact-XXXX", DIR => $dir);
+                $tmp = compact_tmp_shard($wip);
                 setup_signals();
-                $tmp = $ft->dirname;
-                PublicInbox::Syscall::nodatacow_dir($tmp);
-        } else {
-                $tmp = $new;
         }
 
         # like copydatabase(1), be sure we don't overwrite anything in case
         # of other bugs:
-        my $flag = eval($PublicInbox::Search::Xap.'::DB_CREATE()');
-        die if $@;
-        my $XapianWritableDatabase = $PublicInbox::Search::X{WritableDatabase};
-        $flag |= $PublicInbox::SearchIdx::DB_NO_SYNC if !$opt->{fsync};
-        my $dst = $XapianWritableDatabase->new($tmp, $flag);
+        my $new = $wip->dirname;
+        my $dst = $X->{WritableDatabase}->new($tmp->dirname, $flag);
         my $pr = $opt->{-progress};
         my $pfx = $opt->{-progress_pfx} = progress_pfx($new);
         my $pr_data = { pr => $pr, pfx => $pfx, nr => 0 } if $pr;
@@ -467,11 +541,10 @@ sub cpdb ($$) { # cb_spawn callback
                         $dst->set_metadata('last_commit', $lc) if $lc;
 
                         # only the first xapian shard (0) gets 'indexlevel'
-                        if ($new =~ m!(?:xapian[0-9]+|xap[0-9]+/0)\b!) {
+                        if ($new =~ m!/(?:xapian[0-9]+|(?:ei|xap)[0-9]+/0)\b!) {
                                 my $l = $src->get_metadata('indexlevel');
-                                if ($l eq 'medium') {
+                                $l eq 'medium' and
                                         $dst->set_metadata('indexlevel', $l);
-                                }
                         }
                         if ($pr_data) {
                                 my $tot = $src->get_doccount;
@@ -498,7 +571,7 @@ sub cpdb ($$) { # cb_spawn callback
                 # individually.
                 $src = undef;
                 foreach (@$old) {
-                        my $old = $XapianDatabase->new($_);
+                        my $old = $X->{Database}->new($_);
                         cpdb_loop($old, $dst, $pr_data, $cur_shard, $reshard);
                 }
         } else {
@@ -513,7 +586,6 @@ sub cpdb ($$) { # cb_spawn callback
         # this is probably the best place to do xapian-compact
         # since $dst isn't readable by HTTP or NNTP clients, yet:
         compact([ $tmp, $new ], $opt);
-        remove_tree($tmp) or die "failed to remove $tmp: $!\n";
 }
 
 1;
diff --git a/lib/PublicInbox/xap_helper.h b/lib/PublicInbox/xap_helper.h
new file mode 100644
index 00000000..3456910b
--- /dev/null
+++ b/lib/PublicInbox/xap_helper.h
@@ -0,0 +1,1049 @@
+/*
+ * Copyright (C) all contributors <meta@public-inbox.org>
+ * License: GPL-2.0+ <https://www.gnu.org/licenses/gpl-2.0.txt>
+ * Note: GPL-2+ since it'll incorporate approxidate from git someday
+ *
+ * Standalone helper process using C and minimal C++ for Xapian,
+ * this is not linked to Perl in any way.
+ * C (not C++) is used as much as possible to lower the contribution
+ * barrier for hackers who mainly know C (this includes the maintainer).
+ * Yes, that means we use C stdlib stuff like hsearch and open_memstream
+ * instead their equivalents in the C++ stdlib :P
+ * Everything here is an unstable internal API of public-inbox and
+ * NOT intended for ordinary users; only public-inbox hackers
+ */
+#ifndef _ALL_SOURCE
+#        define _ALL_SOURCE
+#endif
+#if defined(__NetBSD__) && !defined(_OPENBSD_SOURCE) // for reallocarray(3)
+#        define _OPENBSD_SOURCE
+#endif
+#include <sys/file.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/wait.h>
+
+#include <assert.h>
+#include <err.h> // BSD, glibc, and musl all have this
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <search.h>
+#include <signal.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sysexits.h>
+#include <unistd.h>
+#include <xapian.h> // our only reason for using C++
+
+#define MY_VER(maj,min,rev) ((maj) << 16 | (min) << 8 | (rev))
+#define XAP_VER \
+        MY_VER(XAPIAN_MAJOR_VERSION,XAPIAN_MINOR_VERSION,XAPIAN_REVISION)
+
+#if XAP_VER >= MY_VER(1,3,6)
+#        define NRP Xapian::NumberRangeProcessor
+#        define ADD_RP add_rangeprocessor
+#        define SET_MAX_EXPANSION set_max_expansion // technically 1.3.3
+#else
+#        define NRP Xapian::NumberValueRangeProcessor
+#        define ADD_RP add_valuerangeprocessor
+#        define SET_MAX_EXPANSION set_max_wildcard_expansion
+#endif
+
+#if defined(__GLIBC__)
+#        define MY_DO_OPTRESET() do { optind = 0; } while (0)
+#else /* FreeBSD, musl, dfly, NetBSD, OpenBSD */
+#        define MY_DO_OPTRESET() do { optind = optreset = 1; } while (0)
+#endif
+
+#if defined(__DragonFly__) || defined(__FreeBSD__) || defined(__GLIBC__)
+#        define STDERR_ASSIGNABLE (1)
+#else
+#        define STDERR_ASSIGNABLE (0)
+#endif
+
+// assert functions are used correctly (e.g. ensure hackers don't
+// cause EINVAL/EFAULT).  Not for stuff that can fail due to HW
+// failures.
+# define CHECK(type, expect, expr) do { \
+        type ckvar______ = (expr); \
+        assert(ckvar______ == (expect) && "BUG" && __FILE__ && __LINE__); \
+} while (0)
+
+// coredump on most usage errors since our only users are internal
+#define ABORT(...) do { warnx(__VA_ARGS__); abort(); } while (0)
+#define EABORT(...) do { warn(__VA_ARGS__); abort(); } while (0)
+
+// sock_fd is modified in signal handler, yes, it's SOCK_SEQPACKET
+static volatile int sock_fd = STDIN_FILENO;
+static sigset_t fullset, workerset;
+static bool alive = true;
+#if STDERR_ASSIGNABLE
+static FILE *orig_err = stderr;
+#endif
+static int orig_err_fd = -1;
+static void *srch_tree; // tsearch + tdelete + twalk
+static pid_t *worker_pids; // nr => pid
+#define WORKER_MAX USHRT_MAX
+static unsigned long nworker, nworker_hwm;
+static int pipefds[2];
+
+// PublicInbox::Search and PublicInbox::CodeSearch generate these:
+static void mail_nrp_init(void);
+static void code_nrp_init(void);
+static void qp_init_mail_search(Xapian::QueryParser *);
+static void qp_init_code_search(Xapian::QueryParser *);
+
+enum exc_iter {
+        ITER_OK = 0,
+        ITER_RETRY,
+        ITER_ABORT
+};
+
+struct srch {
+        int paths_len; // int for comparisons
+        unsigned qp_flags;
+        Xapian::Database *db;
+        Xapian::QueryParser *qp;
+        char paths[]; // $shard_path0\0$shard_path1\0...
+};
+
+#define MY_ARG_MAX 256
+typedef bool (*cmd)(struct req *);
+
+// only one request per-process since we have RLIMIT_CPU timeout
+struct req { // argv and pfxv point into global rbuf
+        char *argv[MY_ARG_MAX];
+        char *pfxv[MY_ARG_MAX]; // -A <prefix>
+        size_t *lenv; // -A <prefix>LENGTH
+        struct srch *srch;
+        char *Pgit_dir;
+        char *Oeidx_key;
+        cmd fn;
+        unsigned long long max;
+        unsigned long long off;
+        unsigned long long threadid;
+        unsigned long timeout_sec;
+        size_t nr_out;
+        long sort_col; // value column, negative means BoolWeight
+        int argc;
+        int pfxc;
+        FILE *fp[2]; // [0] response pipe or sock, [1] status/errors (optional)
+        bool has_input; // fp[0] is bidirectional
+        bool collapse_threads;
+        bool code_search;
+        bool relevance; // sort by relevance before column
+        bool emit_percent;
+        bool emit_docdata;
+        bool asc; // ascending sort
+};
+
+struct worker {
+        pid_t pid;
+        unsigned nr;
+};
+
+struct fbuf {
+        FILE *fp;
+        char *ptr;
+        size_t len;
+};
+
+#define SPLIT2ARGV(dst,buf,len) split2argv(dst,buf,len,MY_ARRAY_SIZE(dst))
+static size_t split2argv(char **dst, char *buf, size_t len, size_t limit)
+{
+        if (buf[0] == 0 || len == 0 || buf[len - 1] != 0)
+                ABORT("bogus argument given");
+        size_t nr = 0;
+        char *c = buf;
+        for (size_t i = 1; i < len; i++) {
+                if (!buf[i]) {
+                        dst[nr++] = c;
+                        c = buf + i + 1;
+                }
+                if (nr == limit)
+                        ABORT("too many args: %zu == %zu", nr, limit);
+        }
+        if (nr == 0) ABORT("no argument given");
+        if ((long)nr < 0) ABORT("too many args: %zu", nr);
+        return (long)nr;
+}
+
+static bool has_threadid(const struct srch *srch)
+{
+        return srch->db->get_metadata("has_threadid") == "1";
+}
+
+static Xapian::Enquire prep_enquire(const struct req *req)
+{
+        Xapian::Enquire enq(*req->srch->db);
+        if (req->sort_col < 0) {
+                enq.set_weighting_scheme(Xapian::BoolWeight());
+                enq.set_docid_order(req->asc ? Xapian::Enquire::ASCENDING
+                                        : Xapian::Enquire::DESCENDING);
+        } else if (req->relevance) {
+                enq.set_sort_by_relevance_then_value(req->sort_col, !req->asc);
+        } else {
+                enq.set_sort_by_value_then_relevance(req->sort_col, !req->asc);
+        }
+        return enq;
+}
+
+static Xapian::MSet enquire_mset(struct req *req, Xapian::Enquire *enq)
+{
+        if (!req->max) {
+                switch (sizeof(Xapian::doccount)) {
+                case 4: req->max = UINT_MAX; break;
+                default: req->max = ULLONG_MAX;
+                }
+        }
+        for (int i = 0; i < 9; i++) {
+                try {
+                        Xapian::MSet mset = enq->get_mset(req->off, req->max);
+                        return mset;
+                } catch (const Xapian::DatabaseModifiedError & e) {
+                        req->srch->db->reopen();
+                }
+        }
+        return enq->get_mset(req->off, req->max);
+}
+
+// for v1, v2, and extindex
+static Xapian::MSet mail_mset(struct req *req, const char *qry_str)
+{
+        struct srch *srch = req->srch;
+        Xapian::Query qry = srch->qp->parse_query(qry_str, srch->qp_flags);
+        if (req->Oeidx_key) {
+                req->Oeidx_key[0] = 'O'; // modifies static rbuf
+                qry = Xapian::Query(Xapian::Query::OP_FILTER, qry,
+                                        Xapian::Query(req->Oeidx_key));
+        }
+        Xapian::Enquire enq = prep_enquire(req);
+        enq.set_query(qry);
+        // THREADID is a CPP macro defined on CLI (see) XapHelperCxx.pm
+        if (req->collapse_threads && has_threadid(srch))
+                enq.set_collapse_key(THREADID);
+
+        return enquire_mset(req, &enq);
+}
+
+static bool starts_with(const std::string *s, const char *pfx, size_t pfx_len)
+{
+        return s->size() >= pfx_len && !memcmp(pfx, s->c_str(), pfx_len);
+}
+
+static void apply_roots_filter(struct req *req, Xapian::Query *qry)
+{
+        if (!req->Pgit_dir) return;
+        req->Pgit_dir[0] = 'P'; // modifies static rbuf
+        Xapian::Database *xdb = req->srch->db;
+        for (int i = 0; i < 9; i++) {
+                try {
+                        std::string P = req->Pgit_dir;
+                        Xapian::PostingIterator p = xdb->postlist_begin(P);
+                        if (p == xdb->postlist_end(P)) {
+                                warnx("W: %s not indexed?", req->Pgit_dir + 1);
+                                return;
+                        }
+                        Xapian::TermIterator cur = xdb->termlist_begin(*p);
+                        Xapian::TermIterator end = xdb->termlist_end(*p);
+                        cur.skip_to("G");
+                        if (cur == end) {
+                                warnx("W: %s has no root commits?",
+                                        req->Pgit_dir + 1);
+                                return;
+                        }
+                        Xapian::Query f = Xapian::Query(*cur);
+                        for (++cur; cur != end; ++cur) {
+                                std::string tn = *cur;
+                                if (!starts_with(&tn, "G", 1))
+                                        continue;
+                                f = Xapian::Query(Xapian::Query::OP_OR, f, tn);
+                        }
+                        *qry = Xapian::Query(Xapian::Query::OP_FILTER, *qry, f);
+                        return;
+                } catch (const Xapian::DatabaseModifiedError & e) {
+                        xdb->reopen();
+                }
+        }
+}
+
+// for cindex
+static Xapian::MSet commit_mset(struct req *req, const char *qry_str)
+{
+        struct srch *srch = req->srch;
+        Xapian::Query qry = srch->qp->parse_query(qry_str, srch->qp_flags);
+        apply_roots_filter(req, &qry);
+
+        // we only want commits:
+        qry = Xapian::Query(Xapian::Query::OP_FILTER, qry,
+                                Xapian::Query("T" "c"));
+        Xapian::Enquire enq = prep_enquire(req);
+        enq.set_query(qry);
+        return enquire_mset(req, &enq);
+}
+
+static void emit_mset_stats(struct req *req, const Xapian::MSet *mset)
+{
+        if (req->fp[1])
+                fprintf(req->fp[1], "mset.size=%llu nr_out=%zu\n",
+                        (unsigned long long)mset->size(), req->nr_out);
+        else
+                ABORT("BUG: %s caller only passed 1 FD", req->argv[0]);
+}
+
+static int my_setlinebuf(FILE *fp) // glibc setlinebuf(3) can't report errors
+{
+        return setvbuf(fp, NULL, _IOLBF, 0);
+}
+
+// n.b. __cleanup__ works fine with C++ exceptions, but not longjmp
+// Only clang and g++ are supported, as AFAIK there's no other
+// relevant Free(-as-in-speech) C++ compilers.
+#define CLEANUP_FBUF __attribute__((__cleanup__(fbuf_ensure)))
+static void fbuf_ensure(void *ptr)
+{
+        struct fbuf *fbuf = (struct fbuf *)ptr;
+        if (fbuf->fp && fclose(fbuf->fp))
+                err(EXIT_FAILURE, "fclose(fbuf->fp)"); // ENOMEM?
+        fbuf->fp = NULL;
+        free(fbuf->ptr);
+}
+
+static void fbuf_init(struct fbuf *fbuf)
+{
+        assert(!fbuf->ptr);
+        fbuf->fp = open_memstream(&fbuf->ptr, &fbuf->len);
+        if (!fbuf->fp) err(EXIT_FAILURE, "open_memstream(fbuf)");
+}
+
+static bool write_all(int fd, const struct fbuf *wbuf, size_t len)
+{
+        const char *p = wbuf->ptr;
+        assert(wbuf->len >= len);
+        do { // write to client FD
+                ssize_t n = write(fd, p, len);
+                if (n > 0) {
+                        len -= n;
+                        p += n;
+                } else {
+                        perror(n ? "write" : "write (zero bytes)");
+                        return false;
+                }
+        } while (len);
+        return true;
+}
+
+#define ERR_FLUSH(f) do { \
+        if (ferror(f) | fflush(f)) err(EXIT_FAILURE, "ferror|fflush "#f); \
+} while (0)
+
+#define ERR_CLOSE(f, e) do { \
+        if (ferror(f) | fclose(f)) \
+                e ? err(e, "ferror|fclose "#f) : perror("ferror|fclose "#f); \
+} while (0)
+
+static void xclose(int fd)
+{
+        if (close(fd) < 0 && errno != EINTR)
+                EABORT("BUG: close");
+}
+
+static size_t off2size(off_t n)
+{
+        if (n < 0 || (uintmax_t)n > SIZE_MAX)
+                ABORT("off_t out of size_t range: %lld\n", (long long)n);
+        return (size_t)n;
+}
+
+static char *hsearch_enter_key(char *s)
+{
+#if defined(__OpenBSD__) || defined(__DragonFly__)
+        // hdestroy frees each key on some platforms,
+        // so give it something to free:
+        char *ret = strdup(s);
+        if (!ret) err(EXIT_FAILURE, "strdup");
+        return ret;
+// AFAIK there's no way to detect musl, assume non-glibc Linux is musl:
+#elif defined(__GLIBC__) || defined(__linux__) || \
+        defined(__FreeBSD__) || defined(__NetBSD__)
+        // do nothing on these platforms
+#else
+#warning untested platform detected, unsure if hdestroy(3) frees keys
+#warning contact us at meta@public-inbox.org if you get segfaults
+#endif
+        return s;
+}
+
+// for test usage only, we need to ensure the compiler supports
+// __cleanup__ when exceptions are thrown
+struct inspect { struct req *req; };
+
+static void inspect_ensure(struct inspect *x)
+{
+        fprintf(x->req->fp[0], "pid=%d has_threadid=%d",
+                (int)getpid(), has_threadid(x->req->srch) ? 1 : 0);
+}
+
+static bool cmd_test_inspect(struct req *req)
+{
+        __attribute__((__cleanup__(inspect_ensure))) struct inspect x;
+        x.req = req;
+        try {
+                throw Xapian::InvalidArgumentError("test");
+        } catch (Xapian::InvalidArgumentError) {
+                return true;
+        }
+        fputs("this should not be printed", req->fp[0]);
+        return false;
+}
+
+#include "xh_mset.h" // read-only (WWW, IMAP, lei) stuff
+#include "xh_cidx.h" // CodeSearchIdx.pm stuff
+
+#define CMD(n) { .fn_len = sizeof(#n) - 1, .fn_name = #n, .fn = cmd_##n }
+static const struct cmd_entry {
+        size_t fn_len;
+        const char *fn_name;
+        cmd fn;
+} cmds[] = { // should be small enough to not need bsearch || gperf
+        // most common commands first
+        CMD(mset), // WWW and IMAP requests
+        CMD(dump_ibx), // many inboxes
+        CMD(dump_roots), // per-cidx shard
+        CMD(test_inspect), // least common commands last
+};
+
+#define MY_ARRAY_SIZE(x)        (sizeof(x)/sizeof((x)[0]))
+#define RECV_FD_CAPA 2
+#define RECV_FD_SPACE        (RECV_FD_CAPA * sizeof(int))
+union my_cmsg {
+        struct cmsghdr hdr;
+        char pad[sizeof(struct cmsghdr) + 16 + RECV_FD_SPACE];
+};
+
+static bool recv_req(struct req *req, char *rbuf, size_t *len)
+{
+        union my_cmsg cmsg = {};
+        struct msghdr msg = {};
+        struct iovec iov;
+        ssize_t r;
+        iov.iov_base = rbuf;
+        iov.iov_len = *len;
+        msg.msg_iov = &iov;
+        msg.msg_iovlen = 1;
+        msg.msg_control = &cmsg.hdr;
+        msg.msg_controllen = CMSG_SPACE(RECV_FD_SPACE);
+
+        // allow SIGTERM to hit
+        CHECK(int, 0, sigprocmask(SIG_SETMASK, &workerset, NULL));
+
+again:
+        r = recvmsg(sock_fd, &msg, 0);
+        if (r == 0) {
+                exit(EX_NOINPUT); /* grandparent went away */
+        } else if (r < 0) {
+                switch (errno) {
+                case EINTR: goto again;
+                case EBADF: if (sock_fd < 0) exit(0);
+                        // fall-through
+                default: err(EXIT_FAILURE, "recvmsg");
+                }
+        }
+
+        // success! no signals for the rest of the request/response cycle
+        CHECK(int, 0, sigprocmask(SIG_SETMASK, &fullset, NULL));
+        if (r > 0 && msg.msg_flags)
+                ABORT("unexpected msg_flags");
+
+        *len = r;
+        if (cmsg.hdr.cmsg_level == SOL_SOCKET &&
+                        cmsg.hdr.cmsg_type == SCM_RIGHTS) {
+                size_t clen = cmsg.hdr.cmsg_len;
+                int *fdp = (int *)CMSG_DATA(&cmsg.hdr);
+                size_t i;
+                for (i = 0; CMSG_LEN((i + 1) * sizeof(int)) <= clen; i++) {
+                        int fd = *fdp++;
+                        const char *mode = NULL;
+                        int fl = fcntl(fd, F_GETFL);
+                        if (fl == -1) {
+                                errx(EXIT_FAILURE, "invalid fd=%d", fd);
+                        } else if (fl & O_WRONLY) {
+                                mode = "w";
+                        } else if (fl & O_RDWR) {
+                                mode = "r+";
+                                if (i == 0) req->has_input = true;
+                        } else {
+                                errx(EXIT_FAILURE,
+                                        "invalid mode from F_GETFL: 0x%x", fl);
+                        }
+                        req->fp[i] = fdopen(fd, mode);
+                        if (!req->fp[i])
+                                err(EXIT_FAILURE, "fdopen(fd=%d)", fd);
+                }
+                return true;
+        }
+        errx(EXIT_FAILURE, "no FD received in %zd-byte request", r);
+        return false;
+}
+
+static int srch_cmp(const void *pa, const void *pb) // for tfind|tsearch
+{
+        const struct srch *a = (const struct srch *)pa;
+        const struct srch *b = (const struct srch *)pb;
+        int diff = a->paths_len - b->paths_len;
+
+        return diff ? diff : memcmp(a->paths, b->paths, (size_t)a->paths_len);
+}
+
+static bool is_chert(const char *dir)
+{
+        char iamchert[PATH_MAX];
+        struct stat sb;
+        int rc = snprintf(iamchert, sizeof(iamchert), "%s/iamchert", dir);
+
+        if (rc <= 0 || rc >= (int)sizeof(iamchert))
+                err(EXIT_FAILURE, "BUG: snprintf(%s/iamchert)", dir);
+        if (stat(iamchert, &sb) == 0 && S_ISREG(sb.st_mode))
+                return true;
+        return false;
+}
+
+static bool srch_init(struct req *req)
+{
+        char *dirv[MY_ARG_MAX];
+        int i;
+        struct srch *srch = req->srch;
+        int dirc = (int)SPLIT2ARGV(dirv, srch->paths, (size_t)srch->paths_len);
+        const unsigned FLAG_PHRASE = Xapian::QueryParser::FLAG_PHRASE;
+        srch->qp_flags = FLAG_PHRASE |
+                        Xapian::QueryParser::FLAG_BOOLEAN |
+                        Xapian::QueryParser::FLAG_LOVEHATE |
+                        Xapian::QueryParser::FLAG_WILDCARD;
+        if (is_chert(dirv[0]))
+                srch->qp_flags &= ~FLAG_PHRASE;
+        try {
+                srch->db = new Xapian::Database(dirv[0]);
+        } catch (...) {
+                warn("E: Xapian::Database(%s)", dirv[0]);
+                return false;
+        }
+        try {
+                for (i = 1; i < dirc; i++) {
+                        if (srch->qp_flags & FLAG_PHRASE && is_chert(dirv[i]))
+                                srch->qp_flags &= ~FLAG_PHRASE;
+                        srch->db->add_database(Xapian::Database(dirv[i]));
+                }
+        } catch (...) {
+                warn("E: add_database(%s)", dirv[i]);
+                return false;
+        }
+        try {
+                srch->qp = new Xapian::QueryParser;
+        } catch (...) {
+                perror("E: Xapian::QueryParser");
+                return false;
+        }
+        srch->qp->set_default_op(Xapian::Query::OP_AND);
+        srch->qp->set_database(*srch->db);
+        try {
+                srch->qp->set_stemmer(Xapian::Stem("english"));
+        } catch (...) {
+                perror("E: Xapian::Stem");
+                return false;
+        }
+        srch->qp->set_stemming_strategy(Xapian::QueryParser::STEM_SOME);
+        srch->qp->SET_MAX_EXPANSION(100);
+
+        if (req->code_search)
+                qp_init_code_search(srch->qp); // CodeSearch.pm
+        else
+                qp_init_mail_search(srch->qp); // Search.pm
+        return true;
+}
+
+static void free_srch(void *p) // tdestroy
+{
+        struct srch *srch = (struct srch *)p;
+        delete srch->qp;
+        delete srch->db;
+        free(srch);
+}
+
+static void dispatch(struct req *req)
+{
+        int c;
+        size_t size = strlen(req->argv[0]);
+        union {
+                struct srch *srch;
+                char *ptr;
+        } kbuf;
+        char *end;
+        FILE *kfp;
+        struct srch **s;
+        req->threadid = ULLONG_MAX;
+        for (c = 0; c < (int)MY_ARRAY_SIZE(cmds); c++) {
+                if (cmds[c].fn_len == size &&
+                        !memcmp(cmds[c].fn_name, req->argv[0], size)) {
+                        req->fn = cmds[c].fn;
+                        break;
+                }
+        }
+        if (!req->fn) ABORT("not handled: `%s'", req->argv[0]);
+
+        kfp = open_memstream(&kbuf.ptr, &size);
+        if (!kfp) err(EXIT_FAILURE, "open_memstream(kbuf)");
+        // write padding, first (contents don't matter)
+        fwrite(&req->argv[0], offsetof(struct srch, paths), 1, kfp);
+
+        // global getopt variables:
+        optopt = 0;
+        optarg = NULL;
+        MY_DO_OPTRESET();
+
+        // XH_SPEC is generated from @PublicInbox::Search::XH_SPEC
+        while ((c = getopt(req->argc, req->argv, XH_SPEC)) != -1) {
+                switch (c) {
+                case 'a': req->asc = true; break;
+                case 'c': req->code_search = true; break;
+                case 'd': fwrite(optarg, strlen(optarg) + 1, 1, kfp); break;
+                case 'g': req->Pgit_dir = optarg - 1; break; // pad "P" prefix
+                case 'k':
+                        req->sort_col = strtol(optarg, &end, 10);
+                        if (*end) ABORT("-k %s", optarg);
+                        switch (req->sort_col) {
+                        case LONG_MAX: case LONG_MIN: ABORT("-k %s", optarg);
+                        }
+                        break;
+                case 'm':
+                        req->max = strtoull(optarg, &end, 10);
+                        if (*end || req->max == ULLONG_MAX)
+                                ABORT("-m %s", optarg);
+                        break;
+                case 'o':
+                        req->off = strtoull(optarg, &end, 10);
+                        if (*end || req->off == ULLONG_MAX)
+                                ABORT("-o %s", optarg);
+                        break;
+                case 'p': req->emit_percent = true; break;
+                case 'r': req->relevance = true; break;
+                case 't': req->collapse_threads = true; break;
+                case 'A':
+                        req->pfxv[req->pfxc++] = optarg;
+                        if (MY_ARG_MAX == req->pfxc)
+                                ABORT("too many -A");
+                        break;
+                case 'D': req->emit_docdata = true; break;
+                case 'K':
+                        req->timeout_sec = strtoul(optarg, &end, 10);
+                        if (*end || req->timeout_sec == ULONG_MAX)
+                                ABORT("-K %s", optarg);
+                        break;
+                case 'O': req->Oeidx_key = optarg - 1; break; // pad "O" prefix
+                case 'T':
+                        req->threadid = strtoull(optarg, &end, 10);
+                        if (*end || req->threadid == ULLONG_MAX)
+                                ABORT("-T %s", optarg);
+                        break;
+                default: ABORT("bad switch `-%c'", c);
+                }
+        }
+        ERR_CLOSE(kfp, EXIT_FAILURE); // may ENOMEM, sets kbuf.srch
+        kbuf.srch->db = NULL;
+        kbuf.srch->qp = NULL;
+        kbuf.srch->paths_len = size - offsetof(struct srch, paths);
+        if (kbuf.srch->paths_len <= 0)
+                ABORT("no -d args");
+        s = (struct srch **)tsearch(kbuf.srch, &srch_tree, srch_cmp);
+        if (!s) err(EXIT_FAILURE, "tsearch"); // likely ENOMEM
+        req->srch = *s;
+        if (req->srch != kbuf.srch) { // reuse existing
+                free_srch(kbuf.srch);
+        } else if (!srch_init(req)) {
+                assert(kbuf.srch == *((struct srch **)tfind(
+                                        kbuf.srch, &srch_tree, srch_cmp)));
+                void *del = tdelete(kbuf.srch, &srch_tree, srch_cmp);
+                assert(del);
+                free_srch(kbuf.srch);
+                goto cmd_err; // srch_init already warned
+        }
+        try {
+                if (!req->fn(req))
+                        warnx("`%s' failed", req->argv[0]);
+        } catch (const Xapian::Error & e) {
+                warnx("Xapian::Error: %s", e.get_description().c_str());
+        } catch (...) {
+                warn("unhandled exception");
+        }
+cmd_err:
+        return; // just be silent on errors, for now
+}
+
+static void cleanup_pids(void)
+{
+        free(worker_pids);
+        worker_pids = NULL;
+}
+
+static void stderr_set(FILE *tmp_err)
+{
+#if STDERR_ASSIGNABLE
+        if (my_setlinebuf(tmp_err))
+                perror("W: setlinebuf(tmp_err)");
+        stderr = tmp_err;
+        return;
+#endif
+        int fd = fileno(tmp_err);
+        if (fd < 0) err(EXIT_FAILURE, "BUG: fileno(tmp_err)");
+        while (dup2(fd, STDERR_FILENO) < 0) {
+                if (errno != EINTR)
+                        err(EXIT_FAILURE, "dup2(%d => 2)", fd);
+        }
+}
+
+static void stderr_restore(FILE *tmp_err)
+{
+#if STDERR_ASSIGNABLE
+        stderr = orig_err;
+        return;
+#endif
+        ERR_FLUSH(stderr);
+        while (dup2(orig_err_fd, STDERR_FILENO) < 0) {
+                if (errno != EINTR)
+                        err(EXIT_FAILURE, "dup2(%d => 2)", orig_err_fd);
+        }
+        clearerr(stderr);
+}
+
+static void sigw(int sig) // SIGTERM handler for worker
+{
+        sock_fd = -1; // break out of recv_loop
+}
+
+#define CLEANUP_REQ __attribute__((__cleanup__(req_cleanup)))
+static void req_cleanup(void *ptr)
+{
+        struct req *req = (struct req *)ptr;
+        free(req->lenv);
+}
+
+static void recv_loop(void) // worker process loop
+{
+        static char rbuf[4096 * 33]; // per-process
+        struct sigaction sa = {};
+        sa.sa_handler = sigw;
+
+        CHECK(int, 0, sigaction(SIGTERM, &sa, NULL));
+
+        while (sock_fd == 0) {
+                size_t len = sizeof(rbuf);
+                CLEANUP_REQ struct req req = {};
+
+                if (!recv_req(&req, rbuf, &len))
+                        continue;
+                if (req.fp[1])
+                        stderr_set(req.fp[1]);
+                req.argc = (int)SPLIT2ARGV(req.argv, rbuf, len);
+                dispatch(&req);
+                ERR_CLOSE(req.fp[0], 0);
+                if (req.fp[1]) {
+                        stderr_restore(req.fp[1]);
+                        ERR_CLOSE(req.fp[1], 0);
+                }
+        }
+}
+
+static void insert_pid(pid_t pid, unsigned nr)
+{
+        assert(!worker_pids[nr]);
+        worker_pids[nr] = pid;
+}
+
+static void start_worker(unsigned nr)
+{
+        pid_t pid = fork();
+        if (pid < 0) {
+                warn("E: fork(worker=%u)", nr);
+        } else if (pid > 0) {
+                insert_pid(pid, nr);
+        } else {
+                cleanup_pids();
+                xclose(pipefds[0]);
+                xclose(pipefds[1]);
+                if (signal(SIGCHLD, SIG_DFL) == SIG_ERR)
+                        err(EXIT_FAILURE, "signal CHLD");
+                if (signal(SIGTTIN, SIG_IGN) == SIG_ERR)
+                        err(EXIT_FAILURE, "signal TTIN");
+                if (signal(SIGTTOU, SIG_IGN) == SIG_ERR)
+                        err(EXIT_FAILURE, "signal TTIN");
+                recv_loop();
+                exit(0);
+        }
+}
+
+static void start_workers(void)
+{
+        sigset_t old;
+
+        CHECK(int, 0, sigprocmask(SIG_SETMASK, &fullset, &old));
+        for (unsigned long nr = 0; nr < nworker; nr++) {
+                if (!worker_pids[nr])
+                        start_worker(nr);
+        }
+        CHECK(int, 0, sigprocmask(SIG_SETMASK, &old, NULL));
+}
+
+static void cleanup_all(void)
+{
+        cleanup_pids();
+#ifdef __GLIBC__
+        tdestroy(srch_tree, free_srch);
+        srch_tree = NULL;
+#endif
+}
+
+static void sigp(int sig) // parent signal handler
+{
+        static const char eagain[] = "signals coming in too fast";
+        static const char bad_sig[] = "BUG: bad sig\n";
+        static const char write_errno[] = "BUG: sigp write (errno)";
+        static const char write_zero[] = "BUG: sigp write wrote zero bytes";
+        char c = 0;
+
+        switch (sig) {
+        case SIGCHLD: c = '.'; break;
+        case SIGTTOU: c = '-'; break;
+        case SIGTTIN: c = '+'; break;
+        default:
+                write(STDERR_FILENO, bad_sig, sizeof(bad_sig) - 1);
+                _exit(EXIT_FAILURE);
+        }
+        ssize_t w = write(pipefds[1], &c, 1);
+        if (w > 0) return;
+        if (w < 0 && errno == EAGAIN) {
+                write(STDERR_FILENO, eagain, sizeof(eagain) - 1);
+                return;
+        } else if (w == 0) {
+                write(STDERR_FILENO, write_zero, sizeof(write_zero) - 1);
+        } else {
+                // strerror isn't technically async-signal-safe, and
+                // strerrordesc_np+strerrorname_np isn't portable
+                write(STDERR_FILENO, write_errno, sizeof(write_errno) - 1);
+        }
+        _exit(EXIT_FAILURE);
+}
+
+static void reaped_worker(pid_t pid, int st)
+{
+        unsigned long nr = 0;
+        for (; nr < nworker_hwm; nr++) {
+                if (worker_pids[nr] == pid) {
+                        worker_pids[nr] = 0;
+                        break;
+                }
+        }
+        if (nr >= nworker_hwm) {
+                warnx("W: unknown pid=%d reaped $?=%d", (int)pid, st);
+                return;
+        }
+        if (WIFEXITED(st) && WEXITSTATUS(st) == EX_NOINPUT)
+                alive = false;
+        else if (st)
+                warnx("worker[%lu] died $?=%d alive=%d", nr, st, (int)alive);
+        if (alive)
+                start_workers();
+}
+
+static void do_sigchld(void)
+{
+        while (1) {
+                int st;
+                pid_t pid = waitpid(-1, &st, WNOHANG);
+                if (pid > 0) {
+                        reaped_worker(pid, st);
+                } else if (pid == 0) {
+                        return;
+                } else {
+                        switch (errno) {
+                        case ECHILD: return;
+                        case EINTR: break; // can it happen w/ WNOHANG?
+                        default: err(EXIT_FAILURE, "BUG: waitpid");
+                        }
+                }
+        }
+}
+
+static void do_sigttin(void)
+{
+        if (!alive) return;
+        if (nworker >= WORKER_MAX) {
+                warnx("workers cannot exceed %zu", (size_t)WORKER_MAX);
+                return;
+        }
+        void *p = realloc(worker_pids, (nworker + 1) * sizeof(pid_t));
+        if (!p) {
+                warn("realloc worker_pids");
+        } else {
+                worker_pids = (pid_t *)p;
+                worker_pids[nworker++] = 0;
+                if (nworker_hwm < nworker)
+                        nworker_hwm = nworker;
+                start_workers();
+        }
+}
+
+static void do_sigttou(void)
+{
+        if (!alive || nworker <= 1) return;
+
+        // worker_pids array does not shrink
+        --nworker;
+        for (unsigned long nr = nworker; nr < nworker_hwm; nr++) {
+                pid_t pid = worker_pids[nr];
+                if (pid != 0 && kill(pid, SIGTERM))
+                        warn("BUG?: kill(%d, SIGTERM)", (int)pid);
+        }
+}
+
+static size_t living_workers(void)
+{
+        size_t ret = 0;
+
+        for (unsigned long nr = 0; nr < nworker_hwm; nr++) {
+                if (worker_pids[nr])
+                        ret++;
+        }
+        return ret;
+}
+
+int main(int argc, char *argv[])
+{
+        int c;
+        socklen_t slen = (socklen_t)sizeof(c);
+
+        if (getsockopt(sock_fd, SOL_SOCKET, SO_TYPE, &c, &slen))
+                err(EXIT_FAILURE, "getsockopt");
+        if (c != SOCK_SEQPACKET)
+                errx(EXIT_FAILURE, "stdin is not SOCK_SEQPACKET");
+
+        mail_nrp_init();
+        code_nrp_init();
+        atexit(cleanup_all);
+
+        if (!STDERR_ASSIGNABLE) {
+                orig_err_fd = dup(STDERR_FILENO);
+                if (orig_err_fd < 0)
+                        err(EXIT_FAILURE, "dup(2)");
+        }
+
+        nworker = 1;
+#ifdef _SC_NPROCESSORS_ONLN
+        long j = sysconf(_SC_NPROCESSORS_ONLN);
+        if (j > 0)
+                nworker = j > WORKER_MAX ? WORKER_MAX : j;
+#endif // _SC_NPROCESSORS_ONLN
+
+        // make warn/warnx/err multi-process friendly:
+        if (my_setlinebuf(stderr))
+                err(EXIT_FAILURE, "setlinebuf(stderr)");
+        // not using -W<workers> like Daemon.pm, since -W is reserved (glibc)
+        while ((c = getopt(argc, argv, "j:")) != -1) {
+                char *end;
+
+                switch (c) {
+                case 'j':
+                        nworker = strtoul(optarg, &end, 10);
+                        if (*end != 0 || nworker > WORKER_MAX)
+                                errx(EXIT_FAILURE, "-j %s invalid", optarg);
+                        break;
+                case ':':
+                        errx(EXIT_FAILURE, "missing argument: `-%c'", optopt);
+                case '?':
+                        errx(EXIT_FAILURE, "unrecognized: `-%c'", optopt);
+                default:
+                        errx(EXIT_FAILURE, "BUG: `-%c'", c);
+                }
+        }
+        sigset_t pset; // parent-only
+        CHECK(int, 0, sigfillset(&pset));
+
+        // global sigsets:
+        CHECK(int, 0, sigfillset(&fullset));
+        CHECK(int, 0, sigfillset(&workerset));
+
+#define DELSET(sig) do { \
+        CHECK(int, 0, sigdelset(&fullset, sig)); \
+        CHECK(int, 0, sigdelset(&workerset, sig)); \
+        CHECK(int, 0, sigdelset(&pset, sig)); \
+} while (0)
+        DELSET(SIGABRT);
+        DELSET(SIGBUS);
+        DELSET(SIGFPE);
+        DELSET(SIGILL);
+        DELSET(SIGSEGV);
+        DELSET(SIGXCPU);
+        DELSET(SIGXFSZ);
+#undef DELSET
+
+        if (nworker == 0) { // no SIGTERM handling w/o workers
+                recv_loop();
+                return 0;
+        }
+        CHECK(int, 0, sigdelset(&workerset, SIGTERM));
+        CHECK(int, 0, sigdelset(&workerset, SIGCHLD));
+        nworker_hwm = nworker;
+        worker_pids = (pid_t *)calloc(nworker, sizeof(pid_t));
+        if (!worker_pids) err(EXIT_FAILURE, "calloc");
+
+        if (pipe(pipefds)) err(EXIT_FAILURE, "pipe");
+        int fl = fcntl(pipefds[1], F_GETFL);
+        if (fl == -1) err(EXIT_FAILURE, "F_GETFL");
+        if (fcntl(pipefds[1], F_SETFL, fl | O_NONBLOCK))
+                err(EXIT_FAILURE, "F_SETFL");
+
+        CHECK(int, 0, sigdelset(&pset, SIGCHLD));
+        CHECK(int, 0, sigdelset(&pset, SIGTTIN));
+        CHECK(int, 0, sigdelset(&pset, SIGTTOU));
+
+        struct sigaction sa = {};
+        sa.sa_handler = sigp;
+
+        CHECK(int, 0, sigaction(SIGTTIN, &sa, NULL));
+        CHECK(int, 0, sigaction(SIGTTOU, &sa, NULL));
+        sa.sa_flags = SA_NOCLDSTOP;
+        CHECK(int, 0, sigaction(SIGCHLD, &sa, NULL));
+
+        CHECK(int, 0, sigprocmask(SIG_SETMASK, &pset, NULL));
+
+        start_workers();
+
+        char sbuf[64];
+        while (alive || living_workers()) {
+                ssize_t n = read(pipefds[0], &sbuf, sizeof(sbuf));
+                if (n < 0) {
+                        if (errno == EINTR) continue;
+                        err(EXIT_FAILURE, "read");
+                } else if (n == 0) {
+                        errx(EXIT_FAILURE, "read EOF");
+                }
+                do_sigchld();
+                for (ssize_t i = 0; i < n; i++) {
+                        switch (sbuf[i]) {
+                        case '.': break; // do_sigchld already called
+                        case '-': do_sigttou(); break;
+                        case '+': do_sigttin(); break;
+                        default: errx(EXIT_FAILURE, "BUG: c=%c", sbuf[i]);
+                        }
+                }
+        }
+
+        return 0;
+}
diff --git a/lib/PublicInbox/xh_cidx.h b/lib/PublicInbox/xh_cidx.h
new file mode 100644
index 00000000..311ca05f
--- /dev/null
+++ b/lib/PublicInbox/xh_cidx.h
@@ -0,0 +1,277 @@
+// Copyright (C) all contributors <meta@public-inbox.org>
+// License: GPL-2.0+ <https://www.gnu.org/licenses/gpl-2.0.txt>
+// This file is only intended to be included by xap_helper.h
+// it implements pieces used by CodeSearchIdx.pm
+
+static void term_length_extract(struct req *req)
+{
+        req->lenv = (size_t *)calloc(req->pfxc, sizeof(size_t));
+        if (!req->lenv)
+                EABORT("lenv = calloc(%d %zu)", req->pfxc, sizeof(size_t));
+        for (int i = 0; i < req->pfxc; i++) {
+                char *pfx = req->pfxv[i];
+                // extract trailing digits as length:
+                // $len = s/([0-9]+)\z// ? ($1+0) : 0
+                for (size_t j = 0; pfx[j]; j++) {
+                        if (pfx[j] < '0' || pfx[j] > '9')
+                                continue;
+                        if (j == 0) {
+                                warnx("W: `%s' not a valid prefix", pfx);
+                                continue;
+                        }
+                        char *end;
+                        unsigned long long tmp = strtoull(pfx + j, &end, 10);
+                        if (*end || tmp >= (unsigned long long)SIZE_MAX) {
+                                warnx("W: `%s' not recognized", pfx);
+                        } else {
+                                req->lenv[i] = (size_t)tmp;
+                                pfx[j] = 0;
+                                break;
+                        }
+                }
+        }
+}
+
+static void dump_ibx_term(struct req *req, int p,
+                        Xapian::Document *doc, const char *ibx_id)
+{
+        Xapian::TermIterator cur = doc->termlist_begin();
+        Xapian::TermIterator end = doc->termlist_end();
+        const char *pfx = req->pfxv[p];
+        size_t pfx_len = strlen(pfx);
+        size_t term_len = req->lenv[p];
+
+        for (cur.skip_to(pfx); cur != end; cur++) {
+                std::string tn = *cur;
+                if (!starts_with(&tn, pfx, pfx_len)) break;
+                if (term_len > 0 && (tn.length() - pfx_len) != term_len)
+                        continue;
+                fprintf(req->fp[0], "%s %s\n", tn.c_str() + pfx_len, ibx_id);
+                ++req->nr_out;
+        }
+}
+
+static enum exc_iter dump_ibx_iter(struct req *req, const char *ibx_id,
+                                Xapian::MSetIterator *i)
+{
+        try {
+                Xapian::Document doc = i->get_document();
+                for (int p = 0; p < req->pfxc; p++)
+                        dump_ibx_term(req, p, &doc, ibx_id);
+        } catch (const Xapian::DatabaseModifiedError & e) {
+                req->srch->db->reopen();
+                return ITER_RETRY;
+        } catch (const Xapian::DocNotFoundError & e) { // oh well...
+                warnx("doc not found: %s", e.get_description().c_str());
+        }
+        return ITER_OK;
+}
+
+static bool cmd_dump_ibx(struct req *req)
+{
+        if ((optind + 1) >= req->argc)
+                ABORT("usage: dump_ibx [OPTIONS] IBX_ID QRY_STR");
+        if (!req->pfxc)
+                ABORT("dump_ibx requires -A PREFIX");
+
+        const char *ibx_id = req->argv[optind];
+        if (my_setlinebuf(req->fp[0])) // for sort(1) pipe
+                EABORT("setlinebuf(fp[0])"); // WTF?
+        req->asc = true;
+        req->sort_col = -1;
+        term_length_extract(req);
+        Xapian::MSet mset = mail_mset(req, req->argv[optind + 1]);
+
+        // @UNIQ_FOLD in CodeSearchIdx.pm can handle duplicate lines fine
+        // in case we need to retry on DB reopens
+        for (Xapian::MSetIterator i = mset.begin(); i != mset.end(); i++) {
+                for (int t = 10; t > 0; --t)
+                        switch (dump_ibx_iter(req, ibx_id, &i)) {
+                        case ITER_OK: t = 0; break; // leave inner loop
+                        case ITER_RETRY: break; // continue for-loop
+                        case ITER_ABORT: return false; // error
+                        }
+        }
+        emit_mset_stats(req, &mset);
+        return true;
+}
+
+struct dump_roots_tmp {
+        struct stat sb;
+        void *mm_ptr;
+        char **entries;
+        struct fbuf wbuf;
+        int root2off_fd;
+};
+
+#define CLEANUP_DUMP_ROOTS __attribute__((__cleanup__(dump_roots_ensure)))
+static void dump_roots_ensure(void *ptr)
+{
+        struct dump_roots_tmp *drt = (struct dump_roots_tmp *)ptr;
+        if (drt->root2off_fd >= 0)
+                xclose(drt->root2off_fd);
+        hdestroy(); // idempotent
+        size_t size = off2size(drt->sb.st_size);
+        if (drt->mm_ptr && munmap(drt->mm_ptr, size))
+                EABORT("BUG: munmap(%p, %zu)", drt->mm_ptr, size);
+        free(drt->entries);
+        fbuf_ensure(&drt->wbuf);
+}
+
+static bool root2offs_str(struct fbuf *root_offs, Xapian::Document *doc)
+{
+        Xapian::TermIterator cur = doc->termlist_begin();
+        Xapian::TermIterator end = doc->termlist_end();
+        ENTRY e, *ep;
+        fbuf_init(root_offs);
+        for (cur.skip_to("G"); cur != end; cur++) {
+                std::string tn = *cur;
+                if (!starts_with(&tn, "G", 1)) break;
+                union { const char *in; char *out; } u;
+                u.in = tn.c_str() + 1;
+                e.key = u.out;
+                ep = hsearch(e, FIND);
+                if (!ep) ABORT("hsearch miss `%s'", e.key);
+                // ep->data is a NUL-terminated string matching /[0-9]+/
+                fputc(' ', root_offs->fp);
+                fputs((const char *)ep->data, root_offs->fp);
+        }
+        fputc('\n', root_offs->fp);
+        ERR_CLOSE(root_offs->fp, EXIT_FAILURE); // ENOMEM
+        root_offs->fp = NULL;
+        return true;
+}
+
+// writes term values matching @pfx for a given @doc, ending the line
+// with the contents of @root_offs
+static void dump_roots_term(struct req *req, int p,
+                                struct dump_roots_tmp *drt,
+                                struct fbuf *root_offs,
+                                Xapian::Document *doc)
+{
+        Xapian::TermIterator cur = doc->termlist_begin();
+        Xapian::TermIterator end = doc->termlist_end();
+        const char *pfx = req->pfxv[p];
+        size_t pfx_len = strlen(pfx);
+        size_t term_len = req->lenv[p];
+
+        for (cur.skip_to(pfx); cur != end; cur++) {
+                std::string tn = *cur;
+                if (!starts_with(&tn, pfx, pfx_len)) break;
+                if (term_len > 0 && (tn.length() - pfx_len) != term_len)
+                        continue;
+                fputs(tn.c_str() + pfx_len, drt->wbuf.fp);
+                fwrite(root_offs->ptr, root_offs->len, 1, drt->wbuf.fp);
+                ++req->nr_out;
+        }
+}
+
+// we may have lines which exceed PIPE_BUF, so we do our own
+// buffering and rely on flock(2), here
+static bool dump_roots_flush(struct req *req, struct dump_roots_tmp *drt)
+{
+        bool ok = true;
+        off_t off = ftello(drt->wbuf.fp);
+        if (off < 0) EABORT("ftello");
+        if (!off) return ok;
+
+        ERR_FLUSH(drt->wbuf.fp); // ENOMEM
+        int fd = fileno(req->fp[0]);
+
+        while (flock(drt->root2off_fd, LOCK_EX)) {
+                if (errno == EINTR) continue;
+                err(EXIT_FAILURE, "LOCK_EX"); // ENOLCK?
+        }
+        ok = write_all(fd, &drt->wbuf, (size_t)off);
+        while (flock(drt->root2off_fd, LOCK_UN)) {
+                if (errno == EINTR) continue;
+                err(EXIT_FAILURE, "LOCK_UN"); // ENOLCK?
+        }
+        if (fseeko(drt->wbuf.fp, 0, SEEK_SET)) EABORT("fseeko");
+        return ok;
+}
+
+static enum exc_iter dump_roots_iter(struct req *req,
+                                struct dump_roots_tmp *drt,
+                                Xapian::MSetIterator *i)
+{
+        CLEANUP_FBUF struct fbuf root_offs = {}; // " $ID0 $ID1 $IDx..\n"
+        try {
+                Xapian::Document doc = i->get_document();
+                if (!root2offs_str(&root_offs, &doc))
+                        return ITER_ABORT; // bad request, abort
+                for (int p = 0; p < req->pfxc; p++)
+                        dump_roots_term(req, p, drt, &root_offs, &doc);
+        } catch (const Xapian::DatabaseModifiedError & e) {
+                req->srch->db->reopen();
+                return ITER_RETRY;
+        } catch (const Xapian::DocNotFoundError & e) { // oh well...
+                warnx("doc not found: %s", e.get_description().c_str());
+        }
+        return ITER_OK;
+}
+
+static bool cmd_dump_roots(struct req *req)
+{
+        CLEANUP_DUMP_ROOTS struct dump_roots_tmp drt = {};
+        drt.root2off_fd = -1;
+        if ((optind + 1) >= req->argc)
+                ABORT("usage: dump_roots [OPTIONS] ROOT2ID_FILE QRY_STR");
+        if (!req->pfxc)
+                ABORT("dump_roots requires -A PREFIX");
+        const char *root2off_file = req->argv[optind];
+        drt.root2off_fd = open(root2off_file, O_RDONLY);
+        if (drt.root2off_fd < 0)
+                EABORT("open(%s)", root2off_file);
+        if (fstat(drt.root2off_fd, &drt.sb)) // ENOMEM?
+                err(EXIT_FAILURE, "fstat(%s)", root2off_file);
+        // each entry is at least 43 bytes ({OIDHEX}\0{INT}\0),
+        // so /32 overestimates the number of expected entries by
+        // ~%25 (as recommended by Linux hcreate(3) manpage)
+        size_t size = off2size(drt.sb.st_size);
+        size_t est = (size / 32) + 1; //+1 for "\0" termination
+        drt.mm_ptr = mmap(NULL, size, PROT_READ,
+                                MAP_PRIVATE, drt.root2off_fd, 0);
+        if (drt.mm_ptr == MAP_FAILED)
+                err(EXIT_FAILURE, "mmap(%zu, %s)", size, root2off_file);
+        size_t asize = est * 2;
+        if (asize < est) ABORT("too many entries: %zu", est);
+        drt.entries = (char **)calloc(asize, sizeof(char *));
+        if (!drt.entries)
+                err(EXIT_FAILURE, "calloc(%zu * 2, %zu)", est, sizeof(char *));
+        size_t tot = split2argv(drt.entries, (char *)drt.mm_ptr, size, asize);
+        if (tot <= 0) return false; // split2argv already warned on error
+        if (!hcreate(est))
+                err(EXIT_FAILURE, "hcreate(%zu)", est);
+        for (size_t i = 0; i < tot; ) {
+                ENTRY e;
+                e.key = hsearch_enter_key(drt.entries[i++]); // dies on ENOMEM
+                e.data = drt.entries[i++];
+                if (!hsearch(e, ENTER))
+                        err(EXIT_FAILURE, "hsearch(%s => %s, ENTER)", e.key,
+                                        (const char *)e.data);
+        }
+        req->asc = true;
+        req->sort_col = -1;
+        Xapian::MSet mset = commit_mset(req, req->argv[optind + 1]);
+        term_length_extract(req);
+
+        fbuf_init(&drt.wbuf);
+
+        // @UNIQ_FOLD in CodeSearchIdx.pm can handle duplicate lines fine
+        // in case we need to retry on DB reopens
+        for (Xapian::MSetIterator i = mset.begin(); i != mset.end(); i++) {
+                for (int t = 10; t > 0; --t)
+                        switch (dump_roots_iter(req, &drt, &i)) {
+                        case ITER_OK: t = 0; break; // leave inner loop
+                        case ITER_RETRY: break; // continue for-loop
+                        case ITER_ABORT: return false; // error
+                        }
+                if (!(req->nr_out & 0x3fff) && !dump_roots_flush(req, &drt))
+                        return false;
+        }
+        if (!dump_roots_flush(req, &drt))
+                return false;
+        emit_mset_stats(req, &mset);
+        return true;
+}
diff --git a/lib/PublicInbox/xh_mset.h b/lib/PublicInbox/xh_mset.h
new file mode 100644
index 00000000..4e97a284
--- /dev/null
+++ b/lib/PublicInbox/xh_mset.h
@@ -0,0 +1,96 @@
+// Copyright (C) all contributors <meta@public-inbox.org>
+// License: GPL-2.0+ <https://www.gnu.org/licenses/gpl-2.0.txt>
+// This file is only intended to be included by xap_helper.h
+// it implements pieces used by WWW, IMAP and lei
+
+static void emit_doc_term(FILE *fp, const char *pfx, Xapian::Document *doc)
+{
+        Xapian::TermIterator cur = doc->termlist_begin();
+        Xapian::TermIterator end = doc->termlist_end();
+        size_t pfx_len = strlen(pfx);
+
+        for (cur.skip_to(pfx); cur != end; cur++) {
+                std::string tn = *cur;
+                if (!starts_with(&tn, pfx, pfx_len)) break;
+                fputc(0, fp);
+                fwrite(tn.data(), tn.size(), 1, fp);
+        }
+}
+
+static enum exc_iter mset_iter(const struct req *req, FILE *fp, off_t off,
+                                Xapian::MSetIterator *i)
+{
+        try {
+                fprintf(fp, "%llu", (unsigned long long)(*(*i))); // get_docid
+                if (req->emit_percent)
+                        fprintf(fp, "%c%d", 0, i->get_percent());
+                if (req->pfxc || req->emit_docdata) {
+                        Xapian::Document doc = i->get_document();
+                        for (int p = 0; p < req->pfxc; p++)
+                                emit_doc_term(fp, req->pfxv[p], &doc);
+                        if (req->emit_docdata) {
+                                std::string d = doc.get_data();
+                                fputc(0, fp);
+                                fwrite(d.data(), d.size(), 1, fp);
+                        }
+                }
+                fputc('\n', fp);
+        } catch (const Xapian::DatabaseModifiedError & e) {
+                req->srch->db->reopen();
+                if (fseeko(fp, off, SEEK_SET) < 0) EABORT("fseeko");
+                return ITER_RETRY;
+        } catch (const Xapian::DocNotFoundError & e) { // oh well...
+                warnx("doc not found: %s", e.get_description().c_str());
+                if (fseeko(fp, off, SEEK_SET) < 0) EABORT("fseeko");
+        }
+        return ITER_OK;
+}
+
+#ifndef WBUF_FLUSH_THRESHOLD
+#        define WBUF_FLUSH_THRESHOLD (BUFSIZ - 1000)
+#endif
+#if WBUF_FLUSH_THRESHOLD < 0
+#        undef WBUF_FLUSH_THRESHOLD
+#        define WBUF_FLUSH_THRESHOLD BUFSIZ
+#endif
+
+static bool cmd_mset(struct req *req)
+{
+        if (optind >= req->argc) ABORT("usage: mset [OPTIONS] WANT QRY_STR");
+        if (req->fp[1]) ABORT("mset only accepts 1 FD");
+        const char *qry_str = req->argv[optind];
+        CLEANUP_FBUF struct fbuf wbuf = {};
+        Xapian::MSet mset = req->code_search ? commit_mset(req, qry_str) :
+                                                mail_mset(req, qry_str);
+        fbuf_init(&wbuf);
+        fprintf(wbuf.fp, "mset.size=%llu\n", (unsigned long long)mset.size());
+        int fd = fileno(req->fp[0]);
+        for (Xapian::MSetIterator i = mset.begin(); i != mset.end(); i++) {
+                off_t off = ftello(wbuf.fp);
+                if (off < 0) EABORT("ftello");
+                /*
+                 * TODO verify our fflush + fseeko use isn't affected by a
+                 * glibc <2.25 bug:
+                 * https://sourceware.org/bugzilla/show_bug.cgi?id=20181
+                 * CentOS 7.x only has glibc 2.17.  In any case, bug #20181
+                 * shouldn't affect us since our use of fseeko is used to
+                 * effectively discard data.
+                 */
+                if (off > WBUF_FLUSH_THRESHOLD) {
+                        ERR_FLUSH(wbuf.fp);
+                        if (!write_all(fd, &wbuf, (size_t)off)) return false;
+                        if (fseeko(wbuf.fp, 0, SEEK_SET)) EABORT("fseeko");
+                        off = 0;
+                }
+                for (int t = 10; t > 0; --t)
+                        switch (mset_iter(req, wbuf.fp, off, &i)) {
+                        case ITER_OK: t = 0; break; // leave inner loop
+                        case ITER_RETRY: break; // continue for-loop
+                        case ITER_ABORT: return false; // error
+                        }
+        }
+        off_t off = ftello(wbuf.fp);
+        if (off < 0) EABORT("ftello");
+        ERR_FLUSH(wbuf.fp);
+        return off > 0 ? write_all(fd, &wbuf, (size_t)off) : true;
+}
diff --git a/sa_config/README b/sa_config/README
index 6703c38f..3705e1e8 100644
--- a/sa_config/README
+++ b/sa_config/README
@@ -4,9 +4,9 @@ SpamAssassin configs for public-inbox.org
 root/ - files for system-wide use (plugins, rule definitions,
         new rules should have a zero score which should be overridden)
 user/ - per-user config (keep as much in here as possible)
-        These files go into the users home directory
+        These files go into the user's home directory.
 
-All files in these example directory are CC0:
+All files in these example directories are CC0:
 To the extent possible under law, Eric Wong has waived all copyright and
 related or neighboring rights to these examples.
 
diff --git a/script/lei b/script/lei
index 5feb7751..087afc33 100755
--- a/script/lei
+++ b/script/lei
@@ -2,7 +2,7 @@
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use v5.12;
-use Socket qw(AF_UNIX SOCK_SEQPACKET MSG_EOR pack_sockaddr_un);
+use Socket qw(AF_UNIX SOCK_SEQPACKET pack_sockaddr_un);
 use PublicInbox::CmdIPC4;
 my $narg = 5;
 my $sock;
@@ -92,8 +92,8 @@ my $addr = pack_sockaddr_un($path);
 socket($sock, AF_UNIX, SOCK_SEQPACKET, 0) or die "socket: $!";
 unless (connect($sock, $addr)) { # start the daemon if not started
         local $ENV{PERL5LIB} = join(':', @INC);
-        open(my $daemon, '-|', $^X, qw[-MPublicInbox::LEI
-                -E PublicInbox::LEI::lazy_start(@ARGV)],
+        open(my $daemon, '-|', $^X, $^W ? ('-w') : (),
+                qw[-MPublicInbox::LEI -e PublicInbox::LEI::lazy_start(@ARGV)],
                 $path, $! + 0, $narg) or die "popen: $!";
         while (<$daemon>) { warn $_ } # EOF when STDERR is redirected
         close($daemon) or warn <<"";
@@ -109,24 +109,21 @@ open my $dh, '<', '.' or die "open(.) $!";
 my $buf = join("\0", scalar(@ARGV), @ARGV);
 while (my ($k, $v) = each %ENV) { $buf .= "\0$k=$v" }
 $buf .= "\0\0";
-$send_cmd->($sock, [0, 1, 2, fileno($dh)], $buf, MSG_EOR) or die "sendmsg: $!";
-$SIG{TSTP} = sub { send($sock, 'STOP', MSG_EOR); kill 'STOP', $$ };
-$SIG{CONT} = sub { send($sock, 'CONT', MSG_EOR) };
+$send_cmd->($sock, [0, 1, 2, fileno($dh)], $buf, 0) or die "sendmsg: $!";
+$SIG{TSTP} = sub { send($sock, 'STOP', 0); kill 'STOP', $$ };
+$SIG{CONT} = sub { send($sock, 'CONT', 0) };
 
 my $x_it_code = 0;
 while (1) {
         my (@fds) = $recv_cmd->($sock, my $buf, 4096 * 33);
-        if (scalar(@fds) == 1 && !defined($fds[0])) {
-                next if $!{EINTR};
-                die "recvmsg: $!";
-        }
+        die "recvmsg: $!" if scalar(@fds) == 1 && !defined($fds[0]);
         last if $buf eq '';
         if ($buf =~ /\Aexec (.+)\z/) {
                 $exec_cmd->(\@fds, split(/\0/, $1));
         } elsif ($buf eq '-WINCH') {
                 kill($buf, @parent); # for MUA
         } elsif ($buf eq 'umask') {
-                send($sock, 'u'.pack('V', umask), MSG_EOR) or die "send: $!"
+                send($sock, 'u'.pack('V', umask), 0) or die "send: $!"
         } elsif ($buf =~ /\Ax_it ([0-9]+)\z/) {
                 $x_it_code ||= $1 + 0;
                 last;
diff --git a/script/public-inbox-cindex b/script/public-inbox-cindex
new file mode 100755
index 00000000..dd00623a
--- /dev/null
+++ b/script/public-inbox-cindex
@@ -0,0 +1,102 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
+my $help = <<EOF; # the following should fit w/o scrolling in 80x24 term:
+usage: public-inbox-cindex [options] -g GIT_DIR [-g GIT_DIR]...
+usage: public-inbox-cindex [options] --project-list=FILE -r PROJECT_ROOT
+
+  Create and update search indices for code repos
+
+  -d EXTDIR           use EXTDIR instead of GIT_DIR/public-inbox-cindex
+  --no-fsync          speed up indexing, risk corruption on power outage
+  -L LEVEL            `medium', or `full' (default: medium)
+  --project-list=FILE use a cgit/gitweb-compatible list of projects
+  --update | -u       update previously-indexed code repos with `-d'
+  --jobs=NUM          set or disable parallelization (NUM=0)
+  --batch-size=BYTES  flush changes to OS after a given number of bytes
+  --max-size=BYTES    do not index commit diffs larger than the given size
+  --prune             prune old repos and commits
+  --reindex           reindex previously indexed repos
+  --verbose | -v      increase verbosity (may be repeated)
+
+BYTES may use `k', `m', and `g' suffixes (e.g. `10m' for 10 megabytes)
+See public-inbox-cindex(1) man page for full documentation.
+EOF
+my $opt = { fsync => 1, scan => 1 }; # --no-scan is hidden
+GetOptions($opt, qw(quiet|q verbose|v+ reindex jobs|j=i fsync|sync! dangerous
+                indexlevel|index-level|L=s join:s@
+                batch_size|batch-size=s max_size|max-size=s
+                include|I=s@ only=s@ all show:s@
+                project-list=s exclude=s@ project-root|r=s
+                git-dir|g=s@
+                sort-parallel=s sort-compress-program=s sort-buffer-size=s
+                d=s update|u scan! prune dry-run|n C=s@ help|h))
+        or die $help;
+if ($opt->{help}) { print $help; exit 0 };
+die "--jobs must be >= 0\n" if defined $opt->{jobs} && $opt->{jobs} < 0;
+require IO::Handle;
+STDOUT->autoflush(1);
+STDERR->autoflush(1);
+$SIG{USR1} = 'IGNORE'; # to be overridden in cidx_sync
+$SIG{PIPE} = 'IGNORE';
+# require lazily to speed up --help
+require PublicInbox::Admin;
+PublicInbox::Admin::do_chdir(delete $opt->{C});
+my $cfg = $opt->{-pi_cfg} = PublicInbox::Config->new;
+my $cidx_dir = $opt->{d};
+PublicInbox::Admin::require_or_die('Xapian');
+PublicInbox::Admin::progress_prepare($opt);
+my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
+%ENV = (%ENV, %$env) if $env;
+
+my @git_dirs;
+require PublicInbox::CodeSearchIdx; # unstable internal API
+if (@ARGV) {
+        my @g = map { "-g $_" } @ARGV;
+        die <<EOM;
+Specify git directories with `-g' (or --git-dir=): @g
+Or use --project-list=... and --project-root=...
+EOM
+} elsif (defined(my $pl = $opt->{'project-list'})) {
+        my $pfx = $opt->{'project-root'} // die <<EOM;
+PROJECT_ROOT required for --project-list
+EOM
+        $opt->{'git-dir'} and die <<EOM;
+--project-list does not accept additional --git-dir directories
+(@{$opt->{'git-dir'}})
+EOM
+        open my $fh, '<', $pl or die "open($pl): $!\n";
+        chomp(@git_dirs = <$fh>);
+        $pfx .= '/';
+        $pfx =~ tr!/!/!s;
+        substr($_, 0, 0, $pfx) for @git_dirs;
+} elsif (my $gd = $opt->{'git-dir'}) {
+        @git_dirs = @$gd;
+} elsif (grep defined, @$opt{qw(show update prune)}) {
+} else {
+        warn "No --git-dir= nor --project-list= + --project-root= specified\n";
+        die $help;
+}
+
+$_ = PublicInbox::Admin::resolve_git_dir($_) for @git_dirs;
+if (defined $cidx_dir) { # external index
+        die "`%' is not allowed in $cidx_dir\n" if $cidx_dir =~ /\%/;
+        my $cidx = PublicInbox::CodeSearchIdx->new($cidx_dir, $opt);
+        @{$cidx->{git_dirs}} = @git_dirs; # may be empty
+        $cidx->cidx_run;
+} elsif (!@git_dirs) {
+        die $help
+} else {
+        die <<EOM if $opt->{update};
+--update requires `-d EXTDIR'
+EOM
+        for my $gd (@git_dirs) {
+                my $cd = "$gd/public-inbox-cindex";
+                my $cidx = PublicInbox::CodeSearchIdx->new($cd, { %$opt });
+                $cidx->{-cidx_internal} = 1;
+                @{$cidx->{git_dirs}} = ($gd);
+                $cidx->cidx_run;
+        }
+}
diff --git a/script/public-inbox-clone b/script/public-inbox-clone
index 54059d03..c3e64485 100755
--- a/script/public-inbox-clone
+++ b/script/public-inbox-clone
@@ -2,14 +2,14 @@
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Wrapper to git clone remote public-inboxes
-use strict;
-use v5.10.1;
+use v5.12;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
 my $opt = {};
 my $help = <<EOF; # the following should fit w/o scrolling in 80x24 term:
-usage: public-inbox-clone INBOX_URL [DESTINATION]
+usage: public-inbox-clone [OPTIONS] INBOX_URL [INBOX_DIR]
+       public-inbox-clone [OPTIONS] ROOT_URL [DESTINATION]
 
-  clone remote public-inboxes
+  clone remote public-inboxes or grokmirror manifests
 
 options:
 
@@ -17,12 +17,23 @@ options:
   --torsocks VAL      whether or not to wrap git and curl commands with
                       torsocks (default: `auto')
                       Must be one of: `auto', `no' or `yes'
+  --dry-run | -n      show what would be cloned without cloning
   --verbose | -v      increase verbosity (may be repeated)
-    --quiet | -q      increase verbosity (may be repeated)
+    --quiet | -q      disable progress reporting
     -C DIR            chdir to specified directory
+
+See public-inbox-clone(1) man page for --manifest, --remote-manifest,
+--objstore, --project-list, --post-update-hook, --include, --exclude,
+--prune, --keep-going, --jobs, --inbox-config
 EOF
-GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@
-                no-torsocks torsocks=s epoch=s)) or die $help;
+
+# cgit calls it `project-list', grokmirror calls it `projectslist',
+# support both :/
+GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@ include|I=s@ exclude=s@
+        inbox-config=s inbox-version=i objstore=s manifest=s
+        remote-manifest=s project-list|projectslist=s post-update-hook=s@
+        prune|p keep-going|k exit-code purge
+        dry-run|n jobs|j=i no-torsocks torsocks=s epoch=s)) or die $help;
 if ($opt->{help}) { print $help; exit };
 require PublicInbox::Admin; # loads Config
 PublicInbox::Admin::do_chdir(delete $opt->{C});
@@ -35,12 +46,10 @@ defined($dst) or ($dst) = ($url =~ m!/([^/]+)/?\z!);
 index($dst, "\n") >= 0 and die "`\\n' not allowed in `$dst'";
 
 # n.b. this is still a truckload of code...
-require URI;
+require File::Spec;
 require PublicInbox::LEI;
 require PublicInbox::LeiExternal;
 require PublicInbox::LeiMirror;
-require PublicInbox::LeiCurl;
-require PublicInbox::Lock;
 
 $url = PublicInbox::LeiExternal::ext_canonicalize($url);
 my $lei = bless {
@@ -52,8 +61,10 @@ open $lei->{3}, '.' or die "open . $!";
 my $mrr = bless {
         lei => $lei,
         src => $url,
-        dst => $dst,
+        dst => File::Spec->canonpath($dst),
 }, 'PublicInbox::LeiMirror';
+
+$? = 0;
 $mrr->do_mirror;
-$mrr->can('_wq_done_wait')->([$mrr, $lei], $$);
+$mrr->can('_wq_done_wait')->($$, $mrr, $lei);
 exit(($lei->{child_error} // 0) >> 8);
diff --git a/script/public-inbox-compact b/script/public-inbox-compact
index 80d0224b..1062be5a 100755
--- a/script/public-inbox-compact
+++ b/script/public-inbox-compact
@@ -1,12 +1,12 @@
 #!perl -w
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
-my $opt = { compact => 1, -coarse_lock => 1, -eidx_ok => 1 };
+my $opt = { compact => 1, -coarse_lock => 1,
+        -eidx_ok => 1, -cidx_ok => 1 };
 my $help = <<EOF; # the following should fit w/o scrolling in 80x24 term:
-usage: public-inbox-compact <INBOX_DIR|EXTINDEX_DIR>
+usage: public-inbox-compact <INBOX_DIR|EXTINDEX_DIR|CINDEX_DIR>
 
   Compact Xapian DBs in an inbox
 
@@ -31,12 +31,14 @@ PublicInbox::Admin::progress_prepare($opt);
 require PublicInbox::InboxWritable;
 require PublicInbox::Xapcmd;
 my $cfg = PublicInbox::Config->new;
-my ($ibxs, $eidxs) = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt, $cfg);
-unless ($ibxs) { print STDERR $help; exit 1 }
+my ($ibxs, $eidxs, $cidxs) =
+        PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt, $cfg);
+unless (@$ibxs || @$eidxs || @$cidxs) { print STDERR $help; exit 1 }
 for my $ibx (@$ibxs) {
         $ibx = PublicInbox::InboxWritable->new($ibx);
         PublicInbox::Xapcmd::run($ibx, 'compact', $opt);
 }
-for my $eidx (@$eidxs) {
-        PublicInbox::Xapcmd::run($eidx, 'compact', $opt);
+for my $ibxish (@$eidxs, @$cidxs) {
+        my $restore = $ibxish->can('prep_umask') ? $ibxish->prep_umask : undef;
+        PublicInbox::Xapcmd::run($ibxish, 'compact', $opt);
 }
diff --git a/script/public-inbox-convert b/script/public-inbox-convert
index 42955a48..713c2881 100755
--- a/script/public-inbox-convert
+++ b/script/public-inbox-convert
@@ -1,5 +1,5 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <http://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
@@ -63,7 +63,7 @@ if (delete $old->{-unconfigured}) {
 }
 die "Only conversion from v1 inboxes is supported\n" if $old->version >= 2;
 
-my $detected = PublicInbox::Admin::detect_indexlevel($old);
+my $detected = $old->detect_indexlevel;
 $old->{indexlevel} //= $detected;
 my $env;
 if ($opt->{'index'}) {
@@ -75,7 +75,7 @@ if ($opt->{'index'}) {
 }
 local %ENV = (%$env, %ENV) if $env;
 my $new = { %$old };
-$new->{inboxdir} = $cfg->rel2abs_collapsed($new_dir);
+$new->{inboxdir} = PublicInbox::Config::rel2abs_collapsed($new_dir);
 $new->{version} = 2;
 $new = PublicInbox::InboxWritable->new($new, { nproc => $opt->{jobs} });
 $new->{-no_fsync} = 1 if !$opt->{fsync};
@@ -89,7 +89,8 @@ sub link_or_copy ($$) {
         File::Copy::cp($src, $dst) or die "cp $src, $dst failed: $!\n";
 }
 
-$old->with_umask(sub {
+{
+        my $restore = $old->with_umask;
         my $old_cfg = "$old->{inboxdir}/config";
         local $ENV{GIT_CONFIG} = $old_cfg;
         my $new_cfg = "$new->{inboxdir}/all.git/config";
@@ -110,18 +111,16 @@ $old->with_umask(sub {
         my $desc = "$old->{inboxdir}/description";
         link_or_copy($desc, "$new->{inboxdir}/description") if -e $desc;
         my $clone = "$old->{inboxdir}/cloneurl";
-        if (-e $clone) {
-                warn <<"";
+        warn <<"" if -e $clone;
 $clone may not be valid after migrating to v2, not copying
 
-        }
-});
+}
 my $state = '';
 my $head = $old->{ref_head} || 'HEAD';
-my ($rd, $pid) = $old->git->popen(qw(fast-export --use-done-feature), $head);
+my $rd = $old->git->popen(qw(fast-export --use-done-feature), $head);
 $v2w->idx_init($opt);
 my $im = $v2w->importer;
-my ($r, $w) = $im->gfi_start;
+my $io = $im->gfi_start;
 my $h = '[0-9a-f]';
 my %D;
 my $last;
@@ -131,23 +130,17 @@ while (<$rd>) {
         } elsif (/^commit /) {
                 $state = 'commit';
         } elsif (/^data ([0-9]+)/) {
-                my $len = $1;
-                print $w $_ or $im->wfail;
-                while ($len) {
-                        my $n = read($rd, my $tmp, $len) or die "read: $!";
-                        warn "$n != $len\n" if $n != $len;
-                        $len -= $n;
-                        print $w $tmp or $im->wfail;
-                }
+                print $io $_ or $im->wfail;
+                print $io PublicInbox::IO::read_all($rd, $1) or $im->wfail;
                 next;
         } elsif ($state eq 'commit') {
                 if (m{^M 100644 :([0-9]+) (${h}{2}/${h}{38})}o) {
                         my ($mark, $path) = ($1, $2);
                         $D{$path} = $mark;
                         if ($last && $last ne 'm') {
-                                print $w "D $last\n" or $im->wfail;
+                                print $io "D $last\n" or $im->wfail;
                         }
-                        print $w "M 100644 :$mark m\n" or $im->wfail;
+                        print $io "M 100644 :$mark m\n" or $im->wfail;
                         $last = 'm';
                         next;
                 }
@@ -155,20 +148,18 @@ while (<$rd>) {
                         my $mark = delete $D{$1};
                         defined $mark or die "undeleted path: $1\n";
                         if ($last && $last ne 'd') {
-                                print $w "D $last\n" or $im->wfail;
+                                print $io "D $last\n" or $im->wfail;
                         }
-                        print $w "M 100644 :$mark d\n" or $im->wfail;
+                        print $io "M 100644 :$mark d\n" or $im->wfail;
                         $last = 'd';
                         next;
                 }
         }
         last if $_ eq "done\n";
-        print $w $_ or $im->wfail;
+        print $io $_ or $im->wfail;
 }
-close $rd or die "close fast-export: $!\n";
-waitpid($pid, 0) or die "waitpid failed: $!\n";
-$? == 0 or die "fast-export failed: $?\n";
-$r = $w = undef; # v2w->done does the actual close and error checking
+$rd->close or die "fast-export: \$?=$? \$!=$!\n";
+$io = undef;
 $v2w->done;
 if (my $old_mm = $old->mm) {
         $old->cleanup;
diff --git a/script/public-inbox-edit b/script/public-inbox-edit
index 1fbaf5a7..88115d7c 100755
--- a/script/public-inbox-edit
+++ b/script/public-inbox-edit
@@ -1,5 +1,5 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Used for editing messages in a public-inbox.
@@ -184,12 +184,10 @@ retry_edit:
         # rename/relink $edit_fn
         open my $new_fh, '<', $edit_fn or
                 die "can't read edited file ($edit_fn): $!\n";
-        defined(my $new_raw = do { local $/; <$new_fh> }) or die
-                "read $edit_fn: $!\n";
+        my $new_raw = PublicInbox::IO::read_all $new_fh;
 
         if (!$opt->{raw}) {
-                # get rid of the From we added
-                $new_raw =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+                PublicInbox::Eml::strip_from($new_raw);
 
                 # check if user forgot to purge (in mutt) after editing
                 if ($new_raw =~ /^From /sm) {
diff --git a/script/public-inbox-fetch b/script/public-inbox-fetch
index f9bac4e3..6fd15328 100755
--- a/script/public-inbox-fetch
+++ b/script/public-inbox-fetch
@@ -2,8 +2,7 @@
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Wrapper to git fetch remote public-inboxes
-use strict;
-use v5.10.1;
+use v5.12;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
 my $opt = {};
 my $help = <<EOF; # the following should fit w/o scrolling in 80x24 term:
@@ -24,6 +23,7 @@ options:
     -C DIR            chdir to specified directory
 EOF
 GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@ try-remote|T=s@
+        prune|p
         no-torsocks torsocks=s exit-code)) or die $help;
 if ($opt->{help}) { print $help; exit };
 require PublicInbox::Fetch; # loads Admin
diff --git a/script/public-inbox-index b/script/public-inbox-index
index a04be9fc..74232ebf 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -66,6 +66,7 @@ $opt->{-use_cwd} = 1;
 my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt, $cfg);
 PublicInbox::Admin::require_or_die('-index');
 unless (@ibxs) { print STDERR $help; exit 1 }
+require PublicInbox::InboxWritable;
 
 my (@eidx, %eidx_seen);
 my $update_extindex = $opt->{'update-extindex'};
@@ -96,8 +97,9 @@ for my $ei_name (@$update_extindex) {
 my $mods = {};
 my @eidx_unconfigured;
 foreach my $ibx (@ibxs) {
+        $ibx = PublicInbox::InboxWritable->new($ibx);
         # detect_indexlevel may also set $ibx->{-skip_docdata}
-        my $detected = PublicInbox::Admin::detect_indexlevel($ibx);
+        my $detected = $ibx->detect_indexlevel;
         # XXX: users can shoot themselves in the foot, with opt->{indexlevel}
         $ibx->{indexlevel} //= $opt->{indexlevel} // ($opt->{xapian_only} ?
                         'full' : $detected);
@@ -111,17 +113,14 @@ The following inboxes are unconfigured and will not be updated in
 @$update_extindex:\n@eidx_unconfigured
 EOF
 
-# "Search::Xapian" includes SWIG "Xapian", too:
-$opt->{compact} = 0 if !$mods->{'Search::Xapian'};
+$opt->{compact} = 0 if !$mods->{'Xapian'}; # (or old Search::Xapian)
 
 PublicInbox::Admin::require_or_die(keys %$mods);
 my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
 local %ENV = (%ENV, %$env) if $env;
-require PublicInbox::InboxWritable;
 PublicInbox::Xapcmd::check_compact() if $opt->{compact};
 PublicInbox::Admin::progress_prepare($opt);
 for my $ibx (@ibxs) {
-        $ibx = PublicInbox::InboxWritable->new($ibx);
         if ($opt->{compact} >= 2) {
                 PublicInbox::Xapcmd::run($ibx, 'compact', $opt->{compact_opt});
         }
diff --git a/script/public-inbox-init b/script/public-inbox-init
index 5de45781..8915cf31 100755
--- a/script/public-inbox-init
+++ b/script/public-inbox-init
@@ -1,9 +1,10 @@
 #!perl -w
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
 use Getopt::Long qw/:config gnu_getopt no_ignore_case auto_abbrev/;
+use autodie qw(open chmod close rename);
 use Fcntl qw(:DEFAULT);
 my $help = <<EOF; # the following should fit w/o scrolling in 80x24 term:
 usage: public-inbox-init NAME INBOX_DIR HTTP_URL ADDRESS [ADDRESS..]
@@ -60,7 +61,7 @@ my $inboxdir = shift @ARGV or $usage_cb->();
 my $http_url = shift @ARGV or $usage_cb->();
 my (@address) = @ARGV;
 @address or $usage_cb->();
-+PublicInbox::Admin::do_chdir(\@chdir);
+PublicInbox::Admin::do_chdir(\@chdir);
 
 @c_extra = map {
         my ($k, $v) = split(/=/, $_, 2);
@@ -122,17 +123,16 @@ sysopen($lockfh, $lockfile, O_RDWR|O_CREAT|O_EXCL) or do {
 };
 require PublicInbox::OnDestroy;
 my $auto_unlink = PublicInbox::OnDestroy->new($$, sub { unlink $lockfile });
-my ($perm, %seen);
+my $perm = 0644 & ~umask;
+my %seen;
 if (-e $pi_config) {
-        open(my $oh, '<', $pi_config) or die "unable to read $pi_config: $!\n";
-        my @st = stat($oh);
+        require PublicInbox::IO;
+        open(my $oh, '<', $pi_config);
+        my @st = stat($oh) or die "(f)stat failed on $pi_config: $!\n";
         $perm = $st[2];
-        defined $perm or die "(f)stat failed on $pi_config: $!\n";
-        chmod($perm & 07777, $fh) or
-                die "(f)chmod failed on future $pi_config: $!\n";
-        defined(my $old = do { local $/; <$oh> }) or die "read $pi_config: $!\n";
-        print $fh $old or die "failed to write: $!\n";
-        close $oh or die "failed to close $pi_config: $!\n";
+        chmod($perm & 07777, $fh);
+        print $fh PublicInbox::IO::read_all($oh);
+        close $oh;
 
         # yes, this conflict checking is racy if multiple instances of this
         # script are run by the same $PI_DIR
@@ -159,7 +159,7 @@ if (-e $pi_config) {
         $indexlevel //= $ibx->{indexlevel} if $ibx;
 }
 my $pi_config_tmp = $fh->filename;
-close($fh) or die "failed to close $pi_config_tmp: $!\n";
+close($fh);
 
 my $pfx = "publicinbox.$name";
 my @x = (qw/git config/, "--file=$pi_config_tmp");
@@ -214,12 +214,12 @@ $ibx->init_inbox(0, $skip_epoch, $skip_artnum);
 
 my $f = "$inboxdir/description";
 if (sysopen $fh, $f, O_CREAT|O_EXCL|O_WRONLY) {
-        print $fh "public inbox for $address[0]\n" or die "print($f): $!";
-        close $fh or die "close($f): $!";
+        print $fh "public inbox for $address[0]\n";
+        close $fh;
 }
 
 # needed for git prior to v2.1.0
-umask(0077) if defined $perm;
+umask(0077);
 
 require PublicInbox::Spawn;
 PublicInbox::Spawn->import(qw(run_die));
@@ -246,11 +246,6 @@ for my $kv (@c_extra) {
 }
 
 # needed for git prior to v2.1.0
-if (defined $perm) {
-        chmod($perm & 07777, $pi_config_tmp) or
-                        die "(f)chmod failed on future $pi_config: $!\n";
-}
-
-rename $pi_config_tmp, $pi_config or
-        die "failed to rename `$pi_config_tmp' to `$pi_config': $!\n";
+chmod($perm & 07777, $pi_config_tmp);
+rename $pi_config_tmp, $pi_config;
 undef $auto_unlink; # trigger ->DESTROY
diff --git a/script/public-inbox-learn b/script/public-inbox-learn
index 8b8e1b77..a955cdf6 100755
--- a/script/public-inbox-learn
+++ b/script/public-inbox-learn
@@ -28,6 +28,7 @@ use PublicInbox::Spamcheck::Spamc;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
 my %opt = (all => 0);
 GetOptions(\%opt, qw(all help|h)) or die $help;
+use PublicInbox::Import;
 
 my $train = shift or die $help;
 if ($train !~ /\A(?:ham|spam|rm)\z/) {
@@ -37,10 +38,12 @@ die "--all only works with `rm'\n" if $opt{all} && $train ne 'rm';
 
 my $spamc = PublicInbox::Spamcheck::Spamc->new;
 my $pi_cfg = PublicInbox::Config->new;
+local $PublicInbox::Import::DROP_UNIQUE_UNSUB;
+PublicInbox::Import::load_config($pi_cfg);
 my $err;
 my $mime = PublicInbox::Eml->new(do{
-        defined(my $data = do { local $/; <STDIN> }) or die "read STDIN: $!\n";
-        $data =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+        my $data = PublicInbox::IO::read_all \*STDIN;
+        PublicInbox::Eml::strip_from($data);
 
         if ($train ne 'rm') {
                 eval {
@@ -64,6 +67,7 @@ sub remove_or_add ($$$$) {
         $ibx->{name} = $ENV{GIT_COMMITTER_NAME} // $ibx->{name};
         $ibx->{-primary_address} = $ENV{GIT_COMMITTER_EMAIL} // $addr;
         $ibx = PublicInbox::InboxWritable->new($ibx);
+        $ibx->{indexlevel} = $ibx->detect_indexlevel;
         my $im = $ibx->importer(0);
 
         if ($train eq "rm") {
@@ -109,12 +113,12 @@ if ($train eq 'spam' || ($train eq 'rm' && $opt{all})) {
         my %seen;
         while (my ($addr, $ibx) = each %dests) {
                 next unless ref($ibx); # $ibx may be 0
-                next if $seen{"$ibx"}++;
+                next if $seen{0 + $ibx}++;
                 remove_or_add($ibx, $train, $mime, $addr);
         }
         my $dests = PublicInbox::MDA->inboxes_for_list_id($pi_cfg, $mime);
         for my $ibx (@$dests) {
-                next if $seen{"$ibx"}++;
+                next if $seen{0 + $ibx}++;
                 remove_or_add($ibx, $train, $mime, $ibx->{-primary_address});
         }
 }
diff --git a/script/public-inbox-mda b/script/public-inbox-mda
index 7e2bee92..b463b07b 100755
--- a/script/public-inbox-mda
+++ b/script/public-inbox-mda
@@ -1,5 +1,5 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2013-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Mail delivery agent for public-inbox, run from your MTA upon mail delivery
@@ -16,8 +16,14 @@ use strict;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
 my ($ems, $emm, $show_help);
 my $precheck = 1;
+use PublicInbox::Import;
+local $PublicInbox::Import::DROP_UNIQUE_UNSUB; # does this need a CLI switch?
 GetOptions('precheck!' => \$precheck, 'help|h' => \$show_help) or
         do { print STDERR $help; exit 1 };
+if ($show_help) {
+        print $help;
+        exit;
+}
 
 my $do_exit = sub {
         my ($code) = shift;
@@ -33,13 +39,13 @@ use PublicInbox::Filter::Base;
 use PublicInbox::InboxWritable;
 use PublicInbox::Spamcheck;
 
-# n.b: hopefully we can setup the emergency path without bailing due to
-# user error, we really want to setup the emergency destination ASAP
+# n.b.: Hopefully we can set up the emergency path without bailing due to
+# user error, we really want to set up the emergency destination ASAP
 # in case there's bugs in our code or user error.
 my $emergency = $ENV{PI_EMERGENCY} || "$ENV{HOME}/.public-inbox/emergency/";
 $ems = PublicInbox::Emergency->new($emergency);
-my $str = do { local $/; <STDIN> };
-$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+my $str = PublicInbox::IO::read_all \*STDIN;
+PublicInbox::Eml::strip_from($str);
 $ems->prepare(\$str);
 my $eml = PublicInbox::Eml->new(\$str);
 my $cfg = PublicInbox::Config->new;
@@ -47,6 +53,8 @@ my $key = 'publicinboxmda.spamcheck';
 my $default = 'PublicInbox::Spamcheck::Spamc';
 my $spamc = PublicInbox::Spamcheck::get($cfg, $key, $default);
 my $dests = [];
+PublicInbox::Import::load_config($cfg, $do_exit);
+
 my $recipient = $ENV{ORIGINAL_RECIPIENT};
 if (defined $recipient) {
         my $ibx = $cfg->lookup($recipient); # first check
@@ -55,7 +63,8 @@ if (defined $recipient) {
 if (!scalar(@$dests)) {
         $dests = PublicInbox::MDA->inboxes_for_list_id($cfg, $eml);
         if (!scalar(@$dests) && !defined($recipient)) {
-                die "ORIGINAL_RECIPIENT not defined in ENV\n";
+                warn "ORIGINAL_RECIPIENT not defined in ENV\n";
+                $do_exit->(67); # EX_NOUSER
         }
         scalar(@$dests) or $do_exit->(67); # EX_NOUSER 5.1.1 user unknown
 }
diff --git a/script/public-inbox-pop3d b/script/public-inbox-pop3d
new file mode 100755
index 00000000..ec944aee
--- /dev/null
+++ b/script/public-inbox-pop3d
@@ -0,0 +1,8 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# Standalone POP3 server for public-inbox.
+use v5.12;
+use PublicInbox::Daemon;
+PublicInbox::Daemon::run('pop3://0.0.0.0:110');
diff --git a/script/public-inbox-purge b/script/public-inbox-purge
index 121027cc..618cfec4 100755
--- a/script/public-inbox-purge
+++ b/script/public-inbox-purge
@@ -1,5 +1,5 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Used for purging messages entirely from a public-inbox.  Currently
@@ -33,8 +33,8 @@ PublicInbox::Admin::do_chdir(delete $opt->{C});
 my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt);
 PublicInbox::AdminEdit::check_editable(\@ibxs);
 
-defined(my $data = do { local $/; <STDIN> }) or die "read STDIN: $!\n";
-$data =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+my $data = PublicInbox::IO::read_all \*STDIN;
+PublicInbox::Eml::strip_from($data);
 my $n_purged = 0;
 
 foreach my $ibx (@ibxs) {
diff --git a/script/public-inbox-watch b/script/public-inbox-watch
index af02d8f3..9bcd42ed 100755
--- a/script/public-inbox-watch
+++ b/script/public-inbox-watch
@@ -1,5 +1,5 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 my $help = <<EOF;
 usage: public-inbox-watch
@@ -11,13 +11,15 @@ use strict;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
 use IO::Handle; # ->autoflush
 use PublicInbox::Watch;
+use PublicInbox::Import;
+local $PublicInbox::Import::DROP_UNIQUE_UNSUB;
 use PublicInbox::Config;
 use PublicInbox::DS;
 my $do_scan = 1;
 GetOptions('scan!' => \$do_scan, # undocumented, testing only
         'help|h' => \(my $show_help)) or do { print STDERR $help; exit 1 };
 if ($show_help) { print $help; exit 0 };
-my $oldset = PublicInbox::DS::block_signals();
+PublicInbox::DS::block_signals();
 STDOUT->autoflush(1);
 STDERR->autoflush(1);
 local $0 = $0; # local since this script may be eval-ed
@@ -27,7 +29,8 @@ my $reload = sub {
         $watch->quit;
         $watch = PublicInbox::Watch->new(PublicInbox::Config->new);
         if ($watch) {
-                warn("I: reloaded\n");
+                $watch->{sig} = $prev->{sig}; # prevent redundant signalfd
+                warn "# reloaded\n";
         } else {
                 warn("E: reloading failed\n");
                 $watch = $prev;
@@ -37,10 +40,10 @@ my $reload = sub {
 if ($watch) {
         my $scan = sub {
                 return if !$watch;
-                warn "I: scanning\n";
+                warn "# scanning\n";
                 $watch->trigger_scan('full');
         };
-        my $quit = sub {
+        my $quit = sub { # may be called in IMAP/NNTP children
                 $watch->quit if $watch;
                 $watch = undef;
                 $0 .= ' quitting';
@@ -51,8 +54,9 @@ if ($watch) {
                 CHLD => \&PublicInbox::DS::enqueue_reap,
         };
         $sig->{QUIT} = $sig->{TERM} = $sig->{INT} = $quit;
+        local @SIG{keys %$sig} = values(%$sig); # for non-signalfd/kqueue
 
         # --no-scan is only intended for testing atm, undocumented.
         PublicInbox::DS::requeue($scan) if $do_scan;
-        $watch->watch($sig, $oldset) while ($watch);
+        $watch->watch($sig) while ($watch);
 }
diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb
index 24fc5a25..fac54559 100755
--- a/script/public-inbox-xcpdb
+++ b/script/public-inbox-xcpdb
@@ -1,11 +1,10 @@
 #!perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
 my $help = <<EOF; # the following should fit w/o scrolling in 80x24 term:
-usage: public-inbox-xcpdb [options] <INBOX_DIR|EXTINDEX_DIR>
+usage: public-inbox-xcpdb [options] <INBOX_DIR|EXTINDEX_DIR|CINDEX_DIR>
 
   upgrade or reshard Xapian DB(s) used by public-inbox
 
@@ -26,7 +25,8 @@ index options (see public-inbox-index(1) man page for full description):
 
 See public-inbox-xcpdb(1) man page for full documentation.
 EOF
-my $opt = { quiet => -1, compact => 0, fsync => 1, -eidx_ok => 1 };
+my $opt = { quiet => -1, compact => 0, fsync => 1,
+        -eidx_ok => 1, -cidx_ok => 1 };
 GetOptions($opt, qw(
         fsync|sync! compact|c reshard|R=i
         max_size|max-size=s batch_size|batch-size=s
@@ -42,8 +42,9 @@ PublicInbox::Admin::do_chdir(delete $opt->{C});
 
 require PublicInbox::Config;
 my $cfg = PublicInbox::Config->new;
-my ($ibxs, $eidxs) = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt, $cfg);
-unless ($ibxs) { print STDERR $help; exit 1 }
+my ($ibxs, $eidxs, $cidxs) =
+        PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt, $cfg);
+unless (@$ibxs || @$eidxs || @$cidxs) { print STDERR $help; exit 1 }
 my $idx_env = PublicInbox::Admin::index_prepare($opt, $cfg);
 
 # we only set XAPIAN_FLUSH_THRESHOLD for index, since cpdb doesn't
@@ -63,6 +64,7 @@ for my $ibx (@$ibxs) {
         PublicInbox::Xapcmd::run($ibx, 'cpdb', $opt);
 }
 
-for my $eidx (@$eidxs) {
-        PublicInbox::Xapcmd::run($eidx, 'cpdb', $opt);
+for my $ibxish (@$eidxs, @$cidxs) {
+        my $restore = $ibxish->can('prep_umask') ? $ibxish->prep_umask : undef;
+        PublicInbox::Xapcmd::run($ibxish, 'cpdb', $opt);
 }
diff --git a/scripts/README b/scripts/README
index 3b9c37da..7ffbd93c 100644
--- a/scripts/README
+++ b/scripts/README
@@ -1,5 +1,5 @@
 This directory contains informal scripts and random tools used
-in the development of public-inbox.  Some only exist only for
+in the development of public-inbox.  Some only exist for
 historical purposes, and some may not work anymore.
 
 See the "script/" directory (not "scripts/") for supported and
diff --git a/scripts/dc-dlvr b/scripts/dc-dlvr
index 935a8312..ef6033b9 100755
--- a/scripts/dc-dlvr
+++ b/scripts/dc-dlvr
@@ -47,9 +47,9 @@ then
         rm_list="$rm_list $PREMSG"
         set +e
         mv -f $TMPMSG $PREMSG
-        $spamc -E --headers <$PREMSG >$TMPMSG
+        $spamc -E <$PREMSG >$TMPMSG
 else
-        $spamc -E --headers <$CDMSG >$TMPMSG
+        $spamc -E <$CDMSG >$TMPMSG
 fi
 err=$?
 
diff --git a/scripts/import_maildir b/scripts/import_maildir
index 269f2550..7228a3ad 100755
--- a/scripts/import_maildir
+++ b/scripts/import_maildir
@@ -1,21 +1,29 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2014, Eric Wong <e@80x24.org> and all contributors
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-#
-# Script to import a Maildir into a public-inbox
 =begin usage
+Ancient script to import a Maildir into a v1 public-inbox
+
+        # this is only if you want a v1 inbox
         export GIT_DIR=/path/to/your/repo.git
         export GIT_AUTHOR_EMAIL='list@example.com'
         export GIT_AUTHOR_NAME='list name'
         ./import_maildir /path/to/maildir/
+
+For v2 (strongly recommended), use:
+
+        lei convert /path/to/maildir -o /path/to/v2-inbox
+        # (and `lei daemon-kill' if you don't want the daemon to linger)
 =cut
-use strict;
-use warnings;
+use v5.12;
 use Date::Parse qw/str2time/;
 use PublicInbox::Eml;
 use PublicInbox::Git;
 use PublicInbox::Import;
-sub usage { "Usage:\n".join('', grep(/\t/, `head -n 24 $0`)) }
+sub usage {
+        open my $fh, '<', __FILE__;
+        ("Usage:\n", grep { /^=begin usage/../^=cut/ and !/^=/m } <$fh>);
+}
 my $dir = shift @ARGV or die usage();
 my $git_dir = `git rev-parse --git-dir`;
 chomp $git_dir;
diff --git a/scripts/import_slrnspool b/scripts/import_slrnspool
index d9a35dfd..81df6c2e 100755
--- a/scripts/import_slrnspool
+++ b/scripts/import_slrnspool
@@ -1,20 +1,30 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-#
-# Incremental (or one-shot) importer of a slrnpull news spool
 =begin usage
+Incremental (or one-shot) importer of a slrnpull news spool.
+
+Since the news spool can appear as an MH folder, you may also use
+lei from public-inbox 2.0+ to convert it:
+
+        lei convert mh:$SLRNPULL_ROOT/news/foo/bar -o v2:/path/to/inbox/
+        # (and `lei daemon-kill' if you don't want the daemon to linger)
+
+But if you want to use this script:
+
         export ORIGINAL_RECIPIENT=address@example.com
-        public-inbox-init $INBOX $GIT_DIR $HTTP_URL $ORIGINAL_RECIPIENT
-        ./import_slrnspool SLRNPULL_ROOT/news/foo/bar
+        public-inbox-init -V2 $INBOX $INBOX_DIR $HTTP_URL $ORIGINAL_RECIPIENT
+        ./import_slrnspool $SLRNPULL_ROOT/news/foo/bar
 =cut
-use strict;
-use warnings;
+use v5.12;
 use PublicInbox::Config;
 use PublicInbox::Eml;
 use PublicInbox::Import;
 use PublicInbox::Git;
-sub usage { "Usage:\n".join('',grep(/\t/, `head -n 10 $0`)) }
+sub usage {
+        open my $fh, '<', __FILE__;
+        ("Usage:\n", grep { /^=begin usage/../^=cut/ and !/^=/m } <$fh>);
+}
 my $exit = 0;
 my $sighandler = sub { $exit = 1 };
 $SIG{INT} = $sighandler;
diff --git a/scripts/import_vger_from_mbox b/scripts/import_vger_from_mbox
index c33e42e4..40ccf50b 100644
--- a/scripts/import_vger_from_mbox
+++ b/scripts/import_vger_from_mbox
@@ -1,8 +1,8 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use warnings;
+# consider `lei convert' instead since it handles more formats
+use v5.12;
 use Getopt::Long qw/:config gnu_getopt no_ignore_case auto_abbrev/;
 use PublicInbox::InboxWritable;
 my $usage = "usage: $0 NAME EMAIL DIR <MBOX\n";
diff --git a/scripts/slrnspool2maildir b/scripts/slrnspool2maildir
index 8e2ba08a..ba0729ec 100755
--- a/scripts/slrnspool2maildir
+++ b/scripts/slrnspool2maildir
@@ -1,51 +1,55 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2013-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-#
-# One-off script to convert an slrnpull news spool to Maildir
 =begin usage
+One-off script to convert an slrnpull spool from gmane to Maildir
+Note: this contains Gmane-specific header munging to workaround
+the munging done by Gmane.
+
         ./slrnspool2maildir SLRNPULL_ROOT/news/foo/bar /path/to/maildir/
-=cut
-use strict;
-use warnings;
-use Email::Filter;
-use Email::LocalDelivery;
-use File::Glob qw(bsd_glob GLOB_NOSORT);
-sub usage { "Usage:\n".join('',grep(/\t/, `head -n 12 $0`)) }
-my $spool = shift @ARGV or die usage();
-my $dir = shift @ARGV or die usage();
--d $dir or die "$dir is not a directory\n";
-$dir .= '/' unless $dir =~ m!/\z!;
-foreach my $sub (qw(cur new tmp)) {
-        my $nd = "$dir/$sub";
-        -d $nd and next;
-        mkdir $nd or die "mkdir $nd failed: $!\n";
-}
 
-foreach my $n (grep(/\d+\z/, bsd_glob("$spool/*", GLOB_NOSORT))) {
-        if (open my $fh, '<', $n) {
-                my $f = Email::Filter->new(data => do { local $/; <$fh> });
-                my $s = $f->simple;
+A generic replacement w/o Gmane-specific munging could treat
+the slrnpull spool as an MH folder with lei:
 
-                # gmane rewrites Received headers, which increases spamminess
-                # Some older archives set Original-To
-                foreach my $x (qw(Received To)) {
-                        my @h = $s->header("Original-$x");
-                        if (@h) {
-                                $s->header_set($x, @h);
-                                $s->header_set("Original-$x");
-                        }
+        lei convert mh:SLRNPULL_ROOT/news/foo/bar -o /path/to/maildir
+        # (and `lei daemon-kill' if you don't want the daemon to linger)
+=cut
+use v5.12;
+use autodie;
+# warning: unstable internal APIs:
+use PublicInbox::Eml;
+use PublicInbox::LeiToMail;
+use PublicInbox::MHreader;
+use PublicInbox::IO qw(read_all);
+use File::Path qw(make_path);
+use File::Spec ();
+sub usage {
+        open my $fh, '<', __FILE__;
+        ("Usage:\n", grep { /^=begin usage/../^=cut/ and !/^=/m } <$fh>);
+}
+my $spool = shift @ARGV or die usage();
+my $dst = shift @ARGV or die usage();
+$dst .= '/' unless $dst =~ m!/\z!;
+File::Path::make_path(map { $dst.$_ } qw(tmp new cur));
+$dst = File::Spec->rel2abs($dst).'/';
+opendir my $cwdfh, '.';
+my $mhr = PublicInbox::MHreader->new($spool, $cwdfh);
+my $smsg;
+$mhr->mh_each_eml(sub {
+        my ($d, $n, $kw, $eml) = @_;
+        # gmane rewrites Received headers, which increases spamminess
+        # Some older archives set Original-To
+        for my $x (qw(Received To)) {
+                my @h = $eml->header_raw("Original-$x");
+                if (@h) {
+                        $eml->header_set($x, @h);
+                        $eml->header_set("Original-$x");
                 }
-
-                # triggers for the SA HEADER_SPAM rule
-                foreach my $drop (qw(Approved)) { $s->header_set($drop) }
-
-                # appears to be an old gmane bug:
-                $s->header_set('connect()');
-
-                $f->exit(0);
-                $f->accept($dir);
-        } else {
-                warn "Failed to open $n: $!\n";
         }
-}
+        # `Approved' triggers the SA HEADER_SPAM rule
+        # `connect()' appears to be an old gmane bug:
+        $eml->header_set($_) for ('Approved', 'connect()');
+        my $buf = $eml->as_string;
+        $smsg->{blob} = $n;
+        PublicInbox::LeiToMail::_buf2maildir($dst, \$buf, $smsg, 'new/');
+});
diff --git a/t/address.t b/t/address.t
index 6aa94628..86f47395 100644
--- a/t/address.t
+++ b/t/address.t
@@ -1,7 +1,7 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use warnings;
+use v5.12;
 use Test::More;
 use_ok 'PublicInbox::Address';
 
@@ -10,6 +10,7 @@ sub test_pkg {
         my $emails = $pkg->can('emails');
         my $names = $pkg->can('names');
         my $pairs = $pkg->can('pairs');
+        my $objects = $pkg->can('objects');
 
         is_deeply([qw(e@example.com e@example.org)],
                 [$emails->('User <e@example.com>, e@example.org')],
@@ -35,6 +36,18 @@ sub test_pkg {
                         [ 'xyz', 'y@x' ], [ 'U Ser', 'u@x' ] ],
                 "pairs extraction works for $pkg");
 
+        # only what's used by PublicInbox::IMAP:
+        my @objs = $objects->($s);
+        my @exp = (qw(User e e), qw(e e e), ('John A. Doe', qw(j d)),
+                qw(x x x), qw(xyz y x), ('U Ser', qw(u x)));
+        for (my $i = 0; $i <= $#objs; $i++) {
+                my $exp_name = shift @exp;
+                my $name = $objs[$i]->name;
+                is $name, $exp_name, "->name #$i matches";
+                is $objs[$i]->user, shift @exp, "->user #$i matches";
+                is $objs[$i]->host , shift @exp, "->host #$i matches";
+        }
+
         @names = $names->('"user@example.com" <user@example.com>');
         is_deeply(['user'], \@names,
                 'address-as-name extraction works as expected');
@@ -64,6 +77,10 @@ sub test_pkg {
         is_deeply([], \@emails , 'no address for local address');
         @names = $emails->('Local User <user>');
         is_deeply([], \@names, 'no address, no name');
+
+        my $p = $pairs->('NAME, a@example, wtf@');
+        is scalar(grep { defined($_->[0] // $_->[1]) } @$p),
+                scalar(@$p), 'something is always defined in bogus pairs';
 }
 
 test_pkg('PublicInbox::Address');
diff --git a/t/admin.t b/t/admin.t
index 8d09bfc1..586938d0 100644
--- a/t/admin.t
+++ b/t/admin.t
@@ -1,11 +1,12 @@
 #!perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
 use PublicInbox::TestCommon;
 use PublicInbox::Import;
 use_ok 'PublicInbox::Admin';
+use autodie;
 my $v1 = create_inbox 'v1', -no_gc => 1, sub {};
 my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = $v1->{inboxdir};
@@ -13,7 +14,7 @@ my ($res, $err, $v);
 my $v2ibx;
 SKIP: {
         require_mods(qw(DBD::SQLite), 5);
-        require_git(2.6, 1) or skip 5, 'git too old';
+        require_git(2.6, 5);
         $v2ibx = create_inbox 'v2', indexlevel => 'basic', version => 2,
                                 -no_gc => 1, sub {
                 my ($v2w, $ibx) = @_;
@@ -23,6 +24,17 @@ SKIP: {
 };
 
 *resolve_inboxdir = \&PublicInbox::Admin::resolve_inboxdir;
+*resolve_git_dir = \&PublicInbox::Admin::resolve_git_dir;
+
+{
+        symlink $git_dir, my $sym = "$tmpdir/v1-symlink.git";
+        for my $d ('') { # TODO: should work inside $sym/objects
+                local $ENV{PWD} = $sym.$d;
+                chdir $sym.$d;
+                is resolve_git_dir('.'), $sym,
+                        "symlink preserved from {SYMLINKDIR}.git$d";
+        }
+}
 
 # v1
 is(resolve_inboxdir($git_dir), $git_dir, 'top-level GIT_DIR resolved');
diff --git a/t/alt.psgi b/t/alt.psgi
new file mode 100644
index 00000000..c7f42979
--- /dev/null
+++ b/t/alt.psgi
@@ -0,0 +1,17 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use warnings;
+use Plack::Builder;
+my $pi_config = $ENV{PI_CONFIG} // 'unset'; # capture ASAP
+my $app = sub {
+        my ($env) = @_;
+        $env->{'psgi.errors'}->print("ALT\n");
+        [ 200, ['Content-Type', 'text/plain'], [ $pi_config ] ]
+};
+
+builder {
+        enable 'ContentLength';
+        enable 'Head';
+        $app;
+}
diff --git a/t/altid.t b/t/altid.t
index 3ce08a6a..2692029e 100644
--- a/t/altid.t
+++ b/t/altid.t
@@ -5,7 +5,7 @@ use strict;
 use v5.10.1;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 use_ok 'PublicInbox::Msgmap';
 use_ok 'PublicInbox::SearchIdx';
 my ($tmpdir, $for_destroy) = tmpdir();
diff --git a/t/altid_v2.t b/t/altid_v2.t
index 281a09d5..6bc90453 100644
--- a/t/altid_v2.t
+++ b/t/altid_v2.t
@@ -6,7 +6,7 @@ use v5.10.1;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
 require_git(2.6);
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 require PublicInbox::Msgmap;
 my $another = 'another-nntp.sqlite3';
 my $altid = [ "serial:gmane:file=$another" ];
@@ -14,9 +14,9 @@ my $ibx = create_inbox 'v2', version => 2, indexlevel => 'medium',
                         altid => $altid, sub {
         my ($im, $ibx) = @_;
         my $mm = PublicInbox::Msgmap->new_file("$ibx->{inboxdir}/$another", 2);
-        $mm->mid_set(1234, 'a@example.com') == 1 or BAIL_OUT 'mid_set once';
-        ok(0 == $mm->mid_set(1234, 'a@example.com'), 'mid_set not idempotent');
-        ok(0 == $mm->mid_set(1, 'a@example.com'), 'mid_set fails with dup MID');
+        is($mm->mid_set(1234, 'a@example.com'), 1, 'mid_set') or xbail 'once';
+        is($mm->mid_set(1234, 'a@example.com')+0, 0, 'mid_set not idempotent');
+        is($mm->mid_set(1, 'a@example.com')+0, 0, 'mid_set fails with dup MID');
         $im->add(PublicInbox::Eml->new(<<'EOF')) or BAIL_OUT;
 From: a@example.com
 To: b@example.com
@@ -27,8 +27,8 @@ hello world gmane:666
 EOF
 };
 my $mm = PublicInbox::Msgmap->new_file("$ibx->{inboxdir}/$another", 2);
-ok(0 == $mm->mid_set(1234, 'a@example.com'), 'mid_set not idempotent');
-ok(0 ==  $mm->mid_set(1, 'a@example.com'), 'mid_set fails with dup MID');
+is($mm->mid_set(1234, 'a@example.com') + 0, 0, 'mid_set not idempotent');
+is($mm->mid_set(1, 'a@example.com') + 0, 0, 'mid_set fails with dup MID');
 my $mset = $ibx->search->mset('gmane:1234');
 my $msgs = $ibx->search->mset_to_smsg($ibx, $mset);
 $msgs = [ map { $_->{mid} } @$msgs ];
diff --git a/t/check-www-inbox.perl b/t/check-www-inbox.perl
index 033b90d1..46f9ce1e 100644
--- a/t/check-www-inbox.perl
+++ b/t/check-www-inbox.perl
@@ -123,7 +123,7 @@ while (keys %workers) { # reacts to SIGCHLD
                 }
         }
         while ($u = shift @queue) {
-                my $s = $todo[1]->send($u, MSG_EOR);
+                my $s = $todo[1]->send($u, 0);
                 if ($!{EAGAIN}) {
                         unshift @queue, $u;
                         last;
@@ -177,7 +177,7 @@ sub worker_loop {
                 foreach my $l (@links, "DONE\t$u") {
                         next if $l eq '' || $l =~ /\.mbox(?:\.gz)\z/;
                         do {
-                                $s = $done_wr->send($l, MSG_EOR);
+                                $s = $done_wr->send($l, 0);
                         } while (!defined $s && $!{EINTR});
                         die "$$ send $!\n" unless defined $s;
                         my $n = length($l);
diff --git a/t/cindex-join.t b/t/cindex-join.t
new file mode 100644
index 00000000..22c67107
--- /dev/null
+++ b/t/cindex-join.t
@@ -0,0 +1,88 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# cindex --join functionality against mwrap, a small projects
+# started as C+Ruby and got forked to C+Perl/XS w/ public inboxes for each
+use v5.12;
+use PublicInbox::TestCommon;
+use PublicInbox::IO qw(write_file);
+use PublicInbox::Import;
+use PublicInbox::Config;
+use autodie;
+use File::Spec;
+$ENV{TEST_REMOTE_JOIN} or plan skip_all => 'TEST_REMOTE_JOIN unset';
+require_cmd 'join';
+local $ENV{TAIL_ALL} = $ENV{TAIL_ALL} // 1; # while features are unstable
+require_mods(qw(json Xapian DBD::SQLite +SCM_RIGHTS));
+my @code = qw(https://80x24.org/mwrap-perl.git
+                https://80x24.org/mwrap.git);
+my @inboxes = qw(https://80x24.org/mwrap-public 2 inbox.comp.lang.ruby.mwrap
+        https://80x24.org/mwrap-perl 2 inbox.comp.lang.perl.mwrap);
+my (%code, %inboxes);
+my $topdir = File::Spec->rel2abs('.');
+my $tmpdir = tmpdir;
+while (my $url = shift @code) {
+        my ($key) = ($url =~ m!/([^/]+\.git)\z!);
+        $code{$key} = create_coderepo $key, sub {
+                PublicInbox::Import::init_bare '.';
+                write_file '>>', 'config', <<EOM;
+[remote "origin"]
+        url = $url
+        fetch = +refs/*:refs/*
+        mirror = true
+EOM
+                if (my $d = $code{'mwrap-perl.git'}) {
+                        $d = File::Spec->abs2rel("$topdir/$d", 'objects');
+                        write_file '>','objects/info/alternates',"$d/objects\n"
+                }
+                diag "mirroring coderepo: $url ...";
+                xsys_e qw(git fetch -q origin);
+        };
+}
+
+while (my ($url, $v, $ng) = splice(@inboxes, 0, 3)) {
+        my ($key) = ($url =~ m!/([^/]+)\z!);
+        my @opt = (version => $v, tmpdir => "$tmpdir/$key", -no_gc => 1);
+        $inboxes{$key} = create_inbox $key, @opt, sub {
+                my ($im, $ibx) = @_;
+                $im->done;
+                diag "cloning public-inbox $url ...";
+                run_script([qw(-clone -q), $url, $ibx->{inboxdir}]) or
+                        xbail "clone: $?";
+                diag "indexing $ibx->{inboxdir} ...";
+                run_script([qw(-index -v -L medium --dangerous),
+                                $ibx->{inboxdir}]) or xbail "index: $?";
+        };
+        $inboxes{$key}->{newsgroup} = $ng;
+};
+my $env = {};
+open my $fh, '>', $env->{PI_CONFIG} = "$tmpdir/pi_config";
+for (sort keys %inboxes) {
+        print $fh <<EOM;
+[publicinbox "$_"]
+        inboxdir = $inboxes{$_}->{inboxdir}
+        address = $_\@80x24.org
+        newsgroup = $inboxes{$_}->{newsgroup}
+EOM
+}
+close $fh;
+my $cidxdir = "$tmpdir/cidx";
+# this should be fast since mwrap* are small
+my $rdr = { 1 => \my $cout, 2 => \my $cerr };
+ok run_script([qw(-cindex -v --all --show=join_data),
+                '--join=aggressive,dt:..2022-12-01',
+                '-d', $cidxdir, map { ('-g', $_) } values %code ],
+                $env, $rdr), 'initial join inboxes w/ coderepos';
+my $out = PublicInbox::Config->json->decode($cout);
+is($out->{join_data}->{dt}->[0], '19700101'.'000000',
+        'dt:..$END_DATE starts from epoch');
+
+ok run_script([qw(-cindex -v --all -u --join --show),
+                '-d', $cidxdir], $env, $rdr), 'incremental --join';
+
+ok run_script([qw(-cindex -v --no-scan --show),
+                '-d', $cidxdir], $env, $rdr), 'show';
+$out = PublicInbox::Config->json->decode($cout);
+is ref($out->{join_data}), 'HASH', 'got hash join data';
+is $cerr, '', 'no warnings or errors in stderr w/ --show';
+done_testing;
diff --git a/t/cindex.t b/t/cindex.t
new file mode 100644
index 00000000..e5f26ec3
--- /dev/null
+++ b/t/cindex.t
@@ -0,0 +1,299 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::TestCommon;
+use Cwd qw(getcwd);
+use List::Util qw(sum);
+use autodie qw(close mkdir open rename);
+require_mods(qw(json Xapian +SCM_RIGHTS));
+use_ok 'PublicInbox::CodeSearchIdx';
+use PublicInbox::Import;
+my ($tmp, $for_destroy) = tmpdir();
+my $pwd = getcwd();
+my @unused_keys = qw(last_commit has_threadid skip_docdata);
+local $ENV{PI_CONFIG} = '/dev/null';
+# local $ENV{TAIL_ALL} = $ENV{TAIL_ALL} // 1; # while features are unstable
+my $opt = { 1 => \(my $cidx_out), 2 => \(my $cidx_err) };
+
+# I reworked CodeSearchIdx->shard_worker to handle empty trees
+# in the initial commit generated by cvs2svn for xapian.git
+create_coderepo 'empty-tree-root-0600', tmpdir => "$tmp/wt0", sub {
+        xsys_e([qw(/bin/sh -c), <<'EOM']);
+git init -q &&
+git config core.sharedRepository 0600
+tree=$(git mktree </dev/null) &&
+head=$(git symbolic-ref HEAD) &&
+cmt=$(echo 'empty root' | git commit-tree $tree) &&
+git update-ref $head $cmt &&
+echo hi >f &&
+git add f &&
+git commit -q -m hi &&
+git gc -q
+EOM
+}; # /create_coderepo
+
+ok(run_script([qw(-cindex --dangerous -q -g), "$tmp/wt0"]), 'cindex internal');
+{
+        my $exists = -e "$tmp/wt0/.git/public-inbox-cindex/cidx.lock";
+        my @st = stat(_);
+        ok($exists, 'internal dir created');
+        is($st[2] & 0600, 0600, 'mode respects core.sharedRepository');
+        @st = stat("$tmp/wt0/.git/public-inbox-cindex");
+        is($st[2] & 0700, 0700, 'dir mode respects core.sharedRepository');
+}
+
+# it's possible for git to emit NUL characters in diffs
+# (see c4201214cbf10636e2c1ab9131573f735b42c8d4 in linux.git)
+my $zp = create_coderepo 'NUL in patch', sub {
+        my $src = PublicInbox::IO::try_cat("$pwd/COPYING");
+        xsys_e([qw(git init -q)]);
+
+        # needs to be further than FIRST_FEW_BYTES (8000) in git.git
+        $src =~ s/\b(Limitation of Liability\.)\n\n/$1\n\0\n/s or
+                xbail "BUG: no `\\n\\n' in $pwd/COPYING";
+
+        PublicInbox::IO::write_file '>', 'f', $src;
+        xsys_e([qw(/bin/sh -c), <<'EOM']);
+git add f &&
+git commit -q -m 'initial with NUL character'
+EOM
+        $src =~ s/\n\0\n/\n\n/ or xbail "BUG: no `\\n\\0\\n'";
+        PublicInbox::IO::write_file '>', 'f', $src;
+        xsys_e([qw(/bin/sh -c), <<'EOM']);
+git add f &&
+git commit -q -m 'remove NUL character' &&
+git gc -q
+EOM
+}; # /create_coderepo
+
+$zp = File::Spec->rel2abs($zp);
+ok(run_script([qw(-cindex --dangerous -q -d), "$tmp/ext",
+                '-g', $zp, '-g', "$tmp/wt0" ]),
+        'cindex external');
+ok(-e "$tmp/ext/cidx.lock", 'external dir created');
+ok(!-d "$zp/.git/public-inbox-cindex", 'no cindex in original coderepo');
+
+ok(run_script([qw(-cindex -L medium --dangerous -q -d),
+        "$tmp/med", '-g', $zp, '-g', "$tmp/wt0"]), 'cindex external medium');
+
+
+SKIP: {
+        have_xapian_compact 2;
+        ok(run_script([qw(-compact -q), "$tmp/ext"]), 'compact on full');
+        ok(run_script([qw(-compact -q), "$tmp/med"]), 'compact on medium');
+}
+
+my $no_metadata_set = sub {
+        my ($i, $extra, $xdb) = @_;
+        for my $xdb (@$xdb) {
+                for my $k (@unused_keys, @$extra) {
+                        is($xdb->get_metadata($k) // '', '',
+                                "metadata $k unset in shard #$i");
+                }
+                ++$i;
+        }
+};
+
+{
+        my $mid_size = sum(map { -s $_ } glob("$tmp/med/cidx*/*/*"));
+        my $full_size = sum(map { -s $_ } glob("$tmp/ext/cidx*/*/*"));
+        ok($full_size > $mid_size, 'full size > mid size') or
+                diag "full=$full_size mid=$mid_size";
+        for my $l (qw(med ext)) {
+                ok(run_script([qw(-cindex -q --reindex -u -d), "$tmp/$l"]),
+                        "reindex $l");
+        }
+        $mid_size = sum(map { -s $_ } glob("$tmp/med/cidx*/*/*"));
+        $full_size = sum(map { -s $_ } glob("$tmp/ext/cidx*/*/*"));
+        ok($full_size > $mid_size, 'full size > mid size after reindex') or
+                diag "full=$full_size mid=$mid_size";
+        my $csrch = PublicInbox::CodeSearch->new("$tmp/med");
+        my ($xdb0, @xdb) = $csrch->xdb_shards_flat;
+        $no_metadata_set->(0, [], [ $xdb0 ]);
+        is($xdb0->get_metadata('indexlevel'), 'medium',
+                'indexlevel set in shard #0');
+        $no_metadata_set->(1, ['indexlevel'], \@xdb);
+
+        ok(run_script([qw(-cindex -q -L full --reindex -u -d), "$tmp/med"]),
+                'reindex medium as full');
+        @xdb = $csrch->xdb_shards_flat;
+        $no_metadata_set->(0, ['indexlevel'], \@xdb);
+}
+
+use_ok 'PublicInbox::CodeSearch';
+
+
+my @xh_args;
+my $exp = [ 'initial with NUL character', 'remove NUL character' ];
+my $zp_git = "$zp/.git";
+if ('multi-repo search') {
+        my $csrch = PublicInbox::CodeSearch->new("$tmp/ext");
+        my $mset = $csrch->mset('NUL');
+        is(scalar($mset->items), 2, 'got results');
+        my @have = sort(map { $_->get_document->get_data } $mset->items);
+        is_xdeeply(\@have, $exp, 'got expected subjects');
+
+        $mset = $csrch->mset('NUL', { git_dir => "$tmp/wt0/.git" });
+        is(scalar($mset->items), 0, 'no results with other GIT_DIR');
+
+        $mset = $csrch->mset('NUL', { git_dir => $zp_git });
+        @have = sort(map { $_->get_document->get_data } $mset->items);
+        is_xdeeply(\@have, $exp, 'got expected subjects w/ GIT_DIR filter');
+        my @xdb = $csrch->xdb_shards_flat;
+        $no_metadata_set->(0, ['indexlevel'], \@xdb);
+        @xh_args = $csrch->xh_args;
+}
+
+my $test_xhc = sub {
+        my ($xhc) = @_;
+        my $impl = $xhc->{impl};
+        my ($r, @l);
+        $r = $xhc->mkreq([], qw(mset -D -c -g), $zp_git, @xh_args, 'NUL');
+        chomp(@l = <$r>);
+        is(shift(@l), 'mset.size=2', "got expected header $impl");
+        my %docid2data;
+        my @got = sort map {
+                my @f = split /\0/;
+                is scalar(@f), 2, 'got 2 entries';
+                $docid2data{$f[0]} = $f[1];
+                $f[1];
+        } @l;
+        is_deeply(\@got, $exp, "expected doc_data $impl");
+
+        $r = $xhc->mkreq([], qw(mset -c -g), "$tmp/wt0/.git", @xh_args, 'NUL');
+        chomp(@l = <$r>);
+        is(shift(@l), 'mset.size=0', "got miss in wrong dir $impl");
+        is_deeply(\@l, [], "no extra lines $impl");
+
+        my $csrch = PublicInbox::CodeSearch->new("$tmp/ext");
+        while (my ($did, $expect) = each %docid2data) {
+                is_deeply($csrch->xdb->get_document($did)->get_data,
+                        $expect, "docid=$did data matches");
+        }
+        ok(!$xhc->{io}->close, "$impl close");
+        is($?, 66 << 8, "got EX_NOINPUT from $impl exit");
+};
+
+SKIP: {
+        require_mods('+SCM_RIGHTS', 1);
+        require PublicInbox::XapClient;
+        my $xhc = PublicInbox::XapClient::start_helper('-j0');
+        $test_xhc->($xhc);
+        skip 'PI_NO_CXX set', 1 if $ENV{PI_NO_CXX};
+        $xhc->{impl} =~ /Cxx/ or
+                skip 'C++ compiler or xapian development libs missing', 1;
+        skip 'TEST_XH_CXX_ONLY set', 1 if $ENV{TEST_XH_CXX_ONLY};
+        local $ENV{PI_NO_CXX} = 1; # force XS or SWIG binding test
+        $xhc = PublicInbox::XapClient::start_helper('-j0');
+        $test_xhc->($xhc);
+}
+
+if ('--update') {
+        my $csrch = PublicInbox::CodeSearch->new("$tmp/ext");
+        my $mset = $csrch->mset('dfn:for-update');
+        is(scalar($mset->items), 0, 'no result before update');
+
+        my $e = \%PublicInbox::TestCommon::COMMIT_ENV;
+        xsys_e([qw(/bin/sh -c), <<'EOM'], $e, { -C => "$tmp/wt0" });
+>for-update && git add for-update && git commit -q -m updated
+EOM
+        ok(run_script([qw(-cindex -qu -d), "$tmp/ext"]), '-cindex -u');
+        $mset = $csrch->reopen->mset('dfn:for-update');
+        is(scalar($mset->items), 1, 'got updated result');
+
+        ok(run_script([qw(-cindex -qu --reindex -d), "$tmp/ext"]), 'reindex');
+        $mset = $csrch->reopen->mset('dfn:for-update');
+        is(scalar($mset->items), 1, 'same result after reindex');
+}
+
+SKIP: { # --prune
+        require_cmd($ENV{XAPIAN_DELVE} || 'xapian-delve', 1);
+        require_git v2.6, 1;
+        my $csrch = PublicInbox::CodeSearch->new("$tmp/ext");
+        is(scalar($csrch->mset('s:hi')->items), 1, 'got hit');
+
+        rename("$tmp/wt0/.git", "$tmp/wt0/.giit");
+        ok(run_script([qw(-cindex -q --prune -d), "$tmp/ext"], undef, $opt),
+                'prune');
+        is(${$opt->{2}}, '', 'nothing in stderr') or diag explain($opt);
+        $csrch->reopen;
+        is(scalar($csrch->mset('s:hi')->items), 0, 'hit pruned');
+
+        rename("$tmp/wt0/.giit", "$tmp/wt0/.git");
+        ok(run_script([qw(-cindex -qu -d), "$tmp/ext"]), 'update');
+        $csrch->reopen;
+        is(scalar($csrch->mset('s:hi')->items), 0,
+                'hit stays pruned since GIT_DIR was previously pruned');
+        isnt(scalar($csrch->mset('s:NUL')->items), 0,
+                'prune did not clobber entire index');
+}
+
+File::Path::remove_tree("$tmp/ext");
+mkdir("$tmp/ext", 0707);
+ok(run_script([qw(-cindex --dangerous -q -d), "$tmp/ext", '-g', $zp]),
+        'external on existing dir');
+{
+        my @st = stat("$tmp/ext/cidx.lock");
+        is($st[2] & 0777, 0604, 'created lock respects odd permissions');
+}
+
+ok(run_script([qw(-xcpdb), "$tmp/ext"]), 'xcpdb upgrade');
+ok(run_script([qw(-xcpdb -R4), "$tmp/ext"]), 'xcpdb reshard');
+
+SKIP: {
+        have_xapian_compact 2;
+        ok(run_script([qw(-xcpdb -R2 --compact), "$tmp/ext"]),
+                'xcpdb reshard+compact');
+        ok(run_script([qw(-xcpdb --compact), "$tmp/ext"]), 'xcpdb compact');
+};
+
+SKIP: {
+        require_cmd('join', 1);
+        my $basic = create_inbox 'basic', indexlevel => 'basic', sub {
+                my ($im, $ibx) = @_;
+                $im->add(eml_load('t/plack-qp.eml'));
+        };
+        my $env = { PI_CONFIG => "$tmp/pi_config" };
+        PublicInbox::IO::write_file '>', $env->{PI_CONFIG}, <<EOM;
+[publicinbox "basictest"]
+        inboxdir = $basic->{inboxdir}
+        address = basic\@example.com
+EOM
+        my $cmd = [ qw(-cindex -u --all -d), "$tmp/ext",
+                '--join=aggressive,dt:19700101000000..now',
+                '-I', $basic->{inboxdir} ];
+        $cidx_out = $cidx_err = '';
+        ok(run_script($cmd, $env, $opt), 'join w/o search');
+        like($cidx_err, qr/W: \Q$basic->{inboxdir}\E not indexed for search/s,
+                'non-Xapian-enabled inbox noted');
+}
+
+# we need to support blank sections for a top-level repos
+# (e.g. <https://example.com/my-project>
+# git.kernel.org could use "pub" as section name, though, since all git repos
+# are currently under //git.kernel.org/pub/**/*
+{
+        mkdir(my $d = "$tmp/blanksection");
+        my $cfg = cfg_new($d, <<EOM);
+[cindex ""]
+        topdir = $tmp/ext
+        localprefix = $tmp
+EOM
+        my $csrch = $cfg->lookup_cindex('');
+        is ref($csrch), 'PublicInbox::CodeSearch', 'codesearch w/ blank name';
+        is_deeply $csrch->{localprefix}, [ "$tmp" ], 'localprefix respected';
+        my $nr = 0;
+        $cfg->each_cindex(sub {
+                my ($cs, @rest) = @_;
+                is $cs->{topdir}, $csrch->{topdir}, 'each_cindex works';
+                is_deeply \@rest, [ '.' ], 'got expected arg';
+                ++$nr;
+        }, '.');
+        is $nr, 1, 'iterated through cindices';
+        my $oid = 'dba13ed2ddf783ee8118c6a581dbf75305f816a3';
+        my $mset = $csrch->mset("dfpost:$oid");
+        is $mset->size, 1, 'got result from full OID search';
+}
+
+done_testing;
diff --git a/t/clone-coderepo-puh1.sh b/t/clone-coderepo-puh1.sh
new file mode 100755
index 00000000..37a52bd4
--- /dev/null
+++ b/t/clone-coderepo-puh1.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+# sample --post-update-hook for t/clone-coderepo.t test
+case $CLONE_CODEREPO_TEST_OUT in
+'') ;;
+*) echo "uno $@" >> "$CLONE_CODEREPO_TEST_OUT" ;;
+esac
diff --git a/t/clone-coderepo-puh2.sh b/t/clone-coderepo-puh2.sh
new file mode 100755
index 00000000..1170a08a
--- /dev/null
+++ b/t/clone-coderepo-puh2.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+# sample --post-update-hook for t/clone-coderepo.t test
+case $CLONE_CODEREPO_TEST_OUT in
+'') ;;
+*) echo "dos $@" >> "$CLONE_CODEREPO_TEST_OUT" ;;
+esac
diff --git a/t/clone-coderepo.psgi b/t/clone-coderepo.psgi
new file mode 100644
index 00000000..77072174
--- /dev/null
+++ b/t/clone-coderepo.psgi
@@ -0,0 +1,21 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# for clone-coderepo.t
+use v5.12;
+use Plack::Builder;
+use PublicInbox::WwwStatic;
+use PublicInbox::WWW;
+my $www = PublicInbox::WWW->new;
+my $static = PublicInbox::WwwStatic->new(docroot => $ENV{TEST_DOCROOT});
+builder {
+        enable 'Head';
+        sub {
+                my ($env) = @_;
+                if ($env->{PATH_INFO} eq '/manifest.js.gz') {
+                        my $res = $static->call($env);
+                        return $res if $res->[0] != 404;
+                }
+                $www->call($env);
+        };
+}
diff --git a/t/clone-coderepo.t b/t/clone-coderepo.t
new file mode 100644
index 00000000..c6180fc4
--- /dev/null
+++ b/t/clone-coderepo.t
@@ -0,0 +1,220 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::TestCommon;
+use PublicInbox::Import;
+use File::Temp;
+use File::Path qw(remove_tree);
+use PublicInbox::SHA qw(sha1_hex);
+use PublicInbox::IO;
+require_mods(qw(json Plack::Builder HTTP::Date HTTP::Status));
+require_git_http_backend;
+require_git '1.8.5';
+require_cmd 'curl';
+require_ok 'PublicInbox::LeiMirror';
+my ($tmpdir, $for_destroy) = tmpdir();
+my $pa = "$tmpdir/src/a.git";
+my $pb = "$tmpdir/src/b.git";
+PublicInbox::Import::init_bare($pa);
+my ($stdout, $stderr) = ("$tmpdir/out.log", "$tmpdir/err.log");
+my $pi_config = "$tmpdir/pi_config";
+my $td;
+my $tcp = tcp_server();
+my $url = 'http://'.tcp_host_port($tcp).'/';
+my $set_manifest = sub {
+        my ($m, $f) = @_;
+        $f //= "$tmpdir/src/manifest.js.gz";
+        my $ft = File::Temp->new(TMPDIR => $tmpdir, UNLINK => 0);
+        PublicInbox::LeiMirror::dump_manifest($m, $ft);
+        PublicInbox::LeiMirror::ft_rename($ft, $f, 0666);
+};
+my $read_manifest = sub {
+        my ($f) = @_;
+        open my $fh, '<', $f or xbail "open($f): $!";
+        PublicInbox::LeiMirror::decode_manifest($fh, $f, $f);
+};
+
+my $t0 = time - 1;
+my $m; # manifest hashref
+
+{
+        my $fi_data = PublicInbox::IO::try_cat './t/git.fast-import-data';
+        my $db = PublicInbox::Import::default_branch;
+        $fi_data =~ s!\brefs/heads/master\b!$db!gs;
+        my $rdr = { 0 => \$fi_data };
+        my @git = ('git', "--git-dir=$pa");
+        xsys_e([@git, qw(fast-import --quiet)], undef, $rdr);
+        xsys_e([qw(/bin/cp -Rp a.git b.git)], undef, { -C => "$tmpdir/src" });
+        open my $fh, '>', $pi_config or xbail "open($pi_config): $!";
+        print $fh <<EOM or xbail "print: $!";
+[publicinbox]
+        cgitrc = $tmpdir/cgitrc
+        cgit = fallback
+EOM
+        close $fh or xbail "close: $!";
+
+        my $f = "$tmpdir/cgitrc";
+        open $fh, '>', $f or xbail "open($f): $!";
+        print $fh <<EOM or xbail "print: $!";
+project-list=$tmpdir/src/projects.list
+scan-path=$tmpdir/src
+EOM
+        close $fh or xbail "close($f): $!";
+
+        my $cmd = [ '-httpd', '-W0', "--stdout=$stdout", "--stderr=$stderr",
+                File::Spec->rel2abs('t/clone-coderepo.psgi') ];
+        my $env = { TEST_DOCROOT => "$tmpdir/src", PI_CONFIG => $pi_config };
+        $td = start_script($cmd, $env, { 3 => $tcp });
+        my $fp = sha1_hex(my $refs = xqx([@git, 'show-ref']));
+        my $alice = "\x{100}lice";
+        $m = {
+                '/a.git' => {
+                        fingerprint => $fp,
+                        modified => 1,
+                        owner => $alice,
+                        description => "${alice}'s repo",
+                },
+                '/b.git' => {
+                        fingerprint => $fp,
+                        modified => 1,
+                        owner => 'Bob',
+                },
+        };
+        $set_manifest->($m);
+        $f = "$tmpdir/src/projects.list";
+        open $fh, '>', $f, or xbail "open($f): $!";
+        print $fh <<EOM or xbail "print($f): $!";
+a.git
+b.git
+EOM
+        close $fh or xbail "close($f): $!";
+}
+
+my $cmd = [qw(-clone --inbox-config=never --manifest= --project-list=
+        --objstore= -p -q), $url, "$tmpdir/dst", '--exit-code'];
+ok(run_script($cmd), 'clone');
+is(xqx([qw(git config gitweb.owner)], { GIT_DIR => "$tmpdir/dst/a.git" }),
+        "\xc4\x80lice\n", 'a.git gitweb.owner set');
+is(xqx([qw(git config gitweb.owner)], { GIT_DIR => "$tmpdir/dst/b.git" }),
+        "Bob\n", 'b.git gitweb.owner set');
+my $desc = PublicInbox::IO::try_cat("$tmpdir/dst/a.git/description");
+is($desc, "\xc4\x80lice's repo\n", 'description set');
+
+my $dst_pl = "$tmpdir/dst/projects.list";
+my $dst_mf = "$tmpdir/dst/manifest.js.gz";
+ok(!-d "$tmpdir/dst/objstore", 'no objstore created w/o forkgroups');
+my $r = $read_manifest->($dst_mf);
+is_deeply($r, $m, 'manifest matches');
+
+is(PublicInbox::IO::try_cat($dst_pl), "a.git\nb.git\n",
+        'wrote projects.list');
+
+{ # check symlinks
+        $m->{'/a.git'}->{symlinks} = [ '/old/a.git' ];
+        $set_manifest->($m);
+        utime($t0, $t0, $dst_mf) or xbail "utime: $!";
+        ok(run_script($cmd), 'clone again +symlinks');
+        ok(-l "$tmpdir/dst/old/a.git", 'symlink created');
+        is(PublicInbox::IO::try_cat($dst_pl), "a.git\nb.git\n",
+                'projects.list does not include symlink by default');
+
+        $r = $read_manifest->($dst_mf);
+        is_deeply($r, $m, 'updated manifest matches');
+}
+{ # cleanup old projects from projects.list
+        open my $fh, '>>', $dst_pl or xbail $!;
+        print $fh "gone.git\n" or xbail $!;
+        close $fh or xbail $!;
+
+        utime($t0, $t0, $dst_mf) or xbail "utime: $!";
+        my $rdr = { 2 => \(my $err = '') };
+        ok(run_script($cmd, undef, $rdr), 'clone again for expired gone.git');
+        is(PublicInbox::IO::try_cat($dst_pl), "a.git\nb.git\n",
+                'project list cleaned');
+        like($err, qr/no longer exist.*\bgone\.git\b/s, 'gone.git noted');
+}
+
+{ # --purge
+        open my $fh, '>>', $dst_pl or xbail $!;
+        print $fh "gone-rdonly.git\n" or xbail $!;
+        close $fh or xbail $!;
+        my $ro = "$tmpdir/dst/gone-rdonly.git";
+        PublicInbox::Import::init_bare($ro);
+        ok(-d $ro, 'gone-rdonly.git created');
+        my @st = stat($ro) or xbail "stat($ro): $!";
+        chmod($st[2] & 0555, $ro) or xbail "chmod($ro): $!";
+
+        utime($t0, $t0, $dst_mf) or xbail "utime: $!";
+        my $rdr = { 2 => \(my $err = '') };
+        my $xcmd = [ @$cmd, '--purge' ];
+        ok(run_script($xcmd, undef, $rdr), 'clone again for expired gone.git');
+        is(PublicInbox::IO::try_cat($dst_pl), "a.git\nb.git\n",
+                'project list cleaned');
+        like($err, qr!ignored/gone.*?\bgone-rdonly\.git\b!s,
+                'gone-rdonly.git noted');
+        ok(!-d $ro, 'gone-rdonly.git dir gone from --purge');
+}
+
+my $test_puh = sub {
+        my (@clone_arg) = @_;
+        my $x = [qw(-clone --inbox-config=never --manifest= --project-list=
+                -q -p), $url, "$tmpdir/dst", @clone_arg,
+                '--post-update-hook=./t/clone-coderepo-puh1.sh',
+                '--post-update-hook=./t/clone-coderepo-puh2.sh' ];
+        my $log = "$tmpdir/puh.log";
+        my $env = { CLONE_CODEREPO_TEST_OUT => $log };
+        remove_tree("$tmpdir/dst");
+        ok(run_script($x, $env), "fresh clone @clone_arg w/ post-update-hook");
+        ok(-e $log, "hooks run on fresh clone @clone_arg");
+        open my $lh, '<', $log or xbail "open $log: $!";
+        chomp(my @l = readline($lh));
+        is(scalar(@l), 4, "4 lines written by hooks on @clone_arg");
+        for my $r (qw(a b)) {
+                is_xdeeply(['uno', 'dos'],
+                        [ (map { s/ .+//; $_ } grep(m!/$r\.git\z!, @l)) ],
+                        "$r.git hooks ran in order") or diag explain(\@l);
+        }
+        unlink($log) or xbail "unlink: $!";
+        ok(run_script($x, $env), "no-op clone @clone_arg w/ post-update-hook");
+        ok(!-e $log, "hooks not run on no-op @clone_arg");
+
+        push @$x, '--exit-code';
+        ok(!run_script($x, $env), 'no-op clone w/ --exit-code fails');
+        is($? >> 8, 127, '--exit-code gave 127');
+};
+$test_puh->();
+ok(!-e "$tmpdir/dst/objstore", 'no objstore, yet');
+
+my $fgrp = 'fgrp';
+$m->{'/a.git'}->{forkgroup} = $m->{'/b.git'}->{forkgroup} = $fgrp;
+$set_manifest->($m);
+$test_puh->('--objstore=');
+ok(-e "$tmpdir/dst/objstore", 'objstore created');
+
+# ensure new repos can be detected
+{
+        xsys_e([qw(/bin/cp -Rp a.git c.git)], undef, { -C => "$tmpdir/src" });
+        open my $fh, '>>', "$tmpdir/src/projects.list" or xbail "open $!";
+        say $fh 'c.git' or xbail "say $!";
+        close $fh or xbail "close $!";
+        xsys_e([qw(git clone -q), "${url}c.git", "$tmpdir/dst/c.git"]);
+        SKIP: {
+                require_mods(qw(Plack::Test::ExternalServer LWP::UserAgent), 1);
+                use_ok($_) for qw(HTTP::Request::Common);
+                chop(my $uri = $url) eq '/' or xbail "BUG: no /";
+                local $ENV{PLACK_TEST_EXTERNALSERVER_URI} = $uri;
+                my %opt = (ua => LWP::UserAgent->new);
+                $opt{ua}->max_redirect(0);
+                $opt{client} = sub {
+                        my ($cb) = @_;
+                        my $res = $cb->(GET('/c.git/'));
+                        is($res->code, 200, 'got 200 response for /');
+                        $res = $cb->(GET('/c.git/tree/'));
+                        is($res->code, 200, 'got 200 response for /tree');
+                };
+                Plack::Test::ExternalServer::test_psgi(%opt);
+        }
+}
+
+done_testing;
diff --git a/t/cmd_ipc.t b/t/cmd_ipc.t
index 75697a15..c973c6f0 100644
--- a/t/cmd_ipc.t
+++ b/t/cmd_ipc.t
@@ -1,23 +1,20 @@
 #!perl -w
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
-use Test::More;
+use v5.12;
 use PublicInbox::TestCommon;
-use Socket qw(AF_UNIX SOCK_STREAM MSG_EOR);
-pipe(my ($r, $w)) or BAIL_OUT;
+use autodie;
+use Socket qw(AF_UNIX SOCK_STREAM SOCK_SEQPACKET);
+pipe(my $r, my $w);
 my ($send, $recv);
 require_ok 'PublicInbox::Spawn';
-my $SOCK_SEQPACKET = eval { Socket::SOCK_SEQPACKET() } // undef;
-use Time::HiRes qw(usleep);
+require POSIX;
 
 my $do_test = sub { SKIP: {
         my ($type, $flag, $desc) = @_;
-        defined $type or skip 'SOCK_SEQPACKET missing', 7;
         my ($s1, $s2);
         my $src = 'some payload' x 40;
-        socketpair($s1, $s2, AF_UNIX, $type, 0) or BAIL_OUT $!;
+        socketpair($s1, $s2, AF_UNIX, $type, 0);
         my $sfds = [ fileno($r), fileno($w), fileno($s1) ];
         $send->($s1, $sfds, $src, $flag);
         my (@fds) = $recv->($s2, my $buf, length($src) + 1);
@@ -38,7 +35,7 @@ my $do_test = sub { SKIP: {
         @exp = stat $s1;
         @cur = stat $s1a;
         is("$exp[0]\0$exp[1]", "$cur[0]\0$cur[1]", '$s1 dev/ino matches');
-        if (defined($SOCK_SEQPACKET) && $type == $SOCK_SEQPACKET) {
+        if ($type == SOCK_SEQPACKET) {
                 $r1 = $w1 = $s1a = undef;
                 $src = (',' x 1023) . '-' .('.' x 1024);
                 $send->($s1, $sfds, $src, $flag);
@@ -50,47 +47,50 @@ my $do_test = sub { SKIP: {
                 $s2->blocking(0);
                 @fds = $recv->($s2, $buf, length($src) + 1);
                 ok($!{EAGAIN}, "EAGAIN set by ($desc)");
+                is($buf, '', "recv buffer emptied on EAGAIN ($desc)");
                 is_deeply(\@fds, [ undef ], "EAGAIN $desc");
                 $s2->blocking(1);
 
-                if ($ENV{TEST_ALRM}) {
+                if ('test ALRM') {
                         my $alrm = 0;
                         local $SIG{ALRM} = sub { $alrm++ };
                         my $tgt = $$;
-                        my $pid = fork // xbail "fork: $!";
+                        my $pid = fork;
                         if ($pid == 0) {
                                 # need to loop since Perl signals are racy
                                 # (the interpreter doesn't self-pipe)
-                                while (usleep(1000)) {
-                                        kill 'ALRM', $tgt;
+                                my $n = 3;
+                                while (tick(0.01 * $n) && --$n) {
+                                        kill('ALRM', $tgt)
                                 }
+                                close $s1;
+                                POSIX::_exit(1);
                         }
+                        close $s1;
                         @fds = $recv->($s2, $buf, length($src) + 1);
-                        ok($!{EINTR}, "EINTR set by ($desc)");
-                        kill('KILL', $pid);
                         waitpid($pid, 0);
-                        is_deeply(\@fds, [ undef ], "EINTR $desc");
+                        is_deeply(\@fds, [], "EINTR->EOF $desc");
                         ok($alrm, 'SIGALRM hit');
                 }
 
-                close $s1;
                 @fds = $recv->($s2, $buf, length($src) + 1);
                 is_deeply(\@fds, [], "no FDs on EOF $desc");
                 is($buf, '', "buffer cleared on EOF ($desc)");
 
-                socketpair($s1, $s2, AF_UNIX, $type, 0) or BAIL_OUT $!;
+                socketpair($s1, $s2, AF_UNIX, $type, 0);
                 $s1->blocking(0);
                 my $nsent = 0;
+                my $srclen = length($src);
                 while (defined(my $n = $send->($s1, $sfds, $src, $flag))) {
                         $nsent += $n;
-                        fail "sent 0 bytes" if $n == 0;
+                        fail "sent $n bytes of $srclen" if $srclen != $n;
                 }
-                ok($!{EAGAIN} || $!{ETOOMANYREFS},
-                        "hit EAGAIN || ETOOMANYREFS on send $desc") or
-                        diag "send failed with: $!";
+                ok($!{EAGAIN} || $!{ETOOMANYREFS} || $!{EMSGSIZE},
+                        "hit EAGAIN || ETOOMANYREFS || EMSGSIZE on send $desc")
+                        or diag "send failed with: $! (nsent=$nsent)";
                 ok($nsent > 0, 'sent some bytes');
 
-                socketpair($s1, $s2, AF_UNIX, $type, 0) or BAIL_OUT $!;
+                socketpair($s1, $s2, AF_UNIX, $type, 0);
                 is($send->($s1, [], $src, $flag), length($src), 'sent w/o FDs');
                 $buf = 'nope';
                 @fds = $recv->($s2, $buf, length($src));
@@ -99,7 +99,7 @@ my $do_test = sub { SKIP: {
 
                 my $nr = 2 * 1024 * 1024;
                 while (1) {
-                        vec(my $vec = '', $nr * 8 - 1, 1) = 1;
+                        vec(my $vec = '', $nr - 1, 8) = 1;
                         my $n = $send->($s1, [], $vec, $flag);
                         if (defined($n)) {
                                 $n == length($vec) or
@@ -123,7 +123,7 @@ SKIP: {
         $send = $send_ic;
         $recv = $recv_ic;
         $do_test->(SOCK_STREAM, 0, 'Inline::C stream');
-        $do_test->($SOCK_SEQPACKET, MSG_EOR, 'Inline::C seqpacket');
+        $do_test->(SOCK_SEQPACKET, 0, 'Inline::C seqpacket');
 }
 
 SKIP: {
@@ -132,25 +132,24 @@ SKIP: {
         $send = PublicInbox::CmdIPC4->can('send_cmd4');
         $recv = PublicInbox::CmdIPC4->can('recv_cmd4');
         $do_test->(SOCK_STREAM, 0, 'MsgHdr stream');
-        $do_test->($SOCK_SEQPACKET, MSG_EOR, 'MsgHdr seqpacket');
+        $do_test->(SOCK_SEQPACKET, 0, 'MsgHdr seqpacket');
         SKIP: {
                 ($send_ic && $recv_ic) or
                         skip 'Inline::C not installed/enabled', 12;
                 $recv = $recv_ic;
                 $do_test->(SOCK_STREAM, 0, 'Inline::C -> MsgHdr stream');
-                $do_test->($SOCK_SEQPACKET, 0, 'Inline::C -> MsgHdr seqpacket');
+                $do_test->(SOCK_SEQPACKET, 0, 'Inline::C -> MsgHdr seqpacket');
         }
 }
 
 SKIP: {
-        skip 'not Linux', 1 if $^O ne 'linux';
         require_ok 'PublicInbox::Syscall';
         $send = PublicInbox::Syscall->can('send_cmd4') or
-                skip 'send_cmd4 not defined for arch';
+                skip "send_cmd4 not defined for $^O arch", 1;
         $recv = PublicInbox::Syscall->can('recv_cmd4') or
-                skip 'recv_cmd4 not defined for arch';
-        $do_test->(SOCK_STREAM, 0, 'PP Linux stream');
-        $do_test->($SOCK_SEQPACKET, MSG_EOR, 'PP Linux seqpacket');
+                skip "recv_cmd4 not defined for $^O arch", 1;
+        $do_test->(SOCK_STREAM, 0, 'pure Perl stream');
+        $do_test->(SOCK_SEQPACKET, 0, 'pure Perl seqpacket');
 }
 
 done_testing;
diff --git a/t/config.t b/t/config.t
index 7e753cbe..c41a42d3 100644
--- a/t/config.t
+++ b/t/config.t
@@ -1,13 +1,31 @@
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::TestCommon;
 use PublicInbox::Import;
 use_ok 'PublicInbox';
 ok(defined(eval('$PublicInbox::VERSION')), 'VERSION defined');
 use_ok 'PublicInbox::Config';
 my ($tmpdir, $for_destroy) = tmpdir();
+use autodie qw(open close);
+my $validate_git_behavior = $ENV{TEST_VALIDATE_GIT_BEHAVIOR};
+
+{
+        my $f = "$tmpdir/bool_config";
+        open my $fh, '>', $f;
+        print $fh <<EOM;
+[imap]
+        debug
+        port = 2
+EOM
+        close $fh;
+        my $cfg = PublicInbox::Config->git_config_dump($f);
+        $validate_git_behavior and
+                is(xqx([qw(git config -f), $f, qw(--bool imap.debug)]),
+                        "true\n", 'git handles key-only as truth');
+        ok($cfg->git_bool($cfg->{'imap.debug'}), 'key-only value handled');
+        is($cfg->{'imap.port'}, 2, 'normal k=v read after key-only');
+}
 
 {
         PublicInbox::Import::init_bare($tmpdir);
@@ -64,28 +82,30 @@ my ($tmpdir, $for_destroy) = tmpdir();
 
 
 {
-        my $cfgpfx = "publicinbox.test";
         my @altid = qw(serial:gmane:file=a serial:enamg:file=b);
-        my $config = PublicInbox::Config->new(\<<EOF);
-$cfgpfx.address=test\@example.com
-$cfgpfx.mainrepo=/path/to/non/existent
-$cfgpfx.altid=serial:gmane:file=a
-$cfgpfx.altid=serial:enamg:file=b
+        my $config = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = test\@example.com
+        inboxdir = /path/to/non/existent
+        altid=serial:gmane:file=a
+        altid=serial:enamg:file=b
 EOF
         my $ibx = $config->lookup_name('test');
         is_deeply($ibx->{altid}, [ @altid ]);
 
-        $config = PublicInbox::Config->new(\<<EOF);
-$cfgpfx.address=test\@example.com
-$cfgpfx.mainrepo=/path/to/non/existent
+        $config = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = test\@example.com
+        inboxdir = /path/to/non/existent
 EOF
         $ibx = $config->lookup_name('test');
         is($ibx->{inboxdir}, '/path/to/non/existent', 'mainrepo still works');
 
-        $config = PublicInbox::Config->new(\<<EOF);
-$cfgpfx.address=test\@example.com
-$cfgpfx.inboxdir=/path/to/non/existent
-$cfgpfx.mainrepo=/path/to/deprecated
+        $config = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = test\@example.com
+        inboxdir = /path/to/non/existent
+        mainrepo = /path/to/deprecated
 EOF
         $ibx = $config->lookup_name('test');
         is($ibx->{inboxdir}, '/path/to/non/existent',
@@ -93,28 +113,29 @@ EOF
 }
 
 {
-        my $pfx = "publicinbox.test";
-        my $str = <<EOF;
-$pfx.address=test\@example.com
-$pfx.inboxdir=/path/to/non/existent
-$pfx.newsgroup=inbox.test
-publicinbox.nntpserver=news.example.com
+        my $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = test\@example.com
+        inboxdir = /path/to/non/existent
+        newsgroup = inbox.test
+[publicinbox]
+        nntpserver = news.example.com
 EOF
-        my $cfg = PublicInbox::Config->new(\$str);
         my $ibx = $cfg->lookup_name('test');
         is_deeply($ibx->nntp_url({ www => { pi_cfg => $cfg }}),
                 [ 'nntp://news.example.com/inbox.test' ],
                 'nntp_url uses global NNTP server');
 
-        $str = <<EOF;
-$pfx.address=test\@example.com
-$pfx.inboxdir=/path/to/non/existent
-$pfx.newsgroup=inbox.test
-$pfx.nntpserver=news.alt.example.com
-publicinbox.nntpserver=news.example.com
-publicinbox.imapserver=imaps://mail.example.com
+        $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = test\@example.com
+        inboxdir = /path/to/non/existent
+        newsgroup = inbox.test
+        nntpserver = news.alt.example.com
+[publicinbox]
+        nntpserver = news.example.com
+        imapserver = imaps://mail.example.com
 EOF
-        $cfg = PublicInbox::Config->new(\$str);
         $ibx = $cfg->lookup_name('test');
         is_deeply($ibx->nntp_url({ www => { pi_cfg => $cfg }}),
                 [ 'nntp://news.alt.example.com/inbox.test' ],
@@ -126,17 +147,18 @@ EOF
 
 # no obfuscate domains
 {
-        my $pfx = "publicinbox.test";
-        my $pfx2 = "publicinbox.foo";
-        my $str = <<EOF;
-$pfx.address=test\@example.com
-$pfx.inboxdir=/path/to/non/existent
-$pfx2.address=foo\@example.com
-$pfx2.inboxdir=/path/to/foo
-publicinbox.noobfuscate=public-inbox.org \@example.com z\@EXAMPLE.com
-$pfx.obfuscate=true
+        my $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = test\@example.com
+        inboxdir = /path/to/non/existent
+[publicinbox "foo"]
+        address = foo\@example.com
+        inboxdir = /path/to/foo
+[publicinbox]
+        noobfuscate = public-inbox.org \@example.com z\@EXAMPLE.com
+[publicinbox "test"]
+        obfuscate = true
 EOF
-        my $cfg = PublicInbox::Config->new(\$str);
         my $ibx = $cfg->lookup_name('test');
         my $re = $ibx->{-no_obfuscate_re};
         like('meta@public-inbox.org', $re,
@@ -208,20 +230,21 @@ for my $s (@valid) {
 }
 
 {
-        my $pfx1 = "publicinbox.test1";
-        my $pfx2 = "publicinbox.test2";
-        my $str = <<EOF;
-$pfx1.address=test\@example.com
-$pfx1.inboxdir=/path/to/non/existent
-$pfx2.address=foo\@example.com
-$pfx2.inboxdir=/path/to/foo
-$pfx1.coderepo=project
-$pfx2.coderepo=project
-coderepo.project.dir=/path/to/project.git
+        my $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "test1"]
+        address = test\@example.com
+        inboxdir = /path/to/non/existent
+        coderepo = project
+[publicinbox "test2"]
+        address = foo\@example.com
+        inboxdir = /path/to/foo
+        coderepo = project
+[coderepo "project"]
+        dir = /path/to/project.git
 EOF
-        my $cfg = PublicInbox::Config->new(\$str);
         my $t1 = $cfg->lookup_name('test1');
         my $t2 = $cfg->lookup_name('test2');
+        ok $cfg->repo_objs($t1)->[0], 'coderepo parsed';
         is($cfg->repo_objs($t1)->[0], $cfg->repo_objs($t2)->[0],
                 'inboxes share ::Git object');
 }
@@ -245,21 +268,69 @@ EOF
 
 SKIP: {
         # XXX wildcard match requires git 2.26+
-        require_git('1.8.5', 2) or
-                skip 'git 1.8.5+ required for --url-match', 2;
-        my $f = "$tmpdir/urlmatch";
-        open my $fh, '>', $f or BAIL_OUT $!;
-        print $fh <<EOF or BAIL_OUT $!;
+        require_git v1.8.5, 2;
+        my $cfg = cfg_new $tmpdir, <<EOF;
 [imap "imap://mail.example.com"]
         pollInterval = 9
 EOF
-        close $fh or BAIL_OUT;
-        local $ENV{PI_CONFIG} = $f;
-        my $cfg = PublicInbox::Config->new;
         my $url = 'imap://mail.example.com/INBOX';
         is($cfg->urlmatch('imap.pollInterval', $url), 9, 'urlmatch hit');
         is($cfg->urlmatch('imap.idleInterval', $url), undef, 'urlmatch miss');
 };
 
+my $glob2re = PublicInbox::Config->can('glob2re');
+is($glob2re->('http://[::1]:1234/foo/'), undef, 'IPv6 URL not globbed');
+is($glob2re->('foo'), undef, 'plain string unchanged');
+is_deeply($glob2re->('[f-o]'), '[f-o]' , 'range accepted');
+is_deeply($glob2re->('*'), '[^/]*?' , 'wildcard accepted');
+is_deeply($glob2re->('{a,b,c}'), '(a|b|c)' , 'braces');
+is_deeply($glob2re->('{,b,c}'), '(|b|c)' , 'brace with empty @ start');
+is_deeply($glob2re->('{a,b,}'), '(a|b|)' , 'brace with empty @ end');
+is_deeply($glob2re->('{a}'), undef, 'ungrouped brace');
+is_deeply($glob2re->('{a'), undef, 'open left brace');
+is_deeply($glob2re->('a}'), undef, 'open right brace');
+is_deeply($glob2re->('*.[ch]'), '[^/]*?\\.[ch]', 'suffix glob');
+is_deeply($glob2re->('{[a-z],9,}'), '([a-z]|9|)' , 'brace with range');
+is_deeply($glob2re->('\\{a,b\\}'), undef, 'escaped brace');
+is_deeply($glob2re->('\\\\{a,b}'), '\\\\\\\\(a|b)', 'fake escape brace');
+is_deeply($glob2re->('**/foo'), '.*/foo', 'double asterisk start');
+is_deeply($glob2re->('foo/**'), 'foo/.*', 'double asterisk end');
+my $re = $glob2re->('a/**/b');
+is_deeply($re, 'a(?:/.*?/|/)b', 'double asterisk middle');
+like($_, qr!$re!, "a/**/b matches $_") for ('a/b', 'a/c/b', 'a/c/a/b');
+unlike($_, qr!$re!, "a/**/b doesn't match $_") for ('a/ab');
+
+{
+        my $w = '';
+        local $SIG{__WARN__} = sub { $w .= "@_"; };
+        my $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "a"]
+        address = a\@example.com
+        inboxdir = $tmpdir/aa
+[publicinbox "b"]
+        address = b\@example.com
+        inboxdir = $tmpdir/aa
+EOF
+        $cfg->fill_all;
+        like $w, qr!`\Q$tmpdir/aa\E' used by both!, 'inboxdir conflict warned';
+}
+
+{
+        my $w = '';
+        local $SIG{__WARN__} = sub { $w .= "@_"; };
+        my $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "a"]
+        address = a\@example.com
+        inboxdir = $tmpdir/a
+        newsgroup = inbox.test
+[publicinbox "b"]
+        address = b\@example.com
+        inboxdir = $tmpdir/b
+        newsgroup = inbox.tesT
+EOF
+        $cfg->fill_all;
+        like $w, qr!`inbox\.test' used by both!, 'newsgroup conflict warned';
+        like $w, qr!`inbox\.tesT' lowercased!, 'upcase warned';
+}
 
-done_testing();
+done_testing;
diff --git a/t/config_limiter.t b/t/config_limiter.t
index 8c83aca8..f4d99080 100644
--- a/t/config_limiter.t
+++ b/t/config_limiter.t
@@ -1,15 +1,14 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use warnings;
-use Test::More;
-use PublicInbox::Config;
-my $cfgpfx = "publicinbox.test";
+use v5.12;
+use PublicInbox::TestCommon;
+my $tmpdir = tmpdir;
 {
-        my $config = PublicInbox::Config->new(\<<EOF);
-$cfgpfx.address=test\@example.com
-$cfgpfx.inboxdir=/path/to/non/existent
-$cfgpfx.httpbackendmax=12
+        my $config = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = test\@example.com
+        inboxdir = /path/to/non/existent
+        httpbackendmax = 12
 EOF
         my $ibx = $config->lookup_name('test');
         my $git = $ibx->git;
@@ -24,11 +23,13 @@ EOF
 }
 
 {
-        my $config = PublicInbox::Config->new(\<<EOF);
-publicinboxlimiter.named.max=3
-$cfgpfx.address=test\@example.com
-$cfgpfx.inboxdir=/path/to/non/existent
-$cfgpfx.httpbackendmax=named
+        my $config = cfg_new $tmpdir, <<EOF;
+[publicinboxlimiter "named"]
+        max = 3
+[publicinbox "test"]
+        address = test\@example.com
+        inboxdir = /path/to/non/existent
+        httpbackendmax = named
 EOF
         my $ibx = $config->lookup_name('test');
         my $git = $ibx->git;
diff --git a/t/convert-compact.t b/t/convert-compact.t
index 7270cab0..b123f17b 100644
--- a/t/convert-compact.t
+++ b/t/convert-compact.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
@@ -7,9 +7,8 @@ use PublicInbox::Eml;
 use PublicInbox::TestCommon;
 use PublicInbox::Import;
 require_git(2.6);
-require_mods(qw(DBD::SQLite Search::Xapian));
-have_xapian_compact or
-        plan skip_all => 'xapian-compact missing for '.__FILE__;
+require_mods(qw(DBD::SQLite Xapian));
+have_xapian_compact;
 my ($tmpdir, $for_destroy) = tmpdir();
 my $ibx = create_inbox 'v1', indexlevel => 'medium', tmpdir => "$tmpdir/v1",
                 pre_cb => sub {
@@ -36,14 +35,14 @@ EOF
         $im->add($eml) or BAIL_OUT '->add';
 };
 umask(077) or BAIL_OUT "umask: $!";
-is(((stat("$ibx->{inboxdir}/public-inbox"))[2]) & 07777, 0755,
+oct_is(((stat("$ibx->{inboxdir}/public-inbox"))[2]) & 05777, 0755,
         'sharedRepository respected for v1');
-is(((stat("$ibx->{inboxdir}/public-inbox/msgmap.sqlite3"))[2]) & 07777, 0644,
-        'sharedRepository respected for v1 msgmap');
+oct_is(((stat("$ibx->{inboxdir}/public-inbox/msgmap.sqlite3"))[2]) & 05777,
+        0644, 'sharedRepository respected for v1 msgmap');
 my @xdir = glob("$ibx->{inboxdir}/public-inbox/xap*/*");
 foreach (@xdir) {
         my @st = stat($_);
-        is($st[2] & 07777, -f _ ? 0644 : 0755,
+        oct_is($st[2] & 05777, -f _ ? 0644 : 0755,
                 'sharedRepository respected on file after convert');
 }
 
@@ -56,7 +55,7 @@ ok(run_script($cmd, undef, $rdr), 'v1 compact works');
 
 @xdir = glob("$ibx->{inboxdir}/public-inbox/xap*");
 is(scalar(@xdir), 1, 'got one xapian directory after compact');
-is(((stat($xdir[0]))[2]) & 07777, 0755,
+oct_is(((stat($xdir[0]))[2]) & 05777, 0755,
         'sharedRepository respected on v1 compact');
 
 my $hwm = do {
@@ -72,9 +71,9 @@ ok(run_script($cmd, undef, $rdr), 'convert --no-index works');
 $cmd = [ '-convert', $ibx->{inboxdir}, "$tmpdir/x/v2" ];
 ok(run_script($cmd, undef, $rdr), 'convert works');
 @xdir = glob("$tmpdir/x/v2/xap*/*");
-foreach (@xdir) {
+for (@xdir) { # TODO: should public-inbox-convert preserve S_ISGID bit?
         my @st = stat($_);
-        is($st[2] & 07777, -f _ ? 0644 : 0755,
+        oct_is($st[2] & 07777, -f _ ? 0644 : 0755,
                 'sharedRepository respected after convert');
 }
 
@@ -88,20 +87,20 @@ is($ibx->mm->num_highwater, $hwm, 'highwater mark unchanged in v2 inbox');
 @xdir = glob("$tmpdir/x/v2/xap*/*");
 foreach (@xdir) {
         my @st = stat($_);
-        is($st[2] & 07777, -f _ ? 0644 : 0755,
+        oct_is($st[2] & 07777, -f _ ? 0644 : 0755,
                 'sharedRepository respected after v2 compact');
 }
-is(((stat("$tmpdir/x/v2/msgmap.sqlite3"))[2]) & 07777, 0644,
+oct_is(((stat("$tmpdir/x/v2/msgmap.sqlite3"))[2]) & 07777, 0644,
         'sharedRepository respected for v2 msgmap');
 
 @xdir = (glob("$tmpdir/x/v2/git/*.git/objects/*/*"),
          glob("$tmpdir/x/v2/git/*.git/objects/pack/*"));
 foreach (@xdir) {
         my @st = stat($_);
-        is($st[2] & 07777, -f _ ? 0444 : 0755,
+        oct_is($st[2] & 07777, -f _ ? 0444 : 0755,
                 'sharedRepository respected after v2 compact');
 }
-my $msgs = $ibx->recent({limit => 1000});
+my $msgs = $ibx->over->recent({limit => 1000});
 is($msgs->[0]->{mid}, 'a-mid@b', 'message exists in history');
 is(scalar @$msgs, 1, 'only one message in history');
 
diff --git a/t/data/attached-mbox-with-utf8.eml b/t/data/attached-mbox-with-utf8.eml
new file mode 100644
index 00000000..53dad830
--- /dev/null
+++ b/t/data/attached-mbox-with-utf8.eml
@@ -0,0 +1,45 @@
+Date: Mon, 24 Sep 2018 09:46:40 -0700 (PDT)
+Message-Id: <attached-mbox-with-utf8@example>
+To: test@example.com
+Subject: [PATCHES] attached mbox with UTF-8 patch
+From: attacher@example.com
+Mime-Version: 1.0
+Content-Type: Multipart/Mixed;
+ boundary="--Next_Part(Mon_Sep_24_09_46_40_2018_110)--"
+Content-Transfer-Encoding: 7bit
+
+----Next_Part(Mon_Sep_24_09_46_40_2018_110)--
+Content-Type: Text/Plain; charset=us-ascii
+Content-Transfer-Encoding: 7bit
+
+hello world
+
+----Next_Part(Mon_Sep_24_09_46_40_2018_110)--
+Content-Type: Application/Octet-Stream
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="foo.mbox"
+
+RnJvbSAzNGRkMWQyNWQ3NmU0NjRjNTM0ZGI0MDllYTdlZDQyNWFiMDVjODI2IE1vbiBTZXAgMTcg
+MDA6MDA6MDAgMjAwMQpGcm9tOiA9P1VURi04P3E/Qmo9QzM9Qjhybj89IDxiam9ybkBleGFtcGxl
+LmNvbT4KRGF0ZTogVGh1LCAxMiBTZXAgMjAxOSAxMDo0MjowMCArMDIwMApNSU1FLVZlcnNpb246
+IDEuMApDb250ZW50LVR5cGU6IHRleHQvcGxhaW47IGNoYXJzZXQ9VVRGLTgKQ29udGVudC1UcmFu
+c2Zlci1FbmNvZGluZzogOGJpdAoKU2lnbmVkLW9mZi1ieTogQmrDuHJuIDxiam9ybkBleGFtcGxl
+LmNvbT4KU2lnbmVkLW9mZi1ieTogaiDFu2VuIDx6QGV4YW1wbGUuY29tPgotLS0KIGZvby5jIHwg
+MSArLQogMSBmaWxlIGNoYW5nZWQsIDEgaW5zZXJ0aW9ucygrKSwgMSBkZWxldGlvbnMoLSkKCmRp
+ZmYgLS1naXQgYS9mb28uYyBiL2Zvby5jCmluZGV4IDVjNDJjZjgxYTA4Yi4uODVmYmE2NGMzZmNm
+IDEwMDY0NAotLS0gYS9mb28uYworKysgYi9mb28uYwpAQCAtMjIxLDkgKzIyMSw5IEBAIGludCBo
+ZWxsbyh2b2lkKQogCQlnb3RvIHBoYWlsOwogCX0KIHNraXA6Ci0JaWYgKAlmb28gJiYKKwl1bmxl
+c3MgKGZvbykKIGJsYWgKIGJsYWgKIGJsYWgKLS0gCkJqw7hybgoKRnJvbSAzNGRkMWQyNWQ3NmU0
+NjRjNTM0ZGI0MDllYTdlZDQyNWFiMDVjODI2IE1vbiBTZXAgMTcgMDA6MDA6MDAgMjAwMQpGcm9t
+OiA9P1VURi04P3E/Qmo9QzM9Qjhybj89IDxiam9ybkBleGFtcGxlLmNvbT4KRGF0ZTogVGh1LCAx
+MiBTZXAgMjAxOSAxMDo0MjowMCArMDIwMApNSU1FLVZlcnNpb246IDEuMApDb250ZW50LVR5cGU6
+IHRleHQvcGxhaW47IGNoYXJzZXQ9VVRGLTgKQ29udGVudC1UcmFuc2Zlci1FbmNvZGluZzogOGJp
+dAoKU2lnbmVkLW9mZi1ieTogQmrDuHJuIDxiam9ybkBleGFtcGxlLmNvbT4KU2lnbmVkLW9mZi1i
+eTogaiDFu2VuIDx6QGV4YW1wbGUuY29tPgotLS0KIGZvby5jIHwgMSArLQogMSBmaWxlIGNoYW5n
+ZWQsIDEgaW5zZXJ0aW9ucygrKSwgMSBkZWxldGlvbnMoLSkKCmRpZmYgLS1naXQgYS9mb28uYyBi
+L2Zvby5jCmluZGV4IDVjNDJjZjgxYTA4Yi4uODVmYmE2NGMzZmNmIDEwMDY0NAotLS0gYS9mb28u
+YworKysgYi9mb28uYwpAQCAtMjIxLDkgKzIyMSw5IEBAIGludCBoZWxsbyh2b2lkKQogCQlnb3Rv
+IHBoYWlsOwogCX0KIHNraXA6Ci0JaWYgKAlmb28gJiYKKwl1bmxlc3MgKGZvbykKIGJsYWgKIGJs
+YWgKIGJsYWgKLS0gCkJqw7hybgo=
+
+----Next_Part(Mon_Sep_24_09_46_40_2018_110)----
diff --git a/t/dir_idle.t b/t/dir_idle.t
index 19e54967..8d085d6e 100644
--- a/t/dir_idle.t
+++ b/t/dir_idle.t
@@ -1,7 +1,7 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use v5.10.1; use strict; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
 use PublicInbox::DS qw(now);
 use File::Path qw(make_path);
 use_ok 'PublicInbox::DirIdle';
@@ -11,26 +11,30 @@ my @x;
 my $cb = sub { push @x, \@_ };
 my $di = PublicInbox::DirIdle->new($cb);
 $di->add_watches(["$tmpdir/a", "$tmpdir/c"], 1);
-PublicInbox::DS->SetLoopTimeout(1000);
+$PublicInbox::DS::loop_timeout = 1000;
 my $end = 3 + now;
-PublicInbox::DS->SetPostLoopCallback(sub { scalar(@x) == 0 && now < $end });
-tick(0.011);
+local @PublicInbox::DS::post_loop_do = (sub { scalar(@x) == 0 && now < $end });
 rmdir("$tmpdir/a/b") or xbail "rmdir $!";
 PublicInbox::DS::event_loop();
-is(scalar(@x), 1, 'got an event') and
+if (is(scalar(@x), 1, 'got an rmdir event')) {
         is($x[0]->[0]->fullname, "$tmpdir/a/b", 'got expected fullname') and
         ok($x[0]->[0]->IN_DELETE, 'IN_DELETE set');
+} else {
+        check_broken_tmpfs;
+        xbail explain(\@x);
+}
 
-tick(0.011);
 rmdir("$tmpdir/a") or xbail "rmdir $!";
 @x = ();
 $end = 3 + now;
 PublicInbox::DS::event_loop();
-is(scalar(@x), 1, 'got an event') and
+if (is(scalar(@x), 1, 'got an event after rmdir')) {
         is($x[0]->[0]->fullname, "$tmpdir/a", 'got expected fullname') and
         ok($x[0]->[0]->IN_DELETE_SELF, 'IN_DELETE_SELF set');
-
-tick(0.011);
+} else {
+        check_broken_tmpfs;
+        diag explain(\@x);
+}
 rename("$tmpdir/c", "$tmpdir/j") or xbail "rmdir $!";
 @x = ();
 $end = 3 + now;
@@ -40,5 +44,4 @@ is(scalar(@x), 1, 'got an event') and
         ok($x[0]->[0]->IN_DELETE_SELF || $x[0]->[0]->IN_MOVE_SELF,
                 'IN_DELETE_SELF set on move');
 
-PublicInbox::DS->Reset;
 done_testing;
diff --git a/t/ds-kqxs.t b/t/ds-kqxs.t
index 43c71fed..87f7199d 100644
--- a/t/ds-kqxs.t
+++ b/t/ds-kqxs.t
@@ -1,13 +1,14 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # Licensed the same as Danga::Socket (and Perl5)
 # License: GPL-1.0+ or Artistic-1.0-Perl
 #  <https://www.gnu.org/licenses/gpl-1.0.txt>
 #  <https://dev.perl.org/licenses/artistic.html>
-use strict;
+use v5.12;
 use Test::More;
 unless (eval { require IO::KQueue }) {
-        my $m = $^O !~ /bsd/ ? 'DSKQXS is only for *BSD systems'
-                                : "no IO::KQueue, skipping $0: $@";
+        my $m = ($^O =~ /bsd/ || $^O eq 'dragonfly') ?
+                "no IO::KQueue, skipping $0: $@" :
+                'DSKQXS is only for *BSD systems';
         plan skip_all => $m;
 }
 
diff --git a/t/ds-leak.t b/t/ds-leak.t
index 4e8d76cd..f39985e0 100644
--- a/t/ds-leak.t
+++ b/t/ds-leak.t
@@ -1,9 +1,9 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # Licensed the same as Danga::Socket (and Perl5)
 # License: GPL-1.0+ or Artistic-1.0-Perl
 #  <https://www.gnu.org/licenses/gpl-1.0.txt>
 #  <https://dev.perl.org/licenses/artistic.html>
-use strict; use v5.10.1; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
 use_ok 'PublicInbox::DS';
 
 if ('close-on-exec for epoll and kqueue') {
@@ -11,8 +11,8 @@ if ('close-on-exec for epoll and kqueue') {
         my $pid;
         my $evfd_re = qr/(?:kqueue|eventpoll)/i;
 
-        PublicInbox::DS->SetLoopTimeout(0);
-        PublicInbox::DS->SetPostLoopCallback(sub { 0 });
+        $PublicInbox::DS::loop_timeout = 0;
+        local @PublicInbox::DS::post_loop_do = (sub { 0 });
 
         # make sure execve closes if we're using fork()
         my ($r, $w);
@@ -29,11 +29,8 @@ if ('close-on-exec for epoll and kqueue') {
         is($l, undef, 'cloexec works and sleep(1) is running');
 
         SKIP: {
-                my $lsof = require_cmd('lsof', 1) or skip 'lsof missing', 1;
                 my $rdr = { 2 => \(my $null) };
-                my @of = grep(/$evfd_re/, xqx([$lsof, '-p', $pid], {}, $rdr));
-                my $err = $?;
-                skip "lsof broken ? (\$?=$err)", 1 if $err;
+                my @of = grep /$evfd_re/, lsof_pid $pid, $rdr;
                 is_deeply(\@of, [], 'no FDs leaked to subprocess');
         };
         if (defined $pid) {
@@ -54,8 +51,8 @@ SKIP: {
         }
         my $cb = sub {};
         for my $i (0..$n) {
-                PublicInbox::DS->SetLoopTimeout(0);
-                PublicInbox::DS->SetPostLoopCallback($cb);
+                $PublicInbox::DS::loop_timeout = 0;
+                local @PublicInbox::DS::post_loop_do = ($cb);
                 PublicInbox::DS::event_loop();
                 PublicInbox::DS->Reset;
         }
diff --git a/t/ds-poll.t b/t/ds-poll.t
index d8861369..22dbc802 100644
--- a/t/ds-poll.t
+++ b/t/ds-poll.t
@@ -1,50 +1,64 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # Licensed the same as Danga::Socket (and Perl5)
 # License: GPL-1.0+ or Artistic-1.0-Perl
 #  <https://www.gnu.org/licenses/gpl-1.0.txt>
 #  <https://dev.perl.org/licenses/artistic.html>
-use strict;
-use warnings;
+use v5.12;
 use Test::More;
-use PublicInbox::Syscall qw(:epoll);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLOUT EPOLLONESHOT);
+use autodie qw(close pipe syswrite);
 my $cls = $ENV{TEST_IOPOLLER} // 'PublicInbox::DSPoll';
 use_ok $cls;
 my $p = $cls->new;
 
 my ($r, $w, $x, $y);
-pipe($r, $w) or die;
-pipe($x, $y) or die;
-is($p->epoll_ctl(EPOLL_CTL_ADD, fileno($r), EPOLLIN), 0, 'add EPOLLIN');
+pipe($r, $w);
+pipe($x, $y);
+is($p->ep_add($r, EPOLLIN), 0, 'add EPOLLIN');
 my $events = [];
-$p->epoll_wait(9, 0, $events);
+$p->ep_wait(0, $events);
 is_deeply($events, [], 'no events set');
-is($p->epoll_ctl(EPOLL_CTL_ADD, fileno($w), EPOLLOUT|EPOLLONESHOT), 0,
-        'add EPOLLOUT|EPOLLONESHOT');
-$p->epoll_wait(9, -1, $events);
+is($p->ep_add($w, EPOLLOUT|EPOLLONESHOT), 0, 'add EPOLLOUT|EPOLLONESHOT');
+$p->ep_wait(-1, $events);
 is(scalar(@$events), 1, 'got POLLOUT event');
 is($events->[0], fileno($w), '$w ready');
 
-$p->epoll_wait(9, 0, $events);
+$p->ep_wait(0, $events);
 is(scalar(@$events), 0, 'nothing ready after oneshot');
 is_deeply($events, [], 'no events set after oneshot');
 
 syswrite($w, '1') == 1 or die;
 for my $t (0..1) {
-        $p->epoll_wait(9, $t, $events);
+        $p->ep_wait($t, $events);
         is($events->[0], fileno($r), "level-trigger POLLIN ready #$t");
         is(scalar(@$events), 1, "only event ready #$t");
 }
 syswrite($y, '1') == 1 or die;
-is($p->epoll_ctl(EPOLL_CTL_ADD, fileno($x), EPOLLIN|EPOLLONESHOT), 0,
-        'EPOLLIN|EPOLLONESHOT add');
-$p->epoll_wait(9, -1, $events);
+is($p->ep_add($x, EPOLLIN|EPOLLONESHOT), 0, 'EPOLLIN|EPOLLONESHOT add');
+$p->ep_wait(-1, $events);
 is(scalar @$events, 2, 'epoll_wait has 2 ready');
 my @fds = sort @$events;
 my @exp = sort((fileno($r), fileno($x)));
 is_deeply(\@fds, \@exp, 'got both ready FDs');
 
-is($p->epoll_ctl(EPOLL_CTL_DEL, fileno($r), 0), 0, 'EPOLL_CTL_DEL OK');
-$p->epoll_wait(9, 0, $events);
+is($p->ep_del($r, 0), 0, 'EPOLL_CTL_DEL OK');
+$p->ep_wait(0, $events);
 is(scalar @$events, 0, 'nothing ready after EPOLL_CTL_DEL');
 
+is($p->ep_add($r, EPOLLIN), 0, 're-add');
+SKIP: {
+        $cls =~ m!::(?:DSPoll|Select)\z! or
+                skip 'EBADF test for select|poll only', 1;
+        my $old_fd = fileno($r);
+        close $r;
+        my @w;
+        eval {
+                local $SIG{__WARN__} = sub { push @w, @_ };
+                $p->ep_wait(0, $events);
+        };
+        ok($@, 'error detected from bad FD');
+        ok($!{EBADF}, 'EBADF errno set');
+        @w and ok(grep(/\bFD=$old_fd invalid/, @w), 'carps invalid FD');
+}
+
 done_testing;
diff --git a/t/edit.t b/t/edit.t
index e6e0f9cf..1621df3b 100644
--- a/t/edit.t
+++ b/t/edit.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # edit frontend behavior test (t/replace.t for backend)
 use strict;
@@ -24,10 +24,11 @@ local $ENV{PI_CONFIG} = $cfgfile;
 my ($in, $out, $err, $cmd, $cur, $t);
 my $git = PublicInbox::Git->new("$ibx->{inboxdir}/git/0.git");
 my $opt = { 0 => \$in, 1 => \$out, 2 => \$err };
+my $ipe = "$^X -w -i -p -e";
 
 $t = '-F FILE'; {
         $in = $out = $err = '';
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/boolean prefix/bool pfx/'";
+        local $ENV{MAIL_EDITOR} = "$ipe 's/boolean prefix/bool pfx/'";
         $cmd = [ '-edit', "-F$file", $inboxdir ];
         ok(run_script($cmd, undef, $opt), "$t edit OK");
         $cur = PublicInbox::Eml->new($ibx->msg_by_mid($mid));
@@ -37,7 +38,7 @@ $t = '-F FILE'; {
 
 $t = '-m MESSAGE_ID'; {
         $in = $out = $err = '';
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/bool pfx/boolean prefix/'";
+        local $ENV{MAIL_EDITOR} = "$ipe 's/bool pfx/boolean prefix/'";
         $cmd = [ '-edit', "-m$mid", $inboxdir ];
         ok(run_script($cmd, undef, $opt), "$t edit OK");
         $cur = PublicInbox::Eml->new($ibx->msg_by_mid($mid));
@@ -48,7 +49,7 @@ $t = '-m MESSAGE_ID'; {
 $t = 'no-op -m MESSAGE_ID'; {
         $in = $out = $err = '';
         my $before = $git->qx(qw(rev-parse HEAD));
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/bool pfx/boolean prefix/'";
+        local $ENV{MAIL_EDITOR} = "$ipe 's/bool pfx/boolean prefix/'";
         $cmd = [ '-edit', "-m$mid", $inboxdir ];
         ok(run_script($cmd, undef, $opt), "$t succeeds");
         my $prev = $cur;
@@ -64,7 +65,7 @@ $t = 'no-op -m MESSAGE_ID'; {
 $t = 'no-op -m MESSAGE_ID w/Status: header'; { # because mutt does it
         $in = $out = $err = '';
         my $before = $git->qx(qw(rev-parse HEAD));
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^Subject:.*/Status: RO\\n\$&/'";
+        local $ENV{MAIL_EDITOR} = "$ipe 's/^Subject:.*/Status: RO\\n\$&/'";
         $cmd = [ '-edit', "-m$mid", $inboxdir ];
         ok(run_script($cmd, undef, $opt), "$t succeeds");
         my $prev = $cur;
@@ -80,7 +81,7 @@ $t = 'no-op -m MESSAGE_ID w/Status: header'; { # because mutt does it
 
 $t = '-m MESSAGE_ID can change Received: headers'; {
         $in = $out = $err = '';
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^Subject:.*/Received: x\\n\$&/'";
+        local $ENV{MAIL_EDITOR} = "$ipe 's/^Subject:.*/Received: x\\n\$&/'";
         $cmd = [ '-edit', "-m$mid", $inboxdir ];
         ok(run_script($cmd, undef, $opt), "$t succeeds");
         $cur = PublicInbox::Eml->new($ibx->msg_by_mid($mid));
@@ -91,7 +92,7 @@ $t = '-m MESSAGE_ID can change Received: headers'; {
 
 $t = '-m miss'; {
         $in = $out = $err = '';
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/boolean/FAIL/'";
+        local $ENV{MAIL_EDITOR} = "$ipe 's/boolean/FAIL/'";
         $cmd = [ '-edit', "-m$mid-miss", $inboxdir ];
         ok(!run_script($cmd, undef, $opt), "$t fails on invalid MID");
         like($err, qr/No message found/, "$t shows error");
@@ -99,7 +100,7 @@ $t = '-m miss'; {
 
 $t = 'non-interactive editor failure'; {
         $in = $out = $err = '';
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 'END { exit 1 }'";
+        local $ENV{MAIL_EDITOR} = "$ipe 'END { exit 1 }'";
         $cmd = [ '-edit', "-m$mid", $inboxdir ];
         ok(!run_script($cmd, undef, $opt), "$t detected");
         like($err, qr/END \{ exit 1 \}' failed:/, "$t shows error");
@@ -109,7 +110,7 @@ $t = 'mailEditor set in config'; {
         $in = $out = $err = '';
         my $rc = xsys(qw(git config), "--file=$cfgfile",
                         'publicinbox.maileditor',
-                        "$^X -i -p -e 's/boolean prefix/bool pfx/'");
+                        "$ipe 's/boolean prefix/bool pfx/'");
         is($rc, 0, 'set publicinbox.mailEditor');
         local $ENV{MAIL_EDITOR};
         delete $ENV{MAIL_EDITOR};
@@ -123,20 +124,20 @@ $t = 'mailEditor set in config'; {
 
 $t = '--raw and mbox escaping'; {
         $in = $out = $err = '';
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^\$/\\nFrom not mbox\\n/'";
+        local $ENV{MAIL_EDITOR} = "$ipe 's/^\$/\\nFrom not mbox\\n/'";
         $cmd = [ '-edit', "-m$mid", '--raw', $inboxdir ];
         ok(run_script($cmd, undef, $opt), "$t succeeds");
         $cur = PublicInbox::Eml->new($ibx->msg_by_mid($mid));
         like($cur->body, qr/^From not mbox/sm, 'put "From " line into body');
 
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^>From not/\$& an/'";
+        local $ENV{MAIL_EDITOR} = "$ipe 's/^>From not/\$& an/'";
         $cmd = [ '-edit', "-m$mid", $inboxdir ];
         ok(run_script($cmd, undef, $opt), "$t succeeds with mbox escaping");
         $cur = PublicInbox::Eml->new($ibx->msg_by_mid($mid));
         like($cur->body, qr/^From not an mbox/sm,
                 'changed "From " line unescaped');
 
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^From not an mbox\\n//s'";
+        local $ENV{MAIL_EDITOR} = "$ipe 's/^From not an mbox\\n//s'";
         $cmd = [ '-edit', "-m$mid", '--raw', $inboxdir ];
         ok(run_script($cmd, undef, $opt), "$t succeeds again");
         $cur = PublicInbox::Eml->new($ibx->msg_by_mid($mid));
@@ -154,7 +155,7 @@ $t = 'reuse Message-ID'; {
 
 $t = 'edit ambiguous Message-ID with -m'; {
         $in = $out = $err = '';
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/bool pfx/boolean prefix/'";
+        local $ENV{MAIL_EDITOR} = "$ipe 's/bool pfx/boolean prefix/'";
         $cmd = [ '-edit', "-m$mid", $inboxdir ];
         ok(!run_script($cmd, undef, $opt), "$t fails w/o --force");
         like($err, qr/Multiple messages with different content found matching/,
@@ -164,7 +165,7 @@ $t = 'edit ambiguous Message-ID with -m'; {
 
 $t .= ' and --force'; {
         $in = $out = $err = '';
-        local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^Subject:.*/Subject:x/i'";
+        local $ENV{MAIL_EDITOR} = "$ipe 's/^Subject:.*/Subject:x/i'";
         $cmd = [ '-edit', "-m$mid", '--force', $inboxdir ];
         ok(run_script($cmd, undef, $opt), "$t succeeds");
         like($err, qr/Will edit all of them/, "$t notes all will be edited");
diff --git a/t/eml.t b/t/eml.t
index 2e6a441f..690ada57 100644
--- a/t/eml.t
+++ b/t/eml.t
@@ -1,8 +1,8 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.10.1; # TODO: check unicode_strings w/ 5.12
 use strict;
-use Test::More;
 use PublicInbox::TestCommon;
 use PublicInbox::MsgIter qw(msg_part_text);
 my @classes = qw(PublicInbox::Eml);
@@ -355,7 +355,7 @@ if ('maxparts is a feature unique to us') {
 }
 
 SKIP: {
-        require_mods('PublicInbox::MIME', 1);
+        require_mods('Email::MIME', 1);
         my $eml = eml_load 't/utf8.eml';
         my $mime = mime_load 't/utf8.eml';
         for my $h (qw(Subject From To)) {
diff --git a/t/epoll.t b/t/epoll.t
index f346b387..ab9ac22a 100644
--- a/t/epoll.t
+++ b/t/epoll.t
@@ -1,25 +1,22 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use Test::More;
-use PublicInbox::Syscall qw(:epoll);
+use autodie;
+use PublicInbox::Syscall qw(EPOLLOUT);
 plan skip_all => 'not Linux' if $^O ne 'linux';
-my $epfd = epoll_create();
-ok($epfd >= 0, 'epoll_create');
-open(my $hnd, '+<&=', $epfd); # for autoclose
-
-pipe(my ($r, $w)) or die "pipe: $!";
-is(epoll_ctl($epfd, EPOLL_CTL_ADD, fileno($w), EPOLLOUT), 0,
-    'epoll_ctl socket EPOLLOUT');
+require_ok 'PublicInbox::Epoll';
+my $ep = PublicInbox::Epoll->new;
+pipe(my $r, my $w);
+is($ep->ep_add($w, EPOLLOUT), 0, 'epoll_ctl pipe EPOLLOUT');
 
 my @events;
-epoll_wait($epfd, 100, 10000, \@events);
+$ep->ep_wait(10000, \@events);
 is(scalar(@events), 1, 'got one event');
 is($events[0], fileno($w), 'got expected FD');
 close $w;
-epoll_wait($epfd, 100, 0, \@events);
+$ep->ep_wait(0, \@events);
 is(scalar(@events), 0, 'epoll_wait timeout');
 
 done_testing;
diff --git a/t/extindex-psgi.t b/t/extindex-psgi.t
index 98dc2e48..896c46ff 100644
--- a/t/extindex-psgi.t
+++ b/t/extindex-psgi.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
@@ -8,7 +8,7 @@ use PublicInbox::Config;
 use File::Copy qw(cp);
 use IO::Handle ();
 require_git(2.6);
-require_mods(qw(json DBD::SQLite Search::Xapian
+require_mods(qw(json DBD::SQLite Xapian
                 HTTP::Request::Common Plack::Test URI::Escape Plack::Builder));
 use_ok($_) for (qw(HTTP::Request::Common Plack::Test));
 use IO::Uncompress::Gunzip qw(gunzip);
@@ -21,7 +21,28 @@ mkdir "$home/.public-inbox" or BAIL_OUT $!;
 my $pi_config = "$home/.public-inbox/config";
 cp($cfg_path, $pi_config) or BAIL_OUT;
 my $env = { HOME => $home };
-run_script([qw(-extindex --all), "$tmpdir/eidx"], $env) or BAIL_OUT;
+my $m2t = create_inbox 'mid2tid', version => 2, indexlevel => 'basic', sub {
+        my ($im, $ibx) = @_;
+        for my $n (1..3) {
+                $im->add(PublicInbox::Eml->new(<<EOM)) or xbail 'add';
+Date: Fri, 02 Oct 1993 00:0$n:00 +0000
+Message-ID: <t\@$n>
+Subject: tid $n
+From: x\@example.com
+References: <a-mid\@b>
+
+$n
+EOM
+                $im->add(PublicInbox::Eml->new(<<EOM)) or xbail 'add';
+Date: Fri, 02 Oct 1993 00:0$n:00 +0000
+Message-ID: <ut\@$n>
+Subject: unrelated tid $n
+From: x\@example.com
+References: <b-mid\@b>
+
+EOM
+        }
+};
 {
         open my $cfgfh, '>>', $pi_config or BAIL_OUT;
         $cfgfh->autoflush(1);
@@ -32,8 +53,14 @@ run_script([qw(-extindex --all), "$tmpdir/eidx"], $env) or BAIL_OUT;
 [publicinbox]
         wwwlisting = all
         grokManifest = all
+[publicinbox "m2t"]
+        inboxdir = $m2t->{inboxdir}
+        address = $m2t->{-primary_address}
 EOM
+        close $cfgfh or xbail "close: $!";
 }
+
+run_script([qw(-extindex --all), "$tmpdir/eidx"], $env) or BAIL_OUT;
 my $www = PublicInbox::WWW->new(PublicInbox::Config->new($pi_config));
 my $client = sub {
         my ($cb) = @_;
@@ -83,6 +110,22 @@ my $client = sub {
                 't2 manifest');
         is_deeply([ sort keys %{$m->{'/t1'}} ], [ '/t1' ],
                 't2 manifest');
+
+        # ensure ibx->{isrch}->{es}->over is used instead of ibx->over:
+        $res = $cb->(POST("/m2t/t\@1/?q=dt:19931002000259..&x=m"));
+        is($res->code, 200, 'hit on mid2tid query');
+        $res = $cb->(POST("/m2t/t\@1/?q=dt:19931002000400..&x=m"));
+        is($res->code, 404, '404 on out-of-range mid2tid query');
+        $res = $cb->(POST("/m2t/t\@1/?q=s:unrelated&x=m"));
+        is($res->code, 404, '404 on cross-thread search');
+
+
+        for my $c (qw(new active)) {
+                $res = $cb->(GET("/m2t/topics_$c.html"));
+                is($res->code, 200, "topics_$c.html on basic v2");
+                $res = $cb->(GET("/all/topics_$c.html"));
+                is($res->code, 200, "topics_$c.html on extindex");
+        }
 };
 test_psgi(sub { $www->call(@_) }, $client);
 %$env = (%$env, TMPDIR => $tmpdir, PI_CONFIG => $pi_config);
diff --git a/t/extsearch.t b/t/extsearch.t
index 2d7375d6..090f6db5 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -1,30 +1,25 @@
 #!perl -w
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use Test::More;
+use v5.12;
 use PublicInbox::TestCommon;
 use PublicInbox::Config;
 use PublicInbox::InboxWritable;
-use Fcntl qw(:seek);
 require_git(2.6);
-require_mods(qw(json DBD::SQLite Search::Xapian));
+require_mods(qw(json DBD::SQLite Xapian));
+use autodie qw(open rename truncate unlink);
 require PublicInbox::Search;
 use_ok 'PublicInbox::ExtSearch';
 use_ok 'PublicInbox::ExtSearchIdx';
 use_ok 'PublicInbox::OverIdx';
-my $sock = tcp_server();
-my $host_port = tcp_host_port($sock);
 my ($home, $for_destroy) = tmpdir();
 local $ENV{HOME} = $home;
 mkdir "$home/.public-inbox" or BAIL_OUT $!;
 my $cfg_path = "$home/.public-inbox/config";
-open my $fh, '>', $cfg_path or BAIL_OUT $!;
-print $fh <<EOF or BAIL_OUT $!;
+PublicInbox::IO::write_file '>', $cfg_path, <<EOF;
 [publicinboxMda]
         spamcheck = none
 EOF
-close $fh or BAIL_OUT $!;
 my $v2addr = 'v2test@example.com';
 my $v1addr = 'v1test@example.com';
 ok(run_script([qw(-init -Lbasic -V2 v2test --newsgroup v2.example),
@@ -33,24 +28,18 @@ my $env = { ORIGINAL_RECIPIENT => $v2addr };
 my $eml = eml_load('t/utf8.eml');
 
 $eml->header_set('List-Id', '<v2.example.com>');
-open($fh, '+>', undef) or BAIL_OUT $!;
-$fh->autoflush(1);
-print $fh $eml->as_string or BAIL_OUT $!;
-seek($fh, 0, SEEK_SET) or BAIL_OUT $!;
 
-run_script(['-mda', '--no-precheck'], $env, { 0 => $fh }) or BAIL_OUT '-mda';
+my $in = \($eml->as_string);
+run_script(['-mda', '--no-precheck'], $env, { 0 => $in }) or BAIL_OUT '-mda';
 
 ok(run_script([qw(-init -V1 v1test --newsgroup v1.example), "$home/v1test",
         'http://example.com/v1test', $v1addr ]), 'v1test init');
 
 $eml->header_set('List-Id', '<v1.example.com>');
-seek($fh, 0, SEEK_SET) or BAIL_OUT $!;
-truncate($fh, 0) or BAIL_OUT $!;
-print $fh $eml->as_string or BAIL_OUT $!;
-seek($fh, 0, SEEK_SET) or BAIL_OUT $!;
+$in = \$eml->as_string;
 
 $env = { ORIGINAL_RECIPIENT => $v1addr };
-run_script(['-mda', '--no-precheck'], $env, { 0 => $fh }) or BAIL_OUT '-mda';
+run_script(['-mda', '--no-precheck'], $env, { 0 => $in }) or BAIL_OUT '-mda';
 
 run_script([qw(-index -Lbasic), "$home/v1test"]) or BAIL_OUT "index $?";
 
@@ -106,14 +95,11 @@ if ('with boost') {
 }
 
 { # TODO: -extindex should write this to config
-        open $fh, '>>', $cfg_path or BAIL_OUT $!;
-        print $fh <<EOF or BAIL_OUT $!;
+        PublicInbox::IO::write_file '>>', $cfg_path, <<EOF;
 ; for ->ALL
 [extindex "all"]
         topdir = $home/extindex
 EOF
-        close $fh or BAIL_OUT $!;
-
         my $pi_cfg = PublicInbox::Config->new;
         $pi_cfg->fill_all;
         ok($pi_cfg->ALL, '->ALL');
@@ -125,6 +111,8 @@ EOF
 
 SKIP: {
         require_mods(qw(Net::NNTP), 1);
+        my $sock = tcp_server();
+        my $host_port = tcp_host_port($sock);
         my ($out, $err) = ("$home/nntpd.out.log", "$home/nntpd.err.log");
         my $cmd = [ '-nntpd', '-W0', "--stdout=$out", "--stderr=$err" ];
         my $td = start_script($cmd, undef, { 3 => $sock });
@@ -152,7 +140,7 @@ if ('inbox edited') {
         my ($in, $out, $err);
         $in = $out = $err = '';
         my $opt = { 0 => \$in, 1 => \$out, 2 => \$err };
-        my $env = { MAIL_EDITOR => "$^X -i -p -e 's/test message/BEST MSG/'" };
+        my $env = { MAIL_EDITOR => "$^X -w -i -p -e 's/test message/BEST MSG/'" };
         my $cmd = [ qw(-edit -Ft/utf8.eml), "$home/v2test" ];
         ok(run_script($cmd, $env, $opt), '-edit');
         ok(run_script([qw(-extindex --all), "$home/extindex"], undef, $opt),
@@ -203,11 +191,7 @@ if ('inbox edited') {
         is_deeply($res, $exp, 'isearch limited results');
         $pi_cfg = $res = $exp = undef;
 
-        open my $rmfh, '+>', undef or BAIL_OUT $!;
-        $rmfh->autoflush(1);
-        print $rmfh $eml2->as_string or BAIL_OUT $!;
-        seek($rmfh, 0, SEEK_SET) or BAIL_OUT $!;
-        $opt->{0} = $rmfh;
+        $opt->{0} = \($eml2->as_string);
         ok(run_script([qw(-learn rm --all)], undef, $opt), '-learn rm');
 
         ok(run_script([qw(-extindex --all), "$home/extindex"], undef, undef),
@@ -246,13 +230,11 @@ if ('inject w/o indexing') {
         isnt($tip, $cmt, '0.git v2 updated');
 
         # inject a message w/o updating index
-        rename("$home/v1test/public-inbox", "$home/v1test/skip-index") or
-                BAIL_OUT $!;
-        open(my $eh, '<', 't/iso-2202-jp.eml') or BAIL_OUT $!;
+        rename("$home/v1test/public-inbox", "$home/v1test/skip-index");
+        open(my $eh, '<', 't/iso-2202-jp.eml');
         run_script(['-mda', '--no-precheck'], $env, { 0 => $eh}) or
                 BAIL_OUT '-mda';
-        rename("$home/v1test/skip-index", "$home/v1test/public-inbox") or
-                BAIL_OUT $!;
+        rename("$home/v1test/skip-index", "$home/v1test/public-inbox");
 
         my ($in, $out, $err);
         $in = $out = $err = '';
@@ -309,7 +291,7 @@ if ('reindex catches missed messages') {
         is($oidx->eidx_meta($lc_key), $cmt_b, 'lc-v2 stays unchanged');
         my @err = split(/^/, $err);
         is(scalar(@err), 1, 'only one warning') or diag "err=$err";
-        like($err[0], qr/I: reindex_unseen/, 'got reindex_unseen message');
+        like($err[0], qr/# reindex_unseen/, 'got reindex_unseen message');
         my $new = $oidx->get_art($max + 1);
         is($new->{subject}, $eml->header('Subject'), 'new message added');
 
@@ -415,8 +397,8 @@ if ('remove v1test and test gc') {
         my $opt = { 2 => \(my $err = '') };
         ok(run_script([qw(-extindex --gc), "$home/extindex"], undef, $opt),
                 'extindex --gc');
-        like($err, qr/^I: remove #1 v1\.example /ms, 'removed v1 message');
-        is(scalar(grep(!/^I:/, split(/^/m, $err))), 0,
+        like($err, qr/^# remove #1 v1\.example /ms, 'removed v1 message');
+        is(scalar(grep(!/^#/, split(/^/m, $err))), 0,
                 'no non-informational messages');
         $misc->{xdb}->reopen;
         @it = $misc->mset('')->items;
@@ -481,7 +463,7 @@ SKIP: {
         for my $i (2..3) {
                 is(grep(m!/ei[0-9]+/$i\z!, @dirs), 0, "no shard [$i]");
         }
-        skip 'xapian-compact missing', 4 unless have_xapian_compact;
+        have_xapian_compact 1;
         ok(run_script([qw(-compact), $d], undef, $o), 'compact');
         # n.b. stderr contains xapian-compact output
 
@@ -501,10 +483,8 @@ SKIP: {
                 "$home/v2tmp", 'http://example.com/v2tmp', $tmp_addr ])
                 or xbail '-init';
         $env = { ORIGINAL_RECIPIENT => $tmp_addr };
-        open $fh, '+>', undef or xbail "open $!";
-        $fh->autoflush(1);
         my $mid = 'tmpmsg@example.com';
-        print $fh <<EOM or xbail "print $!";
+        my $in = \<<EOM;
 From: b\@z
 To: b\@r
 Message-Id: <$mid>
@@ -512,8 +492,7 @@ Subject: tmpmsg
 Date: Tue, 19 Jan 2038 03:14:07 +0000
 
 EOM
-        seek $fh, 0, SEEK_SET or xbail "seek $!";
-        run_script([qw(-mda --no-precheck)], $env, {0 => $fh}) or xbail '-mda';
+        run_script([qw(-mda --no-precheck)], $env, {0 => $in}) or xbail '-mda';
         ok(run_script([qw(-extindex --all), "$home/extindex"]), 'update');
         my $nr;
         {
@@ -526,7 +505,7 @@ EOM
                 $mset = $es->search->mset('z:0..');
                 $nr = $mset->size;
         }
-        truncate($cfg_path, $old_size) or xbail "truncate $!";
+        truncate($cfg_path, $old_size);
         my $rdr = { 2 => \(my $err) };
         ok(run_script([qw(-extindex --gc), "$home/extindex"], undef, $rdr),
                 'gc to get rid of removed inbox');
@@ -554,4 +533,55 @@ EOM
         is_deeply($x, $o, 'xref3 and over docids match');
 }
 
+{
+        my $d = "$home/eidx-med";
+        ok(run_script([qw(-extindex --dangerous --all -L medium -j3), $d]),
+                'extindex medium init');
+        my $es = PublicInbox::ExtSearch->new($d);
+        is($es->xdb->get_metadata('indexlevel'), 'medium',
+                'es indexlevel before');
+        my @xdb = $es->xdb_shards_flat;
+        is($xdb[0]->get_metadata('indexlevel'), 'medium',
+                '0 indexlevel before');
+        shift @xdb;
+        for (@xdb) {
+                ok(!$_->get_metadata('indexlevel'), 'no indexlevel in >0 shard')
+        }
+        is($es->xdb->get_metadata('indexlevel'), 'medium', 'indexlevel before');
+        ok(run_script([qw(-xcpdb -R5), $d]), 'xcpdb R5');
+        $es = PublicInbox::ExtSearch->new($d);
+        is($es->xdb->get_metadata('indexlevel'), 'medium',
+                '0 indexlevel after');
+        @xdb = $es->xdb_shards_flat;
+        is(scalar(@xdb), 5, 'got 5 shards');
+        is($xdb[0]->get_metadata('indexlevel'), 'medium', '0 indexlevel after');
+        shift @xdb;
+        for (@xdb) {
+                ok(!$_->get_metadata('indexlevel'), 'no indexlevel in >0 shard')
+        }
+}
+
+test_lei(sub {
+        my $d = "$home/extindex";
+        lei_ok('convert', '-o', "$home/md1", $d);
+        lei_ok('convert', '-o', "$home/md2", "extindex:$d");
+        my $dst = [];
+        my $cb = sub { push @$dst, $_[2]->as_string };
+        require PublicInbox::MdirReader;
+        PublicInbox::MdirReader->new->maildir_each_eml("$home/md1", $cb);
+        my @md1 = sort { $a cmp $b } @$dst;
+        ok(scalar(@md1), 'dumped messages to md1');
+        $dst = [];
+        PublicInbox::MdirReader->new->maildir_each_eml("$home/md2", $cb);
+        @$dst = sort { $a cmp $b } @$dst;
+        is_deeply($dst, \@md1,
+                "convert from extindex w/ or w/o `extindex' prefix");
+
+        my @o = glob "$home/extindex/ei*/over.sqlite*";
+        unlink(@o);
+        ok(!lei('convert', '-o', "$home/fail", "extindex:$d"));
+        like($lei_err, qr/unindexed .*?not supported/,
+                'noted unindexed extindex is unsupported');
+});
+
 done_testing;
diff --git a/t/fake_inotify.t b/t/fake_inotify.t
index 734ddbfb..8221e092 100644
--- a/t/fake_inotify.t
+++ b/t/fake_inotify.t
@@ -1,13 +1,12 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Ensure FakeInotify can pick up rename(2) and link(2) operations
 # used by Maildir writing tools
-use strict;
+use v5.12;
 use PublicInbox::TestCommon;
 use_ok 'PublicInbox::FakeInotify';
-my $MIN_FS_TICK = 0.011; # for low-res CONFIG_HZ=100 systems
 my ($tmpdir, $for_destroy) = tmpdir();
 mkdir "$tmpdir/new" or BAIL_OUT "mkdir: $!";
 mkdir "$tmpdir/new/rmd" or BAIL_OUT "mkdir: $!";
@@ -18,37 +17,35 @@ my $fi = PublicInbox::FakeInotify->new;
 my $mask = PublicInbox::FakeInotify::MOVED_TO_OR_CREATE();
 my $w = $fi->watch("$tmpdir/new", $mask);
 
-tick $MIN_FS_TICK;
 rename("$tmpdir/tst", "$tmpdir/new/tst") or BAIL_OUT "rename: $!";
 my @events = map { $_->fullname } $fi->read;
-is_deeply(\@events, ["$tmpdir/new/tst"], 'rename(2) detected');
+is_deeply(\@events, ["$tmpdir/new/tst"], 'rename(2) detected') or
+        diag explain(\@events);
 
-tick $MIN_FS_TICK;
 open $fh, '>', "$tmpdir/tst" or BAIL_OUT "open: $!";
 close $fh or BAIL_OUT "close: $!";
 link("$tmpdir/tst", "$tmpdir/new/link") or BAIL_OUT "link: $!";
 @events = map { $_->fullname } $fi->read;
-is_deeply(\@events, ["$tmpdir/new/link"], 'link(2) detected');
+is_deeply(\@events, ["$tmpdir/new/link"], 'link(2) detected') or
+        diag explain(\@events);
 
 $w->cancel;
-tick $MIN_FS_TICK;
 link("$tmpdir/new/tst", "$tmpdir/new/link2") or BAIL_OUT "link: $!";
 @events = map { $_->fullname } $fi->read;
-is_deeply(\@events, [], 'link(2) not detected after cancel');
+is_deeply(\@events, [], 'link(2) not detected after cancel') or
+        diag explain(\@events);
 $fi->watch("$tmpdir/new", PublicInbox::FakeInotify::IN_DELETE());
 
-tick $MIN_FS_TICK;
 rmdir("$tmpdir/new/rmd") or xbail "rmdir: $!";
 @events = $fi->read;
-is_deeply([map{ $_->fullname }@events], ["$tmpdir/new/rmd"], 'rmdir detected');
-ok($events[0]->IN_DELETE, 'IN_DELETE set on rmdir');
+is_deeply([map{ $_->fullname }@events], ["$tmpdir/new/rmd"], 'rmdir detected') or
+        diag explain(\@events);
+ok($events[-1]->IN_DELETE, 'IN_DELETE set on rmdir');
 
-tick $MIN_FS_TICK;
 unlink("$tmpdir/new/tst") or xbail "unlink: $!";
 @events = grep { ref =~ /Gone/ } $fi->read;
-is_deeply([map{ $_->fullname }@events], ["$tmpdir/new/tst"], 'unlink detected');
+is_deeply([map{ $_->fullname }@events], ["$tmpdir/new/tst"], 'unlink detected') or
+        diag explain(\@events);
 ok($events[0]->IN_DELETE, 'IN_DELETE set on unlink');
 
-PublicInbox::DS->Reset;
-
 done_testing;
diff --git a/t/filter_rubylang.t b/t/filter_rubylang.t
index 4e9695e1..490a2154 100644
--- a/t/filter_rubylang.t
+++ b/t/filter_rubylang.t
@@ -1,8 +1,7 @@
-# Copyright (C) 2017-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use warnings;
-use Test::More;
+use v5.12;
 use PublicInbox::Eml;
 use PublicInbox::TestCommon;
 use_ok 'PublicInbox::Filter::RubyLang';
@@ -56,6 +55,15 @@ EOF
         $mime = PublicInbox::Eml->new($msg);
         $ret = $f->delivery($mime);
         is($ret, 100, "delivery rejected without X-Mail-Count");
+
+        $mime = PublicInbox::Eml->new(<<'EOM');
+Message-ID: <new@host>
+Subject: [ruby-core:13] times
+
+EOM
+        $ret = $f->delivery($mime);
+        is($ret, $mime, "delivery successful");
+        is($mm->num_for('new@host'), 13, 'MM entry created based on Subject');
 }
 
 done_testing();
diff --git a/t/gcf2.t b/t/gcf2.t
index d12a4420..33f3bbca 100644
--- a/t/gcf2.t
+++ b/t/gcf2.t
@@ -10,6 +10,7 @@ use POSIX qw(_exit);
 use Cwd qw(abs_path);
 require_mods('PublicInbox::Gcf2');
 use_ok 'PublicInbox::Gcf2';
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
 use PublicInbox::Import;
 my ($tmpdir, $for_destroy) = tmpdir();
 
@@ -109,7 +110,7 @@ SKIP: {
         for my $blk (1, 0) {
                 my ($r, $w);
                 pipe($r, $w) or BAIL_OUT $!;
-                fcntl($w, 1031, 4096) or
+                fcntl($w, $F_SETPIPE_SZ, 4096) or
                         skip('Linux too old for F_SETPIPE_SZ', 14);
                 $w->blocking($blk);
                 seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!";
@@ -129,7 +130,7 @@ SKIP: {
                 $ck_copying->("pipe blocking($blk)");
 
                 pipe($r, $w) or BAIL_OUT $!;
-                fcntl($w, 1031, 4096) or BAIL_OUT $!;
+                fcntl($w, $F_SETPIPE_SZ, 4096) or BAIL_OUT $!;
                 $w->blocking($blk);
                 close $r;
                 local $SIG{PIPE} = 'IGNORE';
diff --git a/t/gcf2_client.t b/t/gcf2_client.t
index 6d059cad..33ee2c91 100644
--- a/t/gcf2_client.t
+++ b/t/gcf2_client.t
@@ -1,10 +1,10 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
+use v5.12;
 use PublicInbox::TestCommon;
-use Test::More;
 use Cwd qw(getcwd);
+use autodie qw(open close);
 use PublicInbox::Import;
 use PublicInbox::DS;
 
@@ -17,7 +17,7 @@ PublicInbox::Import::init_bare($git_a);
 PublicInbox::Import::init_bare($git_b);
 my $fi_data = './t/git.fast-import-data';
 my $rdr = {};
-open $rdr->{0}, '<', $fi_data or BAIL_OUT $!;
+open $rdr->{0}, '<', $fi_data;
 xsys([qw(git fast-import --quiet)], { GIT_DIR => $git_a }, $rdr);
 is($?, 0, 'fast-import succeeded');
 
@@ -26,9 +26,9 @@ my $called = 0;
 my $err_f = "$tmpdir/err";
 {
         PublicInbox::DS->Reset;
-        open my $err, '>>', $err_f or BAIL_OUT $!;
+        open my $err, '>>', $err_f;
         my $gcf2c = PublicInbox::Gcf2Client::new({ 2 => $err });
-        $gcf2c->gcf2_async(\"$tree $git_a\n", sub {
+        $gcf2c->gcf2_async("$tree $git_a\n", sub {
                 my ($bref, $oid, $type, $size, $arg) = @_;
                 is($oid, $tree, 'got expected OID');
                 is($size, 30, 'got expected length');
@@ -39,12 +39,12 @@ my $err_f = "$tmpdir/err";
         }, 'hi');
         $gcf2c->cat_async_step($gcf2c->{inflight});
 
-        open $err, '<', $err_f or BAIL_OUT $!;
+        open $err, '<', $err_f;
         my $estr = do { local $/; <$err> };
         is($estr, '', 'nothing in stderr');
 
         my $trunc = substr($tree, 0, 39);
-        $gcf2c->gcf2_async(\"$trunc $git_a\n", sub {
+        $gcf2c->gcf2_async("$trunc $git_a\n", sub {
                 my ($bref, $oid, $type, $size, $arg) = @_;
                 is(undef, $bref, 'missing bref is undef');
                 is($oid, $trunc, 'truncated OID printed');
@@ -55,30 +55,30 @@ my $err_f = "$tmpdir/err";
         }, 'bye');
         $gcf2c->cat_async_step($gcf2c->{inflight});
 
-        open $err, '<', $err_f or BAIL_OUT $!;
+        open $err, '<', $err_f;
         $estr = do { local $/; <$err> };
         like($estr, qr/retrying/, 'warned about retry');
 
         # try failed alternates lookup
         PublicInbox::DS->Reset;
-        open $err, '>', $err_f or BAIL_OUT $!;
+        open $err, '>', $err_f;
         $gcf2c = PublicInbox::Gcf2Client::new({ 2 => $err });
-        $gcf2c->gcf2_async(\"$tree $git_b\n", sub {
+        $gcf2c->gcf2_async("$tree $git_b\n", sub {
                 my ($bref, $oid, $type, $size, $arg) = @_;
                 is(undef, $bref, 'missing bref from alt is undef');
                 $called++;
         });
         $gcf2c->cat_async_step($gcf2c->{inflight});
-        open $err, '<', $err_f or BAIL_OUT $!;
+        open $err, '<', $err_f;
         $estr = do { local $/; <$err> };
         like($estr, qr/retrying/, 'warned about retry before alt update');
 
         # now try successful alternates lookup
-        open my $alt, '>>', "$git_b/objects/info/alternates" or BAIL_OUT $!;
-        print $alt "$git_a/objects\n" or BAIL_OUT $!;
-        close $alt or BAIL_OUT;
+        open my $alt, '>>', "$git_b/objects/info/alternates";
+        print $alt "$git_a/objects\n";
+        close $alt;
         my $expect = xqx(['git', "--git-dir=$git_a", qw(cat-file tree), $tree]);
-        $gcf2c->gcf2_async(\"$tree $git_a\n", sub {
+        $gcf2c->gcf2_async("$tree $git_a\n", sub {
                 my ($bref, $oid, $type, $size, $arg) = @_;
                 is($oid, $tree, 'oid match on alternates retry');
                 is($$bref, $expect, 'tree content matched');
diff --git a/t/git.t b/t/git.t
index 56fc8d95..b7df6186 100644
--- a/t/git.t
+++ b/t/git.t
@@ -1,12 +1,14 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use Test::More;
+use v5.12;
 use PublicInbox::TestCommon;
 my ($dir, $for_destroy) = tmpdir();
 use PublicInbox::Import;
 use POSIX qw(strftime);
 use PublicInbox::Git;
+is(PublicInbox::Git::MAX_INFLIGHT,
+        int(PublicInbox::Git::MAX_INFLIGHT), 'MAX_INFLIGHT is an integer');
 
 {
         PublicInbox::Import::init_bare($dir, 'master');
@@ -44,7 +46,7 @@ use PublicInbox::Git;
         my $f = 'HEAD:foo.txt';
         my @x = $gcf->check($f);
         is(scalar @x, 3, 'returned 3 element array for existing file');
-        like($x[0], qr/\A[a-f0-9]{40}\z/, 'returns obj ID in 1st element');
+        like($x[0], qr/\A[a-f0-9]{40,64}\z/, 'returns obj ID in 1st element');
         is('blob', $x[1], 'returns obj type in 2nd element');
         like($x[2], qr/\A\d+\z/, 'returns obj size in 3rd element');
 
@@ -134,7 +136,7 @@ if (1) {
 }
 
 SKIP: {
-        require_git(2.6, 7) or skip('need git 2.6+ for --batch-all-objects', 7);
+        require_git(2.6, 7);
         my ($alt, $alt_obj) = tmpdir();
         my $hash_obj = [ 'git', "--git-dir=$alt", qw(hash-object -w --stdin) ];
         PublicInbox::Import::init_bare($alt);
@@ -202,4 +204,5 @@ is(git_quote($s = "hello\nworld"), '"hello\\nworld"', 'quoted LF');
 is(git_quote($s = "hello\x06world"), '"hello\\006world"', 'quoted \\x06');
 is(git_unquote($s = '"hello\\006world"'), "hello\x06world", 'unquoted \\x06');
 
-done_testing();
+diag 'git_version='.sprintf('%vd', PublicInbox::Git::git_version());
+done_testing;
diff --git a/t/gzip_filter.t b/t/gzip_filter.t
index b349ae58..97eac2d0 100644
--- a/t/gzip_filter.t
+++ b/t/gzip_filter.t
@@ -1,7 +1,7 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use Test::More;
+use v5.12;
 use IO::Handle (); # autoflush
 use Fcntl qw(SEEK_SET);
 use PublicInbox::TestCommon;
@@ -31,7 +31,7 @@ require_ok 'PublicInbox::GzipFilter';
         open my $fh, '<', 'COPYING' or die "open(COPYING): $!";
         my $buf = do { local $/; <$fh> };
         while ($filter->write($buf .= rand)) {}
-        ok($sigpipe, 'got SIGPIPE');
+        ok($sigpipe, 'got SIGPIPE') or diag "\$!=$!";
         close $w;
 }
 done_testing;
diff --git a/t/hl_mod.t b/t/hl_mod.t
index a88f6c03..6ddbb778 100644
--- a/t/hl_mod.t
+++ b/t/hl_mod.t
@@ -1,5 +1,5 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon; use IO::Handle; # ->autoflush
 use Fcntl qw(:seek);
@@ -11,7 +11,7 @@ is($hls->_shebang2lang(\"#!/usr/bin/perl -w\n"), 'perl', 'perl shebang OK');
 is($hls->{-ext2lang}->{'pm'}, 'perl', '.pm suffix OK');
 is($hls->{-ext2lang}->{'pl'}, 'perl', '.pl suffix OK');
 like($hls->_path2lang('Makefile'), qr/\Amake/, 'Makefile OK');
-my $str = do { local $/; open(my $fh, __FILE__); <$fh> };
+my $str = do { local $/; open(my $fh, '<', __FILE__); <$fh> };
 my $orig = $str;
 
 {
diff --git a/t/httpd-corner.psgi b/t/httpd-corner.psgi
index e9a3a6b7..e29fd87b 100644
--- a/t/httpd-corner.psgi
+++ b/t/httpd-corner.psgi
@@ -1,11 +1,25 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # corner case tests for the generic PSGI server
 # Usage: plackup [OPTIONS] /path/to/this/file
-use strict;
-use warnings;
+use v5.12;
 use Plack::Builder;
-require Digest::SHA;
+require PublicInbox::SHA;
+if (defined(my $f = $ENV{TEST_OPEN_FIFO})) {
+        open my $fh, '>', $f or die "open($f): $!";
+        say $fh 'hi';
+        close $fh;
+}
+
+END {
+        if (defined(my $f = $ENV{TEST_EXIT_FIFO})) {
+                open my $fh, '>', $f or die "open($f): $!";
+                say $fh "bye from $$";
+                close $fh;
+        }
+}
+
+my $pi_config = $ENV{PI_CONFIG} // 'unset'; # capture ASAP
 my $app = sub {
         my ($env) = @_;
         my $path = $env->{PATH_INFO};
@@ -15,7 +29,7 @@ my $app = sub {
         my $h = [ 'Content-Type' => 'text/plain' ];
         my $body = [];
         if ($path eq '/sha1') {
-                my $sha1 = Digest::SHA->new('SHA-1');
+                my $sha1 = PublicInbox::SHA->new(1);
                 my $buf;
                 while (1) {
                         my $r = $in->read($buf, 4096);
@@ -78,34 +92,34 @@ my $app = sub {
                 my $rdr = { 2 => fileno($null) };
                 my $cmd = [qw(dd if=/dev/zero count=30 bs=1024k)];
                 my $qsp = PublicInbox::Qspawn->new($cmd, undef, $rdr);
-                return $qsp->psgi_return($env, undef, sub {
+                return $qsp->psgi_yield($env, undef, sub {
                         my ($r, $bref) = @_;
                         # make $rd_hdr retry sysread + $parse_hdr in Qspawn:
                         return until length($$bref) > 8000;
                         close $null;
                         [ 200, [ qw(Content-Type application/octet-stream) ]];
                 });
-        } elsif ($path eq '/psgi-return-gzip') {
+        } elsif ($path eq '/psgi-yield-gzip') {
                 require PublicInbox::Qspawn;
                 require PublicInbox::GzipFilter;
                 my $cmd = [qw(echo hello world)];
                 my $qsp = PublicInbox::Qspawn->new($cmd);
                 $env->{'qspawn.filter'} = PublicInbox::GzipFilter->new;
-                return $qsp->psgi_return($env, undef, sub {
+                return $qsp->psgi_yield($env, undef, sub {
                         [ 200, [ qw(Content-Type application/octet-stream)]]
                 });
-        } elsif ($path eq '/psgi-return-compressible') {
+        } elsif ($path eq '/psgi-yield-compressible') {
                 require PublicInbox::Qspawn;
                 my $cmd = [qw(echo goodbye world)];
                 my $qsp = PublicInbox::Qspawn->new($cmd);
-                return $qsp->psgi_return($env, undef, sub {
+                return $qsp->psgi_yield($env, undef, sub {
                         [200, [qw(Content-Type text/plain)]]
                 });
-        } elsif ($path eq '/psgi-return-enoent') {
+        } elsif ($path eq '/psgi-yield-enoent') {
                 require PublicInbox::Qspawn;
                 my $cmd = [ 'this-better-not-exist-in-PATH'.rand ];
                 my $qsp = PublicInbox::Qspawn->new($cmd);
-                return $qsp->psgi_return($env, undef, sub {
+                return $qsp->psgi_yield($env, undef, sub {
                         [ 200, [ qw(Content-Type application/octet-stream)]]
                 });
         } elsif ($path eq '/pid') {
@@ -114,6 +128,13 @@ my $app = sub {
         } elsif ($path eq '/url_scheme') {
                 $code = 200;
                 push @$body, $env->{'psgi.url_scheme'}
+        } elsif ($path eq '/PI_CONFIG') {
+                $code = 200;
+                push @$body, $pi_config; # show value at ->refresh_groups
+        } elsif ($path =~ m!\A/exit-fifo(.+)\z!) {
+                $code = 200;
+                $ENV{TEST_EXIT_FIFO} = $1; # for END {}
+                push @$body, "fifo $1 registered";
         }
         [ $code, $h, $body ]
 };
diff --git a/t/httpd-corner.t b/t/httpd-corner.t
index 0a613a9e..7539573c 100644
--- a/t/httpd-corner.t
+++ b/t/httpd-corner.t
@@ -1,12 +1,14 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # note: our HTTP server should be standalone and capable of running
 # generic PSGI/Plack apps.
-use strict; use v5.10.1; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
 use Time::HiRes qw(gettimeofday tv_interval);
+use autodie qw(getsockopt setsockopt);
 use PublicInbox::Spawn qw(spawn popen_rd);
 require_mods(qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status));
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
 use IO::Handle ();
 use IO::Socket::UNIX;
 use Fcntl qw(:seek);
@@ -19,26 +21,27 @@ ok(defined mkfifo($fifo, 0777), 'created FIFO');
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
 my $psgi = "./t/httpd-corner.psgi";
-my $sock = tcp_server() or die;
+my $sock = tcp_server();
 my @zmods = qw(PublicInbox::GzipFilter IO::Uncompress::Gunzip);
 
 # Make sure we don't clobber socket options set by systemd or similar
 # using socket activation:
 my ($defer_accept_val, $accf_arg, $TCP_DEFER_ACCEPT);
-if ($^O eq 'linux') {
+SKIP: {
+        skip 'TCP_DEFER_ACCEPT is Linux-only', 1 if $^O ne 'linux';
         $TCP_DEFER_ACCEPT = eval { Socket::TCP_DEFER_ACCEPT() } // 9;
-        setsockopt($sock, IPPROTO_TCP, $TCP_DEFER_ACCEPT, 5) or die;
+        setsockopt($sock, IPPROTO_TCP, $TCP_DEFER_ACCEPT, 5);
         my $x = getsockopt($sock, IPPROTO_TCP, $TCP_DEFER_ACCEPT);
-        defined $x or die "getsockopt: $!";
         $defer_accept_val = unpack('i', $x);
-        if ($defer_accept_val <= 0) {
-                die "unexpected TCP_DEFER_ACCEPT value: $defer_accept_val";
-        }
-} elsif ($^O eq 'freebsd' && system('kldstat -m accf_data >/dev/null') == 0) {
+        ok($defer_accept_val > 0, 'TCP_DEFER_ACCEPT val non-zero') or
+                xbail "unexpected TCP_DEFER_ACCEPT value: $defer_accept_val";
+}
+SKIP: {
+        require_mods '+accf_data';
         require PublicInbox::Daemon;
         my $var = $PublicInbox::Daemon::SO_ACCEPTFILTER;
         $accf_arg = pack('a16a240', 'dataready', '');
-        setsockopt($sock, SOL_SOCKET, $var, $accf_arg) or die "setsockopt: $!";
+        setsockopt($sock, SOL_SOCKET, $var, $accf_arg);
 }
 
 sub unix_server ($) {
@@ -53,14 +56,40 @@ sub unix_server ($) {
 
 my $upath = "$tmpdir/s";
 my $unix = unix_server($upath);
+my $alt = tcp_server();
 my $td;
 my $spawn_httpd = sub {
         my (@args) = @_;
-        my $cmd = [ '-httpd', @args, "--stdout=$out", "--stderr=$err", $psgi ];
-        $td = start_script($cmd, undef, { 3 => $sock, 4 => $unix });
+        my $x = tcp_host_port($alt);
+        my $cmd = [ '-httpd', @args, "--stdout=$out", "--stderr=$err", $psgi,
+                '-l', "http://$x/?psgi=t/alt.psgi,env.PI_CONFIG=/path/to/alt".
+                        ",err=$tmpdir/alt.err" ];
+        my $env = { PI_CONFIG => '/dev/null' };
+        $td = start_script($cmd, $env, { 3 => $sock, 4 => $unix, 5 => $alt });
 };
 
 $spawn_httpd->();
+{
+        my $conn = conn_for($alt, 'alt PSGI path');
+        $conn->write("GET / HTTP/1.0\r\n\r\n");
+        $conn->read(my $buf, 4096);
+        like($buf, qr!^/path/to/alt\z!sm,
+                'alt.psgi loaded on alt socket with correct env');
+
+        $conn = conn_for($sock, 'default PSGI path');
+        $conn->write("GET /PI_CONFIG HTTP/1.0\r\n\r\n");
+        $conn->read($buf, 4096);
+        like($buf, qr!^/dev/null\z!sm,
+                'default PSGI on original socket');
+        my $log = capture("$tmpdir/alt.err");
+        ok(grep(/ALT/, @$log), 'alt psgi.errors written to');
+        $log = capture($err);
+        ok(!grep(/ALT/, @$log), 'STDERR not written to');
+        is(unlink($err, "$tmpdir/alt.err"), 2, 'unlinked stderr and alt.err');
+
+        $td->kill('USR1'); # trigger reopen_logs
+}
+
 if ('test worker death') {
         my $conn = conn_for($sock, 'killed worker');
         $conn->write("GET /pid HTTP/1.1\r\nHost:example.com\r\n\r\n");
@@ -82,6 +111,10 @@ if ('test worker death') {
         like($body, qr/\A[0-9]+\z/, '/pid response');
         isnt($body, $pid, 'respawned worker');
 }
+{ # check on prior USR1 signal
+        ok(-e $err, 'stderr recreated after USR1');
+        ok(-e "$tmpdir/alt.err", 'alt.err recreated after USR1');
+}
 {
         my $conn = conn_for($sock, 'Header spaces bogus');
         $conn->write("GET /empty HTTP/1.1\r\nSpaced-Out : 3\r\n\r\n");
@@ -289,7 +322,7 @@ sub conn_for {
         $spawn_httpd->('-W0');
 }
 
-sub delay { select(undef, undef, undef, shift || rand(0.02)) }
+sub delay { tick(shift || rand(0.02)) }
 
 my $str = 'abcdefghijklmnopqrstuvwxyz';
 my $len = length $str;
@@ -310,7 +343,7 @@ SKIP: {
         my $url = "$base/sha1";
         my ($r, $w);
         pipe($r, $w) or die "pipe: $!";
-        my $cmd = [$curl, qw(--tcp-nodelay -T- -HExpect: -sSN), $url];
+        my $cmd = [$curl, qw(--tcp-nodelay -T- -HExpect: -gsSN), $url];
         open my $cout, '+>', undef or die;
         open my $cerr, '>', undef or die;
         my $rdr = { 0 => $r, 1 => $cout, 2 => $cerr };
@@ -327,7 +360,7 @@ SKIP: {
         seek($cout, 0, SEEK_SET);
         is(<$cout>, sha1_hex($str), 'read expected body');
 
-        my $fh = popen_rd([$curl, '-sS', "$base/async-big"]);
+        my $fh = popen_rd([$curl, '-gsS', "$base/async-big"]);
         my $n = 0;
         my $non_zero = 0;
         while (1) {
@@ -335,19 +368,19 @@ SKIP: {
                 $n += $r;
                 $buf =~ /\A\0+\z/ or $non_zero++;
         }
-        close $fh or die "close curl pipe: $!";
+        $fh->close or die "close curl pipe: $!";
         is($?, 0, 'curl succesful');
         is($n, 30 * 1024 * 1024, 'got expected output from curl');
         is($non_zero, 0, 'read all zeros');
 
         require_mods(@zmods, 4);
-        my $buf = xqx([$curl, '-sS', "$base/psgi-return-gzip"]);
+        my $buf = xqx([$curl, '-gsS', "$base/psgi-yield-gzip"]);
         is($?, 0, 'curl succesful');
         IO::Uncompress::Gunzip::gunzip(\$buf => \(my $out));
         is($out, "hello world\n");
         my $curl_rdr = { 2 => \(my $curl_err = '') };
-        $buf = xqx([$curl, qw(-sSv --compressed),
-                        "$base/psgi-return-compressible"], undef, $curl_rdr);
+        $buf = xqx([$curl, qw(-gsSv --compressed),
+                        "$base/psgi-yield-compressible"], undef, $curl_rdr);
         is($?, 0, 'curl --compressed successful');
         is($buf, "goodbye world\n", 'gzipped response as expected');
         like($curl_err, qr/\bContent-Encoding: gzip\b/,
@@ -355,8 +388,8 @@ SKIP: {
 }
 
 {
-        my $conn = conn_for($sock, 'psgi_return ENOENT');
-        print $conn "GET /psgi-return-enoent HTTP/1.1\r\n\r\n" or die;
+        my $conn = conn_for($sock, 'psgi_yield ENOENT');
+        print $conn "GET /psgi-yield-enoent HTTP/1.1\r\n\r\n" or die;
         my $buf = '';
         sysread($conn, $buf, 16384, length($buf)) until $buf =~ /\r\n\r\n/;
         like($buf, qr!HTTP/1\.[01] 500\b!, 'got 500 error on ENOENT');
@@ -594,43 +627,33 @@ SKIP: {
 SKIP: {
         skip 'TCP_DEFER_ACCEPT is Linux-only', 1 if $^O ne 'linux';
         my $var = $TCP_DEFER_ACCEPT;
-        defined(my $x = getsockopt($sock, IPPROTO_TCP, $var)) or die;
+        my $x = getsockopt($sock, IPPROTO_TCP, $var);
         is(unpack('i', $x), $defer_accept_val,
                 'TCP_DEFER_ACCEPT unchanged if previously set');
 };
 SKIP: {
-        skip 'SO_ACCEPTFILTER is FreeBSD-only', 1 if $^O ne 'freebsd';
-        skip 'accf_data not loaded: kldload accf_data' if !defined $accf_arg;
+        require_mods '+accf_data';
         my $var = $PublicInbox::Daemon::SO_ACCEPTFILTER;
-        defined(my $x = getsockopt($sock, SOL_SOCKET, $var)) or die;
+        my $x = getsockopt($sock, SOL_SOCKET, $var);
         is($x, $accf_arg, 'SO_ACCEPTFILTER unchanged if previously set');
 };
 
 SKIP: {
-        skip 'only testing lsof(8) output on Linux', 1 if $^O ne 'linux';
-        my $lsof = require_cmd('lsof', 1) or skip 'no lsof in PATH', 1;
-        my $null_in = '';
-        my $rdr = { 2 => \(my $null_err), 0 => \$null_in };
-        my @lsof = xqx([$lsof, '-p', $td->{pid}], undef, $rdr);
-        my $d = [ grep(/\(deleted\)/, @lsof) ];
-        is_deeply($d, [], 'no lingering deleted inputs') or diag explain($d);
+        skip 'only testing /proc/PID/fd on Linux', 1 if $^O ne 'linux';
+        my $fd_dir = "/proc/$td->{pid}/fd";
+        -d $fd_dir or skip '/proc/$PID/fd missing', 1;
+        my @child = grep defined, map readlink, glob "$fd_dir/*";
+        my @d = grep /\(deleted\)/, @child;
+        is_deeply(\@d, [], 'no lingering deleted inputs') or diag explain(\@d);
 
         # filter out pipes inherited from the parent
-        my @this = xqx([$lsof, '-p', $$], undef, $rdr);
-        my $bad;
-        my $extract_inodes = sub {
-                map {;
-                        my @f = split(' ', $_);
-                        my $inode = $f[-2];
-                        $bad = $_ if $inode !~ /\A[0-9]+\z/;
-                        $inode => 1;
-                } grep (/\bpipe\b/, @_);
-        };
-        my %child = $extract_inodes->(@lsof);
+        my @this = grep defined, map readlink, glob "/proc/$$/fd/*";
+        my $extract_inodes = sub { map { $_ => 1 } grep /\bpipe\b/, @_ };
+        my %child = $extract_inodes->(@child);
         my %parent = $extract_inodes->(@this);
-        skip("inode not in expected format: $bad", 1) if defined($bad);
         delete @child{(keys %parent)};
-        is_deeply([], [keys %child], 'no extra pipes with -W0');
+        is_deeply([], [keys %child], 'no extra pipes with -W0') or
+                diag explain([child => \%child, parent => \%parent]);
 };
 
 # ensure compatibility with other PSGI servers
@@ -646,13 +669,13 @@ SKIP: {
         my $app = require $psgi;
         test_psgi($app, sub {
                 my ($cb) = @_;
-                my $req = GET('http://example.com/psgi-return-gzip');
+                my $req = GET('http://example.com/psgi-yield-gzip');
                 my $res = $cb->($req);
                 my $buf = $res->content;
                 IO::Uncompress::Gunzip::gunzip(\$buf => \(my $out));
                 is($out, "hello world\n", 'got expected output');
 
-                $req = GET('http://example.com/psgi-return-enoent');
+                $req = GET('http://example.com/psgi-yield-enoent');
                 $res = $cb->($req);
                 is($res->code, 500, 'got error on ENOENT');
                 seek($tmperr, 0, SEEK_SET) or die;
diff --git a/t/httpd-https.t b/t/httpd-https.t
index d42d7c50..bf086123 100644
--- a/t/httpd-https.t
+++ b/t/httpd-https.t
@@ -1,15 +1,15 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use warnings;
-use Test::More;
+use v5.12;
 use Socket qw(SOCK_STREAM IPPROTO_TCP SOL_SOCKET);
 use PublicInbox::TestCommon;
+use File::Copy qw(cp);
 # IO::Poll is part of the standard library, but distros may split them off...
 require_mods(qw(IO::Socket::SSL IO::Poll Plack::Util));
-my $cert = 'certs/server-cert.pem';
-my $key = 'certs/server-key.pem';
-unless (-r $key && -r $cert) {
+my @certs = qw(certs/server-cert.pem certs/server-key.pem
+        certs/server2-cert.pem certs/server2-key.pem);
+if (scalar(grep { -r $_ } @certs) != scalar(@certs)) {
         plan skip_all =>
                 "certs/ missing for $0, run $^X ./create-certs.perl in certs/";
 }
@@ -22,6 +22,20 @@ my $out = "$tmpdir/stdout.log";
 my $https = tcp_server();
 my $td;
 my $https_addr = tcp_host_port($https);
+my $cert = "$tmpdir/cert.pem";
+my $key = "$tmpdir/key.pem";
+cp('certs/server-cert.pem', $cert) or xbail $!;
+cp('certs/server-key.pem', $key) or xbail $!;
+
+my $check_url_scheme = sub {
+        my ($s, $line) = @_;
+        $s->print("GET /url_scheme HTTP/1.1\r\n\r\nHost: example.com\r\n\r\n")
+                or xbail "failed to write HTTP request: $! (line $line)";
+        my $buf = '';
+        sysread($s, $buf, 2007, length($buf)) until $buf =~ /\r\n\r\nhttps?/;
+        like($buf, qr!\AHTTP/1\.1 200!, "read HTTPS response (line $line)");
+        like($buf, qr!\r\nhttps\z!, "psgi.url_scheme is 'https' (line $line)");
+};
 
 for my $args (
         [ "-lhttps://$https_addr/?key=$key,cert=$cert" ],
@@ -53,12 +67,7 @@ for my $args (
         # normal HTTPS
         my $c = tcp_connect($https);
         IO::Socket::SSL->start_SSL($c, %o);
-        $c->print("GET /url_scheme HTTP/1.1\r\n\r\nHost: example.com\r\n\r\n")
-                or xbail "failed to write HTTP request: $!";
-        my $buf = '';
-        sysread($c, $buf, 2007, length($buf)) until $buf =~ /\r\n\r\nhttps?/;
-        like($buf, qr!\AHTTP/1\.1 200!, 'read HTTP response');
-        like($buf, qr!\r\nhttps\z!, "psgi.url_scheme is 'https'");
+        $check_url_scheme->($c, __LINE__);
 
         # HTTPS with bad hostname
         $c = tcp_connect($https);
@@ -81,7 +90,7 @@ for my $args (
         $slow->blocking(1);
         ok($slow->print("GET /empty HTTP/1.1\r\n\r\nHost: example.com\r\n\r\n"),
                 'wrote HTTP request from slow');
-        $buf = '';
+        my $buf = '';
         sysread($slow, $buf, 666, length($buf)) until $buf =~ /\r\n\r\n/;
         like($buf, qr!\AHTTP/1\.1 200!, 'read HTTP response from slow');
         $slow = undef;
@@ -93,10 +102,7 @@ for my $args (
                 ok(unpack('i', $x) > 0, 'TCP_DEFER_ACCEPT set on https');
         };
         SKIP: {
-                skip 'SO_ACCEPTFILTER is FreeBSD-only', 2 if $^O ne 'freebsd';
-                if (system('kldstat -m accf_data >/dev/null')) {
-                        skip 'accf_data not loaded? kldload accf_data', 2;
-                }
+                require_mods '+accf_data';
                 require PublicInbox::Daemon;
                 ok(defined($PublicInbox::Daemon::SO_ACCEPTFILTER),
                         'SO_ACCEPTFILTER defined');
@@ -105,7 +111,27 @@ for my $args (
                 like($x, qr/\Adataready\0+\z/, 'got dataready accf for https');
         };
 
-        $c = undef;
+        # switch cert and key:
+        cp('certs/server2-cert.pem', $cert) or xbail $!;
+        cp('certs/server2-key.pem', $key) or xbail $!;
+        $td->kill('HUP') or xbail "kill: $!";
+        tick(); # wait for SIGHUP to take effect (hopefully :x)
+
+        my $d = tcp_connect($https);
+        $d = IO::Socket::SSL->start_SSL($d, %o);
+        is($d, undef, 'HTTPS fails with bad hostname after new cert on HUP');
+
+        $d = tcp_connect($https);
+        $o{SSL_hostname} = $o{SSL_verifycn_name} = 'server2.local';
+        is(IO::Socket::SSL->start_SSL($d, %o), $d,
+                'new hostname to match cert works after HUP');
+        $check_url_scheme->($d, __LINE__);
+
+        # existing connection w/ old cert still works:
+        $check_url_scheme->($c, __LINE__);
+
+        undef $c;
+        undef $d;
         $td->kill;
         $td->join;
         is($?, 0, 'no error in exited process');
diff --git a/t/httpd-unix.t b/t/httpd-unix.t
index fe4a2161..0b620bd6 100644
--- a/t/httpd-unix.t
+++ b/t/httpd-unix.t
@@ -1,15 +1,17 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Tests for binding Unix domain sockets
-use strict;
-use warnings;
-use Test::More;
+use v5.12;
 use PublicInbox::TestCommon;
 use Errno qw(EADDRINUSE);
 use Cwd qw(abs_path);
 use Carp qw(croak);
+use autodie qw(close);
 require_mods(qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status));
 use IO::Socket::UNIX;
+use POSIX qw(mkfifo);
+require PublicInbox::Sigfd;
 my ($tmpdir, $for_destroy) = tmpdir();
 my $unix = "$tmpdir/unix.sock";
 my $psgi = './t/httpd-corner.psgi';
@@ -17,6 +19,15 @@ my $out = "$tmpdir/out.log";
 my $err = "$tmpdir/err.log";
 my $td;
 
+my $register_exit_fifo = sub {
+        my ($s, $f) = @_;
+        my $sock = new_sock($s);
+        ok($sock->write("GET /exit-fifo$f HTTP/1.0\r\n\r\n"),
+                'request exit-fifo');
+        ok($sock->read(my $buf, 4096), 'read exit-fifo response');
+        like($buf, qr!\r\n\r\nfifo \Q$f\E registered\z!, 'set exit fifo');
+};
+
 my $spawn_httpd = sub {
         my (@args) = @_;
         my $cmd = [ '-httpd', @args, "--stdout=$out", "--stderr=$err", $psgi ];
@@ -32,18 +43,24 @@ my $spawn_httpd = sub {
 }
 
 ok(!-S $unix, 'UNIX socket does not exist, yet');
-$spawn_httpd->("-l$unix", '-W0');
-my %o = (Peer => $unix, Type => SOCK_STREAM);
-for (1..1000) {
-        last if -S $unix && IO::Socket::UNIX->new(%o);
-        select undef, undef, undef, 0.02
+my $f1 = "$tmpdir/f1";
+mkfifo($f1, 0600);
+{
+        local $ENV{TEST_OPEN_FIFO} = $f1;
+        $spawn_httpd->("-l$unix", '-W0');
+        open my $fh, '<', $f1 or xbail "open($f1): $!";
+        is(my $hi = <$fh>, "hi\n", 'got FIFO greeting');
 }
-
 ok(-S $unix, 'UNIX socket was bound by -httpd');
+
+sub new_sock ($) {
+        IO::Socket::UNIX->new(Peer => $_[0], Type => SOCK_STREAM)
+                // xbail "E: $! connecting to $_[0]";
+}
+
 sub check_sock ($) {
         my ($unix) = @_;
-        my $sock = IO::Socket::UNIX->new(Peer => $unix, Type => SOCK_STREAM)
-                // BAIL_OUT "E: $! connecting to $unix";
+        my $sock = new_sock($unix);
         ok($sock->write("GET /host-port HTTP/1.0\r\n\r\n"),
                 'wrote req to server');
         ok($sock->read(my $buf, 4096), 'read response');
@@ -82,16 +99,17 @@ check_sock($unix);
 
 # portable Perl can delay or miss signal dispatches due to races,
 # so disable some tests on systems lacking signalfd(2) or EVFILT_SIGNAL
-my $has_sigfd = PublicInbox::Sigfd->new({}, 0) ? 1 : $ENV{TEST_UNRELIABLE};
+my $has_sigfd = PublicInbox::Sigfd->new({}) ? 1 : $ENV{TEST_UNRELIABLE};
+PublicInbox::DS::Reset() if $has_sigfd;
 
 sub delay_until {
-        my $cond = shift;
+        my ($cond, $msg) = @_;
         my $end = time + 30;
         do {
                 return if $cond->();
-                select undef, undef, undef, 0.012;
+                tick(0.012);
         } until (time > $end);
-        Carp::confess('condition failed');
+        Carp::confess($msg // 'condition failed');
 }
 
 SKIP: {
@@ -107,95 +125,110 @@ SKIP: {
         };
 
         for my $w (qw(-W0 -W1)) {
+                my ($p0, $p1) = quit_waiter_pipe;
                 # wait for daemonization
                 $spawn_httpd->("-l$unix", '-D', '-P', $pid_file, $w);
+                close $p1;
                 $td->join;
                 is($?, 0, "daemonized $w process");
                 check_sock($unix);
                 ok(-s $pid_file, "$w pid file written");
                 my $pid = $read_pid->($pid_file);
+                no_pollerfd($pid) if $w eq '-W1';
                 is(kill('TERM', $pid), 1, "signaled daemonized $w process");
-                delay_until(sub { !kill(0, $pid) });
-                is(kill(0, $pid), 0, "daemonized $w process exited");
+                delete $td->{-extra}; # drop tail(1) process
+                wait_for_eof($p0, "httpd $w quit pipe");
                 ok(!-e $pid_file, "$w pid file unlinked at exit");
         }
 
-        # try a USR2 upgrade with workers:
         my $httpd = abs_path('blib/script/public-inbox-httpd');
         $psgi = abs_path($psgi);
         my $opt = { run_mode => 0 };
-
         my @args = ("-l$unix", '-D', '-P', $pid_file, -1, $out, -2, $err);
-        $td = start_script([$httpd, @args, $psgi], undef, $opt);
-        $td->join;
-        is($?, 0, "daemonized process again");
-        check_sock($unix);
-        ok(-s $pid_file, 'pid file written');
-        my $pid = $read_pid->($pid_file);
-
-        # stop worker to ensure check_sock below hits $new_pid
-        kill('TTOU', $pid) or die "TTOU failed: $!";
-
-        kill('USR2', $pid) or die "USR2 failed: $!";
-        delay_until(sub {
-                $pid != (eval { $read_pid->($pid_file) } // $pid)
-        });
-        my $new_pid = $read_pid->($pid_file);
-        isnt($new_pid, $pid, 'new child started');
-        ok($new_pid > 0, '$new_pid valid');
-        delay_until(sub { -s "$pid_file.oldbin" });
-        my $old_pid = $read_pid->("$pid_file.oldbin");
-        is($old_pid, $pid, '.oldbin pid file written');
-        ok($old_pid > 0, '$old_pid valid');
-
-        check_sock($unix); # ensures $new_pid is ready to receive signals
-
-        # first, back out of the upgrade
-        kill('QUIT', $new_pid) or die "kill new PID failed: $!";
-        delay_until(sub {
-                $pid == (eval { $read_pid->($pid_file) } // 0)
-        });
-        is($read_pid->($pid_file), $pid, 'old PID file restored');
-        ok(!-f "$pid_file.oldbin", '.oldbin PID file gone');
-
-        # retry USR2 upgrade
-        kill('USR2', $pid) or die "USR2 failed: $!";
-        delay_until(sub {
-                $pid != (eval { $read_pid->($pid_file) } // $pid)
-        });
-        $new_pid = $read_pid->($pid_file);
-        isnt($new_pid, $pid, 'new child started again');
-        $old_pid = $read_pid->("$pid_file.oldbin");
-        is($old_pid, $pid, '.oldbin pid file written');
-
-        # drop the old parent
-        kill('QUIT', $old_pid) or die "QUIT failed: $!";
-        delay_until(sub { !kill(0, $old_pid) });
-        ok(!-f "$pid_file.oldbin", '.oldbin PID file gone');
-
-        # drop the new child
-        check_sock($unix);
-        kill('QUIT', $new_pid) or die "QUIT failed: $!";
-        delay_until(sub { !kill(0, $new_pid) });
-        ok(!-f $pid_file, 'PID file is gone');
-
-
-        # try USR2 without workers (-W0)
-        $td = start_script([$httpd, @args, '-W0', $psgi], undef, $opt);
-        $td->join;
-        is($?, 0, 'daemonized w/o workers');
-        check_sock($unix);
-        $pid = $read_pid->($pid_file);
-
-        # replace running process
-        kill('USR2', $pid) or die "USR2 failed: $!";
-        delay_until(sub { !kill(0, $pid) });
-
-        check_sock($unix);
-        $pid = $read_pid->($pid_file);
-        kill('QUIT', $pid) or die "USR2 failed: $!";
-        delay_until(sub { !kill(0, $pid) });
-        ok(!-f $pid_file, 'PID file is gone');
+
+        if ('USR2 upgrades with workers') {
+                my ($p0, $p1) = quit_waiter_pipe;
+                $td = start_script([$httpd, @args, $psgi], undef, $opt);
+                close $p1;
+                $td->join;
+                is($?, 0, "daemonized process again");
+                check_sock($unix);
+                ok(-s $pid_file, 'pid file written');
+                my $pid = $read_pid->($pid_file);
+
+                # stop worker to ensure check_sock below hits $new_pid
+                kill('TTOU', $pid) or die "TTOU failed: $!";
+
+                kill('USR2', $pid) or die "USR2 failed: $!";
+                delay_until(sub {
+                        $pid != (eval { $read_pid->($pid_file) } // $pid)
+                });
+                my $new_pid = $read_pid->($pid_file);
+                isnt($new_pid, $pid, 'new child started');
+                ok($new_pid > 0, '$new_pid valid');
+                delay_until(sub { -s "$pid_file.oldbin" });
+                my $old_pid = $read_pid->("$pid_file.oldbin");
+                is($old_pid, $pid, '.oldbin pid file written');
+                ok($old_pid > 0, '$old_pid valid');
+
+                check_sock($unix); # ensures $new_pid is ready to receive signals
+
+                # first, back out of the upgrade
+                kill('QUIT', $new_pid) or die "kill new PID failed: $!";
+                delay_until(sub {
+                        $pid == (eval { $read_pid->($pid_file) } // 0)
+                });
+
+                delay_until(sub { !kill(0, $new_pid) }, 'new PID really died');
+
+                is($read_pid->($pid_file), $pid, 'old PID file restored');
+                ok(!-f "$pid_file.oldbin", '.oldbin PID file gone');
+
+                # retry USR2 upgrade
+                kill('USR2', $pid) or die "USR2 failed: $!";
+                delay_until(sub {
+                        $pid != (eval { $read_pid->($pid_file) } // $pid)
+                });
+                $new_pid = $read_pid->($pid_file);
+                isnt($new_pid, $pid, 'new child started again');
+                $old_pid = $read_pid->("$pid_file.oldbin");
+                is($old_pid, $pid, '.oldbin pid file written');
+
+                # drop the old parent
+                kill('QUIT', $old_pid) or die "QUIT failed: $!";
+                delay_until(sub { !kill(0, $old_pid) }, 'old PID really died');
+
+                ok(!-f "$pid_file.oldbin", '.oldbin PID file gone');
+
+                # drop the new child
+                check_sock($unix);
+                kill('QUIT', $new_pid) or die "QUIT failed: $!";
+
+                wait_for_eof($p0, 'new process');
+                ok(!-f $pid_file, 'PID file is gone');
+        }
+
+        if ('try USR2 without workers (-W0)') {
+                my ($p0, $p1) = quit_waiter_pipe;
+                $td = start_script([$httpd, @args, '-W0', $psgi], undef, $opt);
+                close $p1;
+                $td->join;
+                is($?, 0, 'daemonized w/o workers');
+                $register_exit_fifo->($unix, $f1);
+                my $pid = $read_pid->($pid_file);
+
+                # replace running process
+                kill('USR2', $pid) or xbail "USR2 failed: $!";
+                open my $fh, '<', $f1 or xbail "open($f1): $!";
+                is(my $bye = <$fh>, "bye from $pid\n", 'got FIFO bye');
+
+                check_sock($unix);
+                $pid = $read_pid->($pid_file);
+                kill('QUIT', $pid) or xbail "USR2 failed: $!";
+
+                wait_for_eof($p0, '-W0 USR2 test pipe');
+                ok(!-f $pid_file, 'PID file is gone');
+        }
 }
 
 done_testing();
diff --git a/t/httpd.t b/t/httpd.t
index aa601210..c0fbaa22 100644
--- a/t/httpd.t
+++ b/t/httpd.t
@@ -7,6 +7,7 @@ use PublicInbox::TestCommon;
 use PublicInbox::Eml;
 use Socket qw(IPPROTO_TCP SOL_SOCKET);
 require_mods(qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status));
+require_git_http_backend;
 
 # FIXME: too much setup
 my ($tmpdir, $for_destroy) = tmpdir();
@@ -104,10 +105,7 @@ SKIP: {
         ok(unpack('i', $x) > 0, 'TCP_DEFER_ACCEPT set');
 };
 SKIP: {
-        skip 'SO_ACCEPTFILTER is FreeBSD-only', 1 if $^O ne 'freebsd';
-        if (system('kldstat -m accf_http >/dev/null') != 0) {
-                skip 'accf_http not loaded: kldload accf_http', 1;
-        }
+        require_mods '+accf_http';
         require PublicInbox::Daemon;
         ok(defined($PublicInbox::Daemon::SO_ACCEPTFILTER),
                 'SO_ACCEPTFILTER defined');
diff --git a/t/imap.t b/t/imap.t
index e6efe04f..44b8ef2a 100644
--- a/t/imap.t
+++ b/t/imap.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # unit tests (no network) for IMAP, see t/imapd.t for end-to-end tests
 use strict;
@@ -9,12 +9,12 @@ require_git 2.6;
 require_mods(qw(-imapd));
 require_ok 'PublicInbox::IMAP';
 require_ok 'PublicInbox::IMAPD';
+use PublicInbox::IO qw(write_file);
 
 my ($tmpdir, $for_destroy) = tmpdir();
 my $cfgfile = "$tmpdir/config";
 {
-        open my $fh, '>', $cfgfile or BAIL_OUT $!;
-        print $fh <<EOF or BAIL_OUT $!;
+        write_file '>', $cfgfile, <<EOF;
 [publicinbox "a"]
         inboxdir = $tmpdir/a
         newsgroup = x.y.z
@@ -23,9 +23,8 @@ my $cfgfile = "$tmpdir/config";
         newsgroup = x.z.y
 [publicinbox "c"]
         inboxdir = $tmpdir/c
-        newsgroup = IGNORE.THIS
+        newsgroup = ignore.this.9
 EOF
-        close $fh or BAIL_OUT $!;
         local $ENV{PI_CONFIG} = $cfgfile;
         for my $x (qw(a b c)) {
                 ok(run_script(['-init', '-Lbasic', '-V2', $x, "$tmpdir/$x",
@@ -37,8 +36,8 @@ EOF
         local $SIG{__WARN__} = sub { push @w, @_ };
         $imapd->refresh_groups;
         my $self = { imapd => $imapd };
-        is(scalar(@w), 1, 'got a warning for upper-case');
-        like($w[0], qr/IGNORE\.THIS/, 'warned about upper-case');
+        is(scalar(@w), 1, 'got a warning for slice-like name');
+        like($w[0], qr/ignore\.this\.9/i, 'warned about slice-like name');
         my $res = PublicInbox::IMAP::cmd_list($self, 'tag', 'x', '%');
         is(scalar($$res =~ tr/\n/\n/), 2, 'only one result');
         like($$res, qr/ x\r\ntag OK/, 'saw expected');
diff --git a/t/imap_searchqp.t b/t/imap_searchqp.t
index e2f49e5a..ff1b4535 100644
--- a/t/imap_searchqp.t
+++ b/t/imap_searchqp.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
@@ -28,12 +28,15 @@ $q = $parse->(qq{CHARSET UTF-8 From b});
 is($q->{xap}, 'f:"b"', 'charset handled');
 $q = $parse->(qq{CHARSET WTF-8 From b});
 like($q, qr/\ANO \[/, 'bad charset rejected');
-{
-        # TODO: squelch errors by default? clients could flood logs
-        open my $fh, '>:scalar', \(my $buf) or die;
+
+for my $x ('', ' (try #2)') {
+        open my $fh, '>:scalar', \(my $buf = '') or die;
         local *STDERR = $fh;
         $q = $parse->(qq{CHARSET});
+        last if is($buf, '', "nothing spewed to STDERR on bad query$x");
+        diag 'FIXME: above fails mysteriously sometimes, so we try again...';
 }
+
 like($q, qr/\ABAD /, 'bad charset rejected');
 
 $q = $parse->(qq{HEADER CC B (SENTBEFORE 2-Oct-1993)});
diff --git a/t/imapd-tls.t b/t/imapd-tls.t
index 44ab350c..b95085a2 100644
--- a/t/imapd-tls.t
+++ b/t/imapd-tls.t
@@ -1,8 +1,7 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use Socket qw(IPPROTO_TCP SOL_SOCKET);
 use PublicInbox::TestCommon;
 # IO::Poll is part of the standard library, but distros may split it off...
@@ -158,8 +157,19 @@ for my $args (
         test_lei(sub {
                 lei_ok qw(ls-mail-source), "imap://$starttls_addr",
                         \'STARTTLS not used by default';
-                ok(!lei(qw(ls-mail-source -c imap.starttls=true),
+                my $plain_out = $lei_out;
+                ok(!lei(qw(ls-mail-source -c imap.starttls),
                         "imap://$starttls_addr"), 'STARTTLS verify fails');
+                unlike $lei_err, qr!W: imap\.starttls= .*? is not boolean!i,
+                        'no non-boolean warning';
+                lei_ok qw(-c imap.starttls -c imap.sslVerify= ls-mail-source),
+                        "imap://$starttls_addr",
+                        \'disabling imap.sslVerify works w/ STARTTLS';
+                is $lei_out, $plain_out, 'sslVerify=false w/ STARTTLS output';
+                lei_ok qw(ls-mail-source -c imap.sslVerify=false),
+                        "imaps://$imaps_addr",
+                        \'disabling imap.sslVerify works w/ imaps://';
+                is $lei_out, $plain_out, 'sslVerify=false w/ IMAPS output';
         });
 
         SKIP: {
@@ -171,10 +181,7 @@ for my $args (
                 is(unpack('i', $x), 0, 'TCP_DEFER_ACCEPT is 0 on plain IMAP');
         };
         SKIP: {
-                skip 'SO_ACCEPTFILTER is FreeBSD-only', 2 if $^O ne 'freebsd';
-                if (system('kldstat -m accf_data >/dev/null')) {
-                        skip 'accf_data not loaded? kldload accf_data', 2;
-                }
+                require_mods '+accf_data';
                 require PublicInbox::Daemon;
                 my $x = getsockopt($imaps, SOL_SOCKET,
                                 $PublicInbox::Daemon::SO_ACCEPTFILTER);
diff --git a/t/imapd.t b/t/imapd.t
index 80757a9d..549b8766 100644
--- a/t/imapd.t
+++ b/t/imapd.t
@@ -1,11 +1,12 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # end-to-end IMAP tests, see unit tests in t/imap.t, too
-use strict;
-use Test::More;
+use v5.12;
 use Time::HiRes ();
+use PublicInbox::DS qw(now);
 use PublicInbox::TestCommon;
+use PublicInbox::TailNotify;
 use PublicInbox::Config;
 require_mods(qw(-imapd Mail::IMAPClient));
 my $imap_client = 'Mail::IMAPClient';
@@ -20,7 +21,7 @@ my $first_range = '0';
 
 my $level = 'basic';
 SKIP: {
-        require_mods('Search::Xapian', 1);
+        require_mods('Xapian', 1);
         $level = 'medium';
 };
 
@@ -99,7 +100,8 @@ ok($mic->examine($mailbox1), 'EXAMINE succeeds');
 my @raw = $mic->status($mailbox1, qw(Messages uidnext uidvalidity));
 is(scalar(@raw), 2, 'got status response');
 like($raw[0], qr/\A\*\x20STATUS\x20inbox\.i1\.$first_range\x20
-        \(MESSAGES\x20\d+\x20UIDNEXT\x20\d+\x20UIDVALIDITY\x20\d+\)\r\n/sx);
+        \(MESSAGES\x20[1-9][0-9]*\x20
+        UIDNEXT\x20\d+\x20UIDVALIDITY\x20\d+\)\r\n/sx);
 like($raw[1], qr/\A\S+ OK /, 'finished status response');
 
 my @orig_list = @raw = $mic->list;
@@ -248,7 +250,7 @@ SKIP: {
 
 ok($mic->logout, 'logout works');
 
-my $have_inotify = eval { require Linux::Inotify2; 1 };
+my $have_inotify = eval { require PublicInbox::Inotify; 1 };
 
 for my $ibx (@ibx) {
         my $name = $ibx->{name};
@@ -435,10 +437,52 @@ ok($mic->logout, 'logged out');
         like(<$c>, qr/\Atagonly BAD Error in IMAP command/, 'tag-only line');
 }
 
+{
+        ok(my $ic = $imap_client->new(%mic_opt), 'logged in');
+        my $mb = "$ibx[0]->{newsgroup}.$first_range";
+        ok($ic->examine($mb), "EXAMINE $mb");
+        my $uidnext = $ic->uidnext($mb); # we'll fetch BODYSTRUCTURE on this
+        my $im = $ibx[0]->importer(0);
+        $im->add(PublicInbox::Eml->new(<<EOF)) or BAIL_OUT;
+Subject: test Ævar
+Message-ID: <smtputf8-delivered-mess\@age>
+From: Ævar Arnfjörð Bjarmason <avarab\@example>
+To: git\@vger.kernel.org
+
+EOF
+        $im->done;
+        my $envl = $ic->get_envelope($uidnext);
+        is($envl->{subject}, 'test Ævar', 'UTF-8 subject');
+        is($envl->{sender}->[0]->{personalname}, 'Ævar Arnfjörð Bjarmason',
+                'UTF-8 sender[0].personalname');
+        SKIP: {
+                skip 'need compress for comparisons', 1 if !$can_compress;
+                ok($ic = $imap_client->new(%mic_opt), 'uncompressed logged in');
+                ok($ic && $ic->compress, 'compress enabled');
+                ok($ic->examine($mb), "EXAMINE $mb");
+                my $raw = $ic->get_envelope($uidnext);
+                is_deeply($envl, $raw, 'raw and compressed match');
+        }
+}
+
+my $wait_re = sub {
+        my ($tail_notify, $re) = @_;
+        my $end = now() + 5;
+        my (@l, @all);
+        until (grep(/$re/, @l = $tail_notify->getlines(5)) || now > $end) {
+                push @all, @l;
+                @l = ();
+        }
+        return \@l if @l;
+        diag explain(\@all);
+        xbail "never got `$re' message";
+};
+
+my $watcherr = "$tmpdir/watcherr";
+
 SKIP: {
         use_ok 'PublicInbox::InboxIdle';
-        require_git('1.8.5', 1) or
-                skip('git 1.8.5+ needed for --urlmatch', 4);
+        require_git '1.8.5', 4;
         my $old_env = { HOME => $ENV{HOME} };
         my $home = "$tmpdir/watch_home";
         mkdir $home or BAIL_OUT $!;
@@ -457,26 +501,26 @@ SKIP: {
         my $cfg = PublicInbox::Config->new;
         PublicInbox::DS->Reset;
         my $ii = PublicInbox::InboxIdle->new($cfg);
-        my $cb = sub { PublicInbox::DS->SetPostLoopCallback(sub {}) };
+        my $cb = sub { @PublicInbox::DS::post_loop_do = (sub {}) };
         my $obj = bless \$cb, 'PublicInbox::TestCommon::InboxWakeup';
         $cfg->each_inbox(sub { $_[0]->subscribe_unlock('ident', $obj) });
-        my $watcherr = "$tmpdir/watcherr";
         open my $err_wr, '>>', $watcherr or BAIL_OUT $!;
-        open my $err, '<', $watcherr or BAIL_OUT $!;
+        my $errw = PublicInbox::TailNotify->new($watcherr);
         my $w = start_script(['-watch'], undef, { 2 => $err_wr });
 
         diag 'waiting for initial fetch...';
         PublicInbox::DS::event_loop();
         diag 'inbox unlocked on initial fetch, waiting for IDLE';
 
-        tick until (grep(/I: \S+ idling/, <$err>));
+        $wait_re->($errw, qr/# \S+ idling/);
+
         open my $fh, '<', 't/iso-2202-jp.eml' or BAIL_OUT $!;
         $old_env->{ORIGINAL_RECIPIENT} = $addr;
         ok(run_script([qw(-mda --no-precheck)], $old_env, { 0 => $fh }),
                 'delivered a message for IDLE to kick -watch') or
                 diag "mda error \$?=$?";
         diag 'waiting for IMAP IDLE wakeup';
-        PublicInbox::DS->SetPostLoopCallback(undef);
+        @PublicInbox::DS::post_loop_do = ();
         PublicInbox::DS::event_loop();
         diag 'inbox unlocked on IDLE wakeup';
 
@@ -486,14 +530,15 @@ SKIP: {
                 or BAIL_OUT "git config $?";
         $w->kill('HUP');
         diag 'waiting for -watch reload + initial fetch';
-        tick until (grep(/I: will check/, <$err>));
+
+        $wait_re->($errw, qr/# will check/);
 
         open $fh, '<', 't/psgi_attach.eml' or BAIL_OUT $!;
         ok(run_script([qw(-mda --no-precheck)], $old_env, { 0 => $fh }),
                 'delivered a message for -watch PollInterval');
 
         diag 'waiting for PollInterval wakeup';
-        PublicInbox::DS->SetPostLoopCallback(undef);
+        @PublicInbox::DS::post_loop_do = ();
         PublicInbox::DS::event_loop();
         diag 'inbox unlocked (poll)';
         $w->kill;
@@ -503,19 +548,24 @@ SKIP: {
         $cfg->each_inbox(sub { shift->unsubscribe_unlock('ident') });
         $ii->close;
         PublicInbox::DS->Reset;
-        seek($err, 0, 0);
-        my @err = grep(!/^(?:I:|#)/, <$err>);
+        open my $errfh, '<', $watcherr or xbail "open: $!";
+        my @err = grep(!/^(?:I:|#)/, <$errfh>);
         is(@err, 0, 'no warnings/errors from -watch'.join(' ', @err));
 
-        if ($ENV{TEST_KILL_IMAPD}) { # not sure how reliable this test can be
+        SKIP: { # not sure how reliable this test can be
+                skip 'TEST_KILL_IMAPD not set', 1 if !$ENV{TEST_KILL_IMAPD};
+                $^O eq 'linux' or
+                        diag "TEST_KILL_IMAPD may not be reliable under $^O";
                 xsys(qw(git config), "--file=$home/.public-inbox/config",
                         qw(--unset imap.PollInterval)) == 0
                         or BAIL_OUT "git config $?";
-                truncate($err_wr, 0) or BAIL_OUT $!;
+                unlink $watcherr or xbail $!;
+                open my $err_wr, '>>', $watcherr or xbail $!;
                 my @t0 = times;
                 $w = start_script(['-watch'], undef, { 2 => $err_wr });
-                seek($err, 0, 0);
-                tick until (grep(/I: \S+ idling/, <$err>));
+
+                $wait_re->($errw, qr/# \S+ idling/);
+
                 diag 'killing imapd, waiting for CPU spins';
                 my $delay = 0.11;
                 $td->kill(9);
@@ -528,7 +578,8 @@ SKIP: {
                 my $thresh = (0.9 * $delay);
                 diag "c=$c, threshold=$thresh";
                 ok($c < $thresh, 'did not burn much CPU');
-                is_deeply([grep(/ line \d+$/m, <$err>)], [],
+                open $errfh, '<', $watcherr or xbail "open: $!";
+                is_deeply([grep(/ line \d+$/m, <$errfh>)], [],
                                 'no backtraces from errors');
         }
 }
diff --git a/t/import.t b/t/import.t
index ae76858b..7e2432e7 100644
--- a/t/import.t
+++ b/t/import.t
@@ -1,8 +1,8 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.10.1;
 use strict;
-use warnings;
-use Test::More;
 use PublicInbox::Eml;
 use PublicInbox::Smsg;
 use PublicInbox::Git;
@@ -26,11 +26,12 @@ hello world
 EOF
 
 my $v2 = require_git(2.6, 1);
-my $smsg = bless {}, 'PublicInbox::Smsg' if $v2;
+my $smsg = $v2 ? bless({}, 'PublicInbox::Smsg') : undef;
 like($im->add($mime, undef, $smsg), qr/\A:[0-9]+\z/, 'added one message');
 
-if ($v2) {
-        like($smsg->{blob}, qr/\A[a-f0-9]{40}\z/, 'got last object_id');
+SKIP: {
+        skip 'git 2.6+ required', 3 if !$v2;
+        like($smsg->{blob}, qr/\A[a-f0-9]{40,64}\z/, 'got last object_id');
         my @cmd = ('git', "--git-dir=$git->{git_dir}", qw(hash-object --stdin));
         open my $in, '+<', undef or BAIL_OUT "open(+<): $!";
         print $in $mime->as_string or die "write failed: $!";
@@ -97,7 +98,8 @@ ok($@, 'Import->add fails on non-existent dir');
 
 my @cls = qw(PublicInbox::Eml);
 SKIP: {
-        require_mods('PublicInbox::MIME', 1);
+        require_mods('Email::MIME', 1);
+        require PublicInbox::MIME;
         push @cls, 'PublicInbox::MIME';
 };
 
diff --git a/t/inbox_idle.t b/t/inbox_idle.t
index 51379764..0ccffab7 100644
--- a/t/inbox_idle.t
+++ b/t/inbox_idle.t
@@ -1,10 +1,8 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::TestCommon;
-use PublicInbox::Config;
 require_git 2.6;
 require_mods(qw(DBD::SQLite));
 require PublicInbox::SearchIdx;
@@ -26,10 +24,11 @@ for my $V (1, 2) {
                 $sidx->idx_release; # allow watching on lockfile
         };
         my $obj = InboxIdleTestObj->new;
-        my $pi_cfg = PublicInbox::Config->new(\<<EOF);
-publicinbox.inbox-idle.inboxdir=$inboxdir
-publicinbox.inbox-idle.indexlevel=basic
-publicinbox.inbox-idle.address=$ibx->{-primary_address}
+        my $pi_cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "inbox-idle"]
+        inboxdir = $inboxdir
+        indexlevel = basic
+        address = $ibx->{-primary_address}
 EOF
         my $ident = 'whatever';
         $pi_cfg->each_inbox(sub { shift->subscribe_unlock($ident, $obj) });
diff --git a/t/index-git-times.t b/t/index-git-times.t
index 52173396..eac2d650 100644
--- a/t/index-git-times.t
+++ b/t/index-git-times.t
@@ -1,16 +1,16 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
-use Test::More;
 use PublicInbox::TestCommon;
 use PublicInbox::Config;
 use PublicInbox::Admin;
 use PublicInbox::Import;
 use File::Path qw(remove_tree);
+require PublicInbox::InboxWritable;
 
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 use_ok 'PublicInbox::Over';
 
 my ($tmpdir, $for_destroy) = tmpdir();
@@ -58,7 +58,7 @@ my $smsg;
 {
         my $cfg = PublicInbox::Config->new;
         my $ibx = $cfg->lookup($addr);
-        my $lvl = PublicInbox::Admin::detect_indexlevel($ibx);
+        my $lvl = PublicInbox::InboxWritable::detect_indexlevel($ibx);
         is($lvl, 'medium', 'indexlevel detected');
         is($ibx->{-skip_docdata}, 1, '--skip-docdata flag set on -index');
         $smsg = $ibx->over->get_art(1);
@@ -74,14 +74,14 @@ my $smsg;
         is($res->[0]->{ds}, $smsg->{ds}, 'Xapian search on datestamp');
 }
 SKIP: {
-        require_git(2.6, 1) or skip('git 2.6+ required for v2', 10);
+        require_git(2.6, 10);
         my $v2dir = "$tmpdir/v2";
         run_script(['-convert', $v1dir, $v2dir]) or die 'v2 conversion failed';
 
         my $check_v2 = sub {
                 my $ibx = PublicInbox::Inbox->new({inboxdir => $v2dir,
                                 address => $addr});
-                my $lvl = PublicInbox::Admin::detect_indexlevel($ibx);
+                my $lvl = PublicInbox::InboxWritable::detect_indexlevel($ibx);
                 is($lvl, 'medium', 'indexlevel detected after convert');
                 is($ibx->{-skip_docdata}, 1,
                         '--skip-docdata preserved after convert');
diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index ac85643d..c852f72c 100644
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -5,7 +5,7 @@ use strict;
 use v5.10.1;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
-use PublicInbox::Inbox;
+use PublicInbox::InboxWritable;
 require PublicInbox::Admin;
 my $PI_TEST_VERSION = $ENV{PI_TEST_VERSION} || 2;
 require_git('2.6') if $PI_TEST_VERSION == 2;
@@ -41,7 +41,7 @@ my $import_index_incremental = sub {
                 inboxdir => $ibx->{inboxdir},
                 indexlevel => $level
         });
-        my $msgs = $ro_master->recent;
+        my $msgs = $ro_master->over->recent;
         is(scalar(@$msgs), 1, 'only one message in master, so far');
         is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed');
 
@@ -71,7 +71,7 @@ my $import_index_incremental = sub {
                 inboxdir => $mirror,
                 indexlevel => $level,
         });
-        $msgs = $ro_mirror->recent;
+        $msgs = $ro_mirror->over->recent;
         is(scalar(@$msgs), 1, 'only one message, so far');
         is($msgs->[0]->{mid}, 'm@1', 'read first message');
 
@@ -83,7 +83,7 @@ my $import_index_incremental = sub {
         # mirror updates
         is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
         ok(run_script([qw(-index -j0), $mirror]), "v$v index mirror again OK");
-        $msgs = $ro_mirror->recent;
+        $msgs = $ro_mirror->over->recent;
         is(scalar(@$msgs), 2, '2nd message seen in mirror');
         is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
                 ['m@1','m@2'], 'got both messages in mirror');
@@ -91,7 +91,7 @@ my $import_index_incremental = sub {
         # incremental index master (required for v1)
         ok(run_script([qw(-index -j0), $ibx->{inboxdir}, "-L$level"]),
                 'index master OK');
-        $msgs = $ro_master->recent;
+        $msgs = $ro_master->over->recent;
         is(scalar(@$msgs), 2, '2nd message seen in master');
         is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
                 ['m@1','m@2'], 'got both messages in master');
@@ -110,7 +110,8 @@ my $import_index_incremental = sub {
 
         if ($level ne 'basic') {
                 ok(run_script(['-xcpdb', '-q', $mirror]), "v$v xcpdb OK");
-                is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
+                is(PublicInbox::InboxWritable::detect_indexlevel($ro_mirror),
+                        $level,
                    'indexlevel detectable by Admin after xcpdb v' .$v.$level);
                 delete $ro_mirror->{$_} for (qw(over search));
                 my $mset = $ro_mirror->search->mset('m:m@2');
@@ -120,7 +121,7 @@ my $import_index_incremental = sub {
         # sync the mirror
         is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
         ok(run_script([qw(-index -j0), $mirror]), "v$v index mirror again OK");
-        $msgs = $ro_mirror->recent;
+        $msgs = $ro_mirror->over->recent;
         is(scalar(@$msgs), 1, '2nd message gone from mirror');
         is_deeply([map { $_->{mid} } @$msgs], ['m@1'],
                 'message unavailable in mirror');
@@ -152,7 +153,7 @@ my $import_index_incremental = sub {
         is_deeply(\@rw_nums, \@expect, "v$v master has expected NNTP articles");
         is_deeply(\@ro_nums, \@expect, "v$v mirror matches master articles");
 
-        is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
+        is(PublicInbox::InboxWritable::detect_indexlevel($ro_mirror), $level,
            'indexlevel detectable by Admin '.$v.$level);
 
         SKIP: {
@@ -167,7 +168,7 @@ my $import_index_incremental = sub {
 $import_index_incremental->($PI_TEST_VERSION, 'basic', $mime);
 
 SKIP: {
-        require_mods(qw(Search::Xapian), 2);
+        require_mods(qw(Xapian), 2);
         foreach my $l (qw(medium full)) {
                 $import_index_incremental->($PI_TEST_VERSION, $l, $mime);
         }
diff --git a/t/init.t b/t/init.t
index 6f4c9dce..275192cf 100644
--- a/t/init.t
+++ b/t/init.t
@@ -1,11 +1,12 @@
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
-use warnings;
-use Test::More;
+use v5.10.1;
 use PublicInbox::Config;
 use PublicInbox::TestCommon;
 use PublicInbox::Admin;
+use PublicInbox::InboxWritable;
 my ($tmpdir, $for_destroy) = tmpdir();
 sub quiet_fail {
         my ($cmd, $msg) = @_;
@@ -18,7 +19,11 @@ sub quiet_fail {
         my $cfgfile = "$ENV{PI_DIR}/config";
         my $cmd = [ '-init', 'blist', "$tmpdir/blist",
                    qw(http://example.com/blist blist@example.com) ];
+        my $umask = umask(070) // xbail "umask: $!";
         ok(run_script($cmd), 'public-inbox-init OK');
+        umask($umask) // xbail "umask: $!";
+        my $mode = (stat($cfgfile))[2];
+        is(sprintf('0%03o', $mode & 0777), '0604', 'config respects umask');
 
         is(read_indexlevel('blist'), '', 'indexlevel unset by default');
 
@@ -102,7 +107,7 @@ sub quiet_fail {
         umask($umask) // xbail "umask: $!";
         ok(-d "$tmpdir/a/b/c/d", 'directory created');
         my $desc = "$tmpdir/a/b/c/d/description";
-        is(PublicInbox::Inbox::try_cat($desc),
+        is(PublicInbox::IO::try_cat($desc),
                 "public inbox for abcd\@example.com\n", 'description set');
         my $mode = (stat($desc))[2];
         is(sprintf('0%03o', $mode & 0777), '0644',
@@ -116,8 +121,8 @@ sub quiet_fail {
 }
 
 SKIP: {
-        require_mods(qw(DBD::SQLite Search::Xapian), 2);
-        require_git(2.6, 1) or skip "git 2.6+ required", 2;
+        require_mods(qw(DBD::SQLite Xapian), 2);
+        require_git(2.6, 2);
         use_ok 'PublicInbox::Msgmap';
         local $ENV{PI_DIR} = "$tmpdir/.public-inbox/";
         local $ENV{PI_EMERGENCY} = "$tmpdir/.public-inbox/emergency";
@@ -147,7 +152,7 @@ SKIP: {
                 ok(run_script($cmd), "-init -L $lvl");
                 is(read_indexlevel("v2$lvl"), $lvl, "indexlevel set to '$lvl'");
                 my $ibx = PublicInbox::Inbox->new({ inboxdir => $dir });
-                is(PublicInbox::Admin::detect_indexlevel($ibx), $lvl,
+                is(PublicInbox::InboxWritable::detect_indexlevel($ibx), $lvl,
                         'detected expected level w/o config');
                 ok(!$ibx->{-skip_docdata}, 'docdata written by default');
         }
@@ -159,7 +164,7 @@ SKIP: {
                         "$name\@example.com" ];
                 ok(run_script($cmd), "-init -V$v --skip-docdata");
                 my $ibx = PublicInbox::Inbox->new({ inboxdir => $dir });
-                is(PublicInbox::Admin::detect_indexlevel($ibx), 'full',
+                is(PublicInbox::InboxWritable::detect_indexlevel($ibx), 'full',
                         "detected default indexlevel -V$v");
                 ok($ibx->{-skip_docdata}, "docdata skip set -V$v");
                 ok($ibx->search->has_threadid, 'has_threadid flag set on new inbox');
@@ -211,6 +216,14 @@ SKIP: {
         is($n, 13, 'V1 NNTP article numbers skipped via --skip-artnum');
 }
 
+{
+        local $ENV{PI_DIR} = "$tmpdir/.public-inbox/";
+        my $cmd = [ qw(-init -C), "$tmpdir", qw(chdirlist chdirlist),
+                   qw(http://example.com/chdirlist chdirlist@example.com)];
+        ok(run_script($cmd), '-init with -C (chdir)');
+        ok(-d "$tmpdir/chdirlist", '-C processed correctly');
+}
+
 done_testing();
 
 sub read_indexlevel {
diff --git a/t/inotify3.t b/t/inotify3.t
new file mode 100644
index 00000000..c25c0f42
--- /dev/null
+++ b/t/inotify3.t
@@ -0,0 +1,17 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12; use PublicInbox::TestCommon;
+plan skip_all => 'inotify is Linux-only' if $^O ne 'linux';
+use_ok 'PublicInbox::Inotify3';
+my $in = PublicInbox::Inotify3->new;
+my $tmpdir = tmpdir;
+my $w = $in->watch("$tmpdir", PublicInbox::Inotify3::IN_ALL_EVENTS());
+$in->blocking(0);
+is_xdeeply [ $in->read ], [], 'non-blocking has no events, yet';
+undef $tmpdir;
+my @list = $in->read;
+ok scalar(@list), 'got events';
+ok $w->cancel, 'watch canceled';
+
+done_testing;
diff --git a/t/io.t b/t/io.t
new file mode 100644
index 00000000..91f93526
--- /dev/null
+++ b/t/io.t
@@ -0,0 +1,35 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::TestCommon;
+my $tmpdir = tmpdir;
+use_ok 'PublicInbox::IO';
+use PublicInbox::Spawn qw(which run_qx);
+
+# test failures:
+SKIP: {
+my $strace = strace_inject;
+my $env = { PERL5LIB => join(':', @INC) };
+my $opt = { 1 => \my $out, 2 => \my $err };
+my $dst = "$tmpdir/dst";
+my $tr = "$tmpdir/tr";
+my $cmd = [ $strace, "-o$tr", "-P$dst",
+                '-e', 'inject=writev,write:error=EIO',
+                $^X, qw(-w -MPublicInbox::IO=write_file -e),
+                q[write_file '>', $ARGV[0], 'hello world'], $dst ];
+xsys($cmd, $env, $opt);
+isnt($?, 0, 'write failed');
+like($err, qr/\bclose\b/, 'close error noted');
+is(-s $dst, 0, 'file created and empty after EIO');
+} # /SKIP
+
+PublicInbox::IO::write_file '>:unix', "$tmpdir/f", "HI\n";
+is(-s "$tmpdir/f", 3, 'write_file works w/ IO layer');
+PublicInbox::IO::write_file '>>', "$tmpdir/f", "HI\n";
+is(-s "$tmpdir/f", 6, 'write_file can append');
+
+is PublicInbox::IO::try_cat("$tmpdir/non-existent"), '',
+        "try_cat on non-existent file returns `'";
+
+done_testing;
diff --git a/t/ipc.t b/t/ipc.t
index ce89f94b..23ae2e7b 100644
--- a/t/ipc.t
+++ b/t/ipc.t
@@ -1,19 +1,17 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
-use Test::More;
+use v5.12;
 use PublicInbox::TestCommon;
 use Fcntl qw(SEEK_SET);
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
 require_mods(qw(Storable||Sereal));
 require_ok 'PublicInbox::IPC';
 my ($tmpdir, $for_destroy) = tmpdir();
 state $once = eval <<'';
 package PublicInbox::IPC;
 use strict;
-use Digest::SHA qw(sha1_hex);
+use PublicInbox::SHA qw(sha1_hex);
 sub test_array { qw(test array) }
 sub test_scalar { 'scalar' }
 sub test_scalarref { \'scalarref' }
@@ -90,7 +88,6 @@ $test->('local');
         defined($pid) or BAIL_OUT 'no spawn, no test';
         is($ipc->ipc_do('test_pid'), $pid, 'worker pid returned');
         $test->('worker');
-        $ipc->ipc_lock_init("$tmpdir/lock");
         is($ipc->ipc_do('test_pid'), $pid, 'worker pid returned');
         $ipc->ipc_worker_stop;
         ok(!kill(0, $pid) && $!{ESRCH}, 'worker stopped');
@@ -109,7 +106,9 @@ open my $agpl, '<', 'COPYING' or BAIL_OUT "AGPL-3 missing: $!";
 my $big = do { local $/; <$agpl> } // BAIL_OUT "read: $!";
 close $agpl or BAIL_OUT "close: $!";
 
-for my $t ('local', 'worker', 'worker again') {
+for my $t ('worker', 'worker again') {
+        my $ppid = $ipc->wq_workers_start('wq', 1);
+        push(@ppids, $ppid);
         $ipc->wq_io_do('test_write_each_fd', [ $wa, $wb, $wc ], 'hello world');
         my $i = 0;
         for my $fh ($ra, $rb, $rc) {
@@ -133,14 +132,19 @@ for my $t ('local', 'worker', 'worker again') {
                 $exp = sha1_hex($bigger)."\n";
                 is(readline($rb), $exp, "SHA WQWorker limit ($t)");
         }
-        my $ppid = $ipc->wq_workers_start('wq', 1);
-        push(@ppids, $ppid);
+        SKIP: {
+                $ENV{TEST_EXPENSIVE} or skip 'TEST_EXPENSIVE not set', 1;
+                my $bigger = $big x 75000; # over 2G to trigger partial sendmsg
+                $ipc->wq_io_do('test_sha', [ $wa, $wb ], $bigger);
+                my $exp = sha1_hex($bigger)."\n";
+                is(readline($rb), $exp, "SHA WQWorker sendmsg limit ($t)");
+        }
 }
 
 # wq_io_do works across fork (siblings can feed)
 SKIP: {
         skip 'Socket::MsgHdr or Inline::C missing', 3 if !$ppids[0];
-        is_deeply(\@ppids, [$$, undef, undef],
+        is_xdeeply(\@ppids, [$$, undef],
                 'parent pid returned in wq_workers_start');
         my $pid = fork // BAIL_OUT $!;
         if ($pid == 0) {
@@ -174,10 +178,9 @@ SKIP: {
         skip 'Socket::MsgHdr or Inline::C missing', 11 if !$ppids[0];
         seek($warn, 0, SEEK_SET) or BAIL_OUT;
         my @warn = <$warn>;
-        is(scalar(@warn), 3, 'warned 3 times');
-        like($warn[0], qr/ wq_io_do: /, '1st warned from wq_do');
-        like($warn[1], qr/ wq_worker: /, '2nd warned from wq_worker');
-        is($warn[2], $warn[1], 'worker did not die');
+        is(scalar(@warn), 2, 'warned 3 times');
+        like($warn[0], qr/ wq_worker: /, '2nd warned from wq_worker');
+        is($warn[0], $warn[1], 'worker did not die');
 
         $SIG{__WARN__} = 'DEFAULT';
         is($ipc->wq_workers_start('wq', 2), $$, 'workers started again');
diff --git a/t/kqnotify.t b/t/kqnotify.t
index 902ce0f1..add477a4 100644
--- a/t/kqnotify.t
+++ b/t/kqnotify.t
@@ -1,37 +1,67 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Ensure KQNotify can pick up rename(2) and link(2) operations
 # used by Maildir writing tools
-use strict;
-use Test::More;
+use v5.12;
 use PublicInbox::TestCommon;
-plan skip_all => 'KQNotify is only for *BSD systems' if $^O !~ /bsd/;
+use autodie;
+require_bsd;
 require_mods('IO::KQueue');
 use_ok 'PublicInbox::KQNotify';
 my ($tmpdir, $for_destroy) = tmpdir();
-mkdir "$tmpdir/new" or BAIL_OUT "mkdir: $!";
-open my $fh, '>', "$tmpdir/tst" or BAIL_OUT "open: $!";
-close $fh or BAIL_OUT "close: $!";
+mkdir "$tmpdir/new";
 
 my $kqn = PublicInbox::KQNotify->new;
 my $mask = PublicInbox::KQNotify::MOVED_TO_OR_CREATE();
 my $w = $kqn->watch("$tmpdir/new", $mask);
 
-rename("$tmpdir/tst", "$tmpdir/new/tst") or BAIL_OUT "rename: $!";
+open my $fh, '>', "$tmpdir/tst";
+close $fh;
+rename("$tmpdir/tst", "$tmpdir/new/tst");
 my $hit = [ map { $_->fullname } $kqn->read ];
-is_deeply($hit, ["$tmpdir/new/tst"], 'rename(2) detected (via NOTE_EXTEND)');
+is_deeply($hit, ["$tmpdir/new/tst"],
+                'rename(2) detected (via NOTE_EXTEND)')
+                or diag explain($hit);
 
-open $fh, '>', "$tmpdir/tst" or BAIL_OUT "open: $!";
-close $fh or BAIL_OUT "close: $!";
-link("$tmpdir/tst", "$tmpdir/new/link") or BAIL_OUT "link: $!";
-$hit = [ grep m!/link$!, map { $_->fullname } $kqn->read ];
-is_deeply($hit, ["$tmpdir/new/link"], 'link(2) detected (via NOTE_WRITE)');
+open $fh, '>', "$tmpdir/tst";
+close $fh;
+link("$tmpdir/tst", "$tmpdir/new/link");
+my @read = map { $_->fullname } $kqn->read;
+$hit = [ grep(m!/link$!, @read) ];
+is_deeply($hit, ["$tmpdir/new/link"], 'link(2) detected (via NOTE_WRITE)')
+        or diag explain(\@read);
+
+{
+        my $d = "$tmpdir/new/ANOTHER";
+        mkdir $d;
+        $hit = [ map { $_->fullname } $kqn->read ];
+        is_xdeeply($hit, [ $d ], 'mkdir detected');
+        rmdir $d;
+        # TODO: should we always watch for directory removals?
+}
 
 $w->cancel;
-link("$tmpdir/new/tst", "$tmpdir/new/link2") or BAIL_OUT "link: $!";
+link("$tmpdir/new/tst", "$tmpdir/new/link2");
 $hit = [ map { $_->fullname } $kqn->read ];
 is_deeply($hit, [], 'link(2) not detected after cancel');
 
+# rearm:
+my $GONE = PublicInbox::KQNotify::NOTE_DELETE() |
+        PublicInbox::KQNotify::NOTE_REVOKE() |
+        PublicInbox::KQNotify::NOTE_ATTRIB() |
+        PublicInbox::KQNotify::NOTE_WRITE() |
+        PublicInbox::KQNotify::NOTE_RENAME();
+$w = $kqn->watch("$tmpdir/new", $mask|$GONE);
+my @unlink = sort glob("$tmpdir/new/*");
+unlink(@unlink);
+$hit = [ sort(map { $_->fullname } $kqn->read) ];
+is_xdeeply($hit, \@unlink, 'unlinked files match');
+
+# this is unreliable on Dragonfly tmpfs (fixed post-6.4)
+rmdir "$tmpdir/new";
+$hit = [ sort(map { $_->fullname } $kqn->read) ];
+is(scalar(@$hit), 1, 'detected self removal') or check_broken_tmpfs;
+
 done_testing;
diff --git a/t/lei-auto-watch.t b/t/lei-auto-watch.t
index f871188d..1e190316 100644
--- a/t/lei-auto-watch.t
+++ b/t/lei-auto-watch.t
@@ -4,10 +4,10 @@
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 use File::Basename qw(basename);
 plan skip_all => "TEST_FLAKY not enabled for $0" if !$ENV{TEST_FLAKY};
-my $have_fast_inotify = eval { require Linux::Inotify2 } ||
+my $have_fast_inotify = eval { require PublicInbox::Inotify } ||
         eval { require IO::KQueue };
 $have_fast_inotify or
-        diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
+        diag("$0 IO::KQueue or inotify missing, test will be slow");
 
 test_lei(sub {
         my ($ro_home, $cfg_path) = setup_public_inboxes;
diff --git a/t/lei-convert.t b/t/lei-convert.t
index e1849ff7..4670e47f 100644
--- a/t/lei-convert.t
+++ b/t/lei-convert.t
@@ -1,12 +1,16 @@
 #!perl -w
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict; use v5.10.1; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
 use PublicInbox::MboxReader;
 use PublicInbox::MdirReader;
 use PublicInbox::NetReader;
 use PublicInbox::Eml;
 use IO::Uncompress::Gunzip;
+use File::Path qw(remove_tree);
+use PublicInbox::Spawn qw(which run_qx);
+use File::Compare;
+use autodie qw(open);
 require_mods(qw(lei -imapd -nntpd Mail::IMAPClient Net::NNTP));
 my ($tmpdir, $for_destroy) = tmpdir;
 my $sock = tcp_server;
@@ -25,8 +29,36 @@ test_lei({ tmpdir => $tmpdir }, sub {
         my $d = $ENV{HOME};
         lei_ok('convert', '-o', "mboxrd:$d/foo.mboxrd",
                 "imap://$imap_host_port/t.v2.0");
+        my ($nc0) = ($lei_err =~ /converted (\d+) messages/);
         ok(-f "$d/foo.mboxrd", 'mboxrd created from imap://');
 
+        lei_ok qw(convert -o), "v2:$d/v2-test", "mboxrd:$d/foo.mboxrd";
+        my ($nc) = ($lei_err =~ /converted (\d+) messages/);
+        is $nc, $nc0, 'converted all messages messages';
+        lei_ok qw(q z:0.. -f jsonl --only), "$d/v2-test";
+        is(scalar(split(/^/sm, $lei_out)), $nc, 'got all messages in v2-test');
+
+        lei_ok qw(convert -o), "mboxrd:$d/from-v2.mboxrd", "$d/v2-test";
+        like $lei_err, qr/converted $nc messages/;
+        is(compare("$d/foo.mboxrd", "$d/from-v2.mboxrd"), 0,
+                'convert mboxrd -> v2 ->mboxrd roundtrip') or
+                        diag run_qx([qw(git diff --no-index),
+                                        "$d/foo.mboxrd", "$d/from-v2.mboxrd"]);
+
+        lei_ok [qw(convert -F eml -o), "$d/v2-test"], undef,
+                { 0 => \<<'EOM', %$lei_opt };
+From: f@example.com
+To: t@example.com
+Subject: append-to-v2-on-convert
+Message-ID: <append-to-v2-on-convert@example>
+Date: Fri, 02 Oct 1993 00:00:00 +0000
+EOM
+        like $lei_err, qr/converted 1 messages/, 'only one message added';
+        lei_ok qw(q z:0.. -f jsonl --only), "$d/v2-test";
+        is(scalar(split(/^/sm, $lei_out)), $nc + 1,
+                'got expected number of messages after append convert');
+        like $lei_out, qr/append-to-v2-on-convert/;
+
         lei_ok('convert', '-o', "mboxrd:$d/nntp.mboxrd",
                 "nntp://$nntp_host_port/t.v2");
         ok(-f "$d/nntp.mboxrd", 'mboxrd created from nntp://');
@@ -125,5 +157,65 @@ test_lei({ tmpdir => $tmpdir }, sub {
         like($md[0], qr/:2,S\z/, "`seen' flag set in Maildir");
         lei_ok(qw(convert -o mboxrd:/dev/stdout), "$d/md2");
         like($lei_out, qr/^Status: RO/sm, "`seen' flag preserved");
+
+        SKIP: {
+                my $ok;
+                for my $x (($ENV{GZIP}//''), qw(pigz gzip)) {
+                        $x && (`$x -h 2>&1`//'') =~ /--rsyncable\b/s or next;
+                        $ok = $x;
+                        last;
+                }
+                skip 'pigz || gzip do not support --rsyncable', 1 if !$ok;
+                lei_ok qw(convert --rsyncable), "mboxrd:$d/qp.gz",
+                        '-o', "mboxcl2:$d/qp2.gz";
+                undef $fh; # necessary to make IO::Uncompress::Gunzip happy
+                open $fh, '<', "$d/qp2.gz";
+                $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
+                my @tmp;
+                PublicInbox::MboxReader->mboxcl2($fh, sub {
+                        my ($eml) = @_;
+                        $eml->header_set($_) for qw(Content-Length Lines);
+                        push @tmp, $eml;
+                });
+                is_deeply(\@tmp, \@bar, 'read rsyncable-gzipped mboxcl2');
+        }
+        my $cp = which('cp') or xbail 'cp(1) not available (WTF?)';
+        for my $v (1, 2) {
+                my $ibx_dir = "$ro_home/t$v";
+                lei_ok qw(convert -f mboxrd), $ibx_dir,
+                                \"dump v$v inbox to mboxrd";
+                my $out = $lei_out;
+                lei_ok qw(convert -f mboxrd), "v$v:$ibx_dir",
+                                \"dump v$v inbox to mboxrd w/ v$v:// prefix";
+                is $out, $lei_out, "v$v:// prefix accepted";
+                open my $fh, '<', \$out;
+                my (@mb, @md, @md2);
+                PublicInbox::MboxReader->mboxrd($fh, sub {
+                        $_[0]->header_set('Status');
+                        push @mb, $_[0]->as_string;
+                });
+                undef $out;
+                ok(scalar(@mb), 'got messages output');
+                my $mdir = "$d/v$v-mdir";
+                lei_ok qw(convert -o), $mdir, $ibx_dir,
+                        \"dump v$v inbox to Maildir";
+                PublicInbox::MdirReader->new->maildir_each_eml($mdir, sub {
+                        push @md, $_[2]->as_string;
+                });
+                @md = sort { $a cmp $b } @md;
+                @mb = sort { $a cmp $b } @mb;
+                is_deeply(\@mb, \@md, 'got matching inboxes');
+                xsys_e([$cp, '-Rp', $ibx_dir, "$d/tv$v" ]);
+                remove_tree($mdir, "$d/tv$v/public-inbox",
+                                glob("$d/tv$v/xap*"));
+
+                lei_ok qw(convert -o), $mdir, "$d/tv$v",
+                        \"dump u indexed v$v inbox to Maildir";
+                PublicInbox::MdirReader->new->maildir_each_eml($mdir, sub {
+                        push @md2, $_[2]->as_string;
+                });
+                @md2 = sort { $a cmp $b } @md2;
+                is_deeply(\@md, \@md2, 'got matching inboxes even unindexed');
+        }
 });
 done_testing;
diff --git a/t/lei-daemon.t b/t/lei-daemon.t
index e11105bc..d97e494a 100644
--- a/t/lei-daemon.t
+++ b/t/lei-daemon.t
@@ -2,7 +2,7 @@
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
-use Socket qw(AF_UNIX SOCK_SEQPACKET MSG_EOR pack_sockaddr_un);
+use Socket qw(AF_UNIX SOCK_SEQPACKET pack_sockaddr_un);
 
 test_lei({ daemon_only => 1 }, sub {
         my $send_cmd = PublicInbox::Spawn->can('send_cmd4') // do {
@@ -21,6 +21,7 @@ test_lei({ daemon_only => 1 }, sub {
         is($lei_err, '', 'no error from daemon-pid');
         like($lei_out, qr/\A[0-9]+\n\z/s, 'pid returned') or BAIL_OUT;
         chomp(my $pid = $lei_out);
+        no_pollerfd($pid);
         ok(kill(0, $pid), 'pid is valid');
         ok(-S $sock, 'sock created');
         is(-s $err_log, 0, 'nothing in errors.log');
@@ -31,7 +32,7 @@ test_lei({ daemon_only => 1 }, sub {
         SKIP: {
                 skip 'only testing open files on Linux', 1 if $^O ne 'linux';
                 my $d = "/proc/$pid/fd";
-                skip "no $d on Linux" unless -d $d;
+                skip "no $d on Linux", 1 unless -d $d;
                 my @before = sort(glob("$d/*"));
                 my $addr = pack_sockaddr_un($sock);
                 open my $null, '<', '/dev/null' or BAIL_OUT "/dev/null: $!";
@@ -40,7 +41,7 @@ test_lei({ daemon_only => 1 }, sub {
                         socket(my $c, AF_UNIX, SOCK_SEQPACKET, 0) or
                                                         BAIL_OUT "socket: $!";
                         connect($c, $addr) or BAIL_OUT "connect: $!";
-                        $send_cmd->($c, \@fds, 'hi',  MSG_EOR);
+                        $send_cmd->($c, \@fds, 'hi',  0);
                 }
                 lei_ok('daemon-pid');
                 chomp($pid = $lei_out);
diff --git a/t/lei-externals.t b/t/lei-externals.t
index 284be1b9..4f2dd6ba 100644
--- a/t/lei-externals.t
+++ b/t/lei-externals.t
@@ -4,7 +4,7 @@
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 use Fcntl qw(SEEK_SET);
 require_git 2.6;
-require_mods(qw(json DBD::SQLite Search::Xapian));
+require_mods(qw(json DBD::SQLite Xapian));
 use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
 
 my @onions = map { "http://$_.onion/meta/" } qw(
@@ -48,6 +48,7 @@ SKIP: {
                 $tp->join;
                 ok(WIFSIGNALED($?), "signaled @$out");
                 is(WTERMSIG($?), SIGPIPE, "got SIGPIPE @$out");
+                no_coredump;
                 seek($err, 0, 0);
                 my @err = <$err>;
                 is_deeply(\@err, [], "no errors @$out");
@@ -66,6 +67,7 @@ SKIP: {
                         tick();
                 }
                 ok(!$alive, 'daemon-kill worked');
+                no_coredump;
         }
 } # /SKIP
 }; # /sub
diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t
index eb1ae312..14c644e0 100644
--- a/t/lei-import-nntp.t
+++ b/t/lei-import-nntp.t
@@ -1,9 +1,9 @@
 #!perl -w
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 require_git 2.6;
-require_mods(qw(json DBD::SQLite Search::Xapian Net::NNTP));
+require_mods(qw(lei json DBD::SQLite Xapian Net::NNTP));
 my ($ro_home, $cfg_path) = setup_public_inboxes;
 my ($tmpdir, $for_destroy) = tmpdir;
 my $sock = tcp_server;
@@ -43,7 +43,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
         lei_ok 'ls-mail-sync';
         like($lei_out, qr!\A\Q$url\E\n\z!, 'ls-mail-sync output as-expected');
 
-        ok(!lei(qw(import), "$url/12-1"), 'backwards range rejected');
+        ok(!lei(qw(import), "$url/12-1"), 'backwards range rejected') or
+                diag $lei_err;
 
         # new home
         local $ENV{HOME} = "$tmpdir/h2";
diff --git a/t/lei-import.t b/t/lei-import.t
index 6e9a853c..89eb1492 100644
--- a/t/lei-import.t
+++ b/t/lei-import.t
@@ -1,14 +1,17 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict; use v5.10.1; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
+use PublicInbox::DS qw(now);
+use PublicInbox::IO qw(write_file);
+use autodie qw(open close truncate);
 test_lei(sub {
 ok(!lei(qw(import -F bogus), 't/plack-qp.eml'), 'fails with bogus format');
 like($lei_err, qr/\bis `eml', not --in-format/, 'gave error message');
 
 lei_ok(qw(q s:boolean), \'search miss before import');
 unlike($lei_out, qr/boolean/i, 'no results, yet');
-open my $fh, '<', 't/data/0001.patch' or BAIL_OUT $!;
+open my $fh, '<', 't/data/0001.patch';
 lei_ok([qw(import -F eml -)], undef, { %$lei_opt, 0 => $fh },
         \'import single file from stdin') or diag $lei_err;
 close $fh;
@@ -18,7 +21,7 @@ lei_ok(qw(q s:boolean -f mboxrd), \'blob accessible after import');
         my $expect = [ eml_load('t/data/0001.patch') ];
         require PublicInbox::MboxReader;
         my @cmp;
-        open my $fh, '<', \$lei_out or BAIL_OUT "open :scalar: $!";
+        open my $fh, '<', \$lei_out;
         PublicInbox::MboxReader->mboxrd($fh, sub {
                 my ($eml) = @_;
                 $eml->header_set('Status');
@@ -110,6 +113,119 @@ $res = json_utf8->decode($lei_out);
 is_deeply($res->[0]->{kw}, ['seen'], 'keyword set');
 is_deeply($res->[0]->{L}, ['inbox'], 'label set');
 
+# idempotent import can add label
+lei_ok([qw(import -F eml - +L:boombox)],
+        undef, { %$lei_opt, 0 => \$eml_str });
+lei_ok(qw(q m:inbox@example.com));
+$res = json_utf8->decode($lei_out);
+is_deeply($res->[0]->{kw}, ['seen'], 'keyword remains set');
+is_deeply($res->[0]->{L}, [qw(boombox inbox)], 'new label added');
+
+# idempotent import can add keyword
+lei_ok([qw(import -F eml - +kw:answered)],
+        undef, { %$lei_opt, 0 => \$eml_str });
+lei_ok(qw(q m:inbox@example.com));
+$res = json_utf8->decode($lei_out);
+is_deeply($res->[0]->{kw}, [qw(answered seen)], 'keyword added');
+is_deeply($res->[0]->{L}, [qw(boombox inbox)], 'labels preserved');
+
+# +kw:seen is not a location
+open my $null, '<', '/dev/null';
+ok(!lei([qw(import -F eml +kw:seen)], undef, { %$lei_opt, 0 => $null }),
+        'import fails w/ only kw arg');
+like($lei_err, qr/\bLOCATION\.\.\. or --stdin must be set/s, 'error message');
+
+lei_ok([qw(import -F eml +kw:flagged)], # no lone dash (`-')
+        undef, { %$lei_opt, 0 => \$eml_str },
+        'import succeeds with implicit --stdin');
+lei_ok(qw(q m:inbox@example.com));
+$res = json_utf8->decode($lei_out);
+is_deeply($res->[0]->{kw}, [qw(answered flagged seen)], 'keyword added');
+is_deeply($res->[0]->{L}, [qw(boombox inbox)], 'labels preserved');
+
+lei_ok qw(import --commit-delay=1 +L:bin -F eml t/data/binary.patch);
+lei_ok 'ls-label';
+unlike($lei_out, qr/\bbin\b/, 'commit-delay delays label');
+my $end = now + 10;
+my $n = 1;
+diag 'waiting for lei/store commit...';
+do {
+        tick $n;
+        $n = 0.1;
+} until (!lei('ls-label') || $lei_out =~ /\bbin\b/ || now > $end);
+like($lei_out, qr/\bbin\b/, 'commit-delay eventually commits');
+
+SKIP: {
+        my $strace = strace_inject(1); # skips if strace is old or non-Linux
+        my $tmpdir = tmpdir;
+        my $tr = "$tmpdir/tr";
+        my $cmd = [ $strace, '-q', "-o$tr", '-f',
+                "-P", File::Spec->rel2abs('t/plack-qp.eml'),
+                '-e', 'inject=readv,read:error=EIO'];
+        lei_ok qw(daemon-pid);
+        chomp(my $daemon_pid = $lei_out);
+        push @$cmd, '-p', $daemon_pid;
+        require PublicInbox::Spawn;
+        require PublicInbox::AutoReap;
+        my $pid = PublicInbox::Spawn::spawn($cmd, \%ENV);
+        my $ar = PublicInbox::AutoReap->new($pid);
+        tick; # wait for strace to attach
+        ok(!lei(qw(import -F eml t/plack-qp.eml)),
+                '-F eml import fails on pathname error injection');
+        my $IO = '[Ii](?:nput)?/[Oo](?:utput)?';
+        like($lei_err, qr!error reading t/plack-qp\.eml: .*?$IO error!,
+                'EIO noted in stderr');
+        open $fh, '<', 't/plack-qp.eml';
+        ok(!lei(qw(import -F eml -), undef, { %$lei_opt, 0 => $fh }),
+                '-F eml import fails on stdin error injection');
+        like($lei_err, qr!error reading .*?: .*?$IO error!,
+                'EIO noted in stderr');
+}
+
+{
+        local $ENV{PI_CONFIG} = "$ENV{HOME}/pi_config";
+        write_file '>', $ENV{PI_CONFIG}, <<EOM;
+[publicinboxImport]
+        dropUniqueUnsubscribe
+EOM
+        my $in = <<EOM;
+List-Unsubscribe: <https://example.com/some-UUID-here/test>
+List-Unsubscribe-Post: List-Unsubscribe=One-Click
+Message-ID: <unsubscribe-1\@example>
+Subject: unsubscribe-1 example
+From: u\@example.com
+To: 2\@example.com
+Date: Fri, 02 Oct 1993 00:00:00 +0000
+
+EOM
+        lei_ok [qw(import -F eml +L:unsub)], undef, { %$lei_opt, 0 => \$in },
+                'import succeeds w/ List-Unsubscribe';
+        lei_ok qw(q L:unsub -f mboxrd);
+        like $lei_out, qr/some-UUID-here/,
+                'Unsubscribe header preserved despite PI_CONFIG dropping';
+        lei_ok qw(q L:unsub -o), "v2:$ENV{HOME}/v2-1";
+        lei_ok qw(q s:unsubscribe -f mboxrd --only), "$ENV{HOME}/v2-1";
+        unlike $lei_out, qr/some-UUID-here/,
+                'Unsubscribe header dropped w/ dropUniqueUnsubscribe';
+        like $lei_out, qr/Message-ID: <unsubscribe-1\@example>/,
+                'wrote expected message to v2 output';
+
+        # the default for compatibility:
+        truncate $ENV{PI_CONFIG}, 0;
+        lei_ok qw(q L:unsub -o), "v2:$ENV{HOME}/v2-2";
+        lei_ok qw(q s:unsubscribe -f mboxrd --only), "$ENV{HOME}/v2-2";
+        like $lei_out, qr/some-UUID-here/,
+                'Unsubscribe header preserved by default :<';
+
+        # ensure we can fail
+        write_file '>', $ENV{PI_CONFIG}, <<EOM;
+[publicinboxImport]
+        dropUniqueUnsubscribe = bogus
+EOM
+        ok(!lei(qw(q L:unsub -o), "v2:$ENV{HOME}/v2-3"), 'bad config fails');
+        like $lei_err, qr/is not boolean/, 'non-booleaness noted in stderr';
+        ok !-d "$ENV{HOME}/v2-3", 'v2 directory not created';
+}
 
 # see t/lei_to_mail.t for "import -F mbox*"
 });
diff --git a/t/lei-index.t b/t/lei-index.t
index aab8f7e6..2b28f1be 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 use File::Spec;
@@ -48,9 +48,10 @@ symlink(File::Spec->rel2abs('t/mda-mime.eml'), "$tmpdir/md1/cur/x:2,S") or
 test_lei({ tmpdir => $tmpdir }, sub {
         my $store_path = "$ENV{HOME}/.local/share/lei/store/";
 
-        lei_ok('index', "$tmpdir/md");
+        lei_ok qw(index +L:md), "$tmpdir/md";
         lei_ok(qw(q mid:qp@example.com));
         my $res_a = json_utf8->decode($lei_out);
+        is_deeply $res_a->[0]->{L}, [ 'md' ], 'label set on index';
         my $blob = $res_a->[0]->{'blob'};
         like($blob, qr/\A[0-9a-f]{40,}\z/, 'got blob from qp@example');
         lei_ok(qw(-C / blob), $blob);
@@ -85,6 +86,10 @@ test_lei({ tmpdir => $tmpdir }, sub {
         lei_ok qw(q m:multipart-html-sucks@11);
         is_deeply(json_utf8->decode($lei_out)->[0]->{'kw'},
                 ['seen'], 'keyword set');
+        lei_ok 'reindex';
+        lei_ok qw(q m:multipart-html-sucks@11);
+        is_deeply(json_utf8->decode($lei_out)->[0]->{'kw'},
+                ['seen'], 'keyword still set after reindex');
 
         $srv->{nntpd} and
                 lei_ok('index', "nntp://$srv->{nntp_host_port}/t.v2");
@@ -104,6 +109,12 @@ test_lei({ tmpdir => $tmpdir }, sub {
         my $t = xqx(['git', "--git-dir=$store_path/ALL.git",
                         qw(cat-file -t), $res_b->{blob}]);
         is($t, "blob\n", 'got blob');
+
+        lei_ok('reindex');
+        lei_ok qw(q m:multipart-html-sucks@11);
+        $res_a = json_utf8->decode($lei_out)->[0];
+        is_deeply($res_a->{'kw'}, ['seen'],
+                'keywords still set after reindex');
 });
 
 done_testing;
diff --git a/t/lei-mail-diff.t b/t/lei-mail-diff.t
new file mode 100644
index 00000000..1a896e51
--- /dev/null
+++ b/t/lei-mail-diff.t
@@ -0,0 +1,15 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12; use PublicInbox::TestCommon;
+
+test_lei(sub {
+        ok(!lei('mail-diff', 't/data/0001.patch', 't/data/binary.patch'),
+                'different messages are different');
+        like($lei_out, qr/^\+/m, 'diff shown');
+        unlike $lei_out, qr/No newline at end of file/;
+        lei_ok('mail-diff', 't/data/0001.patch', 't/data/0001.patch');
+        is($lei_out, '', 'no output if identical');
+});
+
+done_testing;
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index 32a5b039..76041b73 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -1,10 +1,12 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict; use v5.10.1; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
 use PublicInbox::Inbox;
 require_mods(qw(-httpd lei DBD::SQLite));
 require_cmd('curl');
+require_git_http_backend;
+use PublicInbox::Spawn qw(which);
 require PublicInbox::Msgmap;
 my $sock = tcp_server();
 my ($tmpdir, $for_destroy) = tmpdir();
@@ -22,9 +24,16 @@ test_lei({ tmpdir => $tmpdir }, sub {
         lei_ok('add-external', $t1, '--mirror', "$http/t1/", \'--mirror v1');
         my $mm_dup = "$t1/public-inbox/msgmap.sqlite3";
         ok(-f $mm_dup, 't1-mirror indexed');
-        is(PublicInbox::Inbox::try_cat("$t1/description"),
+        is(PublicInbox::IO::try_cat("$t1/description"),
                 "mirror of $http/t1/\n", 'description set');
         ok(-f "$t1/Makefile", 'convenience Makefile added (v1)');
+        SKIP: {
+                my $make = require_cmd('make', 1);
+                delete local @ENV{qw(MFLAGS MAKEFLAGS MAKELEVEL)};
+                is(xsys([$make, 'help'], undef, { -C => $t1, 1 => \(my $help) }),
+                        0, "$make handled Makefile without errors");
+                isnt($help, '', 'make help worked');
+        }
         ok(-f "$t1/inbox.config.example", 'inbox.config.example downloaded');
         is((stat(_))[9], $created{v1},
                 'inbox.config.example mtime is ->created_at');
@@ -43,7 +52,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
         ok(-f $mm_dup, 't2-mirror indexed');
         ok(-f "$t2/description", 't2 description');
         ok(-f "$t2/Makefile", 'convenience Makefile added (v2)');
-        is(PublicInbox::Inbox::try_cat("$t2/description"),
+        is(PublicInbox::IO::try_cat("$t2/description"),
                 "mirror of $http/t2/\n", 'description set');
         $tb = PublicInbox::Msgmap->new_file($mm_dup)->created_at;
         is($tb, $created{v2}, 'created_at matched in v2 mirror');
@@ -199,14 +208,14 @@ $td->join;
         my $exp = "mirror of https://example.com/src/\n";
         my $f = "$tmpdir/description";
         PublicInbox::LeiMirror::set_description($mrr);
-        is(PublicInbox::Inbox::try_cat($f), $exp, 'description set on ENOENT');
+        is(PublicInbox::IO::try_cat($f), $exp, 'description set on ENOENT');
 
         my $fh;
         (open($fh, '>', $f) and close($fh)) or xbail $!;
         PublicInbox::LeiMirror::set_description($mrr);
-        is(PublicInbox::Inbox::try_cat($f), $exp, 'description set on empty');
+        is(PublicInbox::IO::try_cat($f), $exp, 'description set on empty');
         (open($fh, '>', $f) and print $fh "x\n" and close($fh)) or xbail $!;
-        is(PublicInbox::Inbox::try_cat($f), "x\n",
+        is(PublicInbox::IO::try_cat($f), "x\n",
                 'description preserved if non-default');
 }
 
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
index bf40a43b..44f37d19 100644
--- a/t/lei-p2q.t
+++ b/t/lei-p2q.t
@@ -3,7 +3,7 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 require_git 2.6;
-require_mods(qw(json DBD::SQLite Search::Xapian));
+require_mods(qw(json DBD::SQLite Xapian));
 
 test_lei(sub {
         ok(!lei(qw(p2q this-better-cause-format-patch-to-fail)),
diff --git a/t/lei-q-kw.t b/t/lei-q-kw.t
index 4edee72a..63e46037 100644
--- a/t/lei-q-kw.t
+++ b/t/lei-q-kw.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 use POSIX qw(mkfifo);
@@ -9,6 +9,8 @@ use IO::Compress::Gzip qw(gzip);
 use PublicInbox::MboxReader;
 use PublicInbox::LeiToMail;
 use PublicInbox::Spawn qw(popen_rd);
+use File::Path qw(make_path);
+use PublicInbox::IO qw(write_file);
 my $exp = {
         '<qp@example.com>' => eml_load('t/plack-qp.eml'),
         '<testmessage@example.com>' => eml_load('t/utf8.eml'),
@@ -42,6 +44,19 @@ lei_ok(qw(q -o), "maildir:$o", qw(m:qp@example.com));
 @fn = glob("$o/cur/*:2,S");
 is(scalar(@fn), 1, "`seen' flag (but not `replied') set on Maildir file");
 
+{
+        $o = "$ENV{HOME}/dst-existing";
+        make_path(map { "$o/$_" } qw(new cur tmp));
+        my $bp = eml_load('t/data/binary.patch');
+        write_file '>', "$o/cur/binary-patch:2,S", $bp->as_string;
+        lei_ok qw(q --no-import-before m:qp@example.com -o), $o;
+        my @g = glob("$o/*/*");
+        is scalar(@g), 1, 'only newly imported message left';
+        is eml_load($g[0])->header_raw('Message-ID'), '<qp@example.com>';
+        lei qw(q m:binary-patch-test@example);
+        is $lei_out, "[null]\n", 'old message not imported';
+}
+
 SKIP: {
         $o = "$ENV{HOME}/fifo";
         mkfifo($o, 0600) or skip("mkfifo not supported: $!", 1);
@@ -51,7 +66,7 @@ SKIP: {
                 '--import-before fails on non-seekable output');
         like($lei_err, qr/not seekable/, 'unseekable noted in error');
         is(do { local $/; <$cat> }, '', 'no output on FIFO');
-        close $cat;
+        $cat->close;
         $cat = popen_rd(['cat', $o]);
         lei_ok(qw(q m:qp@example.com -o), "mboxrd:$o");
         my $buf = do { local $/; <$cat> };
@@ -80,9 +95,7 @@ my $write_file = sub {
         if ($_[0] =~ /\.gz\z/) {
                 gzip(\($_[1]), $_[0]) or BAIL_OUT 'gzip';
         } else {
-                open my $fh, '>', $_[0] or BAIL_OUT $!;
-                print $fh $_[1] or BAIL_OUT $!;
-                close $fh or BAIL_OUT;
+                write_file '>', $_[0], $_[1];
         }
 };
 
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 92d8c9b6..885fa3e1 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -1,7 +1,8 @@
 #!perl -w
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict; use v5.10.1; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
+use autodie qw(open close unlink);
 require_mods(qw(lei -httpd));
 require_cmd 'curl';
 use PublicInbox::MboxReader;
@@ -16,7 +17,7 @@ my $url = "http://$host_port/t2/";
 my $exp1 = [ eml_load('t/plack-qp.eml') ];
 my $exp2 = [ eml_load('t/iso-2202-jp.eml') ];
 my $slurp_emls = sub {
-        open my $fh, '<', $_[0] or BAIL_OUT "open: $!";
+        open my $fh, '<', $_[0];
         my @eml;
         PublicInbox::MboxReader->mboxrd($fh, sub {
                 my $eml = shift;
@@ -31,33 +32,33 @@ test_lei({ tmpdir => $tmpdir }, sub {
         my @cmd = ('q', '-o', "mboxrd:$o", 'm:qp@example.com');
         lei_ok(@cmd);
         ok(-f $o && !-s _, 'output exists but is empty');
-        unlink $o or BAIL_OUT $!;
+        unlink $o;
         lei_ok(@cmd, '-I', $url);
         is_deeply($slurp_emls->($o), $exp1, 'got results after remote search');
-        unlink $o or BAIL_OUT $!;
+        unlink $o;
         lei_ok(@cmd);
         ok(-f $o && -s _, 'output exists after import but is not empty') or
                 diag $lei_err;
         is_deeply($slurp_emls->($o), $exp1, 'got results w/o remote search');
-        unlink $o or BAIL_OUT $!;
+        unlink $o;
 
         $cmd[-1] = 'm:199707281508.AAA24167@hoyogw.example';
         lei_ok(@cmd, '-I', $url, '--no-import-remote');
         is_deeply($slurp_emls->($o), $exp2, 'got another after remote search');
-        unlink $o or BAIL_OUT $!;
+        unlink $o;
         lei_ok(@cmd);
         ok(-f $o && !-s _, '--no-import-remote did not memoize');
 
         open my $fh, '>', "$o.lock";
         $cmd[-1] = 'm:qp@example.com';
-        unlink $o or xbail("unlink $o $! cwd=".Cwd::getcwd());
+        unlink $o;
         lei_ok(@cmd, '--lock=none');
         ok(-f $o && -s _, '--lock=none respected') or diag $lei_err;
-        unlink $o or xbail("unlink $o $! cwd=".Cwd::getcwd());
+        unlink $o;
         ok(!lei(@cmd, '--lock=dotlock,timeout=0.000001'), 'dotlock fails');
         like($lei_err, qr/dotlock timeout/, 'timeout noted');
         ok(-f $o && !-s _, 'nothing output on lock failure');
-        unlink "$o.lock" or BAIL_OUT $!;
+        unlink "$o.lock";
         lei_ok(@cmd, '--lock=dotlock,timeout=0.000001',
                 \'succeeds after lock removal');
 
@@ -76,8 +77,8 @@ test_lei({ tmpdir => $tmpdir }, sub {
                         'm:testmessage@example.com');
         is($lei_out, '', 'message not imported when in local external');
 
-        open $fh, '>', $o or BAIL_OUT;
-        print $fh <<'EOF' or BAIL_OUT;
+        open $fh, '>', $o;
+        print $fh <<'EOF';
 From a@z Mon Sep 17 00:00:00 2001
 From: nobody@localhost
 Date: Sat, 13 Mar 2021 18:23:01 +0600
@@ -86,7 +87,7 @@ Status: OR
 
 whatever
 EOF
-        close $fh or BAIL_OUT;
+        close $fh;
         lei_ok(qw(q -o), "mboxrd:$o", 'm:testmessage@example.com');
         is_deeply($slurp_emls->($o), [$exp],
                 'got expected result after clobber') or diag $lei_err;
@@ -103,5 +104,11 @@ EOF
         lei_ok([qw(edit-search), "$ENV{HOME}/md"], $edit_env);
         like($lei_out, qr/^\Q[external "$url"]\E\n\s*lastresult = \d+/sm,
                 'lastresult set');
+
+        unlink $o;
+        lei_ok qw(q --no-save -q m:never2exist@example.com -o), "mboxrd:$o",
+                '--only', $url,
+                \'404 curl exit (22) does not influence lei(1)';
+        is(-s $o, 0, 'empty result');
 });
 done_testing;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 3d09fe37..0970bc3c 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -1,7 +1,8 @@
 #!perl -w
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict; use v5.10.1; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
+use autodie qw(close open unlink);
 use PublicInbox::Smsg;
 use List::Util qw(sum);
 use File::Path qw(remove_tree);
@@ -12,9 +13,10 @@ my $doc2 = eml_load('t/utf8.eml');
 $doc2->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 4)}));
 my $doc3 = eml_load('t/msg_iter-order.eml');
 $doc3->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 4)}));
-
+my $cat_env = { VISUAL => 'cat', EDITOR => 'cat' };
 my $pre_existing = <<'EOF';
 From x Mon Sep 17 00:00:00 2001
+From: <x@example.com>
 Message-ID: <import-before@example.com>
 Subject: pre-existing
 Date: Sat, 02 Oct 2010 00:00:00 +0000
@@ -88,7 +90,7 @@ test_lei(sub {
         like($lei_out, qr!^\Q$home/mbcl2\E$!sm, 'complete got mbcl2 output');
         like($lei_out, qr!^\Q$home/md\E$!sm, 'complete got maildir output');
 
-        unlink("$home/mbcl2") or xbail "unlink $!";
+        unlink("$home/mbcl2");
         lei_ok qw(_complete lei up);
         like($lei_out, qr!^\Q$home/mbcl2\E$!sm,
                 'mbcl2 output shown despite unlink');
@@ -96,24 +98,24 @@ test_lei(sub {
         ok(-f "$home/mbcl2"  && -s _ == 0, 'up recreates on missing output');
 
         # no --augment
-        open my $mb, '>', "$home/mbrd" or xbail "open $!";
+        open my $mb, '>', "$home/mbrd";
         print $mb $pre_existing;
-        close $mb or xbail "close: $!";
+        close $mb;
         lei_ok(qw(q -o mboxrd:mbrd m:qp@example.com -C), $home);
-        open $mb, '<', "$home/mbrd" or xbail "open $!";
+        open $mb, '<', "$home/mbrd";
         is_deeply([grep(/pre-existing/, <$mb>)], [],
                 'pre-existing messsage gone w/o augment');
-        close $mb;
+        undef $mb;
         lei_ok(qw(q m:import-before@example.com));
         is(json_utf8->decode($lei_out)->[0]->{'s'},
                 'pre-existing', '--save imported before clobbering');
 
         # --augment
-        open $mb, '>', "$home/mbrd-aug" or xbail "open $!";
+        open $mb, '>', "$home/mbrd-aug";
         print $mb $pre_existing;
-        close $mb or xbail "close: $!";
+        close $mb;
         lei_ok(qw(q -a -o mboxrd:mbrd-aug m:qp@example.com -C), $home);
-        open $mb, '<', "$home/mbrd-aug" or xbail "open $!";
+        open $mb, '<', "$home/mbrd-aug";
         $mb = do { local $/; <$mb> };
         like($mb, qr/pre-existing/, 'pre-existing message preserved w/ -a');
         like($mb, qr/<qp\@example\.com>/, 'new result written w/ -a');
@@ -183,7 +185,12 @@ test_lei(sub {
         lei_ok(qw(q z:0.. -o), "v2:$v2");
         like($lei_err, qr/^# ([1-9][0-9]*) written to \Q$v2\E/sm,
                 'non-zero write output to stderr');
-        lei_ok(qw(q z:0.. -o), "mboxrd:$home/before", '--only', $v2, '-j1,1');
+        lei_ok('-C', $v2, qw(q z:0.. -o), "mboxrd:$home/before",
+                '--only', '.', '-j1,1');
+        lei_ok(['edit-search', "$home/before"], $cat_env);
+        like($lei_out, qr/^\tonly = \Q$v2\E$/sm,
+                'relative --only saved to absolute path');
+
         open my $fh, '<', "$home/before";
         PublicInbox::MboxReader->mboxrd($fh, sub { push @before, $_[0] });
         isnt(scalar(@before), 0, 'initial v2 written');
@@ -207,7 +214,7 @@ test_lei(sub {
         ok($shared < $orig, 'fewer bytes stored with --shared') or
                 diag "shared=$shared orig=$orig";
 
-        lei_ok([qw(edit-search), $v2s], { VISUAL => 'cat', EDITOR => 'cat' });
+        lei_ok([qw(edit-search), $v2s], $cat_env);
         like($lei_out, qr/^\[lei/sm, 'edit-search can cat');
 
         lei_ok('-C', "$home/v2s", qw(q -q -o ../s m:testmessage@example.com));
@@ -222,16 +229,14 @@ test_lei(sub {
         my @lss = glob("$home/" .
                 '.local/share/lei/saved-searches/*/lei.saved-search');
         my $out = xqx([qw(git config -f), $lss[0], 'lei.q.output']);
-        xsys($^X, qw(-i -p -e), "s/\\[/\\0/", $lss[0])
-                and xbail "-ipe $lss[0]: $?";
+        xsys_e($^X, qw(-w -i -p -e), "s/\\[/\\0/", $lss[0]);
         lei_ok qw(ls-search);
         like($lei_err, qr/bad config line.*?\Q$lss[0]\E/,
                 'git config parse error shown w/ lei ls-search');
         lei_ok qw(up --all), \'up works with bad config';
         like($lei_err, qr/bad config line.*?\Q$lss[0]\E/,
                 'git config parse error shown w/ lei up');
-        xsys($^X, qw(-i -p -e), "s/\\0/\\[/", $lss[0])
-                and xbail "-ipe $lss[0]: $?";
+        xsys_e($^X, qw(-w -i -p -e), "s/\\0/\\[/", $lss[0]);
         lei_ok qw(ls-search);
         is($lei_err, '', 'no errors w/ fixed config');
 
@@ -243,17 +248,17 @@ test_lei(sub {
 
         my $d = "$home/d";
         lei_ok [qw(import -q -F eml)], undef,
-                {0 => \"Subject: do not call\n\n"};
+                {%$lei_opt, 0 => \"Subject: do not call\n\n"};
         lei_ok qw(q -o), $d, 's:do not call';
 
         my @orig = glob("$d/*/*");
         is(scalar(@orig), 1, 'got one message via argv');
         lei_ok [qw(import -q -Feml)], undef,
-                {0 => \"Subject: do not ever call\n\n"};
+                {%$lei_opt, 0 => \"Subject: do not ever call\n\n"};
         lei_ok 'up', $d;
         is_deeply([glob("$d/*/*")], \@orig, 'nothing written');
         lei_ok [qw(import -q -Feml)], undef,
-                {0 => \"Subject: do not call, ever\n\n"};
+                {%$lei_opt, 0 => \"Subject: do not call, ever\n\n"};
         lei_ok 'up', $d;
         @after = glob("$d/*/*");
         is(scalar(@after), 2, '2 total, messages, now');
@@ -264,14 +269,15 @@ test_lei(sub {
                 'up retrieved correct message');
 
         $d = "$home/d-stdin";
-        lei_ok [ qw(q -q -o), $d ], undef, { 0 => \'s:"do not ever call"' };
+        lei_ok [ qw(q -q -o), $d ], undef,
+                { %$lei_opt, 0 => \'s:"do not ever call"' };
         @orig = glob("$d/*/*");
         is(scalar(@orig), 1, 'got one message via stdin');
 
         lei_ok [qw(import -q -Feml)], undef,
-                {0 => \"Subject: do not fall or ever call\n\n"};
+                {%$lei_opt, 0 => \"Subject: do not fall or ever call\n\n"};
         lei_ok [qw(import -q -Feml)], undef,
-                {0 => \"Subject: do not ever call, again\n\n"};
+                {%$lei_opt, 0 => \"Subject: do not ever call, again\n\n"};
         lei_ok 'up', $d;
         @new = glob("$d/new/*");
         is(scalar(@new), 1, "new message written to `new'") or do {
@@ -281,5 +287,25 @@ test_lei(sub {
         is(eml_load($new[0])->header('Subject'), 'do not ever call, again',
                 'up retrieved correct message');
 
+        # --thread expansion
+        $d = "$home/thread-expand";
+        lei_ok(qw(q --no-external m:import-before@example.com -t -o), $d);
+        @orig = glob("$d/{new,cur}/*");
+        is(scalar(@orig), 1, 'one result so far');
+        lei_ok [ qw(import -Feml) ], undef, { %$lei_opt, 0 => \<<'EOM' };
+Date: Sun, 02 Oct 2023 00:00:00 +0000
+From: <x@example.com>
+In-Reply-To: <import-before@example.com>
+Message-ID: <reply1@example.com>
+Subject: reply1
+EOM
+
+        lei_ok qw(up), $d;
+        @new = glob("$d/{new,cur}/*");
+        is(scalar(@new), 2, 'got new message');
+        is_xdeeply([grep { $_ eq $orig[0] } @new], \@orig,
+                'original message preserved on up w/ threads');
+        lei_ok 'up', "$home/md", $d, \'multiple maildir up';
+        unlike $lei_err, qr! line \d+!s, 'no warnings';
 });
 done_testing;
diff --git a/t/lei-q-thread.t b/t/lei-q-thread.t
index 26d06eec..72d3a565 100644
--- a/t/lei-q-thread.t
+++ b/t/lei-q-thread.t
@@ -3,7 +3,7 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 require_git 2.6;
-require_mods(qw(json DBD::SQLite Search::Xapian));
+require_mods(qw(json DBD::SQLite Xapian));
 use PublicInbox::LeiToMail;
 my ($ro_home, $cfg_path) = setup_public_inboxes;
 test_lei(sub {
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
index ea83a513..8ccc68c6 100644
--- a/t/lei-refresh-mail-sync.t
+++ b/t/lei-refresh-mail-sync.t
@@ -5,17 +5,20 @@ use strict; use v5.10.1; use PublicInbox::TestCommon;
 require_mods(qw(lei));
 use File::Path qw(remove_tree);
 require Socket;
+use Fcntl qw(F_SETFD);
+
+pipe(my ($stop_r, $stop_w)) or xbail "pipe: $!";
+fcntl($stop_w, F_SETFD, 0) or xbail "F_SETFD: $!";
 
 my $stop_daemon = sub { # needed since we don't have inotify
+        close $stop_w or xbail "close \$stop_w: $!";
         lei_ok qw(daemon-pid);
         chomp(my $pid = $lei_out);
         $pid > 0 or xbail "bad pid: $pid";
         kill('TERM', $pid) or xbail "kill: $!";
-        for (0..10) {
-                tick;
-                kill(0, $pid) or last;
-        }
-        kill(0, $pid) and xbail "daemon still running (PID:$pid)";
+        is(sysread($stop_r, my $buf, 1), 0, 'daemon stop pipe read EOF');
+        pipe($stop_r, $stop_w) or xbail "pipe: $!";
+        fcntl($stop_w, F_SETFD, 0) or xbail "F_SETFD: $!";
 };
 
 test_lei({ daemon_only => 1 }, sub {
@@ -88,7 +91,8 @@ SKIP: {
                 $sock_cls //= ref($s);
                 my $cmd = [ "-$x", '-W0', "--stdout=$home/$x.out",
                         "--stderr=$home/$x.err" ];
-                my $td = start_script($cmd, $env, { 3 => $s }) or xbail("-$x");
+                my $opt = { 3 => $s, -CLOFORK => [ $stop_w ] };
+                my $td = start_script($cmd, $env, $opt) or xbail("-$x");
                 my $addr = tcp_host_port($s);
                 $srv->{$x} = { addr => $addr, td => $td, cmd => $cmd, s => $s };
         }
@@ -133,14 +137,18 @@ SKIP: {
         my $ar = PublicInbox::AutoReap->new($pid);
         ok(!(lei 'refresh-mail-sync', $url), 'URL fails on dead -imapd');
         ok(!(lei 'refresh-mail-sync', '--all'), '--all fails on dead -imapd');
-        $ar->kill for qw(avoid sig wake miss-no signalfd or EVFILT_SIG);
-        $ar->join('TERM');
+        {
+                local $SIG{CHLD} = sub { $ar->join('TERM'); undef $ar };
+                do {
+                        eval { $ar->kill and tick(0.01) }
+                } while (defined($ar));
+        }
 
         my $cmd = $srv->{imapd}->{cmd};
         my $s = $srv->{imapd}->{s};
         $s->blocking(0);
-        $srv->{imapd}->{td} = start_script($cmd, $env, { 3 => $s }) or
-                xbail "@$cmd";
+        my $opt = { 3 => $s, -CLOFORK => [ $stop_w ] };
+        $srv->{imapd}->{td} = start_script($cmd, $env, $opt) or xbail "@$cmd";
         lei_ok 'refresh-mail-sync', '--all';
         lei_ok 'inspect', "blob:$oid";
         is($lei_out, $before, 'no changes when server was down');
diff --git a/t/lei-reindex.t b/t/lei-reindex.t
new file mode 100644
index 00000000..73346ee8
--- /dev/null
+++ b/t/lei-reindex.t
@@ -0,0 +1,12 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12; use PublicInbox::TestCommon;
+require_mods(qw(lei));
+my ($tmpdir, $for_destroy) = tmpdir;
+test_lei(sub {
+        ok(!lei('reindex'), 'reindex fails w/o store');
+        like $lei_err, qr/nothing indexed/, "`nothing indexed' noted";
+});
+
+done_testing;
diff --git a/t/lei-sigpipe.t b/t/lei-sigpipe.t
index 55c208e2..72bc6c7d 100644
--- a/t/lei-sigpipe.t
+++ b/t/lei-sigpipe.t
@@ -6,6 +6,20 @@ use v5.10.1;
 use PublicInbox::TestCommon;
 use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
 use PublicInbox::OnDestroy;
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
+use autodie qw(close open pipe seek sysread);
+use PublicInbox::IO qw(write_file);
+my $inboxdir = $ENV{GIANT_INBOX_DIR};
+SKIP: {
+        $inboxdir // skip 'GIANT_INBOX_DIR unset to test large results', 1;
+        require PublicInbox::Inbox;
+        my $ibx = PublicInbox::Inbox->new({
+                name => 'unconfigured-test',
+                address => [ "test\@example.com" ],
+                inboxdir => $inboxdir,
+        });
+        $ibx->search or xbail "GIANT_INBOX_DIR=$inboxdir has no search";
+}
 
 # undo systemd (and similar) ignoring SIGPIPE, since lei expects to be run
 # from an interactive terminal:
@@ -20,32 +34,30 @@ test_lei(sub {
         my $f = "$ENV{HOME}/big.eml";
         my $imported;
         for my $out ([], [qw(-f mboxcl2)], [qw(-f text)]) {
-                pipe(my ($r, $w)) or BAIL_OUT $!;
-                my $size = 65536;
-                if ($^O eq 'linux' && fcntl($w, 1031, 4096)) {
-                        $size = 4096;
-                }
+                pipe(my $r, my $w);
+                my $size = $F_SETPIPE_SZ && fcntl($w, $F_SETPIPE_SZ, 4096) ?
+                        4096 : 65536;
                 unless (-f $f) {
-                        open my $fh, '>', $f or xbail "open $f: $!";
-                        print $fh <<'EOM' or xbail;
+                        my $fh = write_file '>', $f, <<'EOM';
 From: big@example.com
 Message-ID: <big@example.com>
 EOM
                         print $fh 'Subject:';
                         print $fh (' '.('x' x 72)."\n") x (($size / 73) + 1);
                         print $fh "\nbody\n";
-                        close $fh or xbail "close: $!";
+                        close $fh;
                 }
 
                 lei_ok(qw(import), $f) if $imported++ == 0;
-                open my $errfh, '+>>', "$ENV{HOME}/stderr.log" or xbail $!;
+                open my $errfh, '+>>', "$ENV{HOME}/stderr.log";
                 my $opt = { run_mode => 0, 2 => $errfh, 1 => $w };
                 my $cmd = [qw(lei q -q -t), @$out, 'z:1..'];
+                push @$cmd, '--only='.$inboxdir if defined $inboxdir;
                 my $tp = start_script($cmd, undef, $opt);
                 close $w;
                 vec(my $rvec = '', fileno($r), 1) = 1;
                 if (!select($rvec, undef, undef, 30)) {
-                        seek($errfh, 0, 0) or xbail $!;
+                        seek($errfh, 0, 0);
                         my $s = do { local $/; <$errfh> };
                         xbail "lei q had no output after 30s, stderr=$s";
                 }
@@ -54,7 +66,7 @@ EOM
                 $tp->join;
                 ok(WIFSIGNALED($?), "signaled @$out");
                 is(WTERMSIG($?), SIGPIPE, "got SIGPIPE @$out");
-                seek($errfh, 0, 0) or xbail $!;
+                seek($errfh, 0, 0);
                 my $s = do { local $/; <$errfh> };
                 is($s, '', "quiet after sigpipe @$out");
         }
diff --git a/t/lei-store-fail.t b/t/lei-store-fail.t
new file mode 100644
index 00000000..c2f03148
--- /dev/null
+++ b/t/lei-store-fail.t
@@ -0,0 +1,57 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# ensure we detect errors in lei/store
+use v5.12;
+use PublicInbox::TestCommon;
+use autodie qw(pipe open close seek);
+use Fcntl qw(SEEK_SET);
+use File::Path qw(remove_tree);
+
+my $start_home = $ENV{HOME}; # bug guard
+my $utf8_oid = '9bf1002c49eb075df47247b74d69bcd555e23422';
+test_lei(sub {
+        lei_ok qw(import -q t/plack-qp.eml); # start the store
+        ok(!lei(qw(blob --mail), $utf8_oid), 't/utf8.eml not imported, yet');
+
+        my $opt;
+        pipe($opt->{0}, my $in_w);
+        open $opt->{1}, '+>', undef;
+        open $opt->{2}, '+>', undef;
+        $opt->{-CLOFORK} = [ $in_w ];
+        my $cmd = [ qw(lei import -q -F mboxrd) ];
+        my $tp = start_script($cmd, undef, $opt);
+        close $opt->{0};
+        $in_w->autoflush(1);
+        print $in_w <<EOM or xbail "print: $!";
+From k\@y Fri Oct  2 00:00:00 1993
+From: <k\@example.com>
+Date: Sat, 02 Oct 2010 00:00:00 +0000
+Subject: hi
+Message-ID: <0\@t>
+
+will this save?
+EOM
+        # import another message w/ delay while mboxrd import is still running
+        lei_ok qw(import -q --commit-delay=300 t/utf8.eml);
+        lei_ok qw(blob --mail), $utf8_oid,
+                \'blob immediately available despite --commit-delay';
+        lei_ok qw(q m:testmessage@example.com);
+        is($lei_out, "[null]\n", 'delayed commit is unindexed');
+
+        # make immediate ->sto_done_request fail from mboxrd import:
+        remove_tree("$ENV{HOME}/.local/share/lei/store");
+        # subsequent lei commands are undefined behavior,
+        # but we need to make sure the current lei command fails:
+
+        close $in_w; # should trigger ->done
+        $tp->join;
+        isnt($?, 0, 'lei import -F mboxrd error code set on failure');
+        is(-s $opt->{1}, 0, 'nothing in stdout');
+        isnt(-s $opt->{2}, 0, 'stderr not empty');
+        seek($opt->{2}, 0, SEEK_SET);
+        my @err = readline($opt->{2});
+        ok(grep(!/^#/, @err), 'noted error in stderr') or diag "@err";
+});
+
+done_testing;
diff --git a/t/lei-tag.t b/t/lei-tag.t
index 5941cd0f..7278dfcd 100644
--- a/t/lei-tag.t
+++ b/t/lei-tag.t
@@ -1,9 +1,10 @@
 #!perl -w
 # Copyright (C) 2021 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict; use v5.10.1; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
 require_git 2.6;
-require_mods(qw(json DBD::SQLite Search::Xapian));
+require_mods(qw(json DBD::SQLite Xapian));
+use PublicInbox::DS qw(now);
 my ($ro_home, $cfg_path) = setup_public_inboxes;
 my $check_kw = sub {
         my ($exp, %opt) = @_;
@@ -101,5 +102,20 @@ test_lei(sub {
         if (0) { # TODO label+kw search w/ externals
                 lei_ok(qw(q L:qp), "mid:$mid", '--only', "$ro_home/t2");
         }
+        lei_ok qw(tag +L:nope -F eml t/data/binary.patch);
+        like $lei_err, qr/\b1 unimported messages/, 'noted unimported'
+                or diag $lei_err;
+
+        lei_ok qw(tag -F eml --commit-delay=1 t/utf8.eml +L:utf8);
+        lei_ok 'ls-label';
+        unlike($lei_out, qr/\butf8\b/, 'commit-delay delays label');
+        my $end = now + 10;
+        my $n = 1;
+        diag 'waiting for lei/store commit...';
+        do {
+                tick $n;
+                $n = 0.1;
+        } until (!lei('ls-label') || $lei_out =~ /\butf8\b/ || now > $end);
+        like($lei_out, qr/\butf8\b/, 'commit-delay eventually commits');
 });
 done_testing;
diff --git a/t/lei-up.t b/t/lei-up.t
index fc369156..2d3afd82 100644
--- a/t/lei-up.t
+++ b/t/lei-up.t
@@ -5,39 +5,50 @@ use strict; use v5.10.1; use PublicInbox::TestCommon;
 use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
 test_lei(sub {
         my ($ro_home, $cfg_path) = setup_public_inboxes;
-        my $s = eml_load('t/plack-qp.eml')->as_string;
+        my $home = $ENV{HOME};
+        my $qp = eml_load('t/plack-qp.eml');
+        my $s = $qp->as_string;
         lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
-        lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a.mbox.gz";
-        lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/b.mbox.gz";
-        lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a";
-        lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/b";
+        lei_ok qw(q z:0.. -f mboxcl2 -o), "$home/a.mbox.gz";
+        lei_ok qw(q z:0.. -f mboxcl2 -o), "$home/b.mbox.gz";
+        lei_ok qw(q z:0.. -f mboxcl2 -o), "$home/a";
+        lei_ok qw(q z:0.. -f mboxcl2 -o), "$home/b";
+        my $uc;
+        for my $x (qw(a b)) {
+                gunzip("$home/$x.mbox.gz" => \$uc, MultiStream => 1) or
+                                xbail "gunzip $GunzipError";
+                ok(index($uc, $qp->body_raw) >= 0,
+                        "original mail in $x.mbox.gz") or diag $uc;
+                open my $fh, '<', "$home/$x" or xbail $!;
+                $uc = do { local $/; <$fh> } // xbail $!;
+                ok(index($uc, $qp->body_raw) >= 0,
+                        "original mail in uncompressed $x") or diag $uc;
+        }
         lei_ok qw(ls-search);
         $s = eml_load('t/utf8.eml')->as_string;
         lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
         lei_ok qw(up --all=local);
-        open my $fh, "$ENV{HOME}/a.mbox.gz" or xbail "open: $!";
-        my $gz = do { local $/; <$fh> };
-        my $uc;
-        gunzip(\$gz => \$uc, MultiStream => 1) or xbail "gunzip $GunzipError";
-        open $fh, "$ENV{HOME}/a" or xbail "open: $!";
 
+        gunzip("$home/a.mbox.gz" => \$uc, MultiStream => 1) or
+                 xbail "gunzip $GunzipError";
+
+        open my $fh, '<', "$home/a" or xbail "open: $!";
         my $exp = do { local $/; <$fh> };
         is($uc, $exp, 'compressed and uncompressed match (a.gz)');
         like($exp, qr/testmessage\@example.com/, '2nd message added');
-        open $fh, "$ENV{HOME}/b.mbox.gz" or xbail "open: $!";
 
-        $gz = do { local $/; <$fh> };
         undef $uc;
-        gunzip(\$gz => \$uc, MultiStream => 1) or xbail "gunzip $GunzipError";
+        gunzip("$home/b.mbox.gz" => \$uc, MultiStream => 1) or
+                 xbail "gunzip $GunzipError";
         is($uc, $exp, 'compressed and uncompressed match (b.gz)');
 
-        open $fh, "$ENV{HOME}/b" or xbail "open: $!";
+        open $fh, '<', "$home/b" or xbail "open: $!";
         $uc = do { local $/; <$fh> };
         is($uc, $exp, 'uncompressed both match');
 
-        lei_ok [ qw(up -q), "$ENV{HOME}/b", "--mua=touch $ENV{HOME}/c" ],
+        lei_ok [ qw(up -q), "$home/b", "--mua=touch $home/c" ],
                 undef, { run_mode => 0 };
-        ok(-f "$ENV{HOME}/c", '--mua works with single output');
+        ok(-f "$home/c", '--mua works with single output');
 });
 
 done_testing;
diff --git a/t/lei-watch.t b/t/lei-watch.t
index 24d9f5c8..8ad50d13 100644
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -3,17 +3,18 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 use File::Path qw(make_path remove_tree);
+use PublicInbox::IO qw(write_file);
 plan skip_all => "TEST_FLAKY not enabled for $0" if !$ENV{TEST_FLAKY};
 require_mods('lei');
-my $have_fast_inotify = eval { require Linux::Inotify2 } ||
+my $have_fast_inotify = eval { require PublicInbox::Inotify } ||
         eval { require IO::KQueue };
 
 $have_fast_inotify or
-        diag("$0 IO::KQueue or Linux::Inotify2 missing, test will be slow");
+        diag("$0 IO::KQueue or inotify missing, test will be slow");
 
 my ($ro_home, $cfg_path) = setup_public_inboxes;
 test_lei(sub {
-        my $md = "$ENV{HOME}/md";
+        my ($md, $mh1, $mh2) = map { "$ENV{HOME}/$_" } qw(md mh1 mh2);
         my $cfg_f = "$ENV{HOME}/.config/lei/config";
         my $md2 = $md.'2';
         lei_ok 'ls-watch';
@@ -45,13 +46,14 @@ test_lei(sub {
         }
 
         # first, make sure tag-ro works
-        make_path("$md/new", "$md/cur", "$md/tmp");
+        make_path("$md/new", "$md/cur", "$md/tmp", $mh1, $mh2);
         lei_ok qw(add-watch --state=tag-ro), $md;
         lei_ok 'ls-watch';
         like($lei_out, qr/^\Qmaildir:$md\E$/sm, 'maildir shown');
         lei_ok qw(q mid:testmessage@example.com -o), $md, '-I', "$ro_home/t1";
         my @f = glob("$md/cur/*:2,");
         is(scalar(@f), 1, 'got populated maildir with one result');
+
         rename($f[0], "$f[0]S") or xbail "rename $!"; # set (S)een
         tick($have_fast_inotify ? 0.2 : 2.2); # always needed for 1 CPU systems
         lei_ok qw(note-event done); # flushes immediately (instead of 5s)
@@ -94,6 +96,12 @@ test_lei(sub {
                 my $cmp = [ <$fh> ];
                 is_xdeeply($cmp, $ino_contents, 'inotify Maildir watches gone');
         };
+
+        write_file '>', "$mh1/.mh_sequences";
+        lei_ok qw(add-watch --state=tag-ro), $mh1, "mh:$mh2";
+        lei_ok 'ls-watch', \'refresh watches';
+        like $lei_out, qr/^\Qmh:$mh1\E$/sm, 'MH 1 shown';
+        like $lei_out, qr/^\Qmh:$mh2\E$/sm, 'MH 2 shown';
 });
 
 done_testing;
diff --git a/t/lei.t b/t/lei.t
index b10c9b59..1dbc9d4c 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -1,7 +1,8 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
+require_mods 'lei';
 use File::Path qw(rmtree);
 
 # this only tests the basic help/config/init/completion bits of lei;
@@ -39,10 +40,21 @@ my $test_help = sub {
         lei_ok(qw(config -h));
         like($lei_out, qr! \Q$home\E/\.config/lei/config\b!,
                 'actual path shown in config -h');
+        my $exp_help = qr/\Q$lei_out\E/s;
+        ok(!lei('config'), 'config w/o args fails');
+        like($lei_err, $exp_help, 'config w/o args shows our help in stderr');
         lei_ok(qw(config -h), { XDG_CONFIG_HOME => '/XDC' },
                 \'config with XDG_CONFIG_HOME');
         like($lei_out, qr! /XDC/lei/config\b!, 'XDG_CONFIG_HOME in config -h');
         is($lei_err, '', 'no errors from config -h');
+
+        lei_ok(qw(-c foo.bar config dash.c works));
+        lei_ok(qw(config dash.c));
+        is($lei_out, "works\n", 'config set w/ -c');
+
+        lei_ok(qw(-c foo.bar config --add dash.c add-works));
+        lei_ok(qw(config --get-all dash.c));
+        is($lei_out, "works\nadd-works\n", 'config --add w/ -c');
 };
 
 my $ok_err_info = sub {
@@ -100,9 +112,11 @@ my $test_config = sub {
         is($lei_out, "tr00\n", "-c string value passed as-is");
         lei_ok(qw(-c imap.debug=a -c imap.debug=b config --get-all imap.debug));
         is($lei_out, "a\nb\n", '-c and --get-all work together');
-
-        lei_ok([qw(config -e)], { VISUAL => 'cat', EDITOR => 'cat' });
+        my $env = { VISUAL => 'cat', EDITOR => 'cat' };
+        lei_ok([qw(config -e)], $env);
         is($lei_out, "[a]\n\tb = c\n", '--edit works');
+        ok(!lei([qw(-c a.b=c config -e)], $env), '-c conflicts with -e');
+        like($lei_err, qr/not allowed/, 'error message shown');
 };
 
 my $test_completion = sub {
@@ -146,9 +160,18 @@ my $test_fail = sub {
         lei_ok('q', "foo\n");
         like($lei_err, qr/trailing `\\n' removed/s, "noted `\\n' removal");
 
+        lei(qw(q from:infinity..));
+        is($? >> 8, 22, 'combined query fails on invalid range op');
+        lei(qw(q -t from:infinity..));
+        is($? >> 8, 22, 'single query fails on invalid range op');
+
         for my $lk (qw(ei inbox)) {
                 my $d = "$home/newline\n$lk";
-                mkdir $d;
+                my $all = $lk eq 'ei' ? 'ALL' : 'all';
+                { # quiet newline warning on older Perls
+                        local $^W = undef if $^V lt v5.22.0;
+                        File::Path::mkpath("$d/$all.git/objects");
+                }
                 open my $fh, '>', "$d/$lk.lock" or BAIL_OUT "open $d/$lk.lock";
                 for my $fl (qw(-I --only)) {
                         ok(!lei('q', $fl, $d, 'whatever'),
@@ -159,6 +182,11 @@ my $test_fail = sub {
         }
         lei_ok('sucks', \'yes, but hopefully less every day');
         like($lei_out, qr/loaded features/, 'loaded features shown');
+
+        lei_ok([qw(q --stdin -f text)], undef, { 0 => \'', %$lei_opt });
+        is($lei_err, '', 'no errors on empty stdin');
+        is($lei_out, '', 'no output on empty query');
+
 SKIP: {
         skip 'no curl', 3 unless require_cmd('curl', 1);
         lei(qw(q --only http://127.0.0.1:99999/bogus/ t:m));
diff --git a/t/lei_dedupe.t b/t/lei_dedupe.t
index e1944d02..13fc1f3b 100644
--- a/t/lei_dedupe.t
+++ b/t/lei_dedupe.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
@@ -10,6 +10,8 @@ use PublicInbox::Smsg;
 require_mods(qw(DBD::SQLite));
 use_ok 'PublicInbox::LeiDedupe';
 my $eml = eml_load('t/plack-qp.eml');
+my $sameish = eml_load('t/plack-qp.eml');
+$sameish->header_set('Message-ID', '<cuepee@example.com>');
 my $mid = $eml->header_raw('Message-ID');
 my $different = eml_load('t/msg_iter-order.eml');
 $different->header_set('Message-ID', $mid);
@@ -47,6 +49,8 @@ for my $strat (undef, 'content') {
         ok(!$dd->is_dup($different), "different is_dup with $desc dedupe");
         ok(!$dd->is_smsg_dup($smsg), "is_smsg_dup pass w/ $desc dedupe");
         ok($dd->is_smsg_dup($smsg), "is_smsg_dup reject w/ $desc dedupe");
+        ok(!$dd->is_dup($sameish),
+                "Message-ID accounted for w/ same content otherwise");
 }
 $lei->{opt}->{dedupe} = 'bogus';
 eval { PublicInbox::LeiDedupe->new($lei) };
diff --git a/t/lei_external.t b/t/lei_external.t
index 51d0af5c..573cbc60 100644
--- a/t/lei_external.t
+++ b/t/lei_external.t
@@ -1,8 +1,8 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # internal unit test, see t/lei-externals.t for functional tests
-use strict; use v5.10.1; use Test::More;
+use v5.12; use Test::More;
 my $cls = 'PublicInbox::LeiExternal';
 require_ok $cls;
 my $canon = $cls->can('ext_canonicalize');
@@ -16,20 +16,4 @@ is($canon->('/this/path/is/nonexistent/'), '/this/path/is/nonexistent',
 is($canon->('/this//path/'), '/this/path', 'extra slashes gone');
 is($canon->('/ALL/CAPS'), '/ALL/CAPS', 'caps preserved');
 
-my $glob2re = $cls->can('glob2re');
-is($glob2re->('http://[::1]:1234/foo/'), undef, 'IPv6 URL not globbed');
-is($glob2re->('foo'), undef, 'plain string unchanged');
-is_deeply($glob2re->('[f-o]'), '[f-o]' , 'range accepted');
-is_deeply($glob2re->('*'), '[^/]*?' , 'wildcard accepted');
-is_deeply($glob2re->('{a,b,c}'), '(a|b|c)' , 'braces');
-is_deeply($glob2re->('{,b,c}'), '(|b|c)' , 'brace with empty @ start');
-is_deeply($glob2re->('{a,b,}'), '(a|b|)' , 'brace with empty @ end');
-is_deeply($glob2re->('{a}'), undef, 'ungrouped brace');
-is_deeply($glob2re->('{a'), undef, 'open left brace');
-is_deeply($glob2re->('a}'), undef, 'open right brace');
-is_deeply($glob2re->('*.[ch]'), '[^/]*?\\.[ch]', 'suffix glob');
-is_deeply($glob2re->('{[a-z],9,}'), '([a-z]|9|)' , 'brace with range');
-is_deeply($glob2re->('\\{a,b\\}'), undef, 'escaped brace');
-is_deeply($glob2re->('\\\\{a,b}'), '\\\\\\\\(a|b)', 'fake escape brace');
-
 done_testing;
diff --git a/t/lei_overview.t b/t/lei_overview.t
index dd9e2cad..b4181ffd 100644
--- a/t/lei_overview.t
+++ b/t/lei_overview.t
@@ -6,7 +6,7 @@ use v5.10.1;
 use Test::More;
 use PublicInbox::TestCommon;
 use POSIX qw(_exit);
-require_mods(qw(Search::Xapian DBD::SQLite));
+require_mods(qw(Xapian DBD::SQLite));
 require_ok 'PublicInbox::LeiOverview';
 
 my $ovv = bless {}, 'PublicInbox::LeiOverview';
diff --git a/t/lei_store.t b/t/lei_store.t
index 40ad7800..17ee0729 100644
--- a/t/lei_store.t
+++ b/t/lei_store.t
@@ -1,11 +1,11 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
 use Test::More;
 use PublicInbox::TestCommon;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 require_git 2.6;
 require_ok 'PublicInbox::LeiStore';
 require_ok 'PublicInbox::ExtSearch';
@@ -149,4 +149,7 @@ EOM
         is($mset->size, 1, 'rt:1.hour.ago.. works w/ local time');
 }
 
+is_deeply([glob("$store_dir/local/*.git/info/refs")], [],
+        'no info/refs in private lei/store');
+
 done_testing;
diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t
index e8958c64..dbd33909 100644
--- a/t/lei_to_mail.t
+++ b/t/lei_to_mail.t
@@ -1,9 +1,10 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
-use Test::More;
+# tests PublicInbox::LeiToMail internals (unstable API)
+# Not as needed now that lei functionality has been ironed out
+use v5.12;
+use autodie qw(open sysopen unlink);
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
 use Fcntl qw(SEEK_SET O_RDONLY O_NONBLOCK);
@@ -74,7 +75,7 @@ for my $mbox (@MBOX) {
 
 my ($tmpdir, $for_destroy) = tmpdir();
 local $ENV{TMPDIR} = $tmpdir;
-open my $err, '>>', "$tmpdir/lei.err" or BAIL_OUT $!;
+open my $err, '>>', "$tmpdir/lei.err";
 my $lei = bless { 2 => $err, cmd => 'test' }, 'PublicInbox::LEI';
 my $commit = sub {
         $_[0] = undef; # wcb
@@ -114,16 +115,16 @@ my $orig = do {
         ok(-f $fn && !-s _, 'empty file created');
         $wcb->(\(my $dup = $buf), $deadbeef);
         $commit->($wcb);
-        open my $fh, '<', $fn or BAIL_OUT $!;
+        open my $fh, '<', $fn;
         my $raw = do { local $/; <$fh> };
         like($raw, qr/^blah\n/sm, 'wrote content');
-        unlink $fn or BAIL_OUT $!;
+        unlink $fn;
 
         $wcb = $wcb_get->($mbox, $fn);
         ok(-f $fn && !-s _, 'truncated mbox destination');
         $wcb->(\($dup = $buf), $deadbeef);
         $commit->($wcb);
-        open $fh, '<', $fn or BAIL_OUT $!;
+        open $fh, '<', $fn;
         is(do { local $/; <$fh> }, $raw, 'wrote identical content');
         $raw;
 };
@@ -162,7 +163,7 @@ for my $zsfx (qw(gz bz2 xz)) {
                 my $uncompressed = xqx([@$dc_cmd, $f]);
                 is($uncompressed, $orig, "$zsfx works unlocked");
 
-                unlink $f or BAIL_OUT "unlink $!";
+                unlink $f;
                 $wcb = $wcb_get->($mbox, $f);
                 $wcb->(\($dup = $buf), { %$deadbeef });
                 $commit->($wcb);
@@ -201,14 +202,14 @@ my $as_orig = sub {
         $eml->as_string;
 };
 
-unlink $fn or BAIL_OUT $!;
+unlink $fn;
 if ('default deduplication uses content_hash') {
         my $wcb = $wcb_get->('mboxo', $fn);
         $deadbeef->{kw} = [];
         $wcb->(\(my $x = $buf), $deadbeef) for (1..2);
         $commit->($wcb);
         my $cmp = '';
-        open my $fh, '<', $fn or BAIL_OUT $!;
+        open my $fh, '<', $fn;
         PublicInbox::MboxReader->mboxo($fh, sub { $cmp .= $as_orig->(@_) });
         is($cmp, $buf, 'only one message written');
 
@@ -216,7 +217,7 @@ if ('default deduplication uses content_hash') {
         $wcb = $wcb_get->('mboxo', $fn);
         $wcb->(\($x = $buf . "\nx\n"), $deadbeef) for (1..2);
         $commit->($wcb);
-        open $fh, '<', $fn or BAIL_OUT $!;
+        open $fh, '<', $fn;
         my @x;
         PublicInbox::MboxReader->mboxo($fh, sub { push @x, $as_orig->(@_) });
         is(scalar(@x), 2, 'augmented mboxo');
@@ -225,12 +226,12 @@ if ('default deduplication uses content_hash') {
 }
 
 { # stdout support
-        open my $tmp, '+>', undef or BAIL_OUT $!;
+        open my $tmp, '+>', undef;
         local $lei->{1} = $tmp;
         my $wcb = $wcb_get->('mboxrd', '/dev/stdout');
         $wcb->(\(my $x = $buf), $deadbeef);
         $commit->($wcb);
-        seek($tmp, 0, SEEK_SET) or BAIL_OUT $!;
+        seek($tmp, 0, SEEK_SET);
         my $cmp = '';
         PublicInbox::MboxReader->mboxrd($tmp, sub { $cmp .= $as_orig->(@_) });
         is($cmp, $buf, 'message written to stdout');
@@ -240,7 +241,7 @@ SKIP: { # FIFO support
         use POSIX qw(mkfifo);
         my $fn = "$tmpdir/fifo";
         mkfifo($fn, 0600) or skip("mkfifo not supported: $!", 1);
-        sysopen(my $cat, $fn, O_RDONLY|O_NONBLOCK) or BAIL_OUT $!;
+        sysopen(my $cat, $fn, O_RDONLY|O_NONBLOCK);
         my $wcb = $wcb_get->('mboxo', $fn);
         $wcb->(\(my $x = $buf), $deadbeef);
         $commit->($wcb);
@@ -260,7 +261,7 @@ SKIP: { # FIFO support
 
         my @f;
         $mdr->maildir_each_file($md, sub { push @f, shift });
-        open my $fh, $f[0] or BAIL_OUT $!;
+        open my $fh, '<', $f[0];
         is(do { local $/; <$fh> }, $buf, 'wrote to Maildir');
 
         $wcb = $wcb_get->('maildir', $md);
@@ -271,7 +272,7 @@ SKIP: { # FIFO support
         $mdr->maildir_each_file($md, sub { push @x, shift });
         is(scalar(@x), 1, 'wrote one new file');
         ok(!-f $f[0], 'old file clobbered');
-        open $fh, $x[0] or BAIL_OUT $!;
+        open $fh, '<', $x[0];
         is(do { local $/; <$fh> }, $buf."\nx\n", 'wrote new file to Maildir');
 
         local $lei->{opt}->{augment} = 1;
@@ -283,9 +284,9 @@ SKIP: { # FIFO support
         is(scalar grep(/\A\Q$x[0]\E\z/, @f), 1, 'old file still there');
         my @new = grep(!/\A\Q$x[0]\E\z/, @f);
         is(scalar @new, 1, '1 new file written (b4dc0ffee skipped)');
-        open $fh, $x[0] or BAIL_OUT $!;
+        open $fh, '<', $x[0];
         is(do { local $/; <$fh> }, $buf."\nx\n", 'old file untouched');
-        open $fh, $new[0] or BAIL_OUT $!;
+        open $fh, '<', $new[0];
         is(do { local $/; <$fh> }, $buf."\ny\n", 'new file written');
 }
 
diff --git a/t/lei_xsearch.t b/t/lei_xsearch.t
index d9ddb297..977fb1e9 100644
--- a/t/lei_xsearch.t
+++ b/t/lei_xsearch.t
@@ -6,7 +6,7 @@ use v5.10.1;
 use List::Util qw(shuffle);
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 require PublicInbox::ExtSearchIdx;
 require_git 2.6;
 require_ok 'PublicInbox::LeiXSearch';
@@ -61,7 +61,7 @@ for my $mi ($mset->items) {
 }
 is(scalar(@msgs), $nr, 'smsgs retrieved for all');
 
-$mset = $lxs->recent(undef, { limit => 1 });
+$mset = $lxs->mset('z:1..', { relevance => -2, limit => 1 });
 is($mset->size, 1, 'one result');
 
 my @ibxish = $lxs->locals;
diff --git a/t/linkify.t b/t/linkify.t
index e42e1efe..9280fd91 100644
--- a/t/linkify.t
+++ b/t/linkify.t
@@ -144,4 +144,9 @@ href="http://www.$hc.example.com/">http://www.$hc.example.com/</a>};
         is($s, $expect, 'IDN message escaped properly');
 }
 
+{
+        my $false_positive = 'LINKIFY'.('A' x 40);
+        is(PublicInbox::Linkify->new->to_html($false_positive),
+                $false_positive, 'false-positive left as-is');
+}
 done_testing();
diff --git a/t/mbox_reader.t b/t/mbox_reader.t
index 87e8f397..14248a2d 100644
--- a/t/mbox_reader.t
+++ b/t/mbox_reader.t
@@ -113,10 +113,10 @@ EOM
 
 SKIP: {
         use PublicInbox::Spawn qw(popen_rd);
-        my $fh = popen_rd([ $^X, '-E', <<'' ]);
-say "From x@y Fri Oct  2 00:00:00 1993";
+        my $fh = popen_rd([ $^X, qw(-w -Mv5.12 -e), <<'' ]);
+say 'From x@y Fri Oct  2 00:00:00 1993';
 print "a: b\n\n", "x" x 70000, "\n\n";
-say "From x@y Fri Oct  2 00:00:00 2010";
+say 'From x@y Fri Oct  2 00:00:00 2010';
 print "Final: bit\n\n", "Incomplete\n\n";
 exit 1
 
diff --git a/t/mda.t b/t/mda.t
index d20cdb92..1d9e237b 100644
--- a/t/mda.t
+++ b/t/mda.t
@@ -1,14 +1,15 @@
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use warnings;
-use Test::More;
 use Cwd qw(getcwd);
 use PublicInbox::MID qw(mid2path);
 use PublicInbox::Git;
 use PublicInbox::InboxWritable;
 use PublicInbox::TestCommon;
 use PublicInbox::Import;
+use PublicInbox::IO qw(write_file);
+use File::Path qw(remove_tree);
 my ($tmpdir, $for_destroy) = tmpdir();
 my $home = "$tmpdir/pi-home";
 my $pi_home = "$home/.public-inbox";
@@ -49,13 +50,11 @@ my $fail_bad_header = sub ($$$) {
         is(1, mkdir($pi_home, 0755), "setup ~/.public-inbox");
         PublicInbox::Import::init_bare($maindir);
 
-        open my $fh, '>>', $pi_config or die;
-        print $fh <<EOF or die;
+        write_file '>>', $pi_config, <<EOF;
 [publicinbox "test"]
         address = $addr
         inboxdir = $maindir
 EOF
-        close $fh or die;
 }
 
 local $ENV{GIT_COMMITTER_NAME} = eval {
@@ -83,6 +82,13 @@ die $@ if $@;
         local $ENV{PI_EMERGENCY} = $faildir;
         local $ENV{HOME} = $home;
         local $ENV{ORIGINAL_RECIPIENT} = $addr;
+        ok(run_script([qw(-mda --help)], undef,
+                { 1 => \my $out, 2 => \my $err }), '-mda --help');
+        like $out, qr/usage:/, 'usage shown w/ --help';
+        ok(!run_script([qw(-mda --bogus)], undef,
+                { 1 => \$out, 2 => \$err }), '-mda --bogus fails');
+        like $err, qr/usage:/, 'usage shown on bogus switch';
+
         my $in = <<EOF;
 From: Me <me\@example.com>
 To: You <you\@example.com>
@@ -92,12 +98,23 @@ Subject: hihi
 Date: Thu, 01 Jan 1970 00:00:00 +0000
 
 EOF
+        {
+                local $ENV{PATH} = $main_path;
+                ok(!run_script(['-mda'], { ORIGINAL_RECIPIENT => undef },
+                        { 0 => \$in, 2 => \$err }),
+                        'missing ORIGINAL_RECIPIENT fails');
+                is($? >> 8, 67, 'got EX_NOUSER');
+                like $err, qr/\bORIGINAL_RECIPIENT\b/,
+                        'ORIGINAL_RECIPIENT noted in stderr';
+                is unlink(glob("$faildir/*/*")), 1, 'unlinked failed message';
+        }
+
         # ensure successful message delivery
         {
                 local $ENV{PATH} = $main_path;
                 ok(run_script(['-mda'], undef, { 0 => \$in }));
                 my $rev = $git->qx(qw(rev-list HEAD));
-                like($rev, qr/\A[a-f0-9]{40}/, "good revision committed");
+                like($rev, qr/\A[a-f0-9]{40,64}/, "good revision committed");
                 chomp $rev;
                 my $cmt = $git->cat_file($rev);
                 like($$cmt, qr/^author Me <me\@example\.com> 0 \+0000\n/m,
@@ -306,10 +323,79 @@ EOF
         # ensure -learn rm works after inbox address is updated
         ($out, $err) = ('', '');
         xsys(qw(git config --file), $pi_config, "$cfgpfx.address",
-                'updated-address@example.com');
+                $addr = 'updated-address@example.com');
         ok(run_script(['-learn', 'rm'], undef, $rdr), 'rm-ed via -learn');
         $cur = $git->qx(qw(diff HEAD~1..HEAD));
         like($cur, qr/^-Message-ID: <2lids\@example>/sm, 'changed in git');
+
+        # ensure we can strip List-Unsubscribe
+        $in = <<EOF;
+To: You <you\@example.com>
+List-Id: <$list_id>
+Message-ID: <unsubscribe-1\@example>
+Subject: unsubscribe-1
+From: user <user\@example.com>
+To: $addr
+Date: Fri, 02 Oct 1993 00:00:00 +0000
+List-Unsubscribe: <https://example.com/some-UUID-here/listname>
+List-Unsubscribe-Post: List-Unsubscribe=One-Click
+
+List-Unsubscribe should be stripped
+EOF
+        write_file '>>', $pi_config, <<EOM;
+[publicinboxImport]
+        dropUniqueUnsubscribe
+EOM
+        $out = $err = '';
+        ok(run_script([qw(-mda)], undef, $rdr), 'mda w/ dropUniqueUnsubscribe');
+        $cur = join('', grep(/^\+/, $git->qx(qw(diff HEAD~1..HEAD))));
+        like $cur, qr/Message-ID: <unsubscribe-1/, 'imported new message';
+        unlike $cur, qr/some-UUID-here/, 'List-Unsubscribe gone';
+        unlike $cur, qr/List-Unsubscribe-Post/i, 'List-Unsubscribe-Post gone';
+
+        $in =~ s/unsubscribe-1/unsubscribe-2/g or xbail 'BUG: s// fail';
+        ok(run_script([qw(-learn ham)], undef, $rdr),
+                        'learn ham w/ dropUniqueUnsubscribe');
+        $cur = join('', grep(/^\+/, $git->qx(qw(diff HEAD~1..HEAD))));
+        like $cur, qr/Message-ID: <unsubscribe-2/, 'learn ham';
+        unlike $cur, qr/some-UUID-here/, 'List-Unsubscribe gone on learn ham';
+        unlike $cur, qr/List-Unsubscribe-Post/i,
+                'List-Unsubscribe-Post gone on learn ham';
 }
 
+SKIP: {
+        require_mods(qw(DBD::SQLite Xapian), 1);
+        local $ENV{PI_EMERGENCY} = $faildir;
+        local $ENV{HOME} = $home;
+        local $ENV{PATH} = $main_path;
+        my $rdr = { 1 => \(my $out = ''), 2 => \(my $err = '') };
+        ok(run_script([qw(-index -L medium), $maindir], undef, $rdr),
+                'index inbox');
+        my $in = <<'EOM';
+From: a@example.com
+To: updated-address@example.com
+Subject: this is a ham message for learn
+Date: Fri, 02 Oct 1993 00:00:00 +0000
+Message-ID: <medium-ham@example>
+
+yum
+EOM
+        $rdr->{0} = \$in;
+        ok(run_script([qw(-learn ham)], undef, $rdr), 'learn medium ham');
+        is($err, '', 'nothing in stderr after medium -learn');
+        my $msg = $git->cat_file('HEAD:'.mid2path('medium-ham@example'));
+        like($$msg, qr/medium-ham/, 'medium ham added via -learn');
+        my @xap = grep(!m!/over\.sqlite3!,
+                        glob("$maindir/public-inbox/xapian*/*"));
+        ok(remove_tree(@xap), 'rm Xapian files to convert to indexlevel=basic');
+        $in =~ s/medium-ham/basic-ham/g or xbail 'BUG: no s//';
+        ok(run_script([qw(-learn ham)], undef, $rdr), 'learn basic ham');
+        is($err, '', 'nothing in stderr after basic -learn');
+        $msg = $git->cat_file('HEAD:'.mid2path('basic-ham@example'));
+        like($$msg, qr/basic-ham/, 'basic ham added via -learn');
+        @xap = grep(!m!/over\.sqlite3!,
+                        glob("$maindir/public-inbox/xapian*/*"));
+        is_deeply(\@xap, [], 'no Xapian files created by -learn');
+};
+
 done_testing();
diff --git a/t/mda_filter_rubylang.t b/t/mda_filter_rubylang.t
index d05eec25..42fa6101 100644
--- a/t/mda_filter_rubylang.t
+++ b/t/mda_filter_rubylang.t
@@ -7,7 +7,7 @@ use PublicInbox::Eml;
 use PublicInbox::Config;
 use PublicInbox::TestCommon;
 require_git(2.6);
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 use_ok 'PublicInbox::V2Writable';
 my ($tmpdir, $for_destroy) = tmpdir();
 my $pi_config = "$tmpdir/pi_config";
diff --git a/t/mh_reader.t b/t/mh_reader.t
new file mode 100644
index 00000000..c81df32e
--- /dev/null
+++ b/t/mh_reader.t
@@ -0,0 +1,120 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use PublicInbox::TestCommon;
+require_ok 'PublicInbox::MHreader';
+use PublicInbox::IO qw(write_file);
+use PublicInbox::Lock;
+use PublicInbox::OnDestroy;
+use PublicInbox::Eml;
+use File::Path qw(remove_tree);
+use autodie;
+opendir my $cwdfh, '.';
+
+my $normal = create_dir 'normal', sub {
+        write_file '>', 3, "Subject: replied a\n\n";
+        write_file '>', 4, "Subject: replied b\n\n";
+        write_file '>', 1, "Subject: unseen\n\n";
+        write_file '>', 2, "Subject: unseen flagged\n\n";
+        write_file '>', '.mh_sequences', <<EOM;
+unseen: 1 2
+flagged: 2
+replied: 3 4
+EOM
+};
+
+my $for_sort = create_dir 'size', sub {
+        for (1..3) {
+                my $name = 10 - $_;
+                write_file '>', $name, "Subject: ".($_ x $_)."\n\n";
+        }
+};
+
+my $stale = create_dir 'stale', sub {
+        write_file '>', 4, "Subject: msg 4\n\n";
+        write_file '>', '.mh_sequences', <<EOM;
+unseen: 1 2
+EOM
+};
+
+{
+        my $mhr = PublicInbox::MHreader->new("$normal/", $cwdfh);
+        $mhr->{sort} = [ '' ];
+        my @res;
+        $mhr->mh_each_eml(sub { push @res, \@_; }, [ 'bogus' ]);
+        is scalar(@res), 4, 'got 4 messages' or diag explain(\@res);
+        is_deeply [map { $_->[1] } @res], [1, 2, 3, 4],
+                'got messages in expected order';
+        is scalar(grep { $_->[4]->[0] eq 'bogus' } @res), scalar(@res),
+                'cb arg passed to all messages' or diag explain(\@res);
+
+        $mhr = PublicInbox::MHreader->new("$stale/", $cwdfh);
+        @res = ();
+        $mhr->mh_each_eml(sub { push @res, \@_; });
+        is scalar(@res), 1, 'ignored stale messages';
+}
+
+test_lei(sub {
+        lei_ok qw(convert -f mboxrd), $normal;
+        my @msgs = grep /\S/s, split /^From .[^\n]+\n/sm, $lei_out;
+        my @eml = map { PublicInbox::Eml->new($_) } @msgs;
+        my $h = 'Subject';
+        @eml = sort { $a->header_raw($h) cmp $b->header_raw($h) } @eml;
+        my @has = map { scalar $_->header_raw($h) } @eml;
+        is_xdeeply \@has,
+                [ 'replied a', 'replied b', 'unseen', 'unseen flagged' ],
+                'subjects sorted';
+        $h = 'X-Status';
+        @has = map { scalar $_->header_raw($h) } @eml;
+        is_xdeeply \@has, [ 'A', 'A', undef, 'F' ], 'answered and flagged kw';
+        $h = 'Status';
+        @has = map { scalar $_->header_raw($h) } @eml;
+        is_xdeeply \@has, ['RO', 'RO', 'O', 'O'], 'read and old';
+        lei_ok qw(import +L:normal), $normal;
+        lei_ok qw(q L:normal -f mboxrd);
+        @msgs = grep /\S/s, split /^From .[^\n]+\n/sm, $lei_out;
+        my @eml2 = map { PublicInbox::Eml->new($_) } @msgs;
+        $h = 'Subject';
+        @eml2 = sort { $a->header_raw($h) cmp $b->header_raw($h) } @eml2;
+        is_xdeeply \@eml2, \@eml, 'import preserved kw';
+
+        lei_ok 'ls-mail-sync';
+        is $lei_out, 'mh:'.File::Spec->rel2abs($normal)."\n",
+                'mail sync stored';
+
+        lei_ok qw(convert -s size -f mboxrd), "mh:$for_sort";
+        chomp(my @s = grep /^Subject:/, split(/^/sm, $lei_out));
+        s/^Subject: // for @s;
+        is_xdeeply \@s, [ 1, 22, 333 ], 'sorted by size';
+
+        for my $s ([], [ 'name' ], [ 'sequence' ]) {
+                lei_ok qw(convert -f mboxrd), "mh:$for_sort", '-s', @$s;
+                chomp(@s = grep /^Subject:/, split(/^/sm, $lei_out));
+                s/^Subject: // for @s;
+                my $desc = "@$s" || '(default)';
+                is_xdeeply \@s, [ 333, 22, 1 ], "sorted by: $desc";
+        }
+
+        lei_ok qw(import +L:sorttest), "MH:$for_sort";
+        lei_ok 'ls-mail-sync', $for_sort;
+        is $lei_out, 'mh:'.File::Spec->rel2abs($for_sort)."\n",
+                "mail sync stored with `MH' normalized to `mh'";
+        lei_ok qw(index), 'mh:'.$stale;
+        lei qw(q -f mboxrd), 's:msg 4';
+        like $lei_out, qr/^Subject: msg 4\nStatus: RO\n\n\n/ms,
+                "message retrieved after `lei index'";
+
+        lei_ok qw(convert -s none -f text), "mh:$for_sort", \'--sort=none';
+
+        # ensure sort works for _input_ when output disallows sort
+        my $v2out = "$ENV{HOME}/v2-out";
+        for my $sort (['--sort=sequence'], []) { # sequence is the default
+                lei_ok qw(convert), @$sort, "mh:$for_sort", '-o', "v2:$v2out";
+                my $g = PublicInbox::Git->new("$v2out/git/0.git");
+                chomp(my @l = $g->qx(qw(log --pretty=oneline --format=%s)));
+                is_xdeeply \@l, [1, 22, 333], 'sequence order preserved for v2';
+                File::Path::remove_tree $v2out;
+        }
+});
+
+done_testing;
diff --git a/t/mime.t b/t/mime.t
index 471f0efa..bf54118a 100644
--- a/t/mime.t
+++ b/t/mime.t
@@ -1,10 +1,10 @@
 #!perl -w
-# Copyright (C) 2017-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # This library is free software; you can redistribute it and/or modify
 # it under the same terms as Perl itself.
 # Artistic or GPL-1+ <https://www.gnu.org/licenses/gpl-1.0.txt>
+use v5.10.1; # TODO: check unicode_strings w/ v5.12
 use strict;
-use Test::More;
 use PublicInbox::TestCommon;
 use PublicInbox::MsgIter;
 my @classes = qw(PublicInbox::Eml);
diff --git a/t/miscsearch.t b/t/miscsearch.t
index 307812a4..ec837153 100644
--- a/t/miscsearch.t
+++ b/t/miscsearch.t
@@ -5,7 +5,7 @@ use strict;
 use v5.10.1;
 use Test::More;
 use PublicInbox::TestCommon;
-require_mods(qw(Search::Xapian DBD::SQLite));
+require_mods(qw(Xapian DBD::SQLite));
 use_ok 'PublicInbox::MiscSearch';
 use_ok 'PublicInbox::MiscIdx';
 
diff --git a/t/net_reader-imap.t b/t/net_reader-imap.t
index 5de8f92b..7b7f5cbe 100644
--- a/t/net_reader-imap.t
+++ b/t/net_reader-imap.t
@@ -3,7 +3,7 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 require_git 2.6;
-require_mods(qw(-imapd Search::Xapian Mail::IMAPClient));
+require_mods(qw(-imapd Xapian Mail::IMAPClient));
 use PublicInbox::Config;
 my ($tmpdir, $for_destroy) = tmpdir;
 my ($ro_home, $cfg_path) = setup_public_inboxes;
diff --git a/t/nntp.t b/t/nntp.t
index 655af398..42a4ea97 100644
--- a/t/nntp.t
+++ b/t/nntp.t
@@ -1,72 +1,72 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use warnings;
-use Test::More;
+use v5.12;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
-require_mods(qw(DBD::SQLite Data::Dumper));
+require_mods(qw(DBD::SQLite));
 use_ok 'PublicInbox::NNTP';
-use_ok 'PublicInbox::Inbox';
 use PublicInbox::Config;
+use POSIX qw(strftime);
+use Data::Dumper;
 
 {
-        sub quote_str {
-                my (undef, $s) = split(/ = /, Data::Dumper::Dumper($_[0]), 2);
+        my $quote_str = sub {
+                my ($orig) = @_;
+                my (undef, $s) = split(/ = /, Dumper($orig), 2);
+                $s // diag explain(['$s undefined, $orig = ', $orig]);
                 $s =~ s/;\n//;
                 $s;
-        }
+        };
 
-        sub wm_prepare {
+        my $wm_prepare = sub {
                 my ($wm) = @_;
                 my $orig = qq{'$wm'};
                 PublicInbox::NNTP::wildmat2re($_[0]);
-                my $new = quote_str($_[0]);
+                my $new = $quote_str->($_[0]);
                 ($orig, $new);
-        }
+        };
 
-        sub wildmat_like {
+        my $wildmat_like = sub {
                 my ($str, $wm) = @_;
-                my ($orig, $new) = wm_prepare($wm);
+                my ($orig, $new) = $wm_prepare->($wm);
                 like($str, $wm, "$orig matches '$str' using $new");
-        }
+        };
 
-        sub wildmat_unlike {
+        my $wildmat_unlike = sub {
                 my ($str, $wm, $check_ex) = @_;
                 if ($check_ex) {
                         use re 'eval';
                         my $re = qr/$wm/;
                         like($str, $re, "normal re with $wm matches, but ...");
                 }
-                my ($orig, $new) = wm_prepare($wm);
+                my ($orig, $new) = $wm_prepare->($wm);
                 unlike($str, $wm, "$orig does not match '$str' using $new");
-        }
+        };
 
-        wildmat_like('[foo]', '[\[foo\]]');
-        wildmat_like('any', '*');
-        wildmat_unlike('bar.foo.bar', 'foo.*');
+        $wildmat_like->('[foo]', '[\[foo\]]');
+        $wildmat_like->('any', '*');
+        $wildmat_unlike->('bar.foo.bar', 'foo.*');
 
         # no code execution
-        wildmat_unlike('HI', '(?{"HI"})', 1);
-        wildmat_unlike('HI', '[(?{"HI"})]', 1);
+        $wildmat_unlike->('HI', '(?{"HI"})', 1);
+        $wildmat_unlike->('HI', '[(?{"HI"})]', 1);
 }
 
 {
-        sub ngpat_like {
+        my $ngpat_like = sub {
                 my ($str, $pat) = @_;
                 my $orig = $pat;
                 PublicInbox::NNTP::ngpat2re($pat);
                 like($str, $pat, "'$orig' matches '$str' using $pat");
-        }
+        };
 
-        ngpat_like('any', '*');
-        ngpat_like('a.s.r', 'a.t,a.s.r');
-        ngpat_like('a.s.r', 'a.t,a.s.*');
+        $ngpat_like->('any', '*');
+        $ngpat_like->('a.s.r', 'a.t,a.s.r');
+        $ngpat_like->('a.s.r', 'a.t,a.s.*');
 }
 
 {
-        use POSIX qw(strftime);
-        sub time_roundtrip {
+        my $time_roundtrip = sub {
                 my ($date, $time, $gmt) = @_;
                 my $m = join(' ', @_);
                 my $ts = PublicInbox::NNTP::parse_time(@_);
@@ -77,12 +77,12 @@ use PublicInbox::Config;
                 }
                 is_deeply([$d, $t], [$date, $time], "roundtripped: $m");
                 $ts;
-        }
-        my $x1 = time_roundtrip(qw(20141109 060606 GMT));
-        my $x2 = time_roundtrip(qw(141109 060606 GMT));
-        my $x3 = time_roundtrip(qw(930724 060606 GMT));
-        my $x5 = time_roundtrip(qw(710101 000000));
-        my $x6 = time_roundtrip(qw(720101 000000));
+        };
+        my $x1 = $time_roundtrip->(qw(20141109 060606 GMT));
+        my $x2 = $time_roundtrip->(qw(141109 060606 GMT));
+        my $x3 = $time_roundtrip->(qw(930724 060606 GMT));
+        my $x5 = $time_roundtrip->(qw(710101 000000));
+        my $x6 = $time_roundtrip->(qw(720101 000000));
         SKIP: {
                 skip('YYMMDD test needs updating', 6) if (time > 0x7fffffff);
                 # our world probably ends in 2038, but if not we'll try to
@@ -90,7 +90,7 @@ use PublicInbox::Config;
                 is($x1, $x2, 'YYYYMMDD and YYMMDD parse identically');
                 is(strftime('%Y', gmtime($x3)), '1993', '930724 was in 1993');
 
-                my $epoch = time_roundtrip(qw(700101 000000 GMT));
+                my $epoch = $time_roundtrip->(qw(700101 000000 GMT));
                 is($epoch, 0, 'epoch parsed correctly');
                 ok($x6 > $x5, '1972 > 1971');
                 ok($x5 > $epoch, '1971 > Unix epoch');
diff --git a/t/nntpd-tls.t b/t/nntpd-tls.t
index 2a76867a..a16cc015 100644
--- a/t/nntpd-tls.t
+++ b/t/nntpd-tls.t
@@ -1,8 +1,7 @@
 #!perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::TestCommon;
 use Socket qw(SOCK_STREAM IPPROTO_TCP SOL_SOCKET);
 # IO::Poll and Net::NNTP are part of the standard library, but
@@ -149,10 +148,22 @@ for my $args (
         test_lei(sub {
                 lei_ok qw(ls-mail-source), "nntp://$starttls_addr",
                         \'STARTTLS not used by default';
-                ok(!lei(qw(ls-mail-source -c nntp.starttls=true),
+                my $plain_out = $lei_out;
+                ok(!lei(qw(ls-mail-source -c nntp.starttls),
                         "nntp://$starttls_addr"), 'STARTTLS verify fails');
                 like $lei_err, qr/STARTTLS requested/,
                         'STARTTLS noted in stderr';
+                unlike $lei_err, qr!W: nntp\.starttls= .*? is not boolean!i,
+                        'no non-boolean warning';
+                lei_ok qw(-c nntp.starttls -c nntp.sslVerify= ls-mail-source),
+                        "nntp://$starttls_addr",
+                        \'disabling nntp.sslVerify works w/ STARTTLS';
+                is $lei_out, $plain_out, 'sslVerify=false w/ STARTTLS output';
+
+                lei_ok qw(ls-mail-source -c nntp.sslVerify=false),
+                        "nntps://$nntps_addr",
+                        \'disabling nntp.sslVerify works w/ nntps://';
+                is $lei_out, $plain_out, 'sslVerify=false w/ NNTPS output';
         });
 
         SKIP: {
@@ -164,10 +175,7 @@ for my $args (
                 is(unpack('i', $x), 0, 'TCP_DEFER_ACCEPT is 0 on plain NNTP');
         };
         SKIP: {
-                skip 'SO_ACCEPTFILTER is FreeBSD-only', 2 if $^O ne 'freebsd';
-                if (system('kldstat -m accf_data >/dev/null')) {
-                        skip 'accf_data not loaded? kldload accf_data', 2;
-                }
+                require_mods '+accf_data';
                 require PublicInbox::Daemon;
                 my $x = getsockopt($nntps, SOL_SOCKET,
                                 $PublicInbox::Daemon::SO_ACCEPTFILTER);
@@ -177,6 +185,14 @@ for my $args (
                 is($x, undef, 'no BSD accept filter for plain NNTP');
         };
 
+        my $s = tcp_connect($nntps);
+        syswrite($s, '->accept_SSL_ will fail on this!');
+        my @r;
+        do { # some platforms or OpenSSL versions need an extra read
+                push @r, sysread($s, my $rbuf, 128);
+        } while ($r[-1] && @r < 2);
+        ok(!$r[-1], 'EOF or ECONNRESET on ->accept_SSL fail') or
+                diag explain(\@r);
         $c = undef;
         $td->kill;
         $td->join;
@@ -187,6 +203,7 @@ for my $args (
                 <$fh>;
         };
         unlike($eout, qr/wide/i, 'no Wide character warnings');
+        unlike($eout, qr/^E:/, 'no other errors');
 }
 done_testing();
 
diff --git a/t/nntpd.t b/t/nntpd.t
index cf1c44f8..7052cb6a 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -1,21 +1,20 @@
 #!perl -w
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
-require_mods(qw(DBD::SQLite));
+require_mods(qw(DBD::SQLite Net::NNTP));
 use PublicInbox::Eml;
 use Socket qw(IPPROTO_TCP TCP_NODELAY);
-use Net::NNTP;
 use Sys::Hostname;
 use POSIX qw(_exit);
-use Digest::SHA;
+use PublicInbox::SHA;
+use PublicInbox::DS;
 
 # t/nntpd-v2.t wraps this for v2
 my $version = $ENV{PI_TEST_VERSION} || 1;
 require_git('2.6') if $version == 2;
 use_ok 'PublicInbox::Msgmap';
-my $lsof = require_cmd('lsof', 1);
-my $fast_idle = eval { require Linux::Inotify2; 1 } //
+my $fast_idle = eval { require PublicInbox::Inotify; 1 } //
                 eval { require IO::KQueue; 1 };
 
 my ($tmpdir, $for_destroy) = tmpdir();
@@ -94,6 +93,13 @@ close $cfgfh or BAIL_OUT;
         is_deeply([$n->group($group)], [ qw(0 1 1), $group ], 'GROUP works');
         is_deeply($n->listgroup($group), [1], 'listgroup OK');
         # TODO: Net::NNTP::listgroup does not support range at the moment
+        my $s = tcp_connect($sock);
+        sysread($s, my $buf, 4096);
+        is($buf, "201 " . hostname . " ready - post via email\r\n",
+                'got greeting');
+        syswrite($s, "LISTGROUP $group 1-1\r\n");
+        $buf = read_til_dot($s);
+        like($buf, qr/\r\n1\r\n/s, 'LISTGROUP with range works');
 
         {
                 my $expect = [ qw(Subject: From: Date: Message-ID:
@@ -121,8 +127,8 @@ close $cfgfh or BAIL_OUT;
                 'references' => '<reftabsqueezed>',
         );
 
-        my $s = tcp_connect($sock);
-        sysread($s, my $buf, 4096);
+        $s = tcp_connect($sock);
+        sysread($s, $buf, 4096);
         is($buf, "201 " . hostname . " ready - post via email\r\n",
                 'got greeting');
 
@@ -298,7 +304,7 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
                         my %sums;
                         for (1..$nart) {
                                 <$s> =~ /\A220 / or _exit(4);
-                                my $dig = Digest::SHA->new(1);
+                                my $dig = PublicInbox::SHA->new(1);
                                 while (my $l = <$s>) {
                                         last if $l eq ".\r\n";
                                         $dig->add($l);
@@ -321,19 +327,19 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
         }
         my $noerr = { 2 => \(my $null) };
         SKIP: {
-                if ($INC{'Search/Xapian.pm'} && ($ENV{TEST_RUN_MODE}//2)) {
-                        skip 'Search/Xapian.pm pre-loaded (by t/run.perl?)', 1;
+                if ($INC{'Search/Xapian.pm'} || $INC{'Xapian.pm'} &&
+                                ($ENV{TEST_RUN_MODE} // 2)) {
+                        skip 'Xapian.pm pre-loaded (by xt/check-run.t?)', 1;
                 }
-                $lsof or skip 'lsof missing', 1;
-                my @of = xqx([$lsof, '-p', $td->{pid}], undef, $noerr);
-                skip('lsof broken', 1) if (!scalar(@of) || $?);
-                my @xap = grep m!Search/Xapian!, @of;
-                is_deeply(\@xap, [], 'Xapian not loaded in nntpd');
+                my @of = lsof_pid $td->{pid}, $noerr;
+                my @xap = grep m!\bXapian\b!, @of;
+                is_deeply(\@xap, [], 'Xapian not loaded in nntpd') or
+                        diag explain(\@of);
         }
         # -compact requires Xapian
         SKIP: {
-                require_mods('Search::Xapian', 2);
-                have_xapian_compact or skip 'xapian-compact missing', 2;
+                require_mods('Xapian', 1);
+                have_xapian_compact 1;
                 is(xsys(qw(git config), "--file=$home/.public-inbox/config",
                                 "publicinbox.$group.indexlevel", 'medium'),
                         0, 'upgraded indexlevel');
@@ -352,23 +358,24 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
                 }
                 ok(run_script([qw(-index -c -j0 --reindex), $ibx->{inboxdir}],
                                 undef, $noerr), '-compacted');
-                select(undef, undef, undef, $fast_idle ? 0.1 : 2.1);
+                tick($fast_idle ? 0.1 : 2.1);
                 $art = $n->article($ex->header('Message-ID'));
                 ok($art, 'new article retrieved after compact');
-                $lsof or skip 'lsof missing', 1;
-                ($^O =~ /\A(?:linux)\z/) or
+                $^O eq 'linux' or
                         skip "lsof /(deleted)/ check untested on $^O", 1;
-                my @lsof = xqx([$lsof, '-p', $td->{pid}], undef, $noerr);
-                my $d = [ grep(/\(deleted\)/, @lsof) ];
-                is_deeply($d, [], 'no deleted files') or diag explain($d);
+                my $fd = "/proc/$td->{pid}/fd";
+                -d $fd or skip '/proc/PID/fd missing', 1;
+                my @of = map readlink, glob "$fd/*";
+                my @d = grep /\(deleted\)/, grep !/batch-command\.err/, @of;
+                is_deeply(\@d, [], 'no deleted files') or diag explain(\@d);
         };
         SKIP: { test_watch($tmpdir, $host_port, $group) };
         {
                 setsockopt($s, IPPROTO_TCP, TCP_NODELAY, 1);
                 syswrite($s, 'HDR List-id 1-');
-                select(undef, undef, undef, 0.15);
+                tick(0.15);
                 ok($td->kill, 'killed nntpd');
-                select(undef, undef, undef, 0.15);
+                tick(0.15);
                 syswrite($s, "\r\n");
                 $buf = '';
                 do {
@@ -407,7 +414,7 @@ sub test_watch {
         use_ok 'PublicInbox::Watch';
         use_ok 'PublicInbox::InboxIdle';
         use_ok 'PublicInbox::Config';
-        require_git('1.8.5', 1) or skip('git 1.8.5+ needed for --urlmatch', 4);
+        require_git('1.8.5', 4);
         my $old_env = { HOME => $ENV{HOME} };
         my $home = "$tmpdir/watch_home";
         mkdir $home or BAIL_OUT $!;
@@ -430,7 +437,7 @@ sub test_watch {
         my $cfg = PublicInbox::Config->new;
         PublicInbox::DS->Reset;
         my $ii = PublicInbox::InboxIdle->new($cfg);
-        my $cb = sub { PublicInbox::DS->SetPostLoopCallback(sub {}) };
+        my $cb = sub { @PublicInbox::DS::post_loop_do = (sub {}) };
         my $obj = bless \$cb, 'PublicInbox::TestCommon::InboxWakeup';
         $cfg->each_inbox(sub { $_[0]->subscribe_unlock('ident', $obj) });
         my $watcherr = "$tmpdir/watcherr";
diff --git a/t/on_destroy.t b/t/on_destroy.t
index 0de67d0b..e7945100 100644
--- a/t/on_destroy.t
+++ b/t/on_destroy.t
@@ -1,6 +1,5 @@
 #!perl -w
-use strict;
-use v5.10.1;
+use v5.12;
 use Test::More;
 require_ok 'PublicInbox::OnDestroy';
 my @x;
@@ -25,6 +24,11 @@ $od = PublicInbox::OnDestroy->new($$, sub { $tmp = $$ });
 undef $od;
 is($tmp, $$, '$tmp set to $$ by callback');
 
+$od = PublicInbox::OnDestroy->new($$, sub { $tmp = 'foo' });
+$od->cancel;
+$od = undef;
+isnt($tmp, 'foo', '->cancel');
+
 if (my $nr = $ENV{TEST_LEAK_NR}) {
         for (0..$nr) {
                 $od = PublicInbox::OnDestroy->new(sub { @x = @_ }, qw(x y));
diff --git a/t/plack.t b/t/plack.t
index a5fd54c9..07cab12a 100644
--- a/t/plack.t
+++ b/t/plack.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
@@ -9,10 +9,11 @@ my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape);
 require_mods(@mods);
 foreach my $mod (@mods) { use_ok $mod; }
 ok(-f $psgi, "psgi example file found");
+my ($tmpdir, $for_destroy) = tmpdir();
 my $pfx = 'http://example.com/test';
 my $eml = eml_load('t/iso-2202-jp.eml');
 # ensure successful message deliveries
-my $ibx = create_inbox('test-1', sub {
+my $ibx = create_inbox('u8-2', sub {
         my ($im, $ibx) = @_;
         my $addr = $ibx->{-primary_address};
         $im->add($eml) or xbail '->add';
@@ -38,6 +39,8 @@ EOF
         # multipart with attached patch + filename
         $im->add(eml_load('t/plack-attached-patch.eml')) or BAIL_OUT '->add';
 
+        $im->add(eml_load('t/data/attached-mbox-with-utf8.eml')) or xbail 'add';
+
         # multipart collapsed to single quoted-printable text/plain
         $im->add(eml_load('t/plack-qp.eml')) or BAIL_OUT '->add';
         my $crlf = <<EOF;
@@ -71,91 +74,74 @@ EOF
         close $fh or BAIL_OUT "close: $!";
 });
 
-local $ENV{PI_CONFIG} = "$ibx->{inboxdir}/pi_config";
-my $app = require $psgi;
-test_psgi($app, sub {
+my $env = { PI_CONFIG => "$ibx->{inboxdir}/pi_config", TMPDIR => $tmpdir };
+local @ENV{keys %$env} = values %$env;
+my $c1 = sub {
         my ($cb) = @_;
+        my $uri = $ENV{PLACK_TEST_EXTERNALSERVER_URI} // 'http://example.com';
+        $pfx = "$uri/test";
+
         foreach my $u (qw(robots.txt favicon.ico .well-known/foo)) {
-                my $res = $cb->(GET("http://example.com/$u"));
+                my $res = $cb->(GET("$uri/$u"));
                 is($res->code, 404, "$u is missing");
         }
-});
 
-test_psgi($app, sub {
-        my ($cb) = @_;
-        my $res = $cb->(GET('http://example.com/test/crlf@example.com/'));
+        my $res = $cb->(GET("$uri/test/crlf\@example.com/"));
         is($res->code, 200, 'retrieved CRLF as HTML');
         like($res->content, qr/mailto:me\@example/, 'no %40, per RFC 6068');
         unlike($res->content, qr/\r/, 'no CR in HTML');
-        $res = $cb->(GET('http://example.com/test/crlf@example.com/raw'));
+        $res = $cb->(GET("$uri/test/crlf\@example.com/raw"));
         is($res->code, 200, 'retrieved CRLF raw');
         like($res->content, qr/\r/, 'CR preserved in raw message');
-        $res = $cb->(GET('http://example.com/test/bogus@example.com/raw'));
+        $res = $cb->(GET("$uri/test/bogus\@example.com/raw"));
         is($res->code, 404, 'missing /raw is 404');
-});
 
-# redirect with newsgroup
-test_psgi($app, sub {
-        my ($cb) = @_;
-        my $from = 'http://example.com/inbox.test';
-        my $to = 'http://example.com/test/';
-        my $res = $cb->(GET($from));
+        # redirect with newsgroup
+        my $from = "$uri/inbox.test";
+        my $to = "http://example.com/test/";
+        $res = $cb->(GET($from));
         is($res->code, 301, 'newsgroup name is permanent redirect');
         is($to, $res->header('Location'), 'redirect location matches');
         $from .= '/';
         is($res->code, 301, 'newsgroup name/ is permanent redirect');
         is($to, $res->header('Location'), 'redirect location matches');
-});
 
-# redirect with trailing /
-test_psgi($app, sub {
-        my ($cb) = @_;
-        my $from = 'http://example.com/test';
-        my $to = "$from/";
-        my $res = $cb->(GET($from));
+        # redirect with trailing /
+        $from = "$uri/test";
+        $to = "$from/";
+        $res = $cb->(GET($from));
         is(301, $res->code, 'is permanent redirect');
         is($to, $res->header('Location'),
                 'redirect location matches with trailing slash');
-});
 
-foreach my $t (qw(t T)) {
-        test_psgi($app, sub {
-                my ($cb) = @_;
+        for my $t (qw(T t)) {
                 my $u = $pfx . "/blah\@example.com/$t";
-                my $res = $cb->(GET($u));
+                $res = $cb->(GET($u));
                 is(301, $res->code, "redirect for missing /");
                 my $location = $res->header('Location');
                 like($location, qr!/\Q$t\E/#u\z!,
                         'redirected with missing /');
-        });
-}
-foreach my $t (qw(f)) {
-        test_psgi($app, sub {
-                my ($cb) = @_;
+        }
+
+        for my $t (qw(f)) { # legacy redirect
                 my $u = $pfx . "/blah\@example.com/$t";
-                my $res = $cb->(GET($u));
+                $res = $cb->(GET($u));
                 is(301, $res->code, "redirect for legacy /f");
                 my $location = $res->header('Location');
                 like($location, qr!/blah\@example\.com/\z!,
                         'redirected with missing /');
-        });
-}
+        }
 
-test_psgi($app, sub {
-        my ($cb) = @_;
-        my $atomurl = 'http://example.com/test/new.atom';
-        my $res = $cb->(GET('http://example.com/test/new.html'));
+        my $atomurl = "$uri/test/new.atom";
+        $res = $cb->(GET("$uri/test/new.html"));
         is(200, $res->code, 'success response received');
         like($res->content, qr!href="new\.atom"!,
                 'atom URL generated');
         like($res->content, qr!href="blah\@example\.com/"!,
                 'index generated');
         like($res->content, qr!1993-10-02!, 'date set');
-});
 
-test_psgi($app, sub {
-        my ($cb) = @_;
-        my $res = $cb->(GET($pfx . '/atom.xml'));
+        $res = $cb->(GET($pfx . '/atom.xml'));
         is(200, $res->code, 'success response received for atom');
         my $body = $res->content;
         like($body, qr!link\s+href="\Q$pfx\E/blah\@example\.com/"!s,
@@ -165,17 +151,17 @@ test_psgi($app, sub {
         like($body, qr/zzzzzz/, 'body included');
         $res = $cb->(GET($pfx . '/description'));
         like($res->content, qr/test for public-inbox/, 'got description');
-});
 
-test_psgi($app, sub {
-        my ($cb) = @_;
         my $path = '/blah@example.com/';
-        my $res = $cb->(GET($pfx . $path));
+        $res = $cb->(GET($pfx . $path));
         is(200, $res->code, "success for $path");
         my $html = $res->content;
+        like($html, qr!\bhref="\Q../_/text/help/"!, 'help available');
         like($html, qr!<title>hihi - Me</title>!, 'HTML returned');
-        like($html, qr!<a\nhref="raw"!s, 'raw link present');
+        like($html, qr!<a\nhref=raw!s, 'raw link present');
         like($html, qr!&gt; quoted text!s, 'quoted text inline');
+        unlike($html, qr!thread overview!,
+                'thread overview not shown w/o ->over');
 
         $path .= 'f/';
         $res = $cb->(GET($pfx . $path));
@@ -196,11 +182,12 @@ test_psgi($app, sub {
 
         $res = $cb->(GET($pfx . '/qp@example.com/'));
         like($res->content, qr/\bhi = bye\b/, "HTML output decoded QP");
-});
 
-test_psgi($app, sub {
-        my ($cb) = @_;
-        my $res = $cb->(GET($pfx . '/blah@example.com/raw'));
+        $res = $cb->(GET($pfx . '/attached-mbox-with-utf8@example/'));
+        like($res->content, qr/: Bj&#248;rn /, 'UTF-8 in mbox #1');
+        like($res->content, qr/: j &#379;en/, 'UTF-8 in mbox #2');
+
+        $res = $cb->(GET($pfx . '/blah@example.com/raw'));
         is(200, $res->code, 'success response received for /*/raw');
         like($res->content, qr!^From !sm, "mbox returned");
         is($res->header('Content-Type'), 'text/plain; charset=iso-8859-1',
@@ -213,75 +200,65 @@ test_psgi($app, sub {
         $res = $cb->(GET($pfx . '/199707281508.AAA24167@hoyogw.example/raw'));
         is($res->header('Content-Type'), 'text/plain; charset=ISO-2022-JP',
                 'ISO-2002-JP returned');
-        chomp(my $body = $res->content);
+        chomp($body = $res->content);
         my $raw = PublicInbox::Eml->new(\$body);
         is($raw->body_raw, $eml->body_raw, 'ISO-2022-JP body unmodified');
 
-        $res = $cb->(GET($pfx . '/blah@example.com/t.mbox.gz'));
-        is(501, $res->code, '501 when overview missing');
-        like($res->content, qr!\bOverview\b!, 'overview omission noted');
-});
+        for my $u (qw(blah@example.com/t.mbox.gz topics_new.html
+                        topics_active.html)) {
+                $res = $cb->(GET("$pfx/$u"));
+                is(501, $res->code, "501 on /$u when overview missing");
+                like($res->content, qr!\bOverview\b!,
+                        "overview omission noted for /$u");
+        }
 
-# legacy redirects
-foreach my $t (qw(m f)) {
-        test_psgi($app, sub {
-                my ($cb) = @_;
-                my $res = $cb->(GET($pfx . "/$t/blah\@example.com.txt"));
+        # legacy redirects
+        for my $t (qw(m f)) {
+                $res = $cb->(GET($pfx . "/$t/blah\@example.com.txt"));
                 is(301, $res->code, "redirect for old $t .txt link");
-                my $location = $res->header('Location');
+                $location = $res->header('Location');
                 like($location, qr!/blah\@example\.com/raw\z!,
                         ".txt redirected to /raw");
-        });
-}
-
-my %umap = (
-        'm' => '',
-        'f' => '',
-        't' => 't/',
-);
-while (my ($t, $e) = each %umap) {
-        test_psgi($app, sub {
-                my ($cb) = @_;
-                my $res = $cb->(GET($pfx . "/$t/blah\@example.com.html"));
+        }
+
+        my %umap = (
+                'm' => '',
+                'f' => '',
+                't' => 't/',
+        );
+        while (my ($t, $e) = each %umap) {
+                $res = $cb->(GET($pfx . "/$t/blah\@example.com.html"));
                 is(301, $res->code, "redirect for old $t .html link");
-                my $location = $res->header('Location');
-                like($location,
-                        qr!/blah\@example\.com/$e(?:#u)?\z!,
-                        ".html redirected to new location");
-        });
-}
-foreach my $sfx (qw(mbox mbox.gz)) {
-        test_psgi($app, sub {
-                my ($cb) = @_;
-                my $res = $cb->(GET($pfx . "/t/blah\@example.com.$sfx"));
+                $location = $res->header('Location');
+                like($location, qr!/blah\@example\.com/$e(?:#u)?\z!,
+                                ".html redirected to new location");
+        }
+
+        for my $sfx (qw(mbox mbox.gz)) {
+                $res = $cb->(GET($pfx . "/t/blah\@example.com.$sfx"));
                 is(301, $res->code, 'redirect for old thread link');
-                my $location = $res->header('Location');
+                $location = $res->header('Location');
                 like($location,
                      qr!/blah\@example\.com/t\.mbox(?:\.gz)?\z!,
                      "$sfx redirected to /mbox.gz");
-        });
-}
-test_psgi($app, sub {
-        my ($cb) = @_;
+        }
+
         # for a while, we used to support /$INBOX/$X40/
         # when we "compressed" long Message-IDs to SHA-1
         # Now we're stuck supporting them forever :<
-        foreach my $path ('f2912279bd7bcd8b7ab3033234942d58746d56f7') {
-                my $from = "http://example.com/test/$path/";
-                my $res = $cb->(GET($from));
+        for my $path ('f2912279bd7bcd8b7ab3033234942d58746d56f7') {
+                $from = "$uri/test/$path/";
+                $res = $cb->(GET($from));
                 is(301, $res->code, 'is permanent redirect');
                 like($res->header('Location'),
                         qr!/test/blah\@example\.com/!,
                         'redirect from x40 MIDs works');
         }
-});
 
-# dumb HTTP clone/fetch support
-test_psgi($app, sub {
-        my ($cb) = @_;
-        my $path = '/test/info/refs';
+        # dumb HTTP clone/fetch support
+        $path = '/test/info/refs';
         my $req = HTTP::Request->new('GET' => $path);
-        my $res = $cb->($req);
+        $res = $cb->($req);
         is(200, $res->code, 'refs readable');
         my $orig = $res->content;
 
@@ -294,19 +271,14 @@ test_psgi($app, sub {
         $res = $cb->($req);
         is(206, $res->code, 'got partial another response');
         is($res->content, substr($orig, 5), 'partial body OK past end');
-});
 
-# things which should fail
-test_psgi($app, sub {
-        my ($cb) = @_;
 
-        my $res = $cb->(PUT('/'));
+        # things which should fail
+        $res = $cb->(PUT('/'));
         is(405, $res->code, 'no PUT to / allowed');
         $res = $cb->(PUT('/test/'));
         is(405, $res->code, 'no PUT /$INBOX allowed');
-
-        # TODO
-        # $res = $cb->(GET('/'));
-});
-
-done_testing();
+};
+test_psgi(require $psgi, $c1);
+test_httpd($env, $c1);
+done_testing;
diff --git a/t/pop3d-limit.t b/t/pop3d-limit.t
new file mode 100644
index 00000000..f52c8802
--- /dev/null
+++ b/t/pop3d-limit.t
@@ -0,0 +1,144 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::TestCommon;
+require_mods(qw(DBD::SQLite Net::POP3 :fcntl_lock));
+use autodie;
+my ($tmpdir, $for_destroy) = tmpdir();
+mkdir("$tmpdir/p3state");
+use PublicInbox::Eml;
+my $group = 'test.pop3d.limit';
+my $addr = 'pop3d-limit@example.com';
+
+my $add_msg = sub {
+        my ($im, $n) = @_;
+        $im->add(PublicInbox::Eml->new(<<EOM)) or die 'add dup';
+From: $n\@example.com
+Subject: msg $n
+To: $addr
+Message-ID: <mid-$n\@example.com>
+Date: Sat, 02 Oct 2010 00:00:00 +0000
+
+body $n
+EOM
+};
+
+my $ibx = create_inbox 'pop3d-limit', -primary_address => $addr,
+                        indexlevel => 'basic', tmpdir => "$tmpdir/ibx", sub {
+        my ($im, $ibx) = @_;
+        $add_msg->($im, $_) for (1..3);
+        $im->done;
+        diag 'done';
+}; # /create_inbox
+
+my $pi_config = "$tmpdir/pi_config";
+{
+        open my $fh, '>', $pi_config;
+        print $fh <<EOF;
+[publicinbox]
+        pop3state = $tmpdir/p3state
+[publicinbox "pop3"]
+        inboxdir = $ibx->{inboxdir}
+        address = $addr
+        indexlevel = basic
+        newsgroup = $group
+EOF
+        close $fh;
+}
+
+my $plain = tcp_server();
+my $plain_addr = tcp_host_port($plain);
+my $env = { PI_CONFIG => $pi_config };
+my $p3d = start_script([qw(-pop3d -W0),
+        "--stdout=$tmpdir/out.log", "--stderr=$tmpdir/err.log" ],
+        $env, { 3 => $plain });
+my @np3args = ($plain->sockhost, Port => $plain->sockport);
+my $fetch_delete = sub {
+        my ($np3) = @_;
+        map {
+                my $msg = $np3->get($_);
+                $np3->delete($_);
+                PublicInbox::Eml->new(join('', @$msg));
+        } sort { $a <=> $b } keys %{$np3->list};
+};
+
+my $login_a = ('a'x32)."\@$group?initial_limit=2&limit=1";
+my $login_a0 = ('a'x32)."\@$group.0?initial_limit=2&limit=1";
+my $login_b = ('b'x32)."\@$group?limit=1";
+my $login_b0 = ('b'x32)."\@$group.0?limit=1";
+my $login_c = ('c'x32)."\@$group?limit=10";
+my $login_c0 = ('c'x32)."\@$group.0?limit=10";
+my $login_d = ('d'x32)."\@$group?limit=100000";
+my $login_d0 = ('d'x32)."\@$group.0?limit=100000";
+
+for my $login ($login_a, $login_a0) {
+        my $np3 = Net::POP3->new(@np3args) or xbail "Net::POP3 $!";
+        $np3->login($login, 'anonymous') or xbail "login $login ($!)";
+        my @msg = $fetch_delete->($np3);
+        $np3->quit;
+        is_deeply([ map { $_->header('Message-ID') } @msg ],
+                [ qw(<mid-2@example.com> <mid-3@example.com>) ],
+                "initial_limit ($login)") or diag explain(\@msg);
+}
+
+for my $login ($login_b, $login_b0) {
+        my $np3 = Net::POP3->new(@np3args);
+        $np3->login($login, 'anonymous') or xbail "login $login ($!)";
+        my @msg = $fetch_delete->($np3);
+        $np3->quit;
+        is_deeply([ map { $_->header('Message-ID') } @msg ],
+                [ qw(<mid-3@example.com>) ],
+                "limit-only ($login)") or diag explain(\@msg);
+}
+
+for my $login ($login_c, $login_c0, $login_d, $login_d0) {
+        my $np3 = Net::POP3->new(@np3args);
+        $np3->login($login, 'anonymous') or xbail "login $login ($!)";
+        my @msg = $fetch_delete->($np3);
+        $np3->quit;
+        is_deeply([ map { $_->header('Message-ID') } @msg ],
+                [ qw(<mid-1@example.com> <mid-2@example.com>
+                        <mid-3@example.com>) ],
+                "excessive limit ($login)") or diag explain(\@msg);
+}
+
+{ # add some new messages
+        my $im = $ibx->importer(0);
+        $add_msg->($im, $_) for (4..5);
+        $im->done;
+}
+
+for my $login ($login_a, $login_a0) {
+        my $np3 = Net::POP3->new(@np3args);
+        $np3->login($login, 'anonymous') or xbail "login $login ($!)";
+        my @msg = $fetch_delete->($np3);
+        $np3->quit;
+        is_deeply([ map { $_->header('Message-ID') } @msg ],
+                [ qw(<mid-5@example.com>) ],
+                "limit used (initial_limit ignored, $login)") or
+                        diag explain(\@msg);
+}
+
+for my $login ($login_b, $login_b0) {
+        my $np3 = Net::POP3->new(@np3args);
+        $np3->login($login, 'anonymous') or xbail "login $login ($!)";
+        my @msg = $fetch_delete->($np3);
+        $np3->quit;
+        is_deeply([ map { $_->header('Message-ID') } @msg ],
+                [ qw(<mid-5@example.com>) ],
+                "limit-only after new messages ($login)") or
+                diag explain(\@msg);
+}
+
+for my $login ($login_c, $login_c0, $login_d, $login_d0) {
+        my $np3 = Net::POP3->new(@np3args);
+        $np3->login($login, 'anonymous') or xbail "login $login ($!)";
+        my @msg = $fetch_delete->($np3);
+        $np3->quit;
+        is_deeply([ map { $_->header('Message-ID') } @msg ],
+                [ qw(<mid-4@example.com> <mid-5@example.com>) ],
+                "excessive limit ($login)") or diag explain(\@msg);
+}
+
+done_testing;
diff --git a/t/pop3d.t b/t/pop3d.t
new file mode 100644
index 00000000..ee19f2d7
--- /dev/null
+++ b/t/pop3d.t
@@ -0,0 +1,346 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::TestCommon;
+use Socket qw(IPPROTO_TCP SOL_SOCKET);
+my $cert = 'certs/server-cert.pem';
+my $key = 'certs/server-key.pem';
+unless (-r $key && -r $cert) {
+        plan skip_all =>
+                "certs/ missing for $0, run $^X ./create-certs.perl in certs/";
+}
+
+# Net::POP3 is part of the standard library, but distros may split it off...
+require_mods(qw(DBD::SQLite Net::POP3 IO::Socket::SSL :fcntl_lock));
+require_git(v2.6); # for v2
+use_ok 'IO::Socket::SSL';
+use_ok 'PublicInbox::TLS';
+my ($tmpdir, $for_destroy) = tmpdir();
+mkdir("$tmpdir/p3state") or xbail "mkdir: $!";
+my $err = "$tmpdir/stderr.log";
+my $out = "$tmpdir/stdout.log";
+my $olderr = "$tmpdir/plain.err";
+my $group = 'test-pop3';
+my $addr = $group . '@example.com';
+my $stls = tcp_server();
+my $plain = tcp_server();
+my $pop3s = tcp_server();
+my $patch = eml_load('t/data/0001.patch');
+my $ibx = create_inbox 'pop3d', version => 2, -primary_address => $addr,
+                        indexlevel => 'basic', sub {
+        my ($im, $ibx) = @_;
+        $im->add(eml_load('t/plack-qp.eml')) or BAIL_OUT '->add';
+        $im->add($patch) or BAIL_OUT '->add';
+};
+my $pi_config = "$tmpdir/pi_config";
+open my $fh, '>', $pi_config or BAIL_OUT "open: $!";
+print $fh <<EOF or BAIL_OUT "print: $!";
+[publicinbox]
+        pop3state = $tmpdir/p3state
+[publicinbox "pop3"]
+        inboxdir = $ibx->{inboxdir}
+        address = $addr
+        indexlevel = basic
+        newsgroup = $group
+EOF
+close $fh or BAIL_OUT "close: $!\n";
+
+my $pop3s_addr = tcp_host_port($pop3s);
+my $stls_addr = tcp_host_port($stls);
+my $plain_addr = tcp_host_port($plain);
+my $env = { PI_CONFIG => $pi_config };
+my $old = start_script(['-pop3d', '-W0',
+        "--stdout=$tmpdir/plain.out", "--stderr=$olderr" ],
+        $env, { 3 => $plain });
+my @old_args = ($plain->sockhost, Port => $plain->sockport);
+my $oldc = Net::POP3->new(@old_args);
+my $locked_mb = ('e'x32)."\@$group";
+ok($oldc->apop("$locked_mb.0", 'anonymous'), 'APOP to old');
+
+my $dbh = DBI->connect("dbi:SQLite:dbname=$tmpdir/p3state/db.sqlite3",'','', {
+        AutoCommit => 1,
+        RaiseError => 1,
+        PrintError => 0,
+        sqlite_use_immediate_transaction => 1,
+        sqlite_see_if_its_a_number => 1,
+});
+
+{ # locking within the same process
+        my $x = Net::POP3->new(@old_args);
+        ok(!$x->apop("$locked_mb.0", 'anonymous'), 'APOP lock failure');
+        like($x->message, qr/unable to lock/, 'diagnostic message');
+
+        $x = Net::POP3->new(@old_args);
+        ok($x->apop($locked_mb, 'anonymous'), 'APOP lock acquire');
+
+        my $y = Net::POP3->new(@old_args);
+        ok(!$y->apop($locked_mb, 'anonymous'), 'APOP lock fails once');
+
+        undef $x;
+        $y = Net::POP3->new(@old_args);
+        ok($y->apop($locked_mb, 'anonymous'), 'APOP lock works after release');
+}
+
+for my $args (
+        [ "--cert=$cert", "--key=$key",
+                "-lpop3s://$pop3s_addr",
+                "-lpop3://$stls_addr" ],
+) {
+        for ($out, $err) { open my $fh, '>', $_ or BAIL_OUT "truncate: $!" }
+        my $cmd = [ '-netd', '-W0', @$args, "--stdout=$out", "--stderr=$err" ];
+        my $td = start_script($cmd, $env, { 3 => $stls, 4 => $pop3s });
+
+        my %o = (
+                SSL_hostname => 'server.local',
+                SSL_verifycn_name => 'server.local',
+                SSL_verify_mode => SSL_VERIFY_PEER(),
+                SSL_ca_file => 'certs/test-ca.pem',
+        );
+        # start negotiating a slow TLS connection
+        my $slow = tcp_connect($pop3s, Blocking => 0);
+        $slow = IO::Socket::SSL->start_SSL($slow, SSL_startHandshake => 0, %o);
+        my $slow_done = $slow->connect_SSL;
+        my @poll;
+        if ($slow_done) {
+                diag('W: connect_SSL early OK, slow client test invalid');
+                use PublicInbox::Syscall qw(EPOLLIN EPOLLOUT);
+                @poll = (fileno($slow), EPOLLIN | EPOLLOUT);
+        } else {
+                @poll = (fileno($slow), PublicInbox::TLS::epollbit());
+        }
+
+        my @p3s_args = ($pop3s->sockhost,
+                        Port => $pop3s->sockport, SSL => 1, %o);
+        my $p3s = Net::POP3->new(@p3s_args);
+        my $capa = $p3s->capa;
+        ok(!exists $capa->{STLS}, 'no STLS CAPA for POP3S');
+        ok($p3s->quit, 'QUIT works w/POP3S');
+        {
+                $p3s = Net::POP3->new(@p3s_args);
+                ok(!$p3s->apop("$locked_mb.0", 'anonymous'),
+                        'APOP lock failure w/ another daemon');
+                like($p3s->message, qr/unable to lock/, 'diagnostic message');
+        }
+
+        # slow TLS connection did not block the other fast clients while
+        # connecting, finish it off:
+        until ($slow_done) {
+                IO::Poll::_poll(-1, @poll);
+                $slow_done = $slow->connect_SSL and last;
+                @poll = (fileno($slow), PublicInbox::TLS::epollbit());
+        }
+        $slow->blocking(1);
+        ok(sysread($slow, my $greet, 4096) > 0, 'slow got a greeting');
+        my @np3_args = ($stls->sockhost, Port => $stls->sockport);
+        my $np3 = Net::POP3->new(@np3_args);
+        ok($np3->quit, 'plain QUIT works');
+        $np3 = Net::POP3->new(@np3_args, %o);
+        $capa = $np3->capa;
+        ok(exists $capa->{STLS}, 'STLS CAPA advertised before STLS');
+        ok($np3->starttls, 'STLS works');
+        $capa = $np3->capa;
+        ok(!exists $capa->{STLS}, 'STLS CAPA not advertised after STLS');
+        ok($np3->quit, 'QUIT works after STLS');
+
+        for my $mailbox (('x'x32)."\@$group", $group, ('a'x32)."\@z.$group") {
+                $np3 = Net::POP3->new(@np3_args);
+                ok(!$np3->user($mailbox), "USER $mailbox reject");
+                ok($np3->quit, 'QUIT after USER fail');
+
+                $np3 = Net::POP3->new(@np3_args);
+                ok(!$np3->apop($mailbox, 'anonymous'), "APOP $mailbox reject");
+                ok($np3->quit, "QUIT after APOP fail $mailbox");
+        }
+
+        # we do connect+QUIT bumps to try ensuring non-QUIT disconnects
+        # get processed below:
+        for my $mailbox ($group, "$group.0") {
+                my $u = ('f'x32)."\@$mailbox";
+                undef $np3;
+                ok(Net::POP3->new(@np3_args)->quit, 'connect+QUIT bump');
+                $np3 = Net::POP3->new(@np3_args);
+                my $n0 = $dbh->selectrow_array('SELECT COUNT(*) FROM deletes');
+                my $u0 = $dbh->selectrow_array('SELECT COUNT(*) FROM users');
+                ok($np3->user($u), "UUID\@$mailbox accept");
+                ok($np3->pass('anonymous'), 'pass works');
+                my $n1 = $dbh->selectrow_array('SELECT COUNT(*) FROM deletes');
+                is($n1 - $n0, 1, 'deletes bumped while connected');
+                ok($np3->quit, 'client QUIT');
+
+                $n1 = $dbh->selectrow_array('SELECT COUNT(*) FROM deletes');
+                is($n1, $n0, 'deletes row gone on no-op after QUIT');
+                my $u1 = $dbh->selectrow_array('SELECT COUNT(*) FROM users');
+                is($u1, $u0, 'users row gone on no-op after QUIT');
+
+                $np3 = Net::POP3->new(@np3_args);
+                ok($np3->user($u), "UUID\@$mailbox accept");
+                ok($np3->pass('anonymous'), 'pass works');
+
+                my $list = $np3->list;
+                my $uidl = $np3->uidl;
+                is_deeply([sort keys %$list], [sort keys %$uidl],
+                        'LIST and UIDL keys match');
+                ok($_ > 0, 'bytes in LIST result') for values %$list;
+                like($_, qr/\A[a-z0-9]{40,}\z/,
+                        'blob IDs in UIDL result') for values %$uidl;
+                ok($np3->quit, 'QUIT after LIST+UIDL');
+                $n1 = $dbh->selectrow_array('SELECT COUNT(*) FROM deletes');
+                is($n1, $n0, 'deletes row gone on no-op after LIST+UIDL');
+                $n0 = $n1;
+
+                $np3 = Net::POP3->new(@np3_args);
+                ok($np3->user($u), "UUID\@$mailbox accept");
+                ok($np3->pass('anonymous'), 'pass works');
+                undef $np3; # QUIT-less disconnect
+                ok(Net::POP3->new(@np3_args)->quit, 'connect+QUIT bump');
+
+                $u1 = $dbh->selectrow_array('SELECT COUNT(*) FROM users');
+                is($u1, $u0, 'users row gone on QUIT-less disconnect');
+                $n1 = $dbh->selectrow_array('SELECT COUNT(*) FROM deletes');
+                is($n1, $n0, 'deletes row gone on QUIT-less disconnect');
+                $n0 = $n1;
+
+                $np3 = Net::POP3->new(@np3_args);
+                ok(!$np3->apop($u, 'anonumuss'), 'APOP wrong pass reject');
+                $n1 = $dbh->selectrow_array('SELECT COUNT(*) FROM deletes');
+                is($n1, $n0, 'deletes row not bumped w/ wrong pass');
+                undef $np3; # QUIT-less disconnect
+                ok(Net::POP3->new(@np3_args)->quit, 'connect+QUIT bump');
+
+                $n1 = $dbh->selectrow_array('SELECT COUNT(*) FROM deletes');
+                is($n1, $n0, 'deletes row not bumped w/ wrong pass');
+
+                $np3 = Net::POP3->new(@np3_args);
+                ok($np3->apop($u, 'anonymous'), "APOP UUID\@$mailbox");
+                my @res = $np3->popstat;
+                is($res[0], 2, 'STAT knows about 2 messages');
+
+                my $msg = $np3->get(2);
+                $msg = join('', @$msg);
+                $msg =~ s/\r\n/\n/g;
+                is_deeply(PublicInbox::Eml->new($msg), $patch,
+                        't/data/0001.patch round-tripped');
+
+                ok(!$np3->get(22), 'missing message');
+
+                $msg = $np3->top(2, 0);
+                $msg = join('', @$msg);
+                $msg =~ s/\r\n/\n/g;
+                is($msg, $patch->header_obj->as_string . "\n",
+                        'TOP numlines=0');
+
+                ok(!$np3->top(2, -1), 'negative TOP numlines');
+
+                $msg = $np3->top(2, 1);
+                $msg = join('', @$msg);
+                $msg =~ s/\r\n/\n/g;
+                is($msg, $patch->header_obj->as_string . <<EOF,
+
+Filenames within a project tend to be reasonably stable within a
+EOF
+                        'TOP numlines=1');
+
+                $msg = $np3->top(2, 10000);
+                $msg = join('', @$msg);
+                $msg =~ s/\r\n/\n/g;
+                is_deeply(PublicInbox::Eml->new($msg), $patch,
+                        'TOP numlines=10000 (excess)');
+
+                $np3 = Net::POP3->new(@np3_args, %o);
+                ok($np3->starttls, 'STLS works before APOP');
+                ok($np3->apop($u, 'anonymous'), "APOP UUID\@$mailbox w/ STLS");
+
+                # undocumented:
+                ok($np3->_NOOP, 'NOOP works') if $np3->can('_NOOP');
+        }
+
+        SKIP: {
+                skip 'TCP_DEFER_ACCEPT is Linux-only', 2 if $^O ne 'linux';
+                my $var = eval { Socket::TCP_DEFER_ACCEPT() } // 9;
+                my $x = getsockopt($pop3s, IPPROTO_TCP, $var) //
+                        xbail "IPPROTO_TCP: $!";
+                ok(unpack('i', $x) > 0, 'TCP_DEFER_ACCEPT set on POP3S');
+                $x = getsockopt($stls, IPPROTO_TCP, $var) //
+                        xbail "IPPROTO_TCP: $!";
+                is(unpack('i', $x), 0, 'TCP_DEFER_ACCEPT is 0 on plain POP3');
+        };
+        SKIP: {
+                require_mods '+accf_data';
+                require PublicInbox::Daemon;
+                my $x = getsockopt($pop3s, SOL_SOCKET,
+                                $PublicInbox::Daemon::SO_ACCEPTFILTER);
+                like($x, qr/\Adataready\0+\z/, 'got dataready accf for pop3s');
+                $x = getsockopt($stls, IPPROTO_TCP,
+                                $PublicInbox::Daemon::SO_ACCEPTFILTER);
+                is($x, undef, 'no BSD accept filter for plain POP3');
+        };
+
+        $td->kill;
+        $td->join;
+        is($?, 0, 'no error in exited -netd');
+        open my $fh, '<', $err or BAIL_OUT "open $err failed: $!";
+        my $eout = do { local $/; <$fh> };
+        unlike($eout, qr/wide/i, 'no Wide character warnings in -netd');
+}
+
+{
+        my $capa = $oldc->capa;
+        ok(defined($capa->{PIPELINING}), 'pipelining supported by CAPA');
+        is($capa->{EXPIRE}, 0, 'EXPIRE 0 set');
+        ok(!exists $capa->{STLS}, 'STLS unset w/o daemon certs');
+
+        # ensure TOP doesn't trigger "EXPIRE 0" like RETR does (cf. RFC2449)
+        my $list = $oldc->list;
+        ok(scalar keys %$list, 'got a listing of messages');
+        ok($oldc->top($_, 1), "TOP $_ 1") for keys %$list;
+        ok($oldc->quit, 'QUIT after TOP');
+
+        # clients which see "EXPIRE 0" can elide DELE requests
+        $oldc = Net::POP3->new(@old_args);
+        ok($oldc->apop("$locked_mb.0", 'anonymous'), 'APOP for RETR');
+        is_deeply($oldc->capa, $capa, 'CAPA unchanged');
+        is_deeply($oldc->list, $list, 'LIST unchanged by previous TOP');
+        ok($oldc->get($_), "RETR $_") for keys %$list;
+        ok($oldc->quit, 'QUIT after RETR');
+
+        $oldc = Net::POP3->new(@old_args);
+        ok($oldc->apop("$locked_mb.0", 'anonymous'), 'APOP reconnect');
+        my $cont = $oldc->list;
+        is_deeply($cont, {}, 'no messages after implicit DELE from EXPIRE 0');
+        ok($oldc->quit, 'QUIT on noop');
+
+        # test w/o checking CAPA to trigger EXPIRE 0
+        $oldc = Net::POP3->new(@old_args);
+        ok($oldc->apop($locked_mb, 'anonymous'), 'APOP on latest slice');
+        my $l2 = $oldc->list;
+        is_deeply($l2, $list, 'different mailbox, different deletes');
+        ok($oldc->get($_), "RETR $_") for keys %$list;
+        ok($oldc->quit, 'QUIT w/o EXPIRE nor DELE');
+
+        $oldc = Net::POP3->new(@old_args);
+        ok($oldc->apop($locked_mb, 'anonymous'), 'APOP again on latest');
+        $l2 = $oldc->list;
+        is_deeply($l2, $list, 'no DELE nor EXPIRE preserves messages');
+        ok($oldc->delete(2), 'explicit DELE on latest');
+        ok($oldc->quit, 'QUIT w/ highest DELE');
+
+        # this is non-standard behavior, but necessary if we expect hundreds
+        # of thousands of users on cheap HW
+        $oldc = Net::POP3->new(@old_args);
+        ok($oldc->apop($locked_mb, 'anonymous'), 'APOP yet again on latest');
+        is_deeply($oldc->list, {}, 'highest DELE deletes older messages, too');
+}
+
+# TODO: more tests, but mpop was really helpful in helping me
+# figure out bugs with larger newsgroups (>50K messages) which
+# probably isn't suited for this test suite.
+
+$old->kill;
+$old->join;
+is($?, 0, 'no error in exited -pop3d');
+open $fh, '<', $olderr or BAIL_OUT "open $olderr failed: $!";
+my $eout = do { local $/; <$fh> };
+unlike($eout, qr/wide/i, 'no Wide character warnings in -pop3d');
+
+done_testing;
diff --git a/t/pop3d_lock.t b/t/pop3d_lock.t
new file mode 100644
index 00000000..fb305f96
--- /dev/null
+++ b/t/pop3d_lock.t
@@ -0,0 +1,16 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::TestCommon;
+require_mods(qw(DBD::SQLite Net::POP3 :fcntl_lock));
+use autodie;
+my $tmpdir = tmpdir;
+require_ok 'PublicInbox::POP3D';
+my $pop3d = bless {}, 'PublicInbox::POP3D';
+open $pop3d->{txn_fh}, '+>>', "$tmpdir/txn.lock";
+use Fcntl qw(F_SETLK F_UNLCK F_WRLCK);
+
+ok $pop3d->_setlk(l_type => F_WRLCK, l_start => 9, l_len => 1),
+        'locked file (check with ktrace/strace)';
+
+done_testing;
diff --git a/t/psgi_attach.t b/t/psgi_attach.t
index 79665d6f..db551696 100644
--- a/t/psgi_attach.t
+++ b/t/psgi_attach.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
@@ -97,19 +97,12 @@ my $client = sub {
 
 test_psgi(sub { $www->call(@_) }, $client);
 SKIP: {
-        require_mods(qw(DBD::SQLite Plack::Test::ExternalServer), 18);
+        require_mods(qw(DBD::SQLite), 18);
         $ibx = create_inbox 'test-indexed', indexlevel => 'basic', $creat_cb;
         $cfgpath = "$ibx->{inboxdir}/pi_config";
         my $env = { PI_CONFIG => $cfgpath };
         $www = PublicInbox::WWW->new(PublicInbox::Config->new($cfgpath));
         test_psgi(sub { $www->call(@_) }, $client);
-        my $sock = tcp_server() or die;
-        my ($tmpdir, $for_destroy) = tmpdir();
-        my ($out, $err) = map { "$tmpdir/std$_.log" } qw(out err);
-        my $cmd = [ qw(-httpd -W0), "--stdout=$out", "--stderr=$err" ];
-        my $td = start_script($cmd, $env, { 3 => $sock });
-        my ($h, $p) = tcp_host_port($sock);
-        local $ENV{PLACK_TEST_EXTERNALSERVER_URI} = "http://$h:$p";
-        Plack::Test::ExternalServer::test_psgi(client => $client);
+        test_httpd($env, $client);
 }
 done_testing;
diff --git a/t/psgi_bad_mids.t b/t/psgi_bad_mids.t
index 8e531b54..ac0eb3c3 100644
--- a/t/psgi_bad_mids.t
+++ b/t/psgi_bad_mids.t
@@ -1,11 +1,9 @@
 #!perl -w
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
-use PublicInbox::Config;
 my @mods = qw(DBD::SQLite HTTP::Request::Common Plack::Test
                 URI::Escape Plack::Builder);
 require_git 2.6;
@@ -37,12 +35,12 @@ Date: Fri, 02 Oct 1993 00:00:0$i +0000
         }
 };
 
-my $cfgpfx = "publicinbox.bad-mids";
-my $cfg = <<EOF;
-$cfgpfx.address=$ibx->{-primary_address}
-$cfgpfx.inboxdir=$ibx->{inboxdir}
-EOF
-my $config = PublicInbox::Config->new(\$cfg);
+my $tmpdir = tmpdir;
+my $config = cfg_new $tmpdir, <<EOM;
+[publicinbox "bad-mids"]
+        address = $ibx->{-primary_address}
+        inboxdir = $ibx->{inboxdir}
+EOM
 my $www = PublicInbox::WWW->new($config);
 test_psgi(sub { $www->call(@_) }, sub {
         my ($cb) = @_;
diff --git a/t/psgi_mount.t b/t/psgi_mount.t
index 7c5487f3..e43b9f2d 100644
--- a/t/psgi_mount.t
+++ b/t/psgi_mount.t
@@ -1,14 +1,11 @@
 #!perl -w
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::Eml;
 use PublicInbox::TestCommon;
-use PublicInbox::Config;
 my ($tmpdir, $for_destroy) = tmpdir();
 my $v1dir = "$tmpdir/v1.git";
-my $cfgpfx = "publicinbox.test";
 my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape
         Plack::Builder Plack::App::URLMap);
 require_mods(@mods);
@@ -27,9 +24,10 @@ Date: Thu, 01 Jan 1970 00:00:00 +0000
 zzzzzz
 EOF
 };
-my $cfg = PublicInbox::Config->new(\<<EOF);
-$cfgpfx.address=$ibx->{-primary_address}
-$cfgpfx.inboxdir=$v1dir
+my $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = $ibx->{-primary_address}
+        inboxdir = $v1dir
 EOF
 my $www = PublicInbox::WWW->new($cfg);
 my $app = builder(sub {
@@ -69,7 +67,7 @@ test_psgi($app, sub {
 });
 
 SKIP: {
-        require_mods(qw(DBD::SQLite Search::Xapian IO::Uncompress::Gunzip), 3);
+        require_mods(qw(DBD::SQLite Xapian IO::Uncompress::Gunzip), 3);
         require_ok 'PublicInbox::SearchIdx';
         PublicInbox::SearchIdx->new($ibx, 1)->index_sync;
         test_psgi($app, sub {
diff --git a/t/psgi_multipart_not.t b/t/psgi_multipart_not.t
index 5f4c06b7..e7c43abf 100644
--- a/t/psgi_multipart_not.t
+++ b/t/psgi_multipart_not.t
@@ -1,13 +1,11 @@
 #!perl -w
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
-use PublicInbox::Config;
 require_git 2.6;
-my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common
+my @mods = qw(DBD::SQLite Xapian HTTP::Request::Common
               Plack::Test URI::Escape Plack::Builder Plack::Test);
 require_mods(@mods);
 use_ok($_) for (qw(HTTP::Request::Common Plack::Test));
@@ -28,12 +26,12 @@ Freed^Wmultipart ain't what it used to be
 EOF
 
 };
-my $cfgpfx = "publicinbox.v2test";
-my $cfg = <<EOF;
-$cfgpfx.address=$ibx->{-primary_address}
-$cfgpfx.inboxdir=$ibx->{inboxdir}
+my $tmpdir = tmpdir;
+my $www = PublicInbox::WWW->new(cfg_new($tmpdir, <<EOF));
+[publicinbox "v2test"]
+        address = $ibx->{-primary_address}
+        inboxdir = $ibx->{inboxdir}
 EOF
-my $www = PublicInbox::WWW->new(PublicInbox::Config->new(\$cfg));
 my ($res, $raw);
 test_psgi(sub { $www->call(@_) }, sub {
         my ($cb) = @_;
diff --git a/t/psgi_scan_all.t b/t/psgi_scan_all.t
index 09e8eaf9..4c28b553 100644
--- a/t/psgi_scan_all.t
+++ b/t/psgi_scan_all.t
@@ -1,17 +1,15 @@
 #!perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
-use PublicInbox::Config;
-my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape DBD::SQLite);
-require_git 2.6;
-require_mods(@mods);
-use_ok 'PublicInbox::WWW';
-foreach my $mod (@mods) { use_ok $mod; }
-my $cfg = '';
+my @use = qw(HTTP::Request::Common Plack::Test);
+my @req = qw(URI::Escape DBD::SQLite);
+require_git v2.6;
+require_mods(@use, @req, qw(PublicInbox::WWW));
+$_->import for @use;
+my $cfgtxt = '';
 foreach my $i (1..2) {
         my $ibx = create_inbox "test-$i", version => 2, indexlevel => 'basic',
         sub {
@@ -26,13 +24,15 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 hello world
 EOF
         };
-        my $cfgpfx = "publicinbox.test-$i";
-        $cfg .= "$cfgpfx.address=$ibx->{-primary_address}\n";
-        $cfg .= "$cfgpfx.inboxdir=$ibx->{inboxdir}\n";
-        $cfg .= "$cfgpfx.url=http://example.com/$i\n";
-
+        $cfgtxt .= <<EOM;
+[publicinbox "test-$i"]
+        address = $ibx->{-primary_address}
+        inboxdir = $ibx->{inboxdir}
+        url = http://example.com/$i
+EOM
 }
-my $www = PublicInbox::WWW->new(PublicInbox::Config->new(\$cfg));
+my $tmpdir = tmpdir;
+my $www = PublicInbox::WWW->new(cfg_new($tmpdir, $cfgtxt));
 
 test_psgi(sub { $www->call(@_) }, sub {
         my ($cb) = @_;
diff --git a/t/psgi_search.t b/t/psgi_search.t
index 3da93eda..8c981c6c 100644
--- a/t/psgi_search.t
+++ b/t/psgi_search.t
@@ -1,14 +1,12 @@
 #!perl -w
-# Copyright (C) 2017-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::TestCommon;
 use IO::Uncompress::Gunzip qw(gunzip);
 use PublicInbox::Eml;
-use PublicInbox::Config;
 use PublicInbox::Inbox;
-my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test
+my @mods = qw(DBD::SQLite Xapian HTTP::Request::Common Plack::Test
                 URI::Escape Plack::Builder);
 require_mods(@mods);
 use_ok($_) for (qw(HTTP::Request::Common Plack::Test));
@@ -20,7 +18,8 @@ local $ENV{TZ} = 'UTC';
 my $digits = '10010260936330';
 my $ua = 'Pine.LNX.4.10';
 my $mid = "$ua.$digits.2460-100000\@penguin.transmeta.com";
-my $ibx = create_inbox 'git', indexlevel => 'full', tmpdir => "$tmpdir/1", sub {
+my $ibx = create_inbox '26-git', indexlevel => 'full', tmpdir => "$tmpdir/1",
+sub {
         my ($im) = @_;
         # n.b. these headers are not properly RFC2047-encoded
         $im->add(PublicInbox::Eml->new(<<EOF)) or BAIL_OUT;
@@ -51,12 +50,23 @@ From: no subject at all <no-subject-at-all@example.com>
 To: git@vger.kernel.org
 
 EOF
+        $im->add(PublicInbox::Eml->new(<<'EOF')) or BAIL_OUT;
+Message-ID: <ampersand@example.com>
+From: <e@example.com>
+To: git@vger.kernel.org
+Subject: git & ampersand
+
+hi +++ b/foo
+x=y
+s'more
+
+EOF
 };
 
-my $cfgpfx = "publicinbox.test";
-my $cfg = PublicInbox::Config->new(\<<EOF);
-$cfgpfx.address=git\@vger.kernel.org
-$cfgpfx.inboxdir=$ibx->{inboxdir}
+my $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = git\@vger.kernel.org
+        inboxdir = $ibx->{inboxdir}
 EOF
 my $www = PublicInbox::WWW->new($cfg);
 test_psgi(sub { $www->call(@_) }, sub {
@@ -93,6 +103,7 @@ test_psgi(sub { $www->call(@_) }, sub {
 
         $res = $cb->(POST('/test/?q=s:bogus&x=m'));
         is($res->code, 404, 'failed search result gives 404');
+        like($res->content, qr/No results found/, "`No results' shown");
         is_deeply([], $warn, 'no warnings');
 
         my $mid_re = qr/\Q$mid\E/o;
@@ -103,6 +114,11 @@ test_psgi(sub { $www->call(@_) }, sub {
                 like($res->content, $mid_re, 'found mid in response');
                 chop($digits);
         }
+        $res = $cb->(GET("/test/$mid/"));
+        $html = $res->content;
+        like($html, qr/\bFrom: &#198;var /,
+                "displayed Ævar's name properly in permalink From:");
+        unlike($html, qr/&#195;/, 'no raw octets in permalink HTML');
 
         $res = $cb->(GET('/test/'));
         $html = $res->content;
@@ -151,6 +167,19 @@ test_psgi(sub { $www->call(@_) }, sub {
         is($res->code, 200, 'successful mbox download w/ threads');
         gunzip(\($res->content) => \(my $after));
         isnt($before, $after);
+
+        $res = $cb->(GET('/test/?q=git+%26+ampersand&x=A'));
+        is $res->code, 200, 'Atom hit with ampersand';
+        unlike $res->content, qr/git\+&\+ampersand/, '& is HTML-escaped';
+
+        $res = $cb->(GET('/test/?q=%22hi+%2b%2b%2b+b/foo%22&x=A'));
+        is $res->code, 200, 'slashes and plusses search hit';
+        like $res->content, qr!q=%22hi\+(?:%2[bB]){3}\+b/foo%22!,
+                '+ and " escaped, but slash not escaped in query';
+
+        $res = $cb->(GET(q{/test/?q=%22s'more%22&x=A}));
+        is $res->code, 200, 'single quote inside phrase';
+        # TODO: more tests and odd cases
 });
 
 done_testing();
diff --git a/t/psgi_text.t b/t/psgi_text.t
index e4613945..25599dd9 100644
--- a/t/psgi_text.t
+++ b/t/psgi_text.t
@@ -1,8 +1,6 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use warnings;
-use Test::More;
+use v5.12;
 use PublicInbox::Eml;
 use PublicInbox::TestCommon;
 my ($tmpdir, $for_destroy) = tmpdir();
@@ -13,13 +11,12 @@ my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape Plack::Builder);
 require_mods(@mods, 'IO::Uncompress::Gunzip');
 use_ok $_ foreach @mods;
 use PublicInbox::Import;
-use PublicInbox::Git;
-use PublicInbox::Config;
 use_ok 'PublicInbox::WWW';
 use_ok 'PublicInbox::WwwText';
-my $config = PublicInbox::Config->new(\<<EOF);
-$cfgpfx.address=$addr
-$cfgpfx.inboxdir=$maindir
+my $config = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = $addr
+        inboxdir = $maindir
 EOF
 PublicInbox::Import::init_bare($maindir);
 my $www = PublicInbox::WWW->new($config);
@@ -43,11 +40,7 @@ test_psgi(sub { $www->call(@_) }, sub {
         $res = $cb->($req);
         $content = $res->content;
         my $olen = $res->header('Content-Length');
-        my $f = "$tmpdir/cfg";
-        open my $fh, '>', $f or die;
-        print $fh $content or die;
-        close $fh or die;
-        my $cfg = PublicInbox::Config->new($f);
+        my $cfg = cfg_new $tmpdir, $content;
         is($cfg->{"$cfgpfx.address"}, $addr, 'got expected address in config');
 
         $req->header('Accept-Encoding' => 'gzip');
diff --git a/t/psgi_v2.t b/t/psgi_v2.t
index 7d73b606..54faae9b 100644
--- a/t/psgi_v2.t
+++ b/t/psgi_v2.t
@@ -1,18 +1,49 @@
 #!perl -w
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
 use PublicInbox::TestCommon;
+use IO::Uncompress::Gunzip qw(gunzip);
 require_git(2.6);
 use PublicInbox::Eml;
 use PublicInbox::Config;
 use PublicInbox::MID qw(mids);
-require_mods(qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test
+require_mods(qw(DBD::SQLite Xapian HTTP::Request::Common Plack::Test
                 URI::Escape Plack::Builder HTTP::Date));
 use_ok($_) for (qw(HTTP::Request::Common Plack::Test));
 use_ok 'PublicInbox::WWW';
 my ($tmpdir, $for_destroy) = tmpdir();
+my $enc_dup = 'ref-20150309094050.GO3427@x1.example';
+
+my $dibx = create_inbox 'v2-dup', version => 2, indexlevel => 'medium',
+                        tmpdir => "$tmpdir/dup", sub {
+        my ($im, $ibx) = @_;
+        my $common = <<"";
+Date: Mon, 9 Mar 2015 09:40:50 +0000
+From: x\@example.com
+To: y\@example.com
+Subject: re
+Message-ID: <$enc_dup>
+MIME-Version: 1.0
+
+        $im->add(PublicInbox::Eml->new($common.<<EOM)) or BAIL_OUT;
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: inline
+Content-Transfer-Encoding: 8bit
+
+cr_mismatch
+pipe \x{e2}\x{94}\x{82} or not
+EOM
+        $im->add(PublicInbox::Eml->new($common.<<EOM)) or BAIL_OUT;
+Content-Type: text/plain; charset="windows-1252"
+Content-Transfer-Encoding: quoted-printable
+
+cr_mismatch\r
+pipe =E2=94=82 or not
+EOM
+};
+
 my $eml = PublicInbox::Eml->new(<<'EOF');
 From oldbug-pre-a0c07cba0e5d8b6a Fri Oct  2 00:00:00 1993
 From: a@example.com
@@ -46,6 +77,30 @@ $new_mid //= do {
         local $/;
         <$fh>;
 };
+
+my $m2t = create_inbox 'mid2tid-1', version => 2, indexlevel => 'medium', sub {
+        my ($im, $ibx) = @_;
+        for my $n (1..3) {
+                $im->add(PublicInbox::Eml->new(<<EOM)) or xbail 'add';
+Date: Fri, 02 Oct 1993 00:0$n:00 +0000
+Message-ID: <t\@$n>
+Subject: tid $n
+From: x\@example.com
+References: <a-mid\@b>
+
+$n
+EOM
+                $im->add(PublicInbox::Eml->new(<<EOM)) or xbail 'add';
+Date: Fri, 02 Oct 1993 00:0$n:00 +0000
+Message-ID: <ut\@$n>
+Subject: unrelated tid $n
+From: x\@example.com
+References: <b-mid\@b>
+
+EOM
+        }
+};
+
 my $cfgpath = "$ibx->{inboxdir}/pi_config";
 {
         open my $fh, '>', $cfgpath or BAIL_OUT $!;
@@ -53,6 +108,12 @@ my $cfgpath = "$ibx->{inboxdir}/pi_config";
 [publicinbox "v2test"]
         inboxdir = $ibx->{inboxdir}
         address = $ibx->{-primary_address}
+[publicinbox "dup"]
+        inboxdir = $dibx->{inboxdir}
+        address = $dibx->{-primary_address}
+[publicinbox "m2t"]
+        inboxdir = $m2t->{inboxdir}
+        address = $m2t->{-primary_address}
 EOF
         close $fh or BAIL_OUT;
 }
@@ -145,20 +206,18 @@ my $client1 = sub {
         $cfg->each_inbox(sub { $_[0]->search->reopen });
 
         SKIP: {
-                eval { require IO::Uncompress::Gunzip };
-                skip 'IO::Uncompress::Gunzip missing', 6 if $@;
                 my ($in, $out, $status);
                 my $req = GET('/v2test/a-mid@b/raw');
                 $req->header('Accept-Encoding' => 'gzip');
                 $res = $cb->($req);
                 is($res->header('Content-Encoding'), 'gzip', 'gzip encoding');
                 $in = $res->content;
-                IO::Uncompress::Gunzip::gunzip(\$in => \$out);
+                gunzip(\$in => \$out);
                 is($out, $raw, 'gzip response matches');
 
                 $res = $cb->(GET('/v2test/a-mid@b/t.mbox.gz'));
                 $in = $res->content;
-                $status = IO::Uncompress::Gunzip::gunzip(\$in => \$out);
+                $status = gunzip(\$in => \$out);
                 unlike($out, qr/^From oldbug/sm, 'buggy "From_" line omitted');
                 like($out, qr/^hello world$/m, 'got first in t.mbox.gz');
                 like($out, qr/^hello world!$/m, 'got second in t.mbox.gz');
@@ -169,7 +228,7 @@ my $client1 = sub {
                 # search interface
                 $res = $cb->(POST('/v2test/?q=m:a-mid@b&x=m'));
                 $in = $res->content;
-                $status = IO::Uncompress::Gunzip::gunzip(\$in => \$out);
+                $status = gunzip(\$in => \$out);
                 unlike($out, qr/^From oldbug/sm, 'buggy "From_" line omitted');
                 like($out, qr/^hello world$/m, 'got first in mbox POST');
                 like($out, qr/^hello world!$/m, 'got second in mbox POST');
@@ -180,7 +239,7 @@ my $client1 = sub {
                 # all.mbox.gz interface
                 $res = $cb->(GET('/v2test/all.mbox.gz'));
                 $in = $res->content;
-                $status = IO::Uncompress::Gunzip::gunzip(\$in => \$out);
+                $status = gunzip(\$in => \$out);
                 unlike($out, qr/^From oldbug/sm, 'buggy "From_" line omitted');
                 like($out, qr/^hello world$/m, 'got first in all.mbox');
                 like($out, qr/^hello world!$/m, 'got second in all.mbox');
@@ -209,6 +268,8 @@ my $client1 = sub {
         local $SIG{__WARN__} = 'DEFAULT';
         $res = $cb->(GET('/v2test/a-mid@b/'));
         $raw = $res->content;
+        like($raw, qr/WARNING: multiple messages have this Message-ID/,
+                'warned about duplicate Message-IDs');
         like($raw, qr/^hello world$/m, 'got first message');
         like($raw, qr/^hello world!$/m, 'got second message');
         like($raw, qr/^hello ghosts$/m, 'got third message');
@@ -218,6 +279,15 @@ my $client1 = sub {
                 like($raw, qr!>\Q$mid\E</a>!s, "Message-ID $mid shown");
         }
         like($raw, qr/\b3\+ messages\b/, 'thread overview shown');
+
+        $res = $cb->(GET("/dup/$enc_dup/d/"));
+        is($res->code, 200, '/d/ (diff) endpoint works');
+        $raw = $res->content;
+        like($raw, qr!</span> cr_mismatch\n!s,
+                'cr_mismatch is only diff context');
+        like($raw, qr!>\-pipe !s, 'pipe diff del line');
+        like($raw, qr!>\+pipe !s, 'pipe diff ins line');
+        unlike $raw, qr/No newline at end of file/;
 };
 
 test_psgi(sub { $www->call(@_) }, $client1);
@@ -292,6 +362,18 @@ my $client3 = sub {
         local $SIG{__WARN__} = sub { push @warn, @_ };
         $res = $cb->(GET('/v2test/?t=1970'.'01'.'01'));
         is_deeply(\@warn, [], 'no warnings on YYYYMMDD only');
+
+        $res = $cb->(POST("/m2t/t\@1/?q=dt:19931002000300..&x=m"));
+        is($res->code, 200, 'got 200 on mid2tid query');
+        gunzip(\(my $in = $res->content) => \(my $out));
+        my @m = ($out =~ m!^Message-ID: <([^>]+)>\n!gms);
+        is_deeply(\@m, ['t@3'], 'only got latest result from query');
+
+        $res = $cb->(POST("/m2t/t\@1/?q=dt:19931002000400..&x=m"));
+        is($res->code, 404, '404 on out-of-range mid2tid query');
+
+        $res = $cb->(POST("/m2t/t\@1/?q=s:unrelated&x=m"));
+        is($res->code, 404, '404 on cross-thread search');
 };
 test_psgi(sub { $www->call(@_) }, $client3);
 test_httpd($env, $client3, 4);
diff --git a/t/qspawn.t b/t/qspawn.t
index 4b9dc8a5..507f86a5 100644
--- a/t/qspawn.t
+++ b/t/qspawn.t
@@ -1,8 +1,9 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
+use v5.12;
 use Test::More;
 use_ok 'PublicInbox::Qspawn';
+use_ok 'PublicInbox::Limiter';
 
 {
         my $cmd = [qw(sh -c), 'echo >&2 err; echo out'];
@@ -20,12 +21,13 @@ use_ok 'PublicInbox::Qspawn';
 sub finish_err ($) {
         my ($qsp) = @_;
         $qsp->finish;
-        $qsp->{err};
+        $qsp->{qsp_err} && ${$qsp->{qsp_err}};
 }
 
-my $limiter = PublicInbox::Qspawn::Limiter->new(1);
+my $limiter = PublicInbox::Limiter->new(1);
 {
         my $x = PublicInbox::Qspawn->new([qw(true)]);
+        $x->{qsp_err} = \(my $err = '');
         my $run = 0;
         $x->start($limiter, sub {
                 my ($self) = @_;
@@ -37,7 +39,9 @@ my $limiter = PublicInbox::Qspawn::Limiter->new(1);
 }
 
 {
+        my @err; local $SIG{__WARN__} = sub { push @err, @_ };
         my $x = PublicInbox::Qspawn->new([qw(false)]);
+        $x->{qsp_err} = \(my $err = '');
         my $run = 0;
         $x->start($limiter, sub {
                 my ($self) = @_;
@@ -47,10 +51,13 @@ my $limiter = PublicInbox::Qspawn::Limiter->new(1);
                 $run = 1;
         });
         is($run, 1, 'callback ran alright');
+        ok(scalar @err, 'got warning');
 }
 
 foreach my $cmd ([qw(sleep 1)], [qw(sh -c), 'sleep 1; false']) {
+        my @err; local $SIG{__WARN__} = sub { push @err, @_ };
         my $s = PublicInbox::Qspawn->new($cmd);
+        $s->{qsp_err} = \(my $err = '');
         my @run;
         $s->start($limiter, sub {
                 my ($self) = @_;
@@ -70,8 +77,10 @@ foreach my $cmd ([qw(sleep 1)], [qw(sh -c), 'sleep 1; false']) {
 
         if ($cmd->[-1] =~ /false\z/) {
                 ok(finish_err($s), 'got error on false after sleep');
+                ok(scalar @err, 'got warning');
         } else {
                 ok(!finish_err($s), 'no error on sleep');
+                is_deeply([], \@err, 'no warnings');
         }
         ok(!finish_err($_->[0]), "true $_->[1] succeeded") foreach @t;
         is_deeply([qw(sleep 0 1 2)], \@run, 'ran in order');
diff --git a/t/replace.t b/t/replace.t
index 626cbe9b..a61c3ca0 100644
--- a/t/replace.t
+++ b/t/replace.t
@@ -49,7 +49,7 @@ EOF
         $im->done;
         my $thread_a = $ibx->over->get_thread('replace@example.com');
 
-        my %before = map {; delete($_->{blob}) => $_ } @{$ibx->recent};
+        my %before = map {; delete($_->{blob}) => $_ } @{$ibx->over->recent};
         my $reject = PublicInbox::Eml->new($orig->as_string);
         foreach my $mid (['<replace@example.com>', '<extra@example.com>'],
                                 [], ['<replaced@example.com>']) {
@@ -126,7 +126,7 @@ EOF
         }
 
         # check overview matches:
-        my %after = map {; delete($_->{blob}) => $_ } @{$ibx->recent};
+        my %after = map {; delete($_->{blob}) => $_ } @{$ibx->over->recent};
         my @before_blobs = keys %before;
         foreach my $blob (@before_blobs) {
                 delete $before{$blob} if delete $after{$blob};
@@ -187,7 +187,7 @@ test_replace(2, 'basic', $opt = { %$opt, post => \&pad_msgs });
 test_replace(2, 'basic', $opt = { %$opt, rotate_bytes => 1 });
 
 SKIP: {
-        require_mods(qw(Search::Xapian), 8);
+        require_mods(qw(Xapian), 8);
         for my $l (qw(medium)) {
                 test_replace(2, $l, {});
                 $opt = { pre => \&pad_msgs };
diff --git a/t/reply.t b/t/reply.t
index 41d72db2..7319e233 100644
--- a/t/reply.t
+++ b/t/reply.t
@@ -38,7 +38,6 @@ my $exp = [
     '--to=from@example.com',
     '--cc=cc@example.com',
     '--cc=to@example.com',
-    "--subject='Re: hihi'"
 ];
 
 is_deeply($arg, $exp, 'default reply is to :all');
@@ -46,8 +45,7 @@ $ibx->{replyto} = ':all';
 ($arg, $link) = PublicInbox::Reply::mailto_arg_link($ibx, $hdr);
 is_deeply($arg, $exp, '":all" also works');
 
-$exp = [ '--in-reply-to=blah@example.com', '--to=primary@example.com',
-        "--subject='Re: hihi'" ];
+$exp = [ '--in-reply-to=blah@example.com', '--to=primary@example.com' ];
 $ibx->{replyto} = ':list';
 ($arg, $link) = PublicInbox::Reply::mailto_arg_link($ibx, $hdr);
 is_deeply($arg, $exp, '":list" works for centralized lists');
@@ -57,7 +55,6 @@ $exp = [
          '--to=primary@example.com',
          '--cc=cc@example.com',
          '--cc=to@example.com',
-        "--subject='Re: hihi'"
 ];
 $ibx->{replyto} = ':list,Cc,To';
 ($arg, $link) = PublicInbox::Reply::mailto_arg_link($ibx, $hdr);
@@ -65,9 +62,7 @@ is_deeply($arg, $exp, '":list,Cc,To" works for kinda centralized lists');
 
 $ibx->{replyto} = 'new@example.com';
 ($arg, $link) = PublicInbox::Reply::mailto_arg_link($ibx, $hdr);
-$exp = [ '--in-reply-to=blah@example.com', '--to=new@example.com',
-        "--subject='Re: hihi'"
-];
+$exp = [ '--in-reply-to=blah@example.com', '--to=new@example.com' ];
 is_deeply($arg, $exp, 'explicit address works, too');
 
 $ibx->{replyto} = ':all';
@@ -78,7 +73,6 @@ $exp = [
     '--to=from@example$(echo .)com',
     '--cc=cc@example$(echo .)com',
     '--cc=to@example$(echo .)com',
-    "--subject='Re: hihi'"
 ];
 is_deeply($arg, $exp, 'address obfuscation works');
 is($link, '', 'no mailto: link given');
diff --git a/t/search-thr-index.t b/t/search-thr-index.t
index 62745dbc..aecd064f 100644
--- a/t/search-thr-index.t
+++ b/t/search-thr-index.t
@@ -7,7 +7,7 @@ use Test::More;
 use PublicInbox::TestCommon;
 use PublicInbox::MID qw(mids);
 use PublicInbox::Eml;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 require PublicInbox::SearchIdx;
 require PublicInbox::Smsg;
 require PublicInbox::Inbox;
diff --git a/t/search.t b/t/search.t
index 13210ff5..9fda6694 100644
--- a/t/search.t
+++ b/t/search.t
@@ -1,10 +1,10 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
-use warnings;
-use Test::More;
+use v5.10;
 use PublicInbox::TestCommon;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 require PublicInbox::SearchIdx;
 require PublicInbox::Inbox;
 require PublicInbox::InboxWritable;
@@ -34,15 +34,11 @@ my $rw_commit = sub {
         $ibx->search->reopen;
 };
 
-sub oct_is ($$$) {
-        my ($got, $exp, $msg) = @_;
-        is(sprintf('0%03o', $got), sprintf('0%03o', $exp), $msg);
-}
-
 {
         # git repository perms
+        use_ok 'PublicInbox::Umask';
         oct_is($ibx->_git_config_perm(),
-                &PublicInbox::InboxWritable::PERM_GROUP,
+                PublicInbox::Umask::PERM_GROUP(),
                 'undefined permission is group');
         my @t = (
                 [ '0644', 0022, '644 => umask(0022)' ],
@@ -54,8 +50,8 @@ sub oct_is ($$$) {
         );
         for (@t) {
                 my ($perm, $exp, $msg) = @$_;
-                my $got = PublicInbox::InboxWritable::_umask_for(
-                        PublicInbox::InboxWritable->_git_config_perm($perm));
+                my $got = PublicInbox::Umask::_umask_for(
+                        PublicInbox::Umask->_git_config_perm($perm));
                 oct_is($got, $exp, $msg);
         }
 }
@@ -436,9 +432,10 @@ $ibx->with_umask(sub {
 my $all_mask = 07777;
 my $dir_mask = 02770;
 
-# FreeBSD and apparently OpenBSD does not allow non-root users to set S_ISGID,
+# FreeBSD, OpenBSD and NetBSD do not allow non-root users to set S_ISGID,
 # so git doesn't set it, either (see DIR_HAS_BSD_GROUP_SEMANTICS in git.git)
-if ($^O =~ /(?:free|open)bsd/i) {
+# Presumably all *BSDs behave the same way.
+if (require_bsd) {
         $all_mask = 0777;
         $dir_mask = 0770;
 }
@@ -534,7 +531,15 @@ $ibx->with_umask(sub {
                 '20200418222508.GA13918@dcvr',
                 'Subject search reaches inside message/rfc822');
 
-        $doc_id = $rw->add_message(eml_load('t/data/binary.patch'));
+        my $eml = eml_load('t/data/binary.patch');
+        my $body = $eml->body;
+        $rw->add_message($eml);
+
+        $body =~ s/^/> /gsm;
+        $eml = PublicInbox::Eml->new($eml->header_obj->as_string."\n".$body);
+        $eml->header_set('Message-ID', '<binary-patch-reply@example>');
+        $rw->add_message($eml);
+
         $rw->commit_txn_lazy;
         $ibx->search->reopen;
         my $res = $query->('HcmV');
@@ -542,8 +547,9 @@ $ibx->with_umask(sub {
         $res = $query->('IcmZPo000310RR91');
         is_deeply($res, [], 'no results against 1-byte binary patch');
         $res = $query->('"GIT binary patch"');
-        is(scalar(@$res), 1, 'got binary result from "GIT binary patch"');
+        is(scalar(@$res), 2, 'got binary results from "GIT binary patch"');
         is($res->[0]->{mid}, 'binary-patch-test@example', 'msgid for binary');
+        is($res->[1]->{mid}, 'binary-patch-reply@example', 'msgid for reply');
         my $s = $query->('"literal 1"');
         is_deeply($s, $res, 'got binary result from exact literal size');
         $s = $query->('"literal 2"');
@@ -565,10 +571,13 @@ SKIP: {
                 skip 'too close to midnight, time is tricky', 6;
         }
         $q = $s->query_argv_to_string($g, [qw(d:20101002 blah)]);
-        is($q, 'd:20101002..20101003 blah', 'YYYYMMDD expanded to range');
+        is($q, 'dt:20101002000000..20101003000000 blah',
+                'YYYYMMDD expanded to range');
         $q = $s->query_argv_to_string($g, [qw(d:2010-10-02)]);
-        is($q, 'd:20101002..20101003', 'YYYY-MM-DD expanded to range');
+        is($q, 'dt:20101002000000..20101003000000',
+                'YYYY-MM-DD expanded to range');
         $q = $s->query_argv_to_string($g, [qw(rt:2010-10-02.. yy)]);
+        diag "q=$q";
         $q =~ /\Art:(\d+)\.\. yy/ or fail("rt: expansion failed: $q");
         is(strftime('%Y-%m-%d', gmtime($1//0)), '2010-10-02', 'rt: beg expand');
         $q = $s->query_argv_to_string($g, [qw(rt:..2010-10-02 zz)]);
@@ -615,7 +624,7 @@ SKIP: {
 
         $orig = $qs = qq[f:bob "hello world" d:1993-10-02..2010-10-02];
         $s->query_approxidate($g, $qs);
-        is($qs, qq[f:bob "hello world" d:19931002..20101002],
+        is($qs, qq[f:bob "hello world" dt:19931002000000..20101002000000],
                 'post-phrase date corrected');
 
         # Xapian uses "" to escape " inside phrases, we don't explictly
@@ -627,7 +636,7 @@ SKIP: {
                 is($qs, $orig, 'phrases unchanged \x'.ord($x).'-\x'.ord($y));
 
                 $s->query_approxidate($g, my $tmp = "$qs d:..2010-10-02");
-                is($tmp, "$orig d:..20101002",
+                is($tmp, "$orig dt:..20101002000000",
                         'two phrases did not throw off date parsing');
 
                 $orig = $qs = qq[${x}hello d:1993-10-02..$y$x world$y];
@@ -635,7 +644,7 @@ SKIP: {
                 is($qs, $orig, 'phrases unchanged \x'.ord($x).'-\x'.ord($y));
 
                 $s->query_approxidate($g, $tmp = "$qs d:..2010-10-02");
-                is($tmp, "$orig d:..20101002",
+                is($tmp, "$orig dt:..20101002000000",
                         'two phrases did not throw off date parsing');
         }
 
@@ -654,7 +663,7 @@ SKIP: {
                 skip 'TEST_EXPENSIVE not set for argv overflow check', 1;
         my @w;
         local $SIG{__WARN__} = sub { push @w, @_ }; # for pure Perl version
-        my @fail = map { 'd:1993-10-02..2010-10-02' } (1..(4096 * 32));
+        my @fail = map { 'dt:1993-10-02..2010-10-02' } (1..(4096 * 32));
         eval { $s->query_argv_to_string($g, \@fail) };
         ok($@, 'exception raised');
 }
diff --git a/t/select.t b/t/select.t
new file mode 100644
index 00000000..e8032c5a
--- /dev/null
+++ b/t/select.t
@@ -0,0 +1,4 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+use v5.12;
+local $ENV{TEST_IOPOLLER} = 'PublicInbox::Select';
+require './t/ds-poll.t';
diff --git a/t/sha.t b/t/sha.t
new file mode 100644
index 00000000..2e2d5636
--- /dev/null
+++ b/t/sha.t
@@ -0,0 +1,25 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::SHA;
+use Test::More;
+
+{
+        my $dig = PublicInbox::SHA->new(1);
+        open my $fh, '<', 'COPYING' or die "open: $!";
+        $dig->add(do { local $/; <$fh> });
+        is($dig->hexdigest, '78e50e186b04c8fe1defaa098f1c192181b3d837',
+                'AGPL-3 matches');
+}
+
+SKIP: {
+        my $n = $ENV{TEST_LEAK_NR} or skip 'TEST_LEAK_NR unset', 1;
+        for (1..$n) {
+                PublicInbox::SHA->new(1)->add('hello')->digest;
+                PublicInbox::SHA->new(1)->add('hello');
+                PublicInbox::SHA->new(1);
+        }
+}
+
+done_testing;
diff --git a/t/sigfd.t b/t/sigfd.t
index a68b12a6..9a7b947d 100644
--- a/t/sigfd.t
+++ b/t/sigfd.t
@@ -1,11 +1,13 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
-use strict;
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+use v5.12;
 use Test::More;
 use IO::Handle;
 use POSIX qw(:signal_h);
 use Errno qw(ENOSYS);
 require_ok 'PublicInbox::Sigfd';
 use PublicInbox::DS;
+my ($linux_sigfd, $has_sigfd);
 
 SKIP: {
         if ($^O ne 'linux' && !eval { require IO::KQueue }) {
@@ -15,17 +17,26 @@ SKIP: {
         my $old = PublicInbox::DS::block_signals();
         my $hit = {};
         my $sig = {};
+        local $SIG{USR2} = sub { $hit->{USR2}->{normal}++ };
         local $SIG{HUP} = sub { $hit->{HUP}->{normal}++ };
         local $SIG{TERM} = sub { $hit->{TERM}->{normal}++ };
         local $SIG{INT} = sub { $hit->{INT}->{normal}++ };
-        for my $s (qw(HUP TERM INT)) {
+        local $SIG{WINCH} = sub { $hit->{WINCH}->{normal}++ };
+        for my $s (qw(USR2 HUP TERM INT WINCH)) {
                 $sig->{$s} = sub { $hit->{$s}->{sigfd}++ };
         }
-        my $sigfd = PublicInbox::Sigfd->new($sig, 0);
+        kill 'USR2', $$ or die "kill $!";
+        ok(!defined($hit->{USR2}), 'no USR2 yet') or diag explain($hit);
+        PublicInbox::DS->Reset;
+        ok($PublicInbox::Syscall::SIGNUM{WINCH}, 'SIGWINCH number defined');
+        my $sigfd = PublicInbox::Sigfd->new($sig);
         if ($sigfd) {
+                $linux_sigfd = 1 if $^O eq 'linux';
+                $has_sigfd = 1;
                 ok($sigfd, 'Sigfd->new works');
                 kill('HUP', $$) or die "kill $!";
                 kill('INT', $$) or die "kill $!";
+                kill('WINCH', $$) or die "kill $!";
                 my $fd = fileno($sigfd->{sock});
                 ok($fd >= 0, 'fileno(Sigfd->{sock}) works');
                 my $rvec = '';
@@ -35,16 +46,23 @@ SKIP: {
                 for my $s (qw(HUP INT)) {
                         is($hit->{$s}->{sigfd}, 1, "sigfd fired $s");
                         is($hit->{$s}->{normal}, undef,
-                                'normal $SIG{$s} not fired');
+                                "normal \$SIG{$s} not fired");
                 }
+                SKIP: {
+                        skip 'Linux sigfd-only behavior', 1 if !$linux_sigfd;
+                        is($hit->{USR2}->{sigfd}, 1,
+                                'USR2 sent before signalfd created received');
+                }
+                ok(!$hit->{USR2}->{normal}, 'USR2 not fired normally');
+                PublicInbox::DS->Reset;
                 $sigfd = undef;
 
-                my $nbsig = PublicInbox::Sigfd->new($sig, 1);
+                my $nbsig = PublicInbox::Sigfd->new($sig);
                 ok($nbsig, 'Sigfd->new SFD_NONBLOCK works');
                 is($nbsig->wait_once, undef, 'nonblocking ->wait_once');
                 ok($! == Errno::EAGAIN, 'got EAGAIN');
                 kill('HUP', $$) or die "kill $!";
-                PublicInbox::DS->SetPostLoopCallback(sub {}); # loop once
+                local @PublicInbox::DS::post_loop_do = (sub {}); # loop once
                 PublicInbox::DS::event_loop();
                 is($hit->{HUP}->{sigfd}, 2, 'HUP sigfd fired in event loop') or
                         diag explain($hit); # sometimes fails on FreeBSD 11.x
@@ -54,10 +72,18 @@ SKIP: {
                 PublicInbox::DS->Reset;
                 is($hit->{TERM}->{sigfd}, 1, 'TERM sigfd fired in event loop');
                 is($hit->{HUP}->{sigfd}, 3, 'HUP sigfd fired in event loop');
+                ok($hit->{WINCH}->{sigfd}, 'WINCH sigfd fired in event loop');
         } else {
                 skip('signalfd disabled?', 10);
         }
-        sigprocmask(SIG_SETMASK, $old) or die "sigprocmask $!";
+        ok(!$hit->{USR2}->{normal}, 'USR2 still not fired normally');
+        PublicInbox::DS::sig_setmask($old);
+        SKIP: {
+                ($has_sigfd && !$linux_sigfd) or
+                        skip 'EVFILT_SIGNAL-only behavior check', 1;
+                is($hit->{USR2}->{normal}, 1,
+                        "USR2 fired normally after unblocking on $^O");
+        }
 }
 
 done_testing;
diff --git a/t/solver_git.t b/t/solver_git.t
index 1baa012b..db672904 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -1,22 +1,23 @@
 #!perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C)  all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::TestCommon;
 use Cwd qw(abs_path);
-require_git(2.6);
+require_git v2.6;
 use PublicInbox::ContentHash qw(git_sha);
-use PublicInbox::Spawn qw(popen_rd);
-require_mods(qw(DBD::SQLite Search::Xapian Plack::Util));
-my $git_dir = xqx([qw(git rev-parse --git-dir)], undef, {2 => \(my $null)});
+use PublicInbox::Spawn qw(run_qx);
+require_mods(qw(DBD::SQLite Xapian URI::Escape));
+require PublicInbox::SolverGit;
+my $rdr = { 2 => \(my $null) };
+my $git_dir = xqx([qw(git rev-parse --git-common-dir)], undef, $rdr);
+$git_dir = xqx([qw(git rev-parse --git-dir)], undef, $rdr) if $? != 0;
 $? == 0 or plan skip_all => "$0 must be run from a git working tree";
 chomp $git_dir;
 
 # needed for alternates, and --absolute-git-dir is only in git 2.13+
 $git_dir = abs_path($git_dir);
 
-use_ok "PublicInbox::$_" for (qw(Inbox V2Writable Git SolverGit WWW));
 my $patch2 = eml_load 't/solve/0002-rename-with-modifications.patch';
 my $patch2_oid = git_sha(1, $patch2)->hexdigest;
 
@@ -28,14 +29,22 @@ my $ibx = create_inbox 'v2', version => 2,
         $im->add($patch2) or BAIL_OUT;
 };
 my $md = "$tmpdir/md";
-File::Path::mkpath([map { $md.$_ } (qw(/ /cur /new /tmp))]);
+File::Path::make_path(map { $md.$_ } (qw(/cur /new /tmp)));
 symlink(abs_path('t/solve/0001-simple-mod.patch'), "$md/cur/foo:2,") or
         xbail "symlink: $!";
 
+my $v1_0_0_rev = '8a918a8523bc9904123460f85999d75f6d604916';
 my $v1_0_0_tag = 'cb7c42b1e15577ed2215356a2bf925aef59cdd8d';
 my $v1_0_0_tag_short = substr($v1_0_0_tag, 0, 16);
 my $expect = '69df7d565d49fbaaeb0a067910f03dc22cd52bd0';
 my $non_existent = 'ee5e32211bf62ab6531bdf39b84b6920d0b6775a';
+my $stderr_empty = sub {
+        my ($msg) = @_;
+        open my $efh, '<', "$tmpdir/stderr.log" or xbail $!;
+        my @l = <$efh>;
+        @l = grep(!/reverse ?proxy/i, @l);
+        is_xdeeply(\@l, [], $msg // 'stderr.log is empty');
+};
 
 test_lei({tmpdir => "$tmpdir/blob"}, sub {
         lei_ok('blob', '--mail', $patch2_oid, '-I', $ibx->{inboxdir},
@@ -199,10 +208,11 @@ my $hinted = $res;
 shift @$res; shift @$hinted;
 is_deeply($res, $hinted, 'hints work (or did not hurt :P');
 
-my @psgi = qw(HTTP::Request::Common Plack::Test URI::Escape Plack::Builder);
+my @psgi = qw(HTTP::Request::Common Plack::Test Plack::Builder);
 SKIP: {
         require_mods(@psgi, 7 + scalar(@psgi));
         use_ok($_) for @psgi;
+        require PublicInbox::WWW;
         my $binfoo = "$ibx->{inboxdir}/binfoo.git";
         my $l = "$ibx->{inboxdir}/inbox.lock";
         -f $l or BAIL_OUT "BUG: $l missing: $!";
@@ -213,27 +223,48 @@ SKIP: {
         };
         my %bin = (big => $big_size, small => 1);
         my %oid; # (small|big) => OID
-        my $lk = bless { lock_path => $l }, 'PublicInbox::Lock';
+        require PublicInbox::Lock;
+        my $lk = PublicInbox::Lock->new($l);
         my $acq = $lk->lock_for_scope;
-        my $stamp = "$binfoo/stamp";
+        my $stamp = "$binfoo/stamp-";
         if (open my $fh, '<', $stamp) {
                 %oid = map { chomp; split(/=/, $_) } (<$fh>);
         } else {
                 PublicInbox::Import::init_bare($binfoo);
                 my $cmd = [ qw(git hash-object -w --stdin) ];
                 my $env = { GIT_DIR => $binfoo };
-                open my $fh, '>', "$stamp.$$" or BAIL_OUT;
                 while (my ($label, $size) = each %bin) {
-                        pipe(my ($rin, $win)) or BAIL_OUT;
-                        my $rout = popen_rd($cmd , $env, { 0 => $rin });
-                        $rin = undef;
-                        print { $win } ("\0" x $size) or BAIL_OUT;
-                        close $win or BAIL_OUT;
-                        chomp(my $x = <$rout>);
-                        close $rout or BAIL_OUT "$?";
-                        print $fh "$label=$x\n" or BAIL_OUT;
+                        my $rdr = { 0 => \("\0" x $size) };
+                        chomp(my $x = run_qx($cmd , $env, $rdr));
+                        xbail "@$cmd: \$?=$?" if $?;
                         $oid{$label} = $x;
                 }
+
+                open my $null, '<', '/dev/null' or xbail "open /dev/null: $!";
+                my $t = xqx([qw(git mktree)], $env, { 0 => $null });
+                xbail "mktree: $?" if $?;
+                chomp($t);
+                my $non_utf8 = "K\x{e5}g";
+                $env->{GIT_AUTHOR_NAME} = $non_utf8;
+                $env->{GIT_AUTHOR_EMAIL} = 'e@example.com';
+                $env->{GIT_COMMITTER_NAME} = $env->{GIT_AUTHOR_NAME};
+                $env->{GIT_COMMITTER_EMAIL} = $env->{GIT_AUTHOR_EMAIL};
+                my $in = \"$non_utf8\n\nK\x{e5}g\n";
+                my @ct = qw(git -c i18n.commitEncoding=iso-8859-1 commit-tree);
+                my $c = xqx([@ct, $t], $env, { 0 => $in });
+                xbail "commit-tree: $?" if $?;
+                chomp($c);
+                $oid{'iso-8859-1'} = $c;
+
+                $c = xqx([@ct, '-p', $c, $t], $env, { 0 => $in });
+                xbail "commit-tree: $?" if $?;
+                chomp($c);
+                $oid{'8859-parent'} = $c;
+
+                open my $fh, '>', "$stamp.$$" or BAIL_OUT;
+                while (my ($k, $v) = each %oid) {
+                        print $fh "$k=$v\n" or xbail "print: $!";
+                }
                 close $fh or BAIL_OUT;
                 rename("$stamp.$$", $stamp) or BAIL_OUT;
         }
@@ -244,6 +275,8 @@ SKIP: {
         my $cfgpath = "$tmpdir/httpd-config";
         open my $cfgfh, '>', $cfgpath or die;
         print $cfgfh <<EOF or die;
+[coderepo]
+        snapshots = tar.gz
 [publicinbox "$name"]
         address = $ibx->{-primary_address}
         inboxdir = $ibx->{inboxdir}
@@ -258,6 +291,16 @@ SKIP: {
         cgiturl = http://example.com/binfoo
 EOF
         close $cfgfh or die;
+        my $exp_digest;
+        {
+                my $exp = xqx([qw(git archive --format=tar.gz
+                                --prefix=public-inbox-1.0.0/ v1.0.0)],
+                                { GIT_DIR => $git_dir });
+                is($?, 0, 'no error from git archive');
+                ok(length($exp) > 1024, 'expected archive generated');
+                $exp_digest = git_sha(256, \$exp)->hexdigest;
+        };
+
         my $cfg = PublicInbox::Config->new($cfgpath);
         my $www = PublicInbox::WWW->new($cfg);
         my $client = sub {
@@ -278,7 +321,7 @@ EOF
                 is($res->code, 404, 'failure with null OID');
 
                 $res = $cb->(GET("/$name/$non_existent/s/"));
-                is($res->code, 404, 'failure with null OID');
+                is($res->code, 404, 'failure with non-existent OID');
 
                 $res = $cb->(GET("/$name/$v1_0_0_tag/s/"));
                 is($res->code, 200, 'shows commit (unabbreviated)');
@@ -287,38 +330,144 @@ EOF
                 while (my ($label, $size) = each %bin) {
                         $res = $cb->(GET("/$name/$oid{$label}/s/"));
                         is($res->code, 200, "$label binary file");
-                        ok(index($res->content, "blob $size bytes") >= 0,
+                        ok(index($res->content,
+                                "blob $oid{$label} $size bytes") >= 0,
                                 "showed $label binary blob size");
                         $res = $cb->(GET("/$name/$oid{$label}/s/raw"));
                         is($res->code, 200, "$label raw binary download");
                         is($res->content, "\0" x $size,
                                 "$label content matches");
                 }
+                my $utf8 = 'e022d3377fd2c50fd9931bf96394728958a90bf3';
+                $res = $cb->(GET("/$name/$utf8/s/"));
+                is($res->code, 200, 'shows commit w/ utf8.eml');
+                like($res->content, qr/El&#233;anor/,
+                                'UTF-8 commit shown properly');
+
+                # WwwCoderepo
+                my $olderr;
+                if (defined $ENV{PLACK_TEST_EXTERNALSERVER_URI}) {
+                        $stderr_empty->('nothing in stderr.log, yet');
+                } else {
+                        open $olderr, '>&', \*STDERR or xbail "open: $!";
+                        open STDERR, '+>>', "$tmpdir/stderr.log" or
+                                xbail "open: $!";
+                }
+                $res = $cb->(GET('/binfoo/'));
+                defined($ENV{PLACK_TEST_EXTERNALSERVER_URI}) or
+                        open STDERR, '>&', $olderr or xbail "open: $!";
+                is($res->code, 200, 'coderepo summary (binfoo)');
+                $stderr_empty->();
+
+                $res = $cb->(GET("/binfoo/$oid{'iso-8859-1'}/s/"));
+                is($res->code, 200, 'ISO-8859-1 commit');
+                like($res->content, qr/K&#229;g/, 'ISO-8859-1 commit message');
+                $stderr_empty->();
+
+                $res = $cb->(GET("/binfoo/$oid{'8859-parent'}/s/"));
+                is($res->code, 200, 'commit w/ ISO-8859-parent');
+                like($res->content, qr/K&#229;g/, 'ISO-8859-1 commit message');
+                $stderr_empty->();
+
+                $res = $cb->(GET('/public-inbox/'));
+                is($res->code, 200, 'coderepo summary (public-inbox)');
+
+                my $tip = 'invalid-'.int(rand(0xdeadbeef));
+                $res = $cb->(GET('/public-inbox/?h='.$tip));
+                is($res->code, 200, 'coderepo summary on dead branch');
+                like($res->content, qr/no commits in `\Q$tip\E', yet/,
+                        'lack of commits noted');
+
+                $res = $cb->(GET('/public-inbox'));
+                is($res->code, 301, 'redirected');
+
+                my $fn = 'public-inbox-1.0.0.tar.gz';
+                $res = $cb->(GET("/public-inbox/snapshot/$fn"));
+                is($res->code, 200, 'tar.gz snapshot');
+                is($res->header('Content-Disposition'),
+                        qq'inline; filename="$fn"', 'c-d header');
+                is($res->header('ETag'), qq'"$v1_0_0_rev"', 'etag header');
+
+                my $got = $res->content;
+                is(git_sha(256, \$got)->hexdigest, $exp_digest,
+                        "content matches installed `git archive' output");
+                undef $got;
+
+                $fn = 'public-inbox-1.0.2.tar.gz';
+                $res = $cb->(GET("/public-inbox/snapshot/$fn"));
+                is($res->code, 404, '404 on non-existent tag');
+
+                $fn = 'public-inbox-1.0.0.tar.bz2';
+                $res = $cb->(GET("/public-inbox/snapshot/$fn"));
+                is($res->code, 404, '404 on unconfigured snapshot format');
+
+                $res = $cb->(GET('/public-inbox/atom/'));
+                is($res->code, 200, 'Atom feed');
+                SKIP: {
+                        require_mods('XML::TreePP', 1);
+                        my $t = eval { XML::TreePP->new->parse($res->content) }
+                                or diag explain($res);
+                        is(scalar @{$t->{feed}->{entry}}, 50,
+                                'got 50 entries') or diag explain([$t, $res]);
+
+                        $res = $cb->(GET('/public-inbox/atom/COPYING'));
+                        is($res->code, 200, 'file Atom feed');
+                        $t = XML::TreePP->new->parse($res->content);
+                        ok($t->{feed}->{entry}, 'got entry') or
+                                diag explain([ $t, $res ]);
+
+                        $res = $cb->(GET('/public-inbox/atom/README.md'));
+                        is($res->code, 404, '404 on missing file Atom feed');
+
+                        $res = $cb->(GET('/public-inbox/atom/?h=gone'));
+                        is($res->code, 404, '404 on missing Atom feed branch');
+                }
+
+                $res = $cb->(GET('/public-inbox/tree/'));
+                is($res->code, 200, 'got 200 for root listing');
+                $got = $res->content;
+                like($got, qr/\bgit ls-tree\b/, 'ls-tree help shown');
+
+                $res = $cb->(GET('/public-inbox/tree/README'));
+                is($res->code, 200, 'got 200 for regular file');
+                $got = $res->content;
+                like($got, qr/\bgit show\b/, 'git show help shown');
+
+                $res = $cb->(GET('/public-inbox/tree/Documentation'));
+                is($res->code, 200, 'got 200 for a directory');
+                $got = $res->content;
+                like($got, qr/\bgit ls-tree\b/, 'ls-tree help shown');
+
+                $res = $cb->(GET('/public-inbox/tree/?h=no-branch'));
+                is($res->code, 404, 'got 404 for non-existent ref root');
+                $res = $cb->(GET('/public-inbox/tree/README?h=no-file'));
+                is($res->code, 404, 'got 404 for non-existent ref README');
+                $res = $cb->(GET('/public-inbox/tree/Documentation?h=no-dir'));
+                is($res->code, 404, 'got 404 for non-existent ref directory');
+
+                $res = $cb->(GET('/public-inbox/tags.atom'));
+                is($res->code, 200, 'Atom feed');
+                SKIP: {
+                        require_mods('XML::TreePP', 1);
+                        my $t = XML::TreePP->new->parse($res->content);
+                        ok(scalar @{$t->{feed}->{entry}}, 'got tag entries');
+                }
         };
         test_psgi(sub { $www->call(@_) }, $client);
+        my $env = { PI_CONFIG => $cfgpath, TMPDIR => $tmpdir };
+        test_httpd($env, $client, 7, sub {
         SKIP: {
-                require_mods(qw(Plack::Test::ExternalServer), 7);
-                my $env = { PI_CONFIG => $cfgpath };
-                my $sock = tcp_server() or die;
-                my ($out, $err) = map { "$tmpdir/std$_.log" } qw(out err);
-                my $cmd = [ qw(-httpd -W0), "--stdout=$out", "--stderr=$err" ];
-                my $td = start_script($cmd, $env, { 3 => $sock });
-                my ($h, $p) = tcp_host_port($sock);
-                my $url = "http://$h:$p";
-                local $ENV{PLACK_TEST_EXTERNALSERVER_URI} = $url;
-                Plack::Test::ExternalServer::test_psgi(client => $client);
                 require_cmd('curl', 1) or skip 'no curl', 1;
-
                 mkdir "$tmpdir/ext" // xbail "mkdir $!";
+                my $rurl = "$ENV{PLACK_TEST_EXTERNALSERVER_URI}/$name";
                 test_lei({tmpdir => "$tmpdir/ext"}, sub {
-                        my $rurl = "$url/$name";
                         lei_ok(qw(blob --no-mail 69df7d5 -I), $rurl);
                         is(git_sha(1, \$lei_out)->hexdigest, $expect,
                                 'blob contents output');
                         ok(!lei(qw(blob -I), $rurl, $non_existent),
                                         'non-existent blob fails');
                 });
-        }
+        }});
 }
 
 done_testing();
diff --git a/t/spawn.t b/t/spawn.t
index 5fc99a2a..5b17ed38 100644
--- a/t/spawn.t
+++ b/t/spawn.t
@@ -1,11 +1,12 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use warnings;
+use v5.12;
 use Test::More;
-use PublicInbox::Spawn qw(which spawn popen_rd);
-use PublicInbox::Sigfd;
-
+use PublicInbox::Spawn qw(which spawn popen_rd run_qx);
+require PublicInbox::Sigfd;
+require PublicInbox::DS;
+my $rlimit_map = PublicInbox::Spawn->can('rlimit_map');
 {
         my $true = which('true');
         ok($true, "'true' command found with which()");
@@ -18,6 +19,17 @@ use PublicInbox::Sigfd;
         is($?, 0, 'true exited successfully');
 }
 
+{
+        my $opt = { 0 => \'in', 2 => \(my $e) };
+        my $out = run_qx(['sh', '-c', 'echo e >&2; cat'], undef, $opt);
+        is($e, "e\n", 'captured stderr');
+        is($out, 'in', 'stdin read and stdout captured');
+        $opt->{0} = \"IN\n3\nLINES";
+        my @out = run_qx(['sh', '-c', 'echo E >&2; cat'], undef, $opt);
+        is($e, "E\n", 'captured stderr clobbers string');
+        is_deeply(\@out, [ "IN\n", "3\n", 'LINES' ], 'stdout array');
+}
+
 SKIP: {
         my $pid = spawn(['true'], undef, { pgid => 0 });
         ok($pid, 'spawned process with new pgid');
@@ -38,9 +50,8 @@ SKIP: {
         $pid = eval { spawn(['true'], undef, { pgid => $wrong_pgid, 2 => $w }) };
         close $w;
         my $err = do { local $/; <$r> };
-        # diag "$err ($@)";
         if (defined $pid) {
-                waitpid($pid, 0) if defined $pid;
+                waitpid($pid, 0);
                 isnt($?, 0, 'child error (pure-Perl)');
         } else {
                 ok($@, 'exception raised');
@@ -62,14 +73,14 @@ elsif ($pid > 0) {
 }
 EOF
         my $oldset = PublicInbox::DS::block_signals();
-        my $rd = popen_rd([$^X, '-e', $script]);
+        my $rd = popen_rd([$^X, qw(-w -e), $script]);
         diag 'waiting for child to reap grandchild...';
         chomp(my $line = readline($rd));
-        my ($rdy, $pid) = split(' ', $line);
+        my ($rdy, $pid) = split(/ /, $line);
         is($rdy, 'RDY', 'got ready signal, waitpid(-1) works in child');
         ok(kill('CHLD', $pid), 'sent SIGCHLD to child');
         is(readline($rd), "HI\n", '$SIG{CHLD} works in child');
-        ok(close $rd, 'popen_rd close works');
+        ok($rd->close, 'popen_rd close works');
         PublicInbox::DS::sig_setmask($oldset);
 }
 
@@ -96,39 +107,40 @@ EOF
 
 {
         my $fh = popen_rd([qw(echo hello)]);
-        ok(fileno($fh) >= 0, 'tied fileno works');
+        ok(fileno($fh) >= 0, 'fileno works');
         my $l = <$fh>;
-        is($l, "hello\n", 'tied readline works');
+        is($l, "hello\n", 'readline works');
         $l = <$fh>;
-        ok(!$l, 'tied readline works for EOF');
+        ok(!$l, 'readline works for EOF');
 }
 
 {
         my $fh = popen_rd([qw(printf foo\nbar)]);
-        ok(fileno($fh) >= 0, 'tied fileno works');
-        my $tfh = (tied *$fh)->{fh};
-        is($tfh->blocking(0), 1, '->blocking was true');
-        is($tfh->blocking, 0, '->blocking is false');
-        is($tfh->blocking(1), 0, '->blocking was true');
-        is($tfh->blocking, 1, '->blocking is true');
+        ok(fileno($fh) >= 0, 'fileno works');
+        is($fh->blocking(0), 1, '->blocking was true');
+        is($fh->blocking, 0, '->blocking is false');
+        is($fh->blocking(1), 0, '->blocking was true');
+        is($fh->blocking, 1, '->blocking is true');
         my @line = <$fh>;
         is_deeply(\@line, [ "foo\n", 'bar' ], 'wantarray works on readline');
 }
 
 {
         my $fh = popen_rd([qw(echo hello)]);
+        like($fh->attached_pid, qr/\A[0-9]+\z/, 'have a PID');
         my $buf;
         is(sysread($fh, $buf, 6), 6, 'sysread got 6 bytes');
-        is($buf, "hello\n", 'tied gets works');
+        is($buf, "hello\n", 'sysread works');
         is(sysread($fh, $buf, 6), 0, 'sysread got EOF');
         $? = 1;
-        ok(close($fh), 'close succeeds');
+        ok($fh->close, 'close succeeds');
         is($?, 0, '$? set properly');
+        is($fh->attached_pid, undef, 'attached_pid cleared after close');
 }
 
 {
         my $fh = popen_rd([qw(false)]);
-        ok(!close($fh), 'close fails on false');
+        ok(!$fh->close, 'close fails on false');
         isnt($?, 0, '$? set properly: '.$?);
 }
 
@@ -140,25 +152,25 @@ EOF
 
 { # ->CLOSE vs ->DESTROY waitpid caller distinction
         my @c;
-        my $fh = popen_rd(['true'], undef, { cb => sub { @c = caller } });
-        ok(close($fh), '->CLOSE fired and successful');
+        my $fh = popen_rd(['true'], undef, undef, sub { @c = caller });
+        ok($fh->close, '->CLOSE fired and successful');
         ok(scalar(@c), 'callback fired by ->CLOSE');
         ok(grep(!m[/PublicInbox/DS\.pm\z], @c), 'callback not invoked by DS');
 
         @c = ();
-        $fh = popen_rd(['true'], undef, { cb => sub { @c = caller } });
+        $fh = popen_rd(['true'], undef, undef, sub { @c = caller });
         undef $fh; # ->DESTROY
         ok(scalar(@c), 'callback fired by ->DESTROY');
-        ok(grep(!m[/PublicInbox/ProcessPipe\.pm\z], @c),
-                'callback not invoked by ProcessPipe');
+        ok(grep(!m[/PublicInbox/IO\.pm\z], @c),
+                'callback not invoked by PublicInbox::IO');
 }
 
 { # children don't wait on siblings
         use POSIX qw(_exit);
         pipe(my ($r, $w)) or BAIL_OUT $!;
-        my $cb = sub { warn "x=$$\n" };
-        my $fh = popen_rd(['cat'], undef, { 0 => $r, cb => $cb });
-        my $pp = tied *$fh;
+        my @arg;
+        my $fh = popen_rd(['cat'], undef, { 0 => $r },
+                        sub { @arg = @_; warn "x=$$\n" }, 'hi');
         my $pid = fork // BAIL_OUT $!;
         local $SIG{__WARN__} = sub { _exit(1) };
         if ($pid == 0) {
@@ -171,30 +183,86 @@ EOF
         my @w;
         local $SIG{__WARN__} = sub { push @w, @_ };
         close $w;
-        close $fh;
+        $fh->close; # may set $?
         is($?, 0, 'cat exited');
+        is(scalar(@arg), 2, 'callback got args');
+        is($arg[1], 'hi', 'passed arg');
+        like($arg[0], qr/\A\d+\z/, 'PID');
         is_deeply(\@w, [ "x=$$\n" ], 'callback fired from owner');
 }
 
 SKIP: {
-        eval {
-                require BSD::Resource;
-                defined(BSD::Resource::RLIMIT_CPU())
-        } or skip 'BSD::Resource::RLIMIT_CPU missing', 3;
-        my ($r, $w);
-        pipe($r, $w) or die "pipe: $!";
-        my $cmd = ['sh', '-c', 'while true; do :; done'];
+        if ($rlimit_map) { # Inline::C installed
+                my %rlim = $rlimit_map->();
+                ok defined($rlim{RLIMIT_CPU}), 'RLIMIT_CPU defined';
+        } else {
+                eval {
+                        require BSD::Resource;
+                        defined(BSD::Resource::RLIMIT_CPU())
+                } or skip 'BSD::Resource::RLIMIT_CPU missing', 3;
+        }
+        my $cmd = [ $^X, qw(-w -e), <<'EOM' ];
+use POSIX qw(:signal_h);
+use Time::HiRes qw(time); # gettimeofday
+my $have_bsd_resource = eval { require BSD::Resource };
+my $set = POSIX::SigSet->new;
+$set->emptyset; # spawn() defaults to blocking all signals
+sigprocmask(SIG_SETMASK, $set) or die "SIG_SETMASK: $!";
+my $tot = 0;
+$SIG{XCPU} = sub { print "SIGXCPU $tot\n"; exit(1) };
+my $next = time + 1.1;
+while (1) {
+        # OpenBSD needs some syscalls (e.g. `times', `gettimeofday'
+        # and `write' (via Perl warn)) on otherwise idle systems to
+        # hit RLIMIT_CPU and fire signals:
+        # https://marc.info/?i=02A4BB8D-313C-464D-845A-845EB6136B35@gmail.com
+        my @t = $have_bsd_resource ? BSD::Resource::times() : (0, 0);
+        $tot = $t[0] + $t[1];
+        if (time > $next) {
+                warn "# T: @t (utime, ctime, cutime, cstime)\n" if @t;
+                $next = time + 1.1;
+        }
+}
+EOM
+        pipe(my($r, $w)) or die "pipe: $!";
         my $fd = fileno($w);
-        my $opt = { RLIMIT_CPU => [ 1, 1 ], RLIMIT_CORE => [ 0, 0 ], 1 => $fd };
+        my $opt = { RLIMIT_CPU => [ 1, 9 ], RLIMIT_CORE => [ 0, 0 ], 1 => $fd };
         my $pid = spawn($cmd, undef, $opt);
         close $w or die "close(w): $!";
         my $rset = '';
         vec($rset, fileno($r), 1) = 1;
         ok(select($rset, undef, undef, 5), 'child died before timeout');
         is(waitpid($pid, 0), $pid, 'XCPU child process reaped');
-        isnt($?, 0, 'non-zero exit status');
+        my $line;
+        like($line = readline($r), qr/SIGXCPU/, 'SIGXCPU handled') or
+                diag explain($line);
+        is($? >> 8, 1, 'non-zero exit status');
 }
 
-done_testing();
+SKIP: {
+        require PublicInbox::SpawnPP;
+        require File::Temp;
+        my $tmp = File::Temp->newdir('spawnpp-XXXX', TMPDIR => 1);
+        my $cmd = [ qw(/bin/sh -c), 'echo $HI >foo' ];
+        my $env = [ 'HI=hihi' ];
+        my $rlim = [];
+        my $pgid = -1;
+        my $pid = PublicInbox::SpawnPP::pi_fork_exec([], '/bin/sh', $cmd, $env,
+                                                $rlim, "$tmp", $pgid);
+        is(waitpid($pid, 0), $pid, 'spawned process exited');
+        is($?, 0, 'no error');
+        open my $fh, '<', "$tmp/foo" or die "open: $!";
+        is(readline($fh), "hihi\n", 'env+chdir worked for SpawnPP');
+        close $fh;
+        unlink("$tmp/foo") or die "unlink: $!";
+        {
+                local $ENV{MOD_PERL} = 1;
+                $pid = PublicInbox::SpawnPP::pi_fork_exec([],
+                                '/bin/sh', $cmd, $env, $rlim, "$tmp", $pgid);
+        }
+        is(waitpid($pid, 0), $pid, 'spawned process exited');
+        open $fh, '<', "$tmp/foo" or die "open: $!";
+        is(readline($fh), "hihi\n", 'env+chdir SpawnPP under (faked) MOD_PERL');
+}
 
-1;
+done_testing();
diff --git a/t/tail_notify.t b/t/tail_notify.t
new file mode 100644
index 00000000..82480ebc
--- /dev/null
+++ b/t/tail_notify.t
@@ -0,0 +1,38 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::TestCommon;
+use POSIX qw(_exit);
+my ($tmpdir, $for_destroy) = tmpdir();
+use_ok 'PublicInbox::TailNotify';
+my $f = "$tmpdir/log";
+open my $fh, '>>', $f or xbail $!;
+my $tn = PublicInbox::TailNotify->new($f);
+my @x = $tn->getlines(1);
+is_deeply(\@x, [], 'nothing, yet');
+my $pid = fork // xbail "fork: $!";
+if ($pid == 0) {
+        tick;
+        syswrite $fh, "hi\n" // xbail "syswrite: $!";
+        _exit(0);
+}
+@x = $tn->getlines;
+is_deeply(\@x, [ "hi\n" ], 'got line');
+waitpid($pid, 0) // xbail "waitpid: $!";
+is($?, 0, 'writer done');
+
+$pid = fork // xbail "fork: $!";
+if ($pid == 0) {
+        tick;
+        unlink $f // xbail "unlink($f): $!";
+        open $fh, '>>', $f or xbail $!;
+        syswrite $fh, "bye\n" // xbail "syswrite: $!";
+        _exit(0);
+}
+@x = $tn->getlines;
+is_deeply(\@x, [ "bye\n" ], 'got line after reopen');
+waitpid($pid, 0) // xbail "waitpid: $!";
+is($?, 0, 'writer done');
+
+done_testing;
diff --git a/t/v1-add-remove-add.t b/t/v1-add-remove-add.t
index a94bf7fd..50ff8143 100644
--- a/t/v1-add-remove-add.t
+++ b/t/v1-add-remove-add.t
@@ -6,7 +6,7 @@ use Test::More;
 use PublicInbox::Import;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 require PublicInbox::SearchIdx;
 my ($inboxdir, $for_destroy) = tmpdir();
 my $ibx = {
@@ -32,7 +32,7 @@ ok($im->add($mime), 'message added again');
 $im->done;
 my $rw = PublicInbox::SearchIdx->new($ibx, 1);
 $rw->index_sync;
-my $msgs = $ibx->recent({limit => 10});
+my $msgs = $ibx->over->recent({limit => 10});
 is($msgs->[0]->{mid}, 'a-mid@b', 'message exists in history');
 is(scalar @$msgs, 1, 'only one message in history');
 is($ibx->mm->num_for('a-mid@b'), 2, 'exists with second article number');
diff --git a/t/v1reindex.t b/t/v1reindex.t
index f593b323..2d12e3f5 100644
--- a/t/v1reindex.t
+++ b/t/v1reindex.t
@@ -8,7 +8,7 @@ use File::Path qw(remove_tree);
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
 require_git(2.6);
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 use_ok 'PublicInbox::SearchIdx';
 use_ok 'PublicInbox::Import';
 use_ok 'PublicInbox::OverIdx';
diff --git a/t/v2-add-remove-add.t b/t/v2-add-remove-add.t
index 579cdcb6..ddf8d248 100644
--- a/t/v2-add-remove-add.t
+++ b/t/v2-add-remove-add.t
@@ -6,7 +6,7 @@ use Test::More;
 use PublicInbox::Eml;
 use PublicInbox::TestCommon;
 require_git(2.6);
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 use_ok 'PublicInbox::V2Writable';
 my ($inboxdir, $for_destroy) = tmpdir();
 my $ibx = {
@@ -32,7 +32,7 @@ ok($im->add($mime), 'message added');
 ok($im->remove($mime), 'message removed');
 ok($im->add($mime), 'message added again');
 $im->done;
-my $msgs = $ibx->recent({limit => 1000});
+my $msgs = $ibx->over->recent({limit => 1000});
 is($msgs->[0]->{mid}, 'a-mid@b', 'message exists in history');
 is(scalar @$msgs, 1, 'only one message in history');
 
diff --git a/t/v2mda.t b/t/v2mda.t
index 8f2f335d..b7d177b2 100644
--- a/t/v2mda.t
+++ b/t/v2mda.t
@@ -3,15 +3,15 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use v5.10.1;
 use strict;
-use Test::More;
 use Fcntl qw(SEEK_SET);
 use Cwd;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
+use File::Path qw(remove_tree);
 require_git(2.6);
 
 my $V = 2;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 use_ok 'PublicInbox::V2Writable';
 my ($tmpdir, $for_destroy) = tmpdir();
 my $ibx = {
@@ -96,4 +96,29 @@ is($eml->as_string, $mime->as_string, 'injected message');
         is($mset->size, 1, 'patchid search works');
 }
 
+{
+        my @shards = grep(m!/[0-9]+\z!, glob("$ibx->{inboxdir}/xap*/*"));
+        ok(remove_tree(@shards), 'rm shards to convert to indexlevel=basic');
+        $ibx->do_cleanup;
+        $rdr->{2} = \(my $err = '');
+        $rdr->{0} = \<<'EOM';
+From: a@example.com
+To: test@example.com
+Subject: this is a ham message for learn
+Date: Fri, 02 Oct 1993 00:00:00 +0000
+Message-ID: <ham@example>
+
+yum
+EOM
+        my ($id, $prev);
+        is($ibx->over->next_by_mid('ham@example', \$id, \$prev), undef,
+                'no ham@example, yet');
+        ok(run_script([qw(-learn ham)], undef, $rdr), '-learn runs on basic')
+                or diag $err;
+        my $smsg = $ibx->over->next_by_mid('ham@example', \$id, \$prev);
+        ok($smsg, 'ham message learned w/ indexlevel=basic');
+        @shards = grep(m!/[0-9]+\z!, glob("$ibx->{inboxdir}/xap*/*"));
+        is_deeply(\@shards, [], 'not converted to medium/full after learn');
+}
+
 done_testing();
diff --git a/t/v2mirror.t b/t/v2mirror.t
index 37d64e83..b8824182 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
@@ -14,7 +14,7 @@ use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
 
 # Integration tests for HTTP cloning + mirroring
 require_mods(qw(Plack::Util Plack::Builder
-                HTTP::Date HTTP::Status Search::Xapian DBD::SQLite));
+                HTTP::Date HTTP::Status Xapian DBD::SQLite));
 use_ok 'PublicInbox::V2Writable';
 use PublicInbox::InboxWritable;
 use PublicInbox::Eml;
@@ -330,12 +330,12 @@ SKIP: {
         require_mods('Email::MIME', 1); # for legacy revision
         # using plackup to test old PublicInbox::WWW since -httpd from
         # back then relied on some packages we no longer depend on
-        my $plackup = which('plackup') or skip('no plackup in path', 1);
+        my $plackup = require_cmd('plackup', 1) or skip('no plackup in path', 1);
         require PublicInbox::Lock;
         chomp $oldrev;
         my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!);
         my $wt = "t/data-gen/$base.pre-manifest-$oldrev";
-        my $lk = bless { lock_path => __FILE__ }, 'PublicInbox::Lock';
+        my $lk = PublicInbox::Lock->new(__FILE__);
         $lk->lock_acquire;
         my $psgi = "$wt/app.psgi";
         if (!-f $psgi) { # checkout a pre-manifest.js.gz version
@@ -368,10 +368,11 @@ EOM
         # wait for plackup socket()+bind()+listen()
         my %opt = ( Proto => 'tcp', Type => Socket::SOCK_STREAM(),
                 PeerAddr => "$host:$port" );
-        for (0..50) {
+        for (0..100) {
                 tick();
                 last if IO::Socket::INET->new(%opt);
         }
+        IO::Socket::INET->new(%opt) or xbail "connect $host:$port: $!";
         my $dst = "$tmpdir/scrape";
         @cmd = (qw(-clone -q), "http://$host:$port/v2", $dst);
         run_script(\@cmd, undef, { 2 => \($err = '') });
diff --git a/t/v2reindex.t b/t/v2reindex.t
index cafe8648..8c49e154 100644
--- a/t/v2reindex.t
+++ b/t/v2reindex.t
@@ -1,11 +1,11 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 use PublicInbox::Eml;
 use PublicInbox::ContentHash qw(content_digest);
 use File::Path qw(remove_tree);
 require_git(2.6);
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 use_ok 'PublicInbox::V2Writable';
 use_ok 'PublicInbox::OverIdx';
 my ($inboxdir, $for_destroy) = tmpdir();
@@ -549,9 +549,8 @@ is($err, '', 'no errors from --xapian-only');
 undef $for_destroy;
 SKIP: {
         skip 'only testing lsof(8) output on Linux', 1 if $^O ne 'linux';
-        my $lsof = require_cmd('lsof', 1) or skip 'no lsof in PATH', 1;
         my $rdr = { 2 => \(my $null_err) };
-        my @d = grep(m!/xap[0-9]+/!, xqx([$lsof, '-p', $$], undef, $rdr));
+        my @d = grep m!/xap[0-9]+/!, lsof_pid $$, $rdr;
         is_deeply(\@d, [], 'no deleted index files') or diag explain(\@d);
 }
 done_testing();
diff --git a/t/v2writable.t b/t/v2writable.t
index 477621e2..1b7e9e7d 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -8,7 +8,7 @@ use PublicInbox::ContentHash qw(content_digest content_hash);
 use PublicInbox::TestCommon;
 use Cwd qw(abs_path);
 require_git(2.6);
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 local $ENV{HOME} = abs_path('t');
 use_ok 'PublicInbox::V2Writable';
 umask 007;
@@ -149,7 +149,7 @@ SELECT COUNT(*) FROM over WHERE num > 0
 }
 
 {
-        use Net::NNTP;
+        require_mods('Net::NNTP', 1);
         my $err = "$inboxdir/stderr.log";
         my $out = "$inboxdir/stdout.log";
         my $group = 'inbox.comp.test.v2writable';
@@ -283,6 +283,22 @@ EOF
         is($msgs->[1]->{mid}, 'y'x244, 'stored truncated mid(2)');
 }
 
+if ('UTF-8 References') {
+        my @w;
+        local $SIG{__WARN__} = sub { push @w, @_ };
+        my $msg = <<EOM;
+From: a\@example.com
+Subject: b
+Message-ID: <horrible\@example>
+References: <\xc4\x80\@example>
+
+EOM
+        ok($im->add(PublicInbox::Eml->new($msg."a\n")), 'UTF-8 References 1');
+        ok($im->add(PublicInbox::Eml->new($msg."b\n")), 'UTF-8 References 2');
+        $im->done;
+        ok(!grep(/Wide character/, @w), 'no wide characters') or xbail(\@w);
+}
+
 my $tmp = {
         inboxdir => "$inboxdir/non-existent/subdir",
         name => 'nope',
diff --git a/t/watch_filter_rubylang.t b/t/watch_filter_rubylang.t
index 004e794e..f72feb9f 100644
--- a/t/watch_filter_rubylang.t
+++ b/t/watch_filter_rubylang.t
@@ -1,12 +1,9 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use warnings;
+use v5.12;
 use PublicInbox::TestCommon;
-use Test::More;
 use PublicInbox::Eml;
-use PublicInbox::Config;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 use_ok 'PublicInbox::Watch';
 use_ok 'PublicInbox::Emergency';
 my ($tmpdir, $for_destroy) = tmpdir();
@@ -25,7 +22,6 @@ SKIP: {
 for my $v (@v) {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
-        my $cfgpfx = "publicinbox.$v";
         my $inboxdir = "$tmpdir/$v";
         my $maildir = "$tmpdir/md-$v";
         my $spamdir = "$tmpdir/spam-$v";
@@ -60,16 +56,16 @@ Date: Sat, 05 Jan 2019 04:19:17 +0000
 spam
 EOF
         PublicInbox::Emergency->new($maildir)->prepare(\"$spam");
-
-        my $orig = <<EOF;
-$cfgpfx.address=$addr
-$cfgpfx.inboxdir=$inboxdir
-$cfgpfx.watch=maildir:$maildir
-$cfgpfx.filter=PublicInbox::Filter::RubyLang
-$cfgpfx.altid=serial:alerts:file=msgmap.sqlite3
-publicinboxwatch.watchspam=maildir:$spamdir
-EOF
-        my $cfg = PublicInbox::Config->new(\$orig);
+        my $cfg = cfg_new $tmpdir, <<EOM;
+[publicinbox "$v"]
+        address = $addr
+        inboxdir = $inboxdir
+        watch = maildir:$maildir
+        filter = PublicInbox::Filter::RubyLang
+        altid = serial:alerts:file=msgmap.sqlite3
+[publicinboxwatch]
+        watchspam = maildir:$spamdir
+EOM
         my $ibx = $cfg->lookup_name($v);
         $ibx->{-no_fsync} = 1;
         ok($ibx, 'found inbox by name');
@@ -99,7 +95,10 @@ EOF
         }
         $w->scan('full');
 
-        $cfg = PublicInbox::Config->new(\$orig);
+        # ensure orderly destruction to avoid SQLite segfault:
+        PublicInbox::DS->Reset;
+
+        $cfg = PublicInbox::Config->new($cfg->{-f});
         $ibx = $cfg->lookup_name($v);
         $ibx->{-no_fsync} = 1;
         is($ibx->search->reopen->mset('b:spam')->size, 0, 'spam removed');
diff --git a/t/watch_imap.t b/t/watch_imap.t
index eeda29eb..26fd5330 100644
--- a/t/watch_imap.t
+++ b/t/watch_imap.t
@@ -1,16 +1,18 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use Test::More;
-use PublicInbox::Config;
+use v5.12;
+use PublicInbox::TestCommon;
 # see t/imapd*.t for tests against a live IMAP server
 
 use_ok 'PublicInbox::Watch';
-my $cfg = PublicInbox::Config->new(\<<EOF);
-publicinbox.i.address=i\@example.com
-publicinbox.i.inboxdir=/nonexistent
-publicinbox.i.watch=imap://example.com/INBOX.a
-publicinboxlearn.watchspam=imap://example.com/INBOX.spam
+my $tmpdir = tmpdir;
+my $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "i"]
+        address = i\@example.com
+        inboxdir = /nonexistent
+        watch = imap://example.com/INBOX.a
+[publicinboxlearn]
+        watchspam = imap://example.com/INBOX.spam
 EOF
 my $watch = PublicInbox::Watch->new($cfg);
 is($watch->{imap}->{'imap://example.com/INBOX.a'}->[0]->{name}, 'i',
diff --git a/t/watch_maildir.t b/t/watch_maildir.t
index e0719f54..a12ceefd 100644
--- a/t/watch_maildir.t
+++ b/t/watch_maildir.t
@@ -1,23 +1,21 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use Test::More;
+use v5.12;
 use PublicInbox::Eml;
 use Cwd;
-use PublicInbox::Config;
 use PublicInbox::TestCommon;
 use PublicInbox::Import;
+use PublicInbox::IO qw(write_file);
 my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/test.git";
 my $maildir = "$tmpdir/md";
 my $spamdir = "$tmpdir/spam";
 use_ok 'PublicInbox::Watch';
 use_ok 'PublicInbox::Emergency';
-my $cfgpfx = "publicinbox.test";
 my $addr = 'test-public@example.com';
 my $default_branch = PublicInbox::Import::default_branch;
 PublicInbox::Import::init_bare($git_dir);
-
 my $msg = <<EOF;
 From: user\@example.com
 To: $addr
@@ -27,6 +25,9 @@ Date: Sat, 18 Jun 2016 00:00:00 +0000
 
 something
 EOF
+
+my $ibx_ro = create_inbox 'ro', sub { $_[0]->add(PublicInbox::Eml->new($msg)) };
+
 PublicInbox::Emergency->new($maildir)->prepare(\$msg);
 ok(POSIX::mkfifo("$maildir/cur/fifo", 0777),
         'create FIFO to ensure we do not get stuck on it :P');
@@ -35,22 +36,21 @@ my $sem = PublicInbox::Emergency->new($spamdir); # create dirs
 {
         my @w;
         local $SIG{__WARN__} = sub { push @w, @_ };
-        my $cfg = PublicInbox::Config->new(\<<EOF);
-$cfgpfx.address=$addr
-$cfgpfx.inboxdir=$git_dir
-$cfgpfx.watch=maildir:$spamdir
-publicinboxlearn.watchspam=maildir:$spamdir
+        my $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = $addr
+        inboxdir = $git_dir
+        watch = maildir:$spamdir
+[publicinboxlearn]
+        watchspam = maildir:$spamdir
 EOF
         my $wm = PublicInbox::Watch->new($cfg);
         is(scalar grep(/is a spam folder/, @w), 1, 'got warning about spam');
-        is_deeply($wm->{mdmap}, { "$spamdir/cur" => 'watchspam' },
+        is_deeply($wm->{d_map}, { "$spamdir/cur" => 'watchspam' },
                 'only got the spam folder to watch');
 }
 
-my $cfg_path = "$tmpdir/config";
-{
-        open my $fh, '>', $cfg_path or BAIL_OUT $!;
-        print $fh <<EOF or BAIL_OUT $!;
+my $cfg = cfg_new $tmpdir, <<EOF;
 [publicinbox "test"]
         address = $addr
         inboxdir = $git_dir
@@ -58,11 +58,12 @@ my $cfg_path = "$tmpdir/config";
         filter = PublicInbox::Filter::Vger
 [publicinboxlearn]
         watchspam = maildir:$spamdir
+[publicinbox "test-ro"]
+        watch = false
+        inboxdir = $ibx_ro->{inboxdir}
+        address = ro-test\@example.com
 EOF
-        close $fh or BAIL_OUT $!;
-}
-
-my $cfg = PublicInbox::Config->new($cfg_path);
+my $cfg_path = $cfg->{-f};
 PublicInbox::Watch->new($cfg)->scan('full');
 my $git = PublicInbox::Git->new($git_dir);
 my @list = $git->qx('rev-list', $default_branch);
@@ -87,6 +88,10 @@ is(scalar @list, 2, 'two revisions in rev-list');
 is(scalar @list, 0, 'tree is empty');
 is(unlink(glob("$spamdir/cur/*")), 1, 'unlinked trained spam');
 
+@list = $ibx_ro->git->qx(qw(ls-tree -r --name-only), $default_branch);
+undef $ibx_ro;
+is scalar(@list), 1, 'read-only inbox is unchanged';
+
 # check with scrubbing
 {
         $msg .= qq(--
@@ -97,6 +102,7 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
         PublicInbox::Watch->new($cfg)->scan('full');
         @list = $git->qx('ls-tree', '-r', '--name-only', $default_branch);
         is(scalar @list, 1, 'tree has one file');
+        chomp(@list);
         my $mref = $git->cat_file('HEAD:'.$list[0]);
         like($$mref, qr/something\n\z/s, 'message scrubbed on import');
 
@@ -137,10 +143,7 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
         PublicInbox::Watch->new($cfg)->scan('full');
         @list = $git->qx('ls-tree', '-r', '--name-only', $default_branch);
         is(scalar @list, 1, 'tree has one file after spamc checked');
-
-        # XXX: workaround some weird caching/memoization in cat-file,
-        # shouldn't be an issue in real-world use, though...
-        $git = PublicInbox::Git->new($git_dir);
+        chomp(@list);
 
         my $mref = $git->cat_file($default_branch.':'.$list[0]);
         like($$mref, qr/something\n\z/s, 'message scrubbed on import');
@@ -151,8 +154,13 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
         my $env = { PI_CONFIG => $cfg_path };
         $git->cleanup;
 
+        write_file '>>', $cfg_path, <<EOM;
+[publicinboxImport]
+        dropUniqueUnsubscribe
+EOM
         # n.b. --no-scan is only intended for testing atm
         my $wm = start_script([qw(-watch --no-scan)], $env);
+        no_pollerfd($wm->{pid});
         my $eml = eml_load('t/data/0001.patch');
         $eml->header_set('Cc', $addr);
         my $em = PublicInbox::Emergency->new($maildir);
@@ -170,11 +178,11 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
         my $ii = PublicInbox::InboxIdle->new($cfg);
         my $obj = bless \$cb, 'PublicInbox::TestCommon::InboxWakeup';
         $cfg->each_inbox(sub { $_[0]->subscribe_unlock('ident', $obj) });
-        PublicInbox::DS->SetPostLoopCallback(sub { $delivered == 0 });
+        local @PublicInbox::DS::post_loop_do = (sub { $delivered == 0 });
 
         # wait for -watch to setup inotify watches
         my $sleep = 1;
-        if (eval { require Linux::Inotify2 } && -d "/proc/$wm->{pid}/fd") {
+        if (eval { require PublicInbox::Inotify } && -d "/proc/$wm->{pid}/fd") {
                 my $end = time + 2;
                 my (@ino, @ino_info);
                 do {
@@ -201,13 +209,32 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
         $em->commit; # wake -watch up
         diag 'waiting for -watch to import new message';
         PublicInbox::DS::event_loop();
+
+        my $head = $git->qx(qw(cat-file commit HEAD));
+        my $subj = $eml->header('Subject');
+        like($head, qr/^\Q$subj\E/sm, 'new commit made');
+
+        # try dropUniqueUnsubscribe
+        $delivered = 0;
+        $eml->header_set('Message-ID', '<unsubscribe@example>');
+        $eml->header_set('List-Unsubscribe',
+                        '<https://example.com/some-UUID-here/test');
+        $eml->header_set('List-Unsubscribe-Post', 'List-Unsubscribe=One-Click');
+        $em = PublicInbox::Emergency->new($maildir);
+        $em->prepare(\($eml->as_string));
+        $em->commit; # wake -watch up
+        diag 'waiting for -watch to import dropUniqueUnsubscribe message';
+        PublicInbox::DS::event_loop();
+        my $cur = $git->qx(qw(diff HEAD~1..HEAD));
+        like $cur, qr/Message-ID: <unsubscribe\@example>/,
+                'unsubscribe@example imported';
+        unlike $cur, qr/List-Unsubscribe\b/,
+                'List-Unsubscribe-* headers gone w/ dropUniqueUnsubscribe';
+
         $wm->kill;
         $wm->join;
         $ii->close;
         PublicInbox::DS->Reset;
-        my $head = $git->qx(qw(cat-file commit HEAD));
-        my $subj = $eml->header('Subject');
-        like($head, qr/^\Q$subj\E/sm, 'new commit made');
 }
 
 sub is_maildir {
diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t
index 7b46232b..fa86f7bf 100644
--- a/t/watch_maildir_v2.t
+++ b/t/watch_maildir_v2.t
@@ -1,14 +1,12 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use Test::More;
+use v5.12;
 use PublicInbox::Eml;
 use Cwd;
-use PublicInbox::Config;
 use PublicInbox::TestCommon;
 use PublicInbox::Import;
 require_git(2.6);
-require_mods(qw(Search::Xapian DBD::SQLite));
+require_mods(qw(Xapian DBD::SQLite));
 require PublicInbox::V2Writable;
 my ($tmpdir, $for_destroy) = tmpdir();
 my $inboxdir = "$tmpdir/v2";
@@ -38,13 +36,15 @@ ok(POSIX::mkfifo("$maildir/cur/fifo", 0777),
 my $sem = PublicInbox::Emergency->new($spamdir); # create dirs
 
 my $orig = <<EOF;
-$cfgpfx.address=$addr
-$cfgpfx.inboxdir=$inboxdir
-$cfgpfx.watch=maildir:$maildir
-$cfgpfx.filter=PublicInbox::Filter::Vger
-publicinboxlearn.watchspam=maildir:$spamdir
+[publicinbox "test"]
+        address = $addr
+        inboxdir = $inboxdir
+        watch = maildir:$maildir
+        filter = PublicInbox::Filter::Vger
+[publicinboxlearn]
+        watchspam = maildir:$spamdir
 EOF
-my $cfg = PublicInbox::Config->new(\$orig);
+my $cfg = cfg_new $tmpdir, $orig;
 my $ibx = $cfg->lookup_name('test');
 ok($ibx, 'found inbox by name');
 $ibx->{-no_fsync} = 1;
@@ -147,12 +147,13 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
         my $v1pfx = "publicinbox.v1";
         my $v1addr = 'v1-public@example.com';
         PublicInbox::Import::init_bare($v1repo);
-        my $raw = <<EOF;
-$orig$v1pfx.address=$v1addr
-$v1pfx.inboxdir=$v1repo
-$v1pfx.watch=maildir:$maildir
+        my $cfg = cfg_new $tmpdir, <<EOF;
+$orig
+[publicinbox "v1"]
+        address = $v1addr
+        inboxdir = $v1repo
+        watch = maildir:$maildir
 EOF
-        my $cfg = PublicInbox::Config->new(\$raw);
         my $both = <<EOF;
 From: user\@example.com
 To: $addr, $v1addr
@@ -185,19 +186,22 @@ List-Id: <do.not.want>
 X-Mailing-List: no@example.com
 Message-ID: <do.not.want@example.com>
 EOF
-        my $raw = $orig."$cfgpfx.listid=i.want.you.to.want.me\n";
         PublicInbox::Emergency->new($maildir)->prepare(\$want);
         PublicInbox::Emergency->new($maildir)->prepare(\$do_not_want);
-        my $cfg = PublicInbox::Config->new(\$raw);
+        my $raw = <<EOM;
+$orig
+[publicinbox "test"]
+        listid = i.want.you.to.want.me
+EOM
+        my $cfg = cfg_new $tmpdir, $raw;
         PublicInbox::Watch->new($cfg)->scan('full');
         $ibx = $cfg->lookup_name('test');
         my $num = $ibx->mm->num_for('do.want@example.com');
         ok(defined $num, 'List-ID matched for watch');
         $num = $ibx->mm->num_for('do.not.want@example.com');
         is($num, undef, 'unaccepted List-ID matched for watch');
-
-        $raw = $orig."$cfgpfx.watchheader=X-Mailing-List:no\@example.com\n";
-        $cfg = PublicInbox::Config->new(\$raw);
+        $raw .= "\twatchheader = X-Mailing-List:no\@example.com\n";
+        $cfg = cfg_new $tmpdir, $raw;
         PublicInbox::Watch->new($cfg)->scan('full');
         $ibx = $cfg->lookup_name('test');
         $num = $ibx->mm->num_for('do.not.want@example.com');
diff --git a/t/watch_mh.t b/t/watch_mh.t
new file mode 100644
index 00000000..04793750
--- /dev/null
+++ b/t/watch_mh.t
@@ -0,0 +1,120 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::Eml;
+use PublicInbox::TestCommon;
+use PublicInbox::Import;
+use PublicInbox::IO qw(write_file);
+use POSIX qw(mkfifo);
+use File::Copy qw(cp);
+use autodie qw(rename mkdir);
+
+my $tmpdir = tmpdir;
+my $git_dir = "$tmpdir/test.git";
+my $mh = "$tmpdir/mh";
+my $spamdir = "$tmpdir/mh-spam";
+mkdir $_ for ($mh, $spamdir);
+use_ok 'PublicInbox::Watch';
+my $addr = 'test-public@example.com';
+my $default_branch = PublicInbox::Import::default_branch;
+PublicInbox::Import::init_bare($git_dir);
+my $msg = <<EOF;
+From: user\@example.com
+To: $addr
+Subject: spam
+Message-ID: <a\@b.com>
+Date: Sat, 18 Jun 2016 00:00:00 +0000
+
+something
+EOF
+
+cp 't/plack-qp.eml', "$mh/1";
+mkfifo("$mh/5", 0777) or xbail "mkfifo: $!"; # FIFO to ensure no stuckage
+my $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = $addr
+        inboxdir = $git_dir
+        watch = mh:$mh
+[publicinboxlearn]
+        watchspam = mh:$spamdir
+EOF
+PublicInbox::Watch->new($cfg)->scan('full');
+my $git = PublicInbox::Git->new($git_dir);
+{
+        my @list = $git->qx('rev-list', $default_branch);
+        is(scalar @list, 1, 'one revision in rev-list');
+        $git->cleanup;
+}
+
+# end-to-end test which actually uses inotify/kevent
+{
+        my $env = { PI_CONFIG => $cfg->{-f} };
+        # n.b. --no-scan is only intended for testing atm
+        my $wm = start_script([qw(-watch --no-scan)], $env);
+        no_pollerfd($wm->{pid});
+
+        my $eml = eml_load 't/data/binary.patch';
+        $eml->header_set('Cc', $addr);
+        write_file '>', "$mh/2.tmp", $eml->as_string;
+
+        use_ok 'PublicInbox::InboxIdle';
+        use_ok 'PublicInbox::DS';
+        my $delivered = 0;
+        my $cb = sub {
+                my ($ibx) = @_;
+                diag "message delivered to `$ibx->{name}'";
+                $delivered++;
+        };
+        PublicInbox::DS->Reset;
+        my $ii = PublicInbox::InboxIdle->new($cfg);
+        my $obj = bless \$cb, 'PublicInbox::TestCommon::InboxWakeup';
+        $cfg->each_inbox(sub { $_[0]->subscribe_unlock('ident', $obj) });
+        local @PublicInbox::DS::post_loop_do = (sub { $delivered == 0 });
+
+        # wait for -watch to setup inotify watches
+        my $sleep = 1;
+        if (eval { require PublicInbox::Inotify } && -d "/proc/$wm->{pid}/fd") {
+                my $end = time + 2;
+                my (@ino, @ino_info);
+                do {
+                        @ino = grep {
+                                (readlink($_)//'') =~ /\binotify\b/
+                        } glob("/proc/$wm->{pid}/fd/*");
+                } until (@ino || time > $end || !tick);
+                if (scalar(@ino) == 1) {
+                        my $ino_fd = (split(m'/', $ino[0]))[-1];
+                        my $ino_fdinfo = "/proc/$wm->{pid}/fdinfo/$ino_fd";
+                        while (time < $end && open(my $fh, '<', $ino_fdinfo)) {
+                                @ino_info = grep(/^inotify wd:/, <$fh>);
+                                last if @ino_info >= 2;
+                                tick;
+                        }
+                        $sleep = undef if @ino_info >= 2;
+                }
+        }
+        if ($sleep) {
+                diag "waiting ${sleep}s for -watch to start up";
+                sleep $sleep;
+        }
+        rename "$mh/2.tmp", "$mh/2";
+        diag 'waiting for -watch to import new message';
+        PublicInbox::DS::event_loop();
+
+        my $subj = $eml->header_raw('Subject');
+        my $head = $git->qx(qw(cat-file commit HEAD));
+        like $head, qr/^\Q$subj\E/sm, 'new commit made';
+
+        $wm->kill;
+        $wm->join;
+        $ii->close;
+        PublicInbox::DS->Reset;
+}
+
+my $is_mh = sub { PublicInbox::Watch::is_mh(my $val = shift) };
+
+is $is_mh->('mh:/hello//world'), '/hello/world', 'extra slash gone';
+is $is_mh->('MH:/hello/world/'), '/hello/world', 'trailing slash gone';
+is $is_mh->('maildir:/hello/world/'), undef, 'non-MH rejected';
+
+done_testing;
diff --git a/t/watch_multiple_headers.t b/t/watch_multiple_headers.t
index 33ed0770..9585da2b 100644
--- a/t/watch_multiple_headers.t
+++ b/t/watch_multiple_headers.t
@@ -1,11 +1,9 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C)  all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use Test::More;
-use PublicInbox::Config;
+use v5.12;
 use PublicInbox::TestCommon;
 require_git(2.6);
-require_mods(qw(Search::Xapian DBD::SQLite));
+require_mods(qw(Xapian DBD::SQLite));
 my ($tmpdir, $for_destroy) = tmpdir();
 my $inboxdir = "$tmpdir/v2";
 my $maildir = "$tmpdir/md";
@@ -54,14 +52,15 @@ PublicInbox::Emergency->new($maildir)->prepare(\$msg_to);
 PublicInbox::Emergency->new($maildir)->prepare(\$msg_cc);
 PublicInbox::Emergency->new($maildir)->prepare(\$msg_none);
 
-my $raw = <<EOF;
-$cfgpfx.address=$addr
-$cfgpfx.inboxdir=$inboxdir
-$cfgpfx.watch=maildir:$maildir
-$cfgpfx.watchheader=To:$addr
-$cfgpfx.watchheader=Cc:$addr
+my $cfg = cfg_new $tmpdir, <<EOF;
+[publicinbox "test"]
+        address = $addr
+        inboxdir = $inboxdir
+        watch = maildir:$maildir
+        watchheader = To:$addr
+        watchheader = Cc:$addr
 EOF
-my $cfg = PublicInbox::Config->new(\$raw);
+
 PublicInbox::Watch->new($cfg)->scan('full');
 my $ibx = $cfg->lookup_name('test');
 ok($ibx, 'found inbox by name');
diff --git a/t/www_altid.t b/t/www_altid.t
index 94a2e807..de1e6ed6 100644
--- a/t/www_altid.t
+++ b/t/www_altid.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 use PublicInbox::Config;
@@ -59,14 +59,7 @@ my $client = sub {
 };
 test_psgi(sub { $www->call(@_) }, $client);
 SKIP: {
-        require_mods(qw(Plack::Test::ExternalServer), 4);
-        my $env = { PI_CONFIG => $cfgpath };
-        my $sock = tcp_server() or die;
-        my ($out, $err) = map { "$tmpdir/std$_.log" } qw(out err);
-        my $cmd = [ qw(-httpd -W0), "--stdout=$out", "--stderr=$err" ];
-        my $td = start_script($cmd, $env, { 3 => $sock });
-        my ($h, $p) = tcp_host_port($sock);
-        local $ENV{PLACK_TEST_EXTERNALSERVER_URI} = "http://$h:$p";
-        Plack::Test::ExternalServer::test_psgi(client => $client);
+        my $env = { PI_CONFIG => $cfgpath, TMPDIR => $tmpdir };
+        test_httpd($env, $client);
 }
 done_testing;
diff --git a/t/www_listing.t b/t/www_listing.t
index c556a2d7..0a4c79e8 100644
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -1,11 +1,12 @@
 #!perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # manifest.js.gz generation and grok-pull integration test
-use strict; use v5.10.1; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
 use PublicInbox::Import;
 use IO::Uncompress::Gunzip qw(gunzip);
-require_mods(qw(json URI::Escape Plack::Builder Digest::SHA HTTP::Tiny));
+require_mods(qw(json URI::Escape Plack::Builder HTTP::Tiny));
+require_cmd 'curl';
 require PublicInbox::WwwListing;
 require PublicInbox::ManifestJsGz;
 use PublicInbox::Config;
@@ -76,6 +77,7 @@ sub tiny_test {
 
 my $td;
 SKIP: {
+        require_git_http_backend 1;
         my $err = "$tmpdir/stderr.log";
         my $out = "$tmpdir/stdout.log";
         my $alt = "$tmpdir/alt.git";
@@ -91,6 +93,10 @@ SKIP: {
                 is(xsys(@clone, $alt, "$v2/git/$i.git"), 0, "clone epoch $i")
         }
         ok(open(my $fh, '>', "$v2/inbox.lock"), 'mock a v2 inbox');
+        open($fh, '>', "$v2/description") or xbail "open $v2/description: $!";
+        print $fh "a v2 inbox\n" or xbail "print $!";
+        close $fh or xbail "write: $v2/description $!";
+
         open $fh, '>', "$alt/description" or xbail "open $alt/description $!";
         print $fh "we're \xc4\x80ll clones\n" or xbail "print $!";
         close $fh or xbail "write: $alt/description $!";
@@ -115,10 +121,71 @@ SKIP: {
 
         my $env = { PI_CONFIG => $cfgfile };
         my $cmd = [ '-httpd', '-W0', "--stdout=$out", "--stderr=$err" ];
+        my $psgi = "$tmpdir/pfx.psgi";
+        {
+                open my $psgi_fh, '>', $psgi or xbail "open: $!";
+                print $psgi_fh <<'EOM' or xbail "print $!";
+use PublicInbox::WWW;
+use Plack::Builder;
+my $www = PublicInbox::WWW->new;
+builder {
+        enable 'Head';
+        mount '/pfx/' => sub { $www->call(@_) }
+}
+EOM
+                close $psgi_fh or xbail "close: $!";
+        }
+
+        # ensure prefixed mount full clones work:
+        $td = start_script([@$cmd, $psgi], $env, { 3 => $sock });
+        my $opt = { 2 => \(my $clone_err = '') };
+        ok(run_script(['-clone', "http://$host:$port/pfx", "$tmpdir/pfx" ],
+                undef, $opt), 'pfx clone w/pfx') or diag "clone_err=$clone_err";
+
+        open my $mh, '<', "$tmpdir/pfx/manifest.js.gz" or xbail "open: $!";
+        gunzip(\(do { local $/; <$mh> }) => \(my $mjs = ''));
+        my $mf = $json->decode($mjs);
+        is_deeply([sort keys %$mf], [ qw(/alt /bare /v2/git/0.git
+                                        /v2/git/1.git /v2/git/2.git) ],
+                'manifest saved');
+        for (keys %$mf) { ok(-d "$tmpdir/pfx$_", "pfx/$_ cloned") }
+        open my $desc, '<', "$tmpdir/pfx/v2/description" or xbail "open: $!";
+        $desc = <$desc>;
+        is($desc, "a v2 inbox\n", 'v2 description retrieved');
+
+        $clone_err = '';
+        ok(run_script(['-clone', '--include=*/alt',
+                        "http://$host:$port/pfx", "$tmpdir/incl" ],
+                undef, $opt), 'clone w/include') or diag "clone_err=$clone_err";
+        ok(-d "$tmpdir/incl/alt", 'alt cloned');
+        ok(!-d "$tmpdir/incl/v2" && !-d "$tmpdir/incl/bare", 'only alt cloned');
+        is(xqx([qw(git config -f), "$tmpdir/incl/alt/config", 'gitweb.owner']),
+                "lorelei \xc4\x80\n", 'gitweb.owner set by -clone');
+
+        $clone_err = '';
+        ok(run_script(['-clone', '--dry-run',
+                        "http://$host:$port/pfx", "$tmpdir/dry-run" ],
+                undef, $opt), 'clone --dry-run') or diag "clone_err=$clone_err";
+        ok(!-d "$tmpdir/dry-run", 'nothing cloned with --dry-run');
+
+        undef $td;
+
+        open $mh, '<', "$tmpdir/incl/manifest.js.gz" or xbail "open: $!";
+        gunzip(\(do { local $/; <$mh> }) => \($mjs = ''));
+        $mf = $json->decode($mjs);
+        is_deeply([keys %$mf], [ '/alt' ], 'excluded keys skipped in manifest');
+
         $td = start_script($cmd, $env, { 3 => $sock });
 
         # default publicinboxGrokManifest match=domain default
         tiny_test($json, $host, $port);
+
+        # normal full clone on /
+        $clone_err = '';
+        ok(run_script(['-clone', "http://$host:$port/", "$tmpdir/full" ],
+                undef, $opt), 'full clone') or diag "clone_err=$clone_err";
+        ok(-d "$tmpdir/full/$_", "$_ cloned") for qw(alt v2 bare);
+
         undef $td;
 
         print $fh <<"" or xbail "print $!";
@@ -127,9 +194,11 @@ SKIP: {
 
         close $fh or xbail "close $!";
         $td = start_script($cmd, $env, { 3 => $sock });
-        tiny_test($json, $host, $port, 1);
         undef $sock;
+        tiny_test($json, $host, $port, 1);
 
+        # grok-pull sleeps a long while some places:
+        # https://lore.kernel.org/tools/20211013110344.GA10632@dcvr/
         skip 'TEST_GROK unset', 12 unless $ENV{TEST_GROK};
         my $grok_pull = require_cmd('grok-pull', 1) or
                 skip('grok-pull not available', 12);
diff --git a/t/xap_helper.t b/t/xap_helper.t
new file mode 100644
index 00000000..0f474608
--- /dev/null
+++ b/t/xap_helper.t
@@ -0,0 +1,281 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use v5.12;
+use PublicInbox::TestCommon;
+require_mods(qw(DBD::SQLite Xapian +SCM_RIGHTS)); # TODO: FIFO support?
+use PublicInbox::Spawn qw(spawn);
+use Socket qw(AF_UNIX SOCK_SEQPACKET SOCK_STREAM);
+require PublicInbox::AutoReap;
+use PublicInbox::IPC;
+require PublicInbox::XapClient;
+use autodie;
+my ($tmp, $for_destroy) = tmpdir();
+
+my $fi_data = './t/git.fast-import-data';
+open my $fi_fh, '<', $fi_data;
+open my $dh, '<', '.';
+my $crepo = create_coderepo 'for-cindex', sub {
+        my ($d) = @_;
+        xsys_e([qw(git init -q --bare)]);
+        xsys_e([qw(git fast-import --quiet)], undef, { 0 => $fi_fh });
+        chdir($dh);
+        run_script([qw(-cindex --dangerous -L medium --no-fsync -q -j1), '-g', $d])
+                or xbail '-cindex internal';
+        run_script([qw(-cindex --dangerous -L medium --no-fsync -q -j3 -d),
+                "$d/cidx-ext", '-g', $d]) or xbail '-cindex "external"';
+};
+$dh = $fi_fh = undef;
+
+my $v2 = create_inbox 'v2', indexlevel => 'medium', version => 2,
+                        tmpdir => "$tmp/v2", sub {
+        my ($im) = @_;
+        for my $f (qw(t/data/0001.patch t/data/binary.patch
+                        t/data/message_embed.eml
+                        t/solve/0001-simple-mod.patch
+                        t/solve/0002-rename-with-modifications.patch
+                        t/solve/bare.patch)) {
+                $im->add(eml_load($f)) or BAIL_OUT;
+        }
+};
+
+my @ibx_idx = glob("$v2->{inboxdir}/xap*/?");
+my @ibx_shard_args = map { ('-d', $_) } @ibx_idx;
+my (@int) = glob("$crepo/public-inbox-cindex/cidx*/?");
+my (@ext) = glob("$crepo/cidx-ext/cidx*/?");
+is(scalar(@ext), 2, 'have 2 external shards') or diag explain(\@ext);
+is(scalar(@int), 1, 'have 1 internal shard') or diag explain(\@int);
+
+my $doreq = sub {
+        my ($s, @arg) = @_;
+        my $err = ref($arg[-1]) ? pop(@arg) : \*STDERR;
+        pipe(my $x, my $y);
+        my $buf = join("\0", @arg, '');
+        my @fds = (fileno($y), fileno($err));
+        my $n = $PublicInbox::IPC::send_cmd->($s, \@fds, $buf, 0) //
+                xbail "send: $!";
+        my $exp = length($buf);
+        $exp == $n or xbail "req @arg sent short ($n != $exp)";
+        $x;
+};
+
+local $SIG{PIPE} = 'IGNORE';
+my $env = { PERL5LIB => join(':', @INC) };
+my $test = sub {
+        my (@cmd) = @_;
+        socketpair(my $s, my $y, AF_UNIX, SOCK_SEQPACKET, 0);
+        my $pid = spawn(\@cmd, $env, { 0 => $y });
+        my $ar = PublicInbox::AutoReap->new($pid);
+        diag "$cmd[-1] running pid=$pid";
+        close $y;
+        my $r = $doreq->($s, qw(test_inspect -d), $ibx_idx[0]);
+        my %info = map { split(/=/, $_, 2) } split(/ /, do { local $/; <$r> });
+        is($info{has_threadid}, '1', 'has_threadid true for inbox');
+        like($info{pid}, qr/\A\d+\z/, 'got PID from inbox inspect');
+
+        $r = $doreq->($s, qw(test_inspect -d), $int[0]);
+        my %cinfo = map { split(/=/, $_, 2) } split(/ /, do { local $/; <$r> });
+        is($cinfo{has_threadid}, '0', 'has_threadid false for cindex');
+        is($cinfo{pid}, $info{pid}, 'PID unchanged for cindex');
+
+        my @dump = (qw(dump_ibx -A XDFID), @ibx_shard_args, qw(13 rt:0..));
+        $r = $doreq->($s, @dump);
+        my @res;
+        while (sysread($r, my $buf, 512) != 0) { push @res, $buf }
+        is(grep(/\n\z/s, @res), scalar(@res), 'line buffered');
+
+        pipe(my $err_rd, my $err_wr);
+        $r = $doreq->($s, @dump, $err_wr);
+        close $err_wr;
+        my $res = do { local $/; <$r> };
+        is(join('', @res), $res, 'got identical response w/ error pipe');
+        my $stats = do { local $/; <$err_rd> };
+        is($stats, "mset.size=6 nr_out=6\n", 'mset.size reported') or
+                diag "res=$res";
+
+        return wantarray ? ($ar, $s) : $ar if $cinfo{pid} == $pid;
+
+        # test worker management:
+        kill('TERM', $cinfo{pid});
+        my $tries = 0;
+        do {
+                $r = $doreq->($s, qw(test_inspect -d), $ibx_idx[0]);
+                %info = map { split(/=/, $_, 2) }
+                        split(/ /, do { local $/; <$r> });
+        } while ($info{pid} == $cinfo{pid} && ++$tries < 10);
+        isnt($info{pid}, $cinfo{pid}, 'spawned new worker');
+
+        my %pids;
+        $tries = 0;
+        my @ins = ($s, qw(test_inspect -d), $ibx_idx[0]);
+        kill('TTIN', $pid);
+        until (scalar(keys %pids) >= 2 || ++$tries > 100) {
+                tick;
+                my @r = map { $doreq->(@ins) } (0..100);
+                for my $fh (@r) {
+                        my $buf = do { local $/; <$fh> } // die "read: $!";
+                        $buf =~ /\bpid=(\d+)/ and $pids{$1} = undef;
+                }
+        }
+        is(scalar keys %pids, 2, 'have two pids') or
+                diag 'pids='.explain(\%pids);
+
+        kill('TTOU', $pid);
+        %pids = ();
+        my $delay = $tries * 0.11 * ($ENV{VALGRIND} ? 10 : 1);
+        $tries = 0;
+        diag 'waiting '.$delay.'s for SIGTTOU';
+        tick($delay);
+        until (scalar(keys %pids) == 1 || ++$tries > 100) {
+                %pids = ();
+                my @r = map { $doreq->(@ins) } (0..100);
+                for my $fh (@r) {
+                        my $buf = do { local $/; <$fh> } // die "read: $!";
+                        $buf =~ /\bpid=(\d+)/ and $pids{$1} = undef;
+                }
+        }
+        is(scalar keys %pids, 1, 'have one pid') or diag explain(\%pids);
+        is($info{pid}, (keys %pids)[0], 'kept oldest PID after TTOU');
+
+        wantarray ? ($ar, $s) : $ar;
+};
+
+my @NO_CXX = (1);
+unless ($ENV{TEST_XH_CXX_ONLY}) {
+        my $ar = $test->($^X, qw[-w -MPublicInbox::XapHelper -e
+                        PublicInbox::XapHelper::start('-j0')]);
+        ($ar, my $s) = $test->($^X, qw[-w -MPublicInbox::XapHelper -e
+                        PublicInbox::XapHelper::start('-j1')]);
+        no_pollerfd($ar->{pid});
+}
+SKIP: {
+        my $cmd = eval {
+                require PublicInbox::XapHelperCxx;
+                PublicInbox::XapHelperCxx::cmd();
+        };
+        skip "XapHelperCxx build: $@", 1 if $@;
+
+        @NO_CXX = $ENV{TEST_XH_CXX_ONLY} ? (0) : (0, 1);
+        my $ar = $test->(@$cmd, '-j0');
+        $ar = $test->(@$cmd, '-j1');
+};
+
+require PublicInbox::CodeSearch;
+my $cs_int = PublicInbox::CodeSearch->new("$crepo/public-inbox-cindex");
+my $root2id_file = "$tmp/root2id";
+my @id2root;
+{
+        open my $fh, '>', $root2id_file;
+        my $i = -1;
+        for ($cs_int->all_terms('G')) {
+                print $fh $_, "\0", ++$i, "\0";
+                $id2root[$i] = $_;
+        }
+        close $fh;
+}
+
+my $ar;
+for my $n (@NO_CXX) {
+        local $ENV{PI_NO_CXX} = $n;
+        my $xhc = PublicInbox::XapClient::start_helper('-j0');
+        pipe(my $err_r, my $err_w);
+
+        # git patch-id --stable <t/data/0001.patch | awk '{print $1}'
+        my $dfid = '91ee6b761fc7f47cad9f2b09b10489f313eb5b71';
+        my $mid = '20180720072141.GA15957@example';
+        my $r = $xhc->mkreq([ undef, $err_w ], qw(dump_ibx -A XDFID -A Q),
+                                (map { ('-d', $_) } @ibx_idx),
+                                9, "mid:$mid");
+        close $err_w;
+        my $res = do { local $/; <$r> };
+        is($res, "$dfid 9\n$mid 9\n", "got expected result ($xhc->{impl})");
+        my $err = do { local $/; <$err_r> };
+        is($err, "mset.size=1 nr_out=2\n", "got expected status ($xhc->{impl})");
+
+        pipe($err_r, $err_w);
+        $r = $xhc->mkreq([ undef, $err_w ], qw(dump_roots -c -A XDFID),
+                        (map { ('-d', $_) } @int),
+                        $root2id_file, 'dt:19700101'.'000000..');
+        close $err_w;
+        my @res = <$r>;
+        is(scalar(@res), 5, 'got expected rows');
+        is(scalar(@res), scalar(grep(/\A[0-9a-f]{40,} [0-9]+\n\z/, @res)),
+                'entries match format');
+        $err = do { local $/; <$err_r> };
+        is $err, "mset.size=6 nr_out=5\n", "got expected status ($xhc->{impl})";
+
+        $r = $xhc->mkreq([], qw(mset -p -A XDFID -A Q), @ibx_shard_args,
+                                'dfn:lib/PublicInbox/Search.pm');
+        chomp((my $hdr, @res) = readline($r));
+        is $hdr, 'mset.size=1', "got expected header via mset ($xhc->{impl}";
+        is scalar(@res), 1, 'got one result';
+        @res = split /\0/, $res[0];
+        {
+                my $doc = $v2->search->xdb->get_document($res[0]);
+                my @q = PublicInbox::Search::xap_terms('Q', $doc);
+                is_deeply \@q, [ $mid ], 'docid usable';
+        }
+        ok $res[1] > 0 && $res[1] <= 100, 'pct > 0 && <= 100';
+        is $res[2], 'XDFID'.$dfid, 'XDFID result matches';
+        is $res[3], 'Q'.$mid, 'Q (msgid) mset result matches';
+        is scalar(@res), 4, 'only 4 columns in result';
+
+        $r = $xhc->mkreq([], qw(mset -p -A XDFID -A Q), @ibx_shard_args,
+                                'dt:19700101'.'000000..');
+        chomp(($hdr, @res) = readline($r));
+        is $hdr, 'mset.size=6',
+                "got expected header via multi-result mset ($xhc->{impl}";
+        is(scalar(@res), 6, 'got 6 rows');
+        for my $r (@res) {
+                my ($docid, $pct, @rest) = split /\0/, $r;
+                my $doc = $v2->search->xdb->get_document($docid);
+                ok $pct > 0 && $pct <= 100,
+                        "pct > 0 && <= 100 #$docid ($xhc->{impl})";
+                my %terms;
+                for (@rest) {
+                        s/\A([A-Z]+)// or xbail 'no prefix=', \@rest;
+                        push @{$terms{$1}}, $_;
+                }
+                while (my ($pfx, $vals) = each %terms) {
+                        @$vals = sort @$vals;
+                        my @q = PublicInbox::Search::xap_terms($pfx, $doc);
+                        is_deeply $vals, \@q,
+                                "#$docid $pfx as expected ($xhc->{impl})";
+                }
+        }
+        my $nr;
+        for my $i (7, 8, 39, 40) {
+                pipe($err_r, $err_w);
+                $r = $xhc->mkreq([ undef, $err_w ], qw(dump_roots -c -A),
+                                "XDFPOST$i", (map { ('-d', $_) } @int),
+                                $root2id_file, 'dt:19700101'.'000000..');
+                close $err_w;
+                @res = <$r>;
+                my @err = <$err_r>;
+                if (defined $nr) {
+                        is scalar(@res), $nr,
+                                "got expected results ($xhc->{impl})";
+                } else {
+                        $nr //= scalar @res;
+                        ok $nr, "got initial results ($xhc->{impl})";
+                }
+                my @oids = (join('', @res) =~ /^([a-f0-9]+) /gms);
+                is_deeply [grep { length == $i } @oids], \@oids,
+                        "all OIDs match expected length ($xhc->{impl})";
+                my ($nr_out) = ("@err" =~ /nr_out=(\d+)/);
+                is $nr_out, scalar(@oids), "output count matches $xhc->{impl}"
+                        or diag explain(\@res, \@err);
+        }
+        pipe($err_r, $err_w);
+        $r = $xhc->mkreq([ undef, $err_w ], qw(dump_ibx -A XDFPOST7),
+                        @ibx_shard_args, qw(13 rt:0..));
+        close $err_w;
+        @res = <$r>;
+        my @err = <$err_r>;
+        my ($nr_out) = ("@err" =~ /nr_out=(\d+)/);
+        my @oids = (join('', @res) =~ /^([a-f0-9]{7}) /gms);
+        is $nr_out, scalar(@oids), "output count matches $xhc->{impl}" or
+                diag explain(\@res, \@err);
+}
+
+done_testing;
diff --git a/t/xcpdb-reshard.t b/t/xcpdb-reshard.t
index 8516b907..7797aaaf 100644
--- a/t/xcpdb-reshard.t
+++ b/t/xcpdb-reshard.t
@@ -4,7 +4,7 @@
 use strict;
 use v5.10.1;
 use PublicInbox::TestCommon;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 require_git('2.6');
 use PublicInbox::Eml;
 require PublicInbox::Search;
@@ -43,7 +43,7 @@ my $XapianDatabase = do {
 for my $R (qw(2 4 1 3 3)) {
         delete $ibx->{search}; # release old handles
         my $cmd = [@xcpdb, "-R$R", $ibx->{inboxdir}];
-        push @$cmd, '--compact' if $R == 1 && have_xapian_compact;
+        push @$cmd, '--compact' if $R == 1 && have_xapian_compact(1);
         ok(run_script($cmd, $env), "xcpdb -R$R");
         my @new_shards = grep(m!/\d+\z!, glob("$ibx->{inboxdir}/xap*/*"));
         is(scalar(@new_shards), $R, 'resharded to two shards');
diff --git a/xt/check-debris.t b/xt/check-debris.t
new file mode 100644
index 00000000..0bb5091d
--- /dev/null
+++ b/xt/check-debris.t
@@ -0,0 +1,30 @@
+#!perl -w
+use v5.12;
+use autodie qw(open);
+use PublicInbox::TestCommon;
+use File::Spec;
+my $tmpdir = File::Spec->tmpdir;
+
+diag "note: writes to `$tmpdir' by others results in false-positives";
+
+my %cur = map { $_ => 1 } glob("$tmpdir/*");
+for my $t (@ARGV ? @ARGV : glob('t/*.t')) {
+        open my $fh, '-|', $^X, '-w', $t;
+        my @out;
+        while (<$fh>) {
+                chomp;
+                push @out, $_;
+                next if /^ok / || /\A[0-9]+\.\.[0-9]+\z/;
+                diag $_;
+        }
+        ok(close($fh), $t) or diag(explain(\@out));
+
+        no_coredump($tmpdir);
+
+        my @remain = grep { !$cur{$_}++ } glob("$tmpdir/*");
+        next if !@remain;
+        is_deeply(\@remain, [], "$t has no leftovers") or
+                diag "$t added: ",explain(\@remain);
+}
+
+done_testing;
diff --git a/t/run.perl b/xt/check-run.t
index cf80a8a1..d12b925d 100755
--- a/t/run.perl
+++ b/xt/check-run.t
@@ -1,5 +1,5 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Parallel test runner which preloads code and reuses worker processes
@@ -8,13 +8,13 @@
 #
 # *.t files run by this should not rely on global state.
 #
-# Usage: $PERL -I lib -w t/run.perl -j4
-# Or via prove(1): prove -lvw t/run.perl :: -j4
-use strict;
-use v5.10.1;
+# Usage: $PERL -I lib -w xt/check-run.t -j4
+# Or via prove(1): prove -lvw xt/check-run.t :: -j4
+use v5.12;
 use IO::Handle; # ->autoflush
 use PublicInbox::TestCommon;
 use PublicInbox::Spawn;
+use PublicInbox::DS; # already loaded by Spawn via PublicInbox::IO
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
 use Errno qw(EINTR);
 use Fcntl qw(:seek);
@@ -86,6 +86,14 @@ if ($shuffle) {
         @tests = sort {
                 ($t->{$b}->{elapsed} // 0) <=> ($t->{$a}->{elapsed} // 0)
         } @tests;
+        if (scalar(@tests) > 1) {
+                my $end = $#tests > 9 ? 9 : $#tests;
+                my $nr = $end + 1;
+                say "# top $nr longest tests (`make check' regenerates)";
+                for (grep defined, @tests[0..$end]) {
+                        printf "# %0.6f %s\n", $t->{$_}->{elapsed}, $_;
+                }
+        }
 }
 
 our $tb = Test::More->builder;
@@ -167,7 +175,7 @@ my $start_worker = sub {
         my ($j, $rd, $wr, $todo) = @_;
         my $pid = fork // DIE "fork: $!";
         if ($pid == 0) {
-                close $wr if $wr;
+                close $wr;
                 $SIG{USR1} = undef; # undo parent $SIG{USR1}
                 $worker = $$;
                 while (1) {
@@ -180,6 +188,7 @@ my $start_worker = sub {
                         DIE "short read $r" if $r != UINT_SIZE;
                         my $t = unpack('I', $buf);
                         run_test($todo->[$t]);
+                        PublicInbox::DS->Reset;
                         $tb->reset;
                 }
                 kill 'USR1', $producer if !$eof; # sets $eof in $producer
@@ -203,15 +212,11 @@ for (my $i = $repeat; $i != 0; $i--) {
         pipe(my ($rd, $wr)) or DIE "pipe: $!";
 
         # fill the queue before forking so children can start earlier
-        my $n = (POSIX::PIPE_BUF / UINT_SIZE);
-        if ($n >= $#todo) {
-                print $wr join('', map { pack('I', $_) } (0..$#todo)) or DIE;
-                undef $wr;
-        } else { # write what we can...
-                $wr->autoflush(1);
-                print $wr join('', map { pack('I', $_) } (0..$n)) or DIE;
-                $n += 1; # and send more ($n..$#todo), later
-        }
+        $wr->autoflush(1);
+        $wr->blocking(0);
+        my $todo_buf = pack('I*', 0..$#todo);
+        my $woff = syswrite($wr, $todo_buf) // DIE "syswrite: $!";
+        substr($todo_buf, 0, $woff, '');
         $eof = undef;
         local $SIG{USR1} = sub { $eof = 1 };
         my $sigchld = sub {
@@ -243,12 +248,13 @@ for (my $i = $repeat; $i != 0; $i--) {
         for (my $j = 0; $j < $jobs; $j++) {
                 $start_worker->($j, $rd, $wr, \@todo);
         }
-        if ($wr) {
+        {
                 local $SIG{CHLD} = $sigchld;
                 # too many tests to fit in the pipe before starting workers,
                 # send the rest now the workers are running
-                print $wr join('', map { pack('I', $_) } ($n..$#todo)) or DIE;
-                undef $wr;
+                $wr->blocking(1);
+                print $wr $todo_buf or DIE;
+                close $wr;
         }
 
         $sigchld->(0) while scalar(keys(%pids));
diff --git a/xt/cmp-msgview.t b/xt/cmp-msgview.t
deleted file mode 100644
index 9b06f88d..00000000
--- a/xt/cmp-msgview.t
+++ /dev/null
@@ -1,94 +0,0 @@
-#!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use Test::More;
-use Benchmark qw(:all);
-use PublicInbox::Inbox;
-use PublicInbox::View;
-use PublicInbox::TestCommon;
-use PublicInbox::Eml;
-use Digest::MD5;
-require_git(2.19);
-require_mods qw(Data::Dumper Email::MIME Plack::Util);
-Data::Dumper->import('Dumper');
-require PublicInbox::MIME;
-my ($tmpdir, $for_destroy) = tmpdir();
-my $inboxdir = $ENV{GIANT_INBOX_DIR};
-plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inboxdir;
-my @cat = qw(cat-file --buffer --batch-check --batch-all-objects --unordered);
-my $ibx = PublicInbox::Inbox->new({ inboxdir => $inboxdir, name => 'perf' });
-my $git = $ibx->git;
-my $fh = $git->popen(@cat);
-vec(my $vec = '', fileno($fh), 1) = 1;
-select($vec, undef, undef, 60) or die "timed out waiting for --batch-check";
-my $mime_ctx = {
-        env => { HTTP_HOST => 'example.com', 'psgi.url_scheme' => 'https' },
-        ibx => $ibx,
-        www => Plack::Util::inline_object(style => sub {''}),
-        obuf => \(my $mime_buf = ''),
-        mhref => '../',
-};
-my $eml_ctx = { %$mime_ctx, obuf => \(my $eml_buf = '') };
-my $n = 0;
-my $m = 0;
-my $ndiff_html = 0;
-my $dig_cls = 'Digest::MD5';
-my $digest_attach = sub { # ensure ->body (not ->body_raw) matches
-        my ($p, $cmp_arg) = @_;
-        my $part = shift @$p;
-        my $dig = $cmp_arg->[0] //= $dig_cls->new;
-        $dig->add($part->body_raw);
-        push @$cmp_arg, join(', ', @$p);
-};
-
-my $git_cb = sub {
-        my ($bref, $oid) = @_;
-        local $SIG{__WARN__} = sub { diag "$inboxdir $oid ", @_ };
-        ++$m;
-        my $mime = PublicInbox::MIME->new($$bref);
-        PublicInbox::View::multipart_text_as_html($mime, $mime_ctx);
-        my $eml = PublicInbox::Eml->new($$bref);
-        PublicInbox::View::multipart_text_as_html($eml, $eml_ctx);
-        if ($eml_buf ne $mime_buf) {
-                ++$ndiff_html;
-                open my $fh, '>', "$tmpdir/mime" or die $!;
-                print $fh $mime_buf or die $!;
-                close $fh or die $!;
-                open $fh, '>', "$tmpdir/eml" or die $!;
-                print $fh $eml_buf or die $!;
-                close $fh or die $!;
-                # using `git diff', diff(1) may not be installed
-                diag "$inboxdir $oid differs";
-                diag xqx([qw(git diff), "$tmpdir/mime", "$tmpdir/eml"]);
-        }
-        $eml_buf = $mime_buf = '';
-
-        # don't tolerate differences in attachment downloads
-        $mime = PublicInbox::MIME->new($$bref);
-        $mime->each_part($digest_attach, my $mime_cmp = [], 1);
-        $eml = PublicInbox::Eml->new($$bref);
-        $eml->each_part($digest_attach, my $eml_cmp = [], 1);
-        $mime_cmp->[0] = $mime_cmp->[0]->hexdigest;
-        $eml_cmp->[0] = $eml_cmp->[0]->hexdigest;
-        # don't have millions of "ok" lines
-        if (join("\0", @$eml_cmp) ne join("\0", @$mime_cmp)) {
-                diag Dumper([ $oid, eml => $eml_cmp, mime =>$mime_cmp ]);
-                is_deeply($eml_cmp, $mime_cmp, "$inboxdir $oid match");
-        }
-};
-my $t = timeit(1, sub {
-        while (<$fh>) {
-                my ($oid, $type) = split / /;
-                next if $type ne 'blob';
-                ++$n;
-                $git->cat_async($oid, $git_cb);
-        }
-        $git->async_wait_all;
-});
-is($m, $n, 'rendered all messages');
-
-# we'll tolerate minor differences in HTML rendering
-diag "$ndiff_html HTML differences";
-
-done_testing();
diff --git a/xt/create-many-inboxes.t b/xt/create-many-inboxes.t
index d22803e3..3d8932b7 100644
--- a/xt/create-many-inboxes.t
+++ b/xt/create-many-inboxes.t
@@ -19,7 +19,7 @@ mkpath($many_root);
 $many_root = abs_path($many_root);
 $many_root =~ m!\A\Q$cwd\E/! and BAIL_OUT "$many_root must not be in $cwd";
 require_git 2.6;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(DBD::SQLite Xapian));
 use_ok 'PublicInbox::V2Writable';
 my $nr_inbox = $ENV{NR_INBOX} // 10;
 my $nproc = $ENV{NPROC} || PublicInbox::IPC::detect_nproc() || 2;
diff --git a/xt/eml_check_limits.t b/xt/eml_check_limits.t
index a6d010af..1f89c6d4 100644
--- a/xt/eml_check_limits.t
+++ b/xt/eml_check_limits.t
@@ -1,15 +1,13 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use v5.10.1;
-use Test::More;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
 use PublicInbox::Inbox;
 use List::Util qw(max);
 use Benchmark qw(:all :hireswallclock);
-use PublicInbox::Spawn qw(popen_rd);
 use Carp ();
 require_git(2.19); # for --unordered
 require_mods(qw(BSD::Resource));
diff --git a/xt/git-http-backend.t b/xt/git-http-backend.t
index adadebb0..6c384faf 100644
--- a/xt/git-http-backend.t
+++ b/xt/git-http-backend.t
@@ -1,19 +1,18 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Ensure buffering behavior in -httpd doesn't cause runaway memory use
 # or data corruption
 use strict;
-use warnings;
-use Test::More;
+use v5.10.1;
 use POSIX qw(setsid);
 use PublicInbox::TestCommon;
-use PublicInbox::Spawn qw(which);
 
 my $git_dir = $ENV{GIANT_GIT_DIR};
 plan 'skip_all' => 'GIANT_GIT_DIR not defined' unless $git_dir;
 require_mods(qw(BSD::Resource Plack::Util Plack::Builder
-                HTTP::Date HTTP::Status Net::HTTP));
+                HTTP::Date HTTP::Status HTTP::Tiny));
 my $psgi = "./t/git-http-backend.psgi";
 my ($tmpdir, $for_destroy) = tmpdir();
 my $err = "$tmpdir/stderr.log";
@@ -21,15 +20,12 @@ my $out = "$tmpdir/stdout.log";
 my $sock = tcp_server();
 my ($host, $port) = tcp_host_port($sock);
 my $td;
+my $http = HTTP::Tiny->new;
 
 my $get_maxrss = sub {
-        my $http = Net::HTTP->new(Host => "$host:$port");
-        ok($http, 'Net::HTTP object created for maxrss');
-        $http->write_request(GET => '/');
-        my ($code, $mess, %h) = $http->read_response_headers;
-        is($code, 200, 'success reading maxrss');
-        my $n = $http->read_entity_body(my $buf, 256);
-        ok(defined $n, 'read response body');
+        my $res = $http->get("http://$host:$port/");
+        is($res->{status}, 200, 'success reading maxrss');
+        my $buf = $res->{content};
         like($buf, qr/\A\d+\n\z/, 'got memory response');
         ok(int($buf) > 0, 'got non-zero memory response');
         int($buf);
@@ -53,19 +49,18 @@ SKIP: {
                 }
         }
         skip "no packs found in $git_dir" unless defined $pack;
-        if ($pack !~ m!(/objects/pack/pack-[a-f0-9]{40}.pack)\z!) {
+        if ($pack !~ m!(/objects/pack/pack-[a-f0-9]{40,64}.pack)\z!) {
                 skip "bad pack name: $pack";
         }
-        my $url = $1;
-        my $http = Net::HTTP->new(Host => "$host:$port");
-        ok($http, 'Net::HTTP object created');
-        $http->write_request(GET => $url);
-        my ($code, $mess, %h) = $http->read_response_headers;
-        is(200, $code, 'got 200 success for pack');
-        is($max, $h{'Content-Length'}, 'got expected Content-Length for pack');
+        my $s = tcp_connect($sock);
+        print $s "GET $1 HTTP/1.1\r\nHost: $host:$port\r\n\r\n" or xbail $!;
+        my $hdr = do { local $/ = "\r\n\r\n"; readline($s) };
+        like $hdr, qr!\AHTTP/1\.1\s+200\b!, 'got 200 success for pack';
+        like $hdr, qr/^content-length:\s*$max\r\n/ims,
+                'got expected Content-Length for pack';
 
-        # no $http->read_entity_body, here, since we want to force buffering
-        foreach my $i (1..3) {
+        # don't read the body
+        for my $i (1..3) {
                 sleep 1;
                 my $diff = $get_maxrss->() - $mem_a;
                 note "${diff}K memory increase after $i seconds";
@@ -77,8 +72,7 @@ SKIP: { # make sure Last-Modified + If-Modified-Since works with curl
         my $nr = 6;
         skip 'no description', $nr unless -f "$git_dir/description";
         my $mtime = (stat(_))[9];
-        my $curl = which('curl');
-        skip 'curl(1) not found', $nr unless $curl;
+        my $curl = require_cmd('curl', 1) or skip 'curl(1) not found', $nr;
         my $url = "http://$host:$port/description";
         my $dst = "$tmpdir/desc";
         is(xsys($curl, qw(-RsSf), '-o', $dst, $url), 0, 'curl -R');
diff --git a/xt/git_async_cmp.t b/xt/git_async_cmp.t
index d66b371f..4038898b 100644
--- a/xt/git_async_cmp.t
+++ b/xt/git_async_cmp.t
@@ -1,10 +1,10 @@
 #!perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use Test::More;
 use Benchmark qw(:all);
-use Digest::SHA;
+use PublicInbox::SHA;
 use PublicInbox::TestCommon;
 my $git_dir = $ENV{GIANT_GIT_DIR};
 plan 'skip_all' => "GIANT_GIT_DIR not defined for $0" unless defined($git_dir);
@@ -20,7 +20,7 @@ my @dig;
 my $nr = $ENV{NR} || 1;
 diag "NR=$nr";
 my $async = timeit($nr, sub {
-        my $dig = Digest::SHA->new(1);
+        my $dig = PublicInbox::SHA->new(1);
         my $cb = sub {
                 my ($bref) = @_;
                 $dig->add($$bref);
@@ -31,27 +31,27 @@ my $async = timeit($nr, sub {
                 my ($oid, undef, undef) = split(/ /);
                 $git->cat_async($oid, $cb);
         }
-        close $cat or die "cat: $?";
+        $cat->close or xbail "cat: $?";
         $git->async_wait_all;
         push @dig, ['async', $dig->hexdigest ];
 });
 
 my $sync = timeit($nr, sub {
-        my $dig = Digest::SHA->new(1);
+        my $dig = PublicInbox::SHA->new(1);
         my $cat = $git->popen(@cat);
         while (<$cat>) {
                 my ($oid, undef, undef) = split(/ /);
                 my $bref = $git->cat_file($oid);
                 $dig->add($$bref);
         }
-        close $cat or die "cat: $?";
+        $cat->close or xbail "cat: $?";
         push @dig, ['sync', $dig->hexdigest ];
 });
 
 ok(scalar(@dig) >= 2, 'got some digests');
 my $ref = shift @dig;
 my $exp = $ref->[1];
-isnt($exp, Digest::SHA->new(1)->hexdigest, 'not empty');
+isnt($exp, PublicInbox::SHA->new(1)->hexdigest, 'not empty');
 foreach (@dig) {
         is($_->[1], $exp, "digest matches $_->[0] <=> $ref->[0]");
 }
diff --git a/xt/httpd-async-stream.t b/xt/httpd-async-stream.t
index c7039f3e..21d09331 100644
--- a/xt/httpd-async-stream.t
+++ b/xt/httpd-async-stream.t
@@ -1,17 +1,19 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Expensive test to validate compression and TLS.
-use strict;
-use Test::More;
+use v5.12;
+use autodie;
+use PublicInbox::IO qw(write_file);
+use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
 use PublicInbox::TestCommon;
 use PublicInbox::DS qw(now);
-use PublicInbox::Spawn qw(which popen_rd);
+use PublicInbox::Spawn qw(popen_rd);
 use Digest::MD5;
 use POSIX qw(_exit);
 my $inboxdir = $ENV{GIANT_INBOX_DIR};
 plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inboxdir;
-my $curl = which('curl') or plan skip_all => "curl(1) missing for $0";
+my $curl = require_cmd('curl');
 my ($tmpdir, $for_destroy) = tmpdir();
 require_mods(qw(DBD::SQLite));
 my $JOBS = $ENV{TEST_JOBS} // 4;
@@ -23,20 +25,15 @@ diag "TEST_JOBS=$JOBS TEST_ENDPOINT=$endpoint TEST_CURL_OPT=$curl_opt";
 my @CURL_OPT = (qw(-HHost:example.com -sSf), split(' ', $curl_opt));
 
 my $make_local_server = sub {
+        my ($http) = @_;
         my $pi_config = "$tmpdir/config";
-        open my $fh, '>', $pi_config or die "open($pi_config): $!";
-        print $fh <<"" or die "print $pi_config: $!";
+        write_file '>', $pi_config, <<"";
 [publicinbox "test"]
 inboxdir = $inboxdir
 address = test\@example.com
 
-        close $fh or die "close($pi_config): $!";
         my ($out, $err) = ("$tmpdir/out", "$tmpdir/err");
-        for ($out, $err) {
-                open my $fh, '>', $_ or die "truncate: $!";
-        }
-        my $http = tcp_server();
-        my $rdr = { 3 => $http };
+        for ($out, $err) { open my $fh, '>', $_ }
 
         # not using multiple workers, here, since we want to increase
         # the chance of tripping concurrency bugs within PublicInbox/HTTP*.pm
@@ -46,10 +43,22 @@ address = test\@example.com
         my $url = "$host_port/test/$endpoint";
         print STDERR "# CMD ". join(' ', @$cmd). "\n";
         my $env = { PI_CONFIG => $pi_config };
-        (start_script($cmd, $env, $rdr), $url);
+        (start_script($cmd, $env, { 3 => $http }), $url)
 };
 
-my ($td, $url) = $make_local_server->();
+my ($td, $url) = $make_local_server->(my $http = tcp_server());
+
+my $s1 = tcp_connect($http);
+my $rbuf = do { # pipeline while reading long response
+        my $req = <<EOM;
+GET /test/$endpoint HTTP/1.1\r
+Host: example.com\r
+\r
+EOM
+        is syswrite($s1, $req), length($req), 'initial long req';
+        <$s1>;
+};
+like $rbuf, qr!\AHTTP/1\.1 200\b!, 'started reading 200 response';
 
 my $do_get_all = sub {
         my ($job) = @_;
@@ -58,7 +67,7 @@ my $do_get_all = sub {
         my ($buf, $nr);
         my $bytes = 0;
         my $t0 = now();
-        my ($rd, $pid) = popen_rd([$curl, @CURL_OPT, $url]);
+        my $rd = popen_rd([$curl, @CURL_OPT, $url]);
         while (1) {
                 $nr = sysread($rd, $buf, 65536);
                 last if !$nr;
@@ -67,25 +76,23 @@ my $do_get_all = sub {
         }
         my $res = $dig->hexdigest;
         my $elapsed = sprintf('%0.3f', now() - $t0);
-        close $rd or die "close curl failed: $!\n";
-        waitpid($pid, 0) == $pid or die "waitpid failed: $!\n";
-        $? == 0 or die "curl failed: $?\n";
+        $rd->close or xbail "close curl failed: $! \$?=$?\n";
         print STDERR "# $job $$ ($?) $res (${elapsed}s) $bytes bytes\n";
         $res;
 };
 
 my (%pids, %res);
 for my $job (1..$JOBS) {
-        pipe(my ($r, $w)) or die;
+        pipe(my $r, my $w);
         my $pid = fork;
         if ($pid == 0) {
-                close $r or die;
+                close $r;
                 my $res = $do_get_all->($job);
-                print $w $res or die;
-                close $w or die;
+                print $w $res;
+                close $w;
                 _exit(0);
         }
-        close $w or die;
+        close $w;
         $pids{$pid} = [ $job, $r ];
 }
 
@@ -98,6 +105,31 @@ while (scalar keys %pids) {
         push @{$res{$sum}}, $job;
 }
 is(scalar keys %res, 1, 'all got the same result');
+{
+        my $req = <<EOM;
+GET /test/manifest.js.gz HTTP/1.1\r
+Host: example.com\r
+Connection: close\r
+\r
+EOM
+        is syswrite($s1, $req), length($req),
+                'pipeline another request while reading long response';
+        diag 'reading remainder of slow response';
+        my $res = do { local $/ = "\r\n\r\n"; <$s1> };
+        like $res, qr/^Transfer-Encoding: chunked\r\n/sm, 'chunked response';
+        {
+                local $/ = "\r\n"; # get to final chunk
+                while (defined(my $l = <$s1>)) { last if $l eq "0\r\n" }
+        };
+        is scalar(readline($s1)), "\r\n", 'got final CRLF from 1st response';
+        diag "second response:";
+        $res = do { local $/ = "\r\n\r\n"; <$s1> };
+        like $res, qr!\AHTTP/1\.1 200 !, 'response for pipelined req';
+        gunzip($s1 => \my $json) or xbail "gunzip $GunzipError";
+        my $m = PublicInbox::Config::json()->decode($json);
+        like $m->{'/test'}->{fingerprint}, qr/\A[0-9a-f]{40,}\z/,
+                'acceptable fingerprint in response';
+}
 $td->kill;
 $td->join;
 is($?, 0, 'no error on -httpd exit');
diff --git a/xt/imapd-mbsync-oimap.t b/xt/imapd-mbsync-oimap.t
index 0baf5b4c..f99779a1 100644
--- a/xt/imapd-mbsync-oimap.t
+++ b/xt/imapd-mbsync-oimap.t
@@ -1,12 +1,12 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # ensure mbsync and offlineimap compatibility
 use strict;
 use v5.10.1;
-use File::Path qw(mkpath);
+use File::Path qw(make_path);
 use PublicInbox::TestCommon;
-use PublicInbox::Spawn qw(which spawn);
+use PublicInbox::Spawn qw(spawn);
 require_mods(qw(-imapd));
 my $inboxdir = $ENV{GIANT_INBOX_DIR};
 (defined($inboxdir) && -d $inboxdir) or
@@ -41,8 +41,9 @@ my ($host, $port) = ($sock->sockhost, $sock->sockport);
 my %pids;
 
 SKIP: {
-        mkpath([map { "$tmpdir/oimapdir/$_" } qw(cur new tmp)]);
-        my $oimap = which('offlineimap') or skip 'no offlineimap(1)', 1;
+        make_path(map { "$tmpdir/oimapdir/$_" } qw(cur new tmp));
+        my $oimap = require_cmd('offlineimap', 1) or
+                skip 'no offlineimap(1)', 1;
         open my $fh, '>', "$tmpdir/.offlineimaprc" or BAIL_OUT "open: $!";
         print $fh <<EOF or BAIL_OUT "print: $!";
 [general]
@@ -77,8 +78,8 @@ EOF
 }
 
 SKIP: {
-        mkpath([map { "$tmpdir/mbsyncdir/test/$_" } qw(cur new tmp)]);
-        my $mbsync = which('mbsync') or skip 'no mbsync(1)', 1;
+        make_path(map { "$tmpdir/mbsyncdir/test/$_" } qw(cur new tmp));
+        my $mbsync = require_cmd('mbsync', 1) or skip 'no mbsync(1)', 1;
         open my $fh, '>', "$tmpdir/.mbsyncrc" or BAIL_OUT "open: $!";
         print $fh <<EOF or BAIL_OUT "print: $!";
 Create Slave
diff --git a/xt/imapd-validate.t b/xt/imapd-validate.t
index 5d27d2a0..5d665fa9 100644
--- a/xt/imapd-validate.t
+++ b/xt/imapd-validate.t
@@ -1,11 +1,12 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Expensive test to validate compression and TLS.
 use strict;
 use v5.10.1;
 use Symbol qw(gensym);
 use PublicInbox::DS qw(now);
+use PublicInbox::SHA;
 use POSIX qw(_exit);
 use PublicInbox::TestCommon;
 my $inbox_dir = $ENV{GIANT_INBOX_DIR};
@@ -64,7 +65,7 @@ my $do_get_all = sub {
         my ($desc, $opt) = @_;
         local $SIG{__DIE__} = sub { print STDERR $desc, ': ', @_; _exit(1) };
         my $t0 = now();
-        my $dig = Digest::SHA->new(1);
+        my $dig = PublicInbox::SHA->new(1);
         my $mic = $imap_client->new(%$opt);
         $mic->examine($mailbox) or die "examine: $!";
         my $uid_base = 1;
diff --git a/xt/lei-auth-fail.t b/xt/lei-auth-fail.t
index 06cb8533..1ccc2ab2 100644
--- a/xt/lei-auth-fail.t
+++ b/xt/lei-auth-fail.t
@@ -1,7 +1,8 @@
 #!perl -w
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict; use v5.10.1; use PublicInbox::TestCommon;
+use v5.12;
+use PublicInbox::TestCommon;
 require_mods(qw(Mail::IMAPClient lei));
 
 # TODO: mock IMAP server which fails at authentication so we don't
@@ -13,7 +14,7 @@ test_lei(sub {
         for my $pfx ([qw(q z:0.. --only), "$ro_home/t1", '-o'],
                         [qw(convert -o mboxrd:/dev/stdout)],
                         [qw(convert t/utf8.eml -o), $imap_fail],
-                        ['import'], [qw(tag +L:INBOX)]) {
+                        ['import'], [qw(tag +L:inbox)]) {
                 ok(!lei(@$pfx, $imap_fail), "IMAP auth failure on @$pfx");
                 like($lei_err, qr!\bE:.*?imaps?://.*?!sm, 'error shown');
                 unlike($lei_err, qr!Hunter2!s, 'password not shown');
diff --git a/xt/lei-onion-convert.t b/xt/lei-onion-convert.t
index 6dd17065..d3afbbb9 100644
--- a/xt/lei-onion-convert.t
+++ b/xt/lei-onion-convert.t
@@ -1,10 +1,12 @@
 #!perl -w
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict; use v5.10; use PublicInbox::TestCommon;
+use v5.12; use PublicInbox::TestCommon;
 use PublicInbox::MboxReader;
+use autodie qw(pipe close);
 my $test_tor = $ENV{TEST_TOR};
 plan skip_all => "TEST_TOR unset" unless $test_tor;
+require_mods qw(IO::Socket::Socks IO::Socket::SSL Mail::IMAPClient Net::NNTP);
 unless ($test_tor =~ m!\Asocks5h://!i) {
         my $default = 'socks5h://127.0.0.1:9050';
         diag "using $default (set TEST_TOR=socks5h://ADDR:PORT to override)";
@@ -19,11 +21,24 @@ my @cnv = qw(lei convert -o mboxrd:/dev/stdout);
 my @proxy_cli = ("--proxy=$test_tor");
 my $proxy_cfg = "proxy=$test_tor";
 test_lei(sub {
+        # ensure TLS + SOCKS works
+        ok !lei(qw(ls-mail-source imaps://mews.public-inbox.org/
+                -c), "imap.$proxy_cfg"),
+                'imaps fails on wrong hostname w/ Tor';
+        ok !lei(qw(ls-mail-source nntps://mews.public-inbox.org/
+                -c), "nntp.$proxy_cfg"),
+                'nntps fails on wrong hostname w/ Tor';
+
+        lei_ok qw(ls-mail-source imaps://news.public-inbox.org/
+                -c), "imap.$proxy_cfg";
+        lei_ok qw(ls-mail-source nntps://news.public-inbox.org/
+                -c), "nntp.$proxy_cfg";
+
         my $run = {};
         for my $args ([$nntp_url, @proxy_cli], [$imap_url, @proxy_cli],
                         [ $nntp_url, '-c', "nntp.$proxy_cfg" ],
                         [ $imap_url, '-c', "imap.$proxy_cfg" ]) {
-                pipe(my ($r, $w)) or xbail "pipe: $!";
+                pipe(my $r, my $w);
                 my $cmd = [@cnv, @$args];
                 my $td = start_script($cmd, undef, { 1 => $w, run_mode => 0 });
                 $args->[0] =~ s!\A(.+?://).*!$1...!;
diff --git a/xt/mem-imapd-tls.t b/xt/mem-imapd-tls.t
index 8992a6fc..53adb11b 100644
--- a/xt/mem-imapd-tls.t
+++ b/xt/mem-imapd-tls.t
@@ -1,13 +1,13 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Idle client memory usage test, particularly after EXAMINE when
 # Message Sequence Numbers are loaded
 use strict;
 use v5.10.1;
 use Socket qw(SOCK_STREAM IPPROTO_TCP SOL_SOCKET);
+use PublicInbox::Spawn qw(which);
 use PublicInbox::TestCommon;
-use PublicInbox::Syscall qw(:epoll);
 use PublicInbox::DS;
 require_mods(qw(-imapd));
 my $inboxdir = $ENV{GIANT_INBOX_DIR};
@@ -72,7 +72,7 @@ if ($TEST_TLS) {
         $ssl_opt{SSL_startHandshake} = 0;
 }
 chomp(my $nfd = `/bin/sh -c 'ulimit -n'`);
-$nfd -= 10;
+$nfd -= 20;
 ok($nfd > 0, 'positive FD count');
 my $MAX_FD = 10000;
 $nfd = $MAX_FD if $nfd >= $MAX_FD;
@@ -81,8 +81,8 @@ sub once { 0 }; # stops event loop
 
 # setup the event loop so that it exits at every step
 # while we're still doing connect(2)
-PublicInbox::DS->SetLoopTimeout(0);
-PublicInbox::DS->SetPostLoopCallback(\&once);
+$PublicInbox::DS::loop_timeout = 0;
+local @PublicInbox::DS::post_loop_do = (\&once);
 my $pid = $td->{pid};
 if ($^O eq 'linux' && open(my $f, '<', "/proc/$pid/status")) {
         diag(grep(/RssAnon/, <$f>));
@@ -100,30 +100,30 @@ foreach my $n (1..$nfd) {
         # try not to overflow the listen() backlog:
         if (!($n % 128) && $DONE != $n) {
                 diag("nr: ($n) $DONE/$nfd");
-                PublicInbox::DS->SetLoopTimeout(-1);
-                PublicInbox::DS->SetPostLoopCallback(sub { $DONE != $n });
+                $PublicInbox::DS::loop_timeout = -1;
+                local @PublicInbox::DS::post_loop_do = (sub { $DONE != $n });
 
                 # clear the backlog:
                 PublicInbox::DS::event_loop();
 
                 # resume looping
-                PublicInbox::DS->SetLoopTimeout(0);
-                PublicInbox::DS->SetPostLoopCallback(\&once);
+                $PublicInbox::DS::loop_timeout = 0;
         }
 }
 
 # run the event loop normally, now:
 diag "done?: @".time." $DONE/$nfd";
 if ($DONE != $nfd) {
-        PublicInbox::DS->SetLoopTimeout(-1);
-        PublicInbox::DS->SetPostLoopCallback(sub { $DONE != $nfd });
+        $PublicInbox::DS::loop_timeout = -1;
+        local @PublicInbox::DS::post_loop_do = (sub { $DONE != $nfd });
         PublicInbox::DS::event_loop();
 }
 is($nfd, $DONE, "$nfd/$DONE done");
-if ($^O eq 'linux' && open(my $f, '<', "/proc/$pid/status")) {
+my $lsof = which('lsof');
+if ($^O eq 'linux' && $lsof && open(my $f, '<', "/proc/$pid/status")) {
         diag(grep(/RssAnon/, <$f>));
-        diag "  SELF lsof | wc -l ".`lsof -p $$ |wc -l`;
-        diag "SERVER lsof | wc -l ".`lsof -p $pid |wc -l`;
+        diag "  SELF lsof | wc -l ".`$lsof -p $$ |wc -l`;
+        diag "SERVER lsof | wc -l ".`$lsof -p $pid |wc -l`;
 }
 PublicInbox::DS->Reset;
 $td->kill;
@@ -135,7 +135,7 @@ package IMAPC;
 use strict;
 use parent qw(PublicInbox::DS);
 # fields: step: state machine, zin: Zlib inflate context
-use PublicInbox::Syscall qw(EPOLLIN EPOLLOUT EPOLLONESHOT);
+use PublicInbox::Syscall qw(EPOLLOUT EPOLLONESHOT);
 use Errno qw(EAGAIN);
 # determines where we start event_step
 use constant FIRST_STEP => ($ENV{TEST_COMPRESS} // 1) ? -2 : 0;
@@ -221,13 +221,13 @@ package IMAPCdeflate;
 use strict;
 our @ISA;
 use Compress::Raw::Zlib;
-use PublicInbox::IMAPdeflate;
+use PublicInbox::IMAP;
 my %ZIN_OPT;
 BEGIN {
         @ISA = qw(IMAPC);
         %ZIN_OPT = ( -WindowBits => -15, -AppendOutput => 1 );
-        *write = \&PublicInbox::IMAPdeflate::write;
-        *do_read = \&PublicInbox::IMAPdeflate::do_read;
+        *write = \&PublicInbox::DSdeflate::write;
+        *do_read = \&PublicInbox::DSdeflate::do_read;
 };
 
 sub enable {
diff --git a/xt/mem-nntpd-tls.t b/xt/mem-nntpd-tls.t
new file mode 100644
index 00000000..ec639a8b
--- /dev/null
+++ b/xt/mem-nntpd-tls.t
@@ -0,0 +1,254 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Idle client memory usage test
+use v5.12.1;
+use PublicInbox::TestCommon;
+use File::Temp qw(tempdir);
+use Socket qw(SOCK_STREAM IPPROTO_TCP SOL_SOCKET);
+require_mods(qw(-nntpd));
+require PublicInbox::InboxWritable;
+require PublicInbox::SearchIdx;
+use PublicInbox::Syscall;
+use PublicInbox::DS;
+my $version = 2; # v2 needs newer git
+require_git('2.6') if $version >= 2;
+use_ok 'IO::Socket::SSL';
+my ($cert, $key) = qw(certs/server-cert.pem certs/server-key.pem);
+unless (-r $key && -r $cert) {
+        plan skip_all =>
+                "certs/ missing for $0, run ./certs/create-certs.perl";
+}
+use_ok 'PublicInbox::TLS';
+my ($tmpdir, $for_destroy) = tmpdir();
+my $err = "$tmpdir/stderr.log";
+my $out = "$tmpdir/stdout.log";
+my $mainrepo = $tmpdir;
+my $pi_config = "$tmpdir/pi_config";
+my $group = 'test-nntpd-tls';
+my $addr = $group . '@example.com';
+local $SIG{PIPE} = 'IGNORE'; # for NNTPC (below)
+my $nntps = tcp_server();
+my $ibx = PublicInbox::Inbox->new({
+        inboxdir => $mainrepo,
+        name => 'nntpd-tls',
+        version => $version,
+        -primary_address => $addr,
+        indexlevel => 'basic',
+});
+$ibx = PublicInbox::InboxWritable->new($ibx, {nproc=>1});
+$ibx->init_inbox(0);
+{
+        open my $fh, '>', $pi_config or die "open: $!\n";
+        print $fh <<EOF
+[publicinbox "nntpd-tls"]
+        mainrepo = $mainrepo
+        address = $addr
+        indexlevel = basic
+        newsgroup = $group
+EOF
+        ;
+        close $fh or die "close: $!\n";
+}
+
+{
+        my $im = $ibx->importer(0);
+        my $eml = eml_load('t/data/0001.patch');
+        ok($im->add($eml), 'message added');
+        $im->done;
+        if ($version == 1) {
+                my $s = PublicInbox::SearchIdx->new($ibx, 1);
+                $s->index_sync;
+        }
+}
+
+my $nntps_addr = tcp_host_port($nntps);
+my $env = { PI_CONFIG => $pi_config };
+my $tls = $ENV{TLS} // 1;
+my $args = $tls ? ["--cert=$cert", "--key=$key", "-lnntps://$nntps_addr"] : [];
+my $cmd = [ '-nntpd', '-W0', @$args, "--stdout=$out", "--stderr=$err" ];
+
+# run_mode=0 ensures Test::More FDs don't get shared
+my $td = start_script($cmd, $env, { 3 => $nntps, run_mode => 0 });
+my %ssl_opt = (
+        SSL_hostname => 'server.local',
+        SSL_verifycn_name => 'server.local',
+        SSL_verify_mode => SSL_VERIFY_PEER(),
+        SSL_ca_file => 'certs/test-ca.pem',
+);
+my $ctx = IO::Socket::SSL::SSL_Context->new(%ssl_opt);
+
+# cf. https://rt.cpan.org/Ticket/Display.html?id=129463
+my $mode = eval { Net::SSLeay::MODE_RELEASE_BUFFERS() };
+if ($mode && $ctx->{context}) {
+        eval { Net::SSLeay::CTX_set_mode($ctx->{context}, $mode) };
+        warn "W: $@ (setting SSL_MODE_RELEASE_BUFFERS)\n" if $@;
+}
+
+$ssl_opt{SSL_reuse_ctx} = $ctx;
+$ssl_opt{SSL_startHandshake} = 0;
+
+my %opt = (
+        Proto => 'tcp',
+        PeerAddr => $nntps_addr,
+        Type => SOCK_STREAM,
+        Blocking => 0
+);
+chomp(my $nfd = `/bin/sh -c 'ulimit -n'`);
+$nfd -= 10;
+ok($nfd > 0, 'positive FD count');
+my $MAX_FD = 10000;
+$nfd = $MAX_FD if $nfd >= $MAX_FD;
+our $DONE = 0;
+sub once { 0 }; # stops event loop
+
+# setup the event loop so that it exits at every step
+# while we're still doing connect(2)
+$PublicInbox::DS::loop_timeout = 0;
+local @PublicInbox::DS::post_loop_do = (\&once);
+
+foreach my $n (1..$nfd) {
+        my $io = tcp_connect($nntps, Blocking => 0);
+        $io = IO::Socket::SSL->start_SSL($io, %ssl_opt) if $tls;
+        NNTPC->new($io);
+
+        # one step through the event loop
+        # do a little work as we connect:
+        PublicInbox::DS::event_loop();
+
+        # try not to overflow the listen() backlog:
+        if (!($n % 128) && $n != $DONE) {
+                diag("nr: ($n) $DONE/$nfd");
+                $PublicInbox::DS::loop_timeout = -1;
+                @PublicInbox::DS::post_loop_do = (sub { $DONE != $n });
+
+                # clear the backlog:
+                PublicInbox::DS::event_loop();
+
+                # resume looping
+                $PublicInbox::DS::loop_timeout = 0;
+                @PublicInbox::DS::post_loop_do = (\&once);
+        }
+}
+my $pid = $td->{pid};
+my $dump_rss = sub {
+        return if $^O ne 'linux';
+        open(my $f, '<', "/proc/$pid/status") or return;
+        diag(grep(/RssAnon/, <$f>));
+};
+$dump_rss->();
+
+# run the event loop normally, now:
+if ($DONE != $nfd) {
+        $PublicInbox::DS::loop_timeout = -1;
+        @PublicInbox::DS::post_loop_do = (sub {
+                diag "done: ".time." $DONE";
+                $DONE != $nfd;
+        });
+        PublicInbox::DS::event_loop();
+}
+
+is($nfd, $DONE, 'done');
+$dump_rss->();
+if ($^O eq 'linux') {
+        diag "  SELF lsof | wc -l ".`lsof -p $$ |wc -l`;
+        diag "SERVER lsof | wc -l ".`lsof -p $pid |wc -l`;
+}
+PublicInbox::DS->Reset;
+$td->kill;
+$td->join;
+is($?, 0, 'no error in exited process');
+done_testing();
+
+package NNTPC;
+use v5.12;
+use parent qw(PublicInbox::DS);
+use PublicInbox::Syscall qw(EPOLLOUT EPOLLONESHOT);
+use Data::Dumper;
+
+# return true if complete, false if incomplete (or failure)
+sub connect_tls_step ($) {
+        my ($self) = @_;
+        my $sock = $self->{sock} or return;
+        return 1 if $sock->connect_SSL;
+        return $self->drop("$!") unless $!{EAGAIN};
+        if (my $ev = PublicInbox::TLS::epollbit()) {
+                unshift @{$self->{wbuf}}, \&connect_tls_step;
+                PublicInbox::DS::epwait($self->{sock}, $ev | EPOLLONESHOT);
+                0;
+        } else {
+                $self->drop('BUG? EAGAIN but '.PublicInbox::TLS::err());
+        }
+}
+
+sub event_step ($) {
+        my ($self) = @_;
+
+        # TLS negotiation happens in flush_write via {wbuf}
+        return unless $self->flush_write && $self->{sock};
+
+        if ($self->{step} == -2) {
+                $self->do_read(\(my $buf = ''), 128) or return;
+                $buf =~ /\A201 / or die "no greeting";
+                $self->{step} = -1;
+                $self->write(\"COMPRESS DEFLATE\r\n");
+        }
+        if ($self->{step} == -1) {
+                $self->do_read(\(my $buf = ''), 128) or return;
+                $buf =~ /\A20[0-9] / or die "no compression $buf";
+                NNTPCdeflate->enable($self);
+                $self->{step} = 1;
+                $self->write(\"DATE\r\n");
+        }
+        if ($self->{step} == 0) {
+                $self->do_read(\(my $buf = ''), 128) or return;
+                $buf =~ /\A201 / or die "no greeting";
+                $self->{step} = 1;
+                $self->write(\"DATE\r\n");
+        }
+        if ($self->{step} == 1) {
+                $self->do_read(\(my $buf = ''), 128) or return;
+                $buf =~ /\A111 / or die 'no date';
+                no warnings 'once';
+                $::DONE++;
+                $self->{step} = 2; # all done
+        } else {
+                die "$self->{step} Should never get here ". Dumper($self);
+        }
+}
+
+sub new {
+        my ($class, $io) = @_;
+        my $self = bless {}, $class;
+
+        # wait for connect(), and maybe SSL_connect()
+        $self->SUPER::new($io, EPOLLOUT|EPOLLONESHOT);
+        $self->{wbuf} = [ \&connect_tls_step ] if $io->can('connect_SSL');
+        $self->{step} = -2; # determines where we start event_step
+        $self;
+};
+
+1;
+package NNTPCdeflate;
+use v5.12;
+our @ISA = qw(NNTPC PublicInbox::DS);
+use Compress::Raw::Zlib;
+use PublicInbox::DSdeflate;
+BEGIN {
+        *write = \&PublicInbox::DSdeflate::write;
+        *do_read = \&PublicInbox::DSdeflate::do_read;
+        *event_step = \&NNTPC::event_step;
+        *flush_write = \&PublicInbox::DS::flush_write;
+        *close = \&PublicInbox::DS::close;
+}
+
+sub enable {
+        my ($class, $self) = @_;
+        my %ZIN_OPT = ( -WindowBits => -15, -AppendOutput => 1 );
+        my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%ZIN_OPT);
+        die "Inflate->new failed: $err" if $err != Z_OK;
+        bless $self, $class;
+        $self->{zin} = $in;
+}
+
+1;
diff --git a/xt/msgtime_cmp.t b/xt/msgtime_cmp.t
index a7ef5245..c63f785e 100644
--- a/xt/msgtime_cmp.t
+++ b/xt/msgtime_cmp.t
@@ -36,7 +36,7 @@ sub quiet_is_deeply ($$$$$) {
                         ($old->[0] != $cur->[0]) ||
                         ($old->[1] != $cur->[1]))) {
                 for ($cur, $old) {
-                        $_->[2] = strftime('%Y-%m-%d %k:%M:%S', gmtime($_->[0]))
+                        $_->[2] = strftime('%F %T', gmtime($_->[0]))
                 }
                 is_deeply($cur, $old, "$func $oid");
                 diag('got: ', explain($cur));
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index 333e0e3b..f7796e8e 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -1,5 +1,5 @@
 #!perl -w
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 use Sys::Hostname qw(hostname);
@@ -233,7 +233,7 @@ EOM
         my $pub_cfg = PublicInbox::Config->new;
         PublicInbox::DS->Reset;
         my $ii = PublicInbox::InboxIdle->new($pub_cfg);
-        my $cb = sub { PublicInbox::DS->SetPostLoopCallback(sub {}) };
+        my $cb = sub { @PublicInbox::DS::post_loop_do = (sub {}) };
         my $obj = bless \$cb, 'PublicInbox::TestCommon::InboxWakeup';
         $pub_cfg->each_inbox(sub { $_[0]->subscribe_unlock('ident', $obj) });
         my $w = start_script(['-watch'], undef, { 2 => $err_wr });
diff --git a/xt/nntpd-validate.t b/xt/nntpd-validate.t
index 83f024f9..a6f3980e 100644
--- a/xt/nntpd-validate.t
+++ b/xt/nntpd-validate.t
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Integration test to validate compression.
@@ -9,6 +9,7 @@ use Symbol qw(gensym);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
 use POSIX qw(_exit);
 use PublicInbox::TestCommon;
+use PublicInbox::SHA;
 my $inbox_dir = $ENV{GIANT_INBOX_DIR};
 plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inbox_dir;
 my $mid = $ENV{TEST_MID};
@@ -55,7 +56,7 @@ sub do_get_all {
         my ($methods) = @_;
         my $desc = join(',', @$methods);
         my $t0 = clock_gettime(CLOCK_MONOTONIC);
-        my $dig = Digest::SHA->new(1);
+        my $dig = PublicInbox::SHA->new(1);
         my $digfh = gensym;
         my $tmpfh;
         if ($File::Temp::KEEP_ALL) {
diff --git a/xt/perf-msgview.t b/xt/perf-msgview.t
index cf550c1a..ef261359 100644
--- a/xt/perf-msgview.t
+++ b/xt/perf-msgview.t
@@ -7,10 +7,12 @@ use PublicInbox::TestCommon;
 use Benchmark qw(:all);
 use PublicInbox::Inbox;
 use PublicInbox::View;
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::WwwStream;
 
 my $inboxdir = $ENV{GIANT_INBOX_DIR} // $ENV{GIANT_PI_DIR};
 my $blob = $ENV{TEST_BLOB};
+my $obfuscate = $ENV{PI_OBFUSCATE} ? 1 : 0;
+diag "PI_OBFUSCATE=$obfuscate";
 plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inboxdir;
 
 my @cat = qw(cat-file --buffer --batch-check --batch-all-objects);
@@ -21,7 +23,8 @@ if (require_git(2.19, 1)) {
 "git <2.19, cat-file lacks --unordered, locality suffers\n";
 }
 require_mods qw(Plack::Util);
-my $ibx = PublicInbox::Inbox->new({ inboxdir => $inboxdir, name => 'name' });
+my $ibx = PublicInbox::Inbox->new({ inboxdir => $inboxdir, name => 'name',
+                                    obfuscate => $obfuscate});
 my $git = $ibx->git;
 my $fh = $blob ? undef : $git->popen(@cat);
 if ($fh) {
@@ -31,26 +34,29 @@ if ($fh) {
                 die "timed out waiting for --batch-check";
 }
 
-my $ctx = {
+my $ctx = bless {
         env => { HTTP_HOST => 'example.com', 'psgi.url_scheme' => 'https' },
         ibx => $ibx,
         www => Plack::Util::inline_object(style => sub {''}),
-};
-my ($mime, $res, $oid, $type);
+        gz => PublicInbox::GzipFilter::gzip_or_die(),
+}, 'PublicInbox::WwwStream';
+my ($eml, $res, $oid, $type);
 my $n = 0;
-my $obuf = '';
 my $m = 0;
+${$ctx->{obuf}} = '';
+$ctx->{mhref} = '../';
 
 my $cb = sub {
-        $mime = PublicInbox::Eml->new(shift);
-        PublicInbox::View::multipart_text_as_html($mime, $ctx);
+        $eml = PublicInbox::Eml->new(shift);
+        $eml->each_part(\&PublicInbox::View::add_text_body, $ctx, 1);
+        $ctx->zflush(grep defined, delete @$ctx{'obuf'}); # compat
         ++$m;
-        $obuf = '';
+        delete $ctx->{zbuf};
+        ${$ctx->{obuf}} = ''; # compat
+        $ctx->{gz} = PublicInbox::GzipFilter::gzip_or_die();
 };
 
 my $t = timeit(1, sub {
-        $ctx->{obuf} = \$obuf;
-        $ctx->{mhref} = '../';
         if (defined $blob) {
                 my $nr = $ENV{NR} // 10000;
                 for (1..$nr) {
@@ -67,6 +73,6 @@ my $t = timeit(1, sub {
         }
         $git->async_wait_all;
 });
-diag 'multipart_text_as_html took '.timestr($t)." for $n <=> $m messages";
+diag 'add_text_body took '.timestr($t)." for $n <=> $m messages";
 is($m, $n, 'rendered all messages');
 done_testing();
diff --git a/xt/perf-obfuscate.t b/xt/perf-obfuscate.t
deleted file mode 100644
index 640309d2..00000000
--- a/xt/perf-obfuscate.t
+++ /dev/null
@@ -1,64 +0,0 @@
-#!perl -w
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use v5.10.1;
-use PublicInbox::TestCommon;
-use Benchmark qw(:all);
-use PublicInbox::Inbox;
-use PublicInbox::View;
-
-my $inboxdir = $ENV{GIANT_INBOX_DIR};
-plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inboxdir;
-
-my $obfuscate = $ENV{PI_OBFUSCATE} ? 1 : 0;
-diag "obfuscate=$obfuscate\n";
-
-my @cat = qw(cat-file --buffer --batch-check --batch-all-objects);
-if (require_git(2.19, 1)) {
-        push @cat, '--unordered';
-} else {
-        warn
-"git <2.19, cat-file lacks --unordered, locality suffers\n";
-}
-require_mods qw(Plack::Util);
-use_ok 'Plack::Util';
-my $ibx = PublicInbox::Inbox->new({ inboxdir => $inboxdir, name => 'name' ,
-                                    obfuscate => $obfuscate});
-my $git = $ibx->git;
-my $fh = $git->popen(@cat);
-my $vec = '';
-vec($vec, fileno($fh), 1) = 1;
-select($vec, undef, undef, 60) or die "timed out waiting for --batch-check";
-
-my $ctx = {
-        env => { HTTP_HOST => 'example.com', 'psgi.url_scheme' => 'https' },
-        ibx => $ibx,
-        www => Plack::Util::inline_object(style => sub {''}),
-};
-my ($mime, $res, $oid, $type);
-my $n = 0;
-my $obuf = '';
-my $m = 0;
-
-my $cb = sub {
-        $mime = PublicInbox::Eml->new(shift);
-        PublicInbox::View::multipart_text_as_html($mime, $ctx);
-        ++$m;
-        $obuf = '';
-};
-
-my $t = timeit(1, sub {
-        $ctx->{obuf} = \$obuf;
-        $ctx->{mhref} = '../';
-        while (<$fh>) {
-                ($oid, $type) = split / /;
-                next if $type ne 'blob';
-                ++$n;
-                $git->cat_async($oid, $cb);
-        }
-        $git->async_wait_all;
-});
-diag 'multipart_text_as_html took '.timestr($t)." for $n <=> $m messages";
-is($m, $n, 'rendered all messages');
-done_testing();
diff --git a/xt/pop3d-mpop.t b/xt/pop3d-mpop.t
new file mode 100644
index 00000000..ff8bb5dc
--- /dev/null
+++ b/xt/pop3d-mpop.t
@@ -0,0 +1,76 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# ensure mpop compatibility
+use v5.12;
+use File::Path qw(make_path);
+use PublicInbox::TestCommon;
+use PublicInbox::Spawn qw(spawn);
+my $inboxdir = $ENV{GIANT_INBOX_DIR};
+(defined($inboxdir) && -d $inboxdir) or
+        plan skip_all => "GIANT_INBOX_DIR not defined for $0";
+plan skip_all => "bad characters in $inboxdir" if $inboxdir =~ m![^\w\.\-/]!;
+my $uuidgen = require_cmd('uuidgen');
+my $mpop = require_cmd('mpop');
+require_mods(qw(DBD::SQLite :fcntl_lock));
+require_git(v2.6); # for v2
+
+my ($tmpdir, $for_destroy) = tmpdir();
+my $cfg = "$tmpdir/cfg";
+my $newsgroup = 'inbox.test';
+my %pids;
+{
+        open my $fh, '>', $cfg or xbail "open: $!";
+        print $fh <<EOF or xbail "print: $!";
+[publicinbox]
+        pop3state = $tmpdir/p3s
+[publicinbox "test"]
+        newsgroup = $newsgroup
+        address = mpop-test\@example.com
+        inboxdir = $inboxdir
+EOF
+        close $fh or xbail "close: $!";
+}
+my ($out, $err) = ("$tmpdir/stdout.log", "$tmpdir/stderr.log");
+my $sock = tcp_server();
+my $cmd = [ '-pop3d', '-W0', "--stdout=$out", "--stderr=$err" ];
+my $env = { PI_CONFIG => $cfg };
+my $td = start_script($cmd, $env, { 3 => $sock }) or xbail "-xbail $?";
+chomp(my $uuid = xqx([$uuidgen]));
+
+make_path("$tmpdir/home/.config/mpop",
+        map { "$tmpdir/md/$_" } qw(new cur tmp));
+
+{
+        open my $fh, '>', "$tmpdir/home/.config/mpop/config"
+                or xbail "open $!";
+        chmod 0600, $fh;
+        print $fh <<EOM or xbail "print $!";
+defaults
+tls off
+delivery maildir $tmpdir/md
+account default
+host ${\$sock->sockhost}
+port ${\$sock->sockport}
+user $uuid\@$newsgroup?limit=10000
+auth user
+password anonymous
+received_header off
+EOM
+        close $fh or xbail "close $!";
+        delete local $ENV{XDG_CONFIG_HOME}; # mpop uses this
+        local $ENV{HOME} = "$tmpdir/home";
+        my $cmd = [ $mpop, '-q' ];
+        my $pid = spawn($cmd, undef, { 1 => 2 });
+        $pids{$pid} = $cmd;
+}
+diag "mpop is writing to $tmpdir/md ...";
+while (scalar keys %pids) {
+        my $pid = waitpid(-1, 0) or next;
+        my $cmd = delete $pids{$pid} or next;
+        is($?, 0, join(' ', @$cmd, 'done'));
+}
+$td->kill;
+$td->join;
+is($?, 0, 'no error on -pop3d exit');
+done_testing;
diff --git a/xt/solver.t b/xt/solver.t
index 880458fb..372d003b 100644
--- a/xt/solver.t
+++ b/xt/solver.t
@@ -1,16 +1,16 @@
 #!perl -w
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use Test::More;
+use v5.12;
 use PublicInbox::TestCommon;
 use PublicInbox::Config; # this relies on PI_CONFIG // ~/.public-inbox/config
 my @psgi = qw(HTTP::Request::Common Plack::Test URI::Escape Plack::Builder);
-require_mods(qw(DBD::SQLite Search::Xapian), @psgi);
+require_mods(qw(DBD::SQLite Xapian), @psgi);
 use_ok($_) for @psgi;
 use_ok 'PublicInbox::WWW';
 my $cfg = PublicInbox::Config->new;
 my $www = PublicInbox::WWW->new($cfg);
+$www->preload;
 my $app = sub {
         my $env = shift;
         $env->{'psgi.errors'} = \*STDERR;
@@ -30,48 +30,52 @@ my $todo = {
                 '6aa8857a11/s/?b=protocol.c',
                 '96f1c7f/s/', # TODO: b=contrib/completion/git-completion.bash
                 'b76f2c0/s/?b=po/zh_CN.po',
+                'c2f3bf071ee90b01f2d629921bb04c4f798f02fa/s/', # tag
+                '7eb93c89651c47c8095d476251f2e4314656b292/s/', # non-UTF-8
         ],
+        'sox-devel' => [
+                'c38987e8d20505621b8d872863afa7d233ed1096/s/', # non-UTF-8
+        ]
 };
 
-my ($ibx_name, $urls, @gone);
+my @gone;
 my $client = sub {
         my ($cb) = @_;
-        for (@$urls) {
-                my $url = "/$ibx_name/$_";
-                my $res = $cb->(GET($url));
-                is($res->code, 200, $url);
-                next if $res->code == 200;
-                # diag $res->content;
-                diag "$url failed";
+        for my $ibx_name (sort keys %$todo) {
+                diag "testing $ibx_name";
+                my $urls = $todo->{$ibx_name};
+                for my $u (@$urls) {
+                        my $url = "/$ibx_name/$u";
+                        my $res = $cb->(GET($url));
+                        is($res->code, 200, $url);
+                        next if $res->code == 200;
+                        diag "$url failed";
+                        diag $res->content;
+                }
         }
 };
 
 my $nr = 0;
-while (($ibx_name, $urls) = each %$todo) {
+while (my ($ibx_name, $urls) = each %$todo) {
         SKIP: {
-                if (!$cfg->lookup_name($ibx_name)) {
+                my $ibx = $cfg->lookup_name($ibx_name);
+                if (!$ibx) {
+                        push @gone, $ibx_name;
+                        skip(qq{[publicinbox "$ibx_name"] not configured},
+                                scalar(@$urls));
+                }
+                if (!defined($ibx->{-repo_objs})) {
                         push @gone, $ibx_name;
-                        skip("$ibx_name not configured", scalar(@$urls));
+                        skip(qq{publicinbox.$ibx_name.coderepo not configured},
+                                scalar(@$urls));
                 }
-                test_psgi($app, $client);
                 $nr++;
         }
 }
 
-SKIP: {
-        require_mods(qw(Plack::Test::ExternalServer), $nr);
-        delete @$todo{@gone};
-
-        my $sock = tcp_server() or BAIL_OUT $!;
-        my ($tmpdir, $for_destroy) = tmpdir();
-        my ($out, $err) = map { "$tmpdir/std$_.log" } qw(out err);
-        my $cmd = [ qw(-httpd -W0), "--stdout=$out", "--stderr=$err" ];
-        my $td = start_script($cmd, undef, { 3 => $sock });
-        my ($h, $p) = tcp_host_port($sock);
-        local $ENV{PLACK_TEST_EXTERNALSERVER_URI} = "http://$h:$p";
-        while (($ibx_name, $urls) = each %$todo) {
-                Plack::Test::ExternalServer::test_psgi(client => $client);
-        }
-}
+delete @$todo{@gone};
+test_psgi($app, $client);
+my $env = { PI_CONFIG => PublicInbox::Config->default_file };
+test_httpd($env, $client, $nr);
 
 done_testing();