ds: use object-oriented API for epoll

This allows us to cut down on imports and reduce code.
This also makes it easier (in the next commit) to provide an option
to disable epoll/kqueue when saving an FD is valued over scalability.

5 files changed, 69 insertions, 77 deletions

diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm
index d6e3d10e..9300ac77 100644
--- a/lib/PublicInbox/DS.pm
+++ b/lib/PublicInbox/DS.pm
@@ -28,7 +28,8 @@ use POSIX qw(WNOHANG sigprocmask SIG_SETMASK SIG_UNBLOCK);
 use Fcntl qw(SEEK_SET :DEFAULT O_APPEND);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
 use Scalar::Util qw(blessed);
-use PublicInbox::Syscall qw(:epoll %SIGNUM);
+use PublicInbox::Syscall qw(%SIGNUM
+        EPOLLIN EPOLLOUT EPOLLONESHOT EPOLLEXCLUSIVE);
 use PublicInbox::Tmpfile;
 use Errno qw(EAGAIN EINVAL ECHILD EINTR);
 use Carp qw(carp croak);
@@ -41,8 +42,7 @@ my $reap_armed;
 my $ToClose; # sockets to close when event loop is done
 our (
      %DescriptorMap,             # fd (num) -> PublicInbox::DS object
-     $Epoll,                     # Global epoll fd (or DSKQXS ref)
-     $ep_io,                     # IO::Handle for Epoll
+     $Epoll,  # global Epoll, DSPoll, or DSKQXS ref
 
      @post_loop_do,              # subref + args to call at the end of each loop
 
@@ -75,7 +75,6 @@ sub Reset {
                 my @q = delete @Stack{keys %Stack};
                 for my $q (@q) { @$q = () }
                 $AWAIT_PIDS = $nextq = $ToClose = undef;
-                $ep_io = undef; # closes real $Epoll FD
                 $Epoll = undef; # may call DSKQXS::DESTROY
         } while (@Timers || keys(%Stack) || $nextq || $AWAIT_PIDS ||
                 $ToClose || keys(%DescriptorMap) ||
@@ -126,21 +125,13 @@ sub add_uniq_timer { # ($name, $secs, $coderef, @args) = @_;
 
 # caller sets return value to $Epoll
 sub _InitPoller () {
-        if (defined $PublicInbox::Syscall::SYS_epoll_create)  {
-                my $fd = epoll_create();
-                die "epoll_create: $!" if $fd < 0;
-                open($ep_io, '+<&=', $fd) or return;
-                fcntl($ep_io, F_SETFD, FD_CLOEXEC);
-                $fd;
-        } else {
-                my $cls;
-                for (qw(DSKQXS DSPoll)) {
-                        $cls = "PublicInbox::$_";
-                        last if eval "require $cls";
-                }
-                $cls->import(qw(epoll_ctl epoll_wait));
-                $cls->new;
+        my @try = ($^O eq 'linux' ? 'Epoll' : 'DSKQXS');
+        my $cls;
+        for (@try, 'DSPoll') {
+                $cls = "PublicInbox::$_";
+                last if eval "require $cls";
         }
+        $cls->new;
 }
 
 sub now () { clock_gettime(CLOCK_MONOTONIC) }
@@ -307,7 +298,7 @@ sub event_loop (;$$) {
                 my $timeout = RunTimers();
 
                 # get up to 1000 events
-                epoll_wait($Epoll, 1000, $timeout, \@events);
+                $Epoll->ep_wait(1000, $timeout, \@events);
                 for my $fd (@events) {
                         # it's possible epoll_wait returned many events,
                         # including some at the end that ones in the front
@@ -345,7 +336,7 @@ sub new {
 
     $Epoll //= _InitPoller();
 retry:
-    if (epoll_ctl($Epoll, EPOLL_CTL_ADD, $fd, $ev)) {
+    if ($Epoll->ep_add($sock, $ev)) {
         if ($! == EINVAL && ($ev & EPOLLEXCLUSIVE)) {
             $ev &= ~EPOLLEXCLUSIVE;
             goto retry;
@@ -399,9 +390,7 @@ sub close {
 
     # if we're using epoll, we have to remove this from our epoll fd so we stop getting
     # notifications about it
-    my $fd = fileno($sock);
-    epoll_ctl($Epoll, EPOLL_CTL_DEL, $fd, 0) and
-        croak("EPOLL_CTL_DEL($self/$sock): $!");
+    $Epoll->ep_del($sock) and croak("EPOLL_CTL_DEL($self/$sock): $!");
 
     # we explicitly don't delete from DescriptorMap here until we
     # actually close the socket, as we might be in the middle of
@@ -619,9 +608,8 @@ sub msg_more ($$) {
 }
 
 sub epwait ($$) {
-    my ($sock, $ev) = @_;
-    epoll_ctl($Epoll, EPOLL_CTL_MOD, fileno($sock), $ev) and
-        croak("EPOLL_CTL_MOD($sock): $!");
+        my ($io, $ev) = @_;
+        $Epoll->ep_mod($io, $ev) and croak("EPOLL_CTL_MOD($io): $!");
 }
 
 # return true if complete, false if incomplete (or failure)
diff --git a/lib/PublicInbox/DSKQXS.pm b/lib/PublicInbox/DSKQXS.pm
index b6e5c4e9..8ef8ffb6 100644
--- a/lib/PublicInbox/DSKQXS.pm
+++ b/lib/PublicInbox/DSKQXS.pm
@@ -12,13 +12,10 @@
 # It also implements signalfd(2) emulation via "tie".
 package PublicInbox::DSKQXS;
 use v5.12;
-use parent qw(Exporter);
 use Symbol qw(gensym);
 use IO::KQueue;
 use Errno qw(EAGAIN);
-use PublicInbox::Syscall qw(EPOLLONESHOT EPOLLIN EPOLLOUT EPOLLET
-        EPOLL_CTL_ADD EPOLL_CTL_MOD EPOLL_CTL_DEL);
-our @EXPORT_OK = qw(epoll_ctl epoll_wait);
+use PublicInbox::Syscall qw(EPOLLONESHOT EPOLLIN EPOLLOUT EPOLLET);
 
 sub EV_DISPATCH () { 0x0080 }
 
@@ -97,30 +94,29 @@ sub READ { # called by sysread() for signalfd compatibility
 # for fileno() calls in PublicInbox::DS
 sub FILENO { ${$_[0]->{kq}} }
 
-sub epoll_ctl {
-        my ($self, $op, $fd, $ev) = @_;
-        my $kq = $self->{kq};
-        if ($op == EPOLL_CTL_MOD) {
-                $kq->EV_SET($fd, EVFILT_READ, kq_flag(EPOLLIN, $ev));
-                eval { $kq->EV_SET($fd, EVFILT_WRITE, kq_flag(EPOLLOUT, $ev)) };
-        } elsif ($op == EPOLL_CTL_DEL) {
-                $kq // return; # called in cleanup
-                $kq->EV_SET($fd, EVFILT_READ, EV_DISABLE);
-                eval { $kq->EV_SET($fd, EVFILT_WRITE, EV_DISABLE) };
-        } else { # EPOLL_CTL_ADD
-                $kq->EV_SET($fd, EVFILT_READ, EV_ADD|kq_flag(EPOLLIN, $ev));
-
-                # we call this blindly for read-only FDs such as tied
-                # DSKQXS (signalfd emulation) and Listeners
-                eval {
-                        $kq->EV_SET($fd, EVFILT_WRITE, EV_ADD |
-                                                        kq_flag(EPOLLOUT, $ev));
-                };
-        }
+sub _ep_mod_add ($$$$) {
+        my ($kq, $fd, $ev, $add) = @_;
+        $kq->EV_SET($fd, EVFILT_READ, $add|kq_flag(EPOLLIN, $ev));
+
+        # we call this blindly for read-only FDs such as tied
+        # DSKQXS (signalfd emulation) and Listeners
+        eval { $kq->EV_SET($fd, EVFILT_WRITE, $add|kq_flag(EPOLLOUT, $ev)) };
+        0;
+}
+
+sub ep_add { _ep_mod_add($_[0]->{kq}, fileno($_[1]), $_[2], EV_ADD) };
+sub ep_mod { _ep_mod_add($_[0]->{kq}, fileno($_[1]), $_[2], 0) };
+
+sub ep_del {
+        my ($self, $io, $ev) = @_;
+        my $kq = $_[0]->{kq} // return; # called in cleanup
+        my $fd = fileno($io);
+        $kq->EV_SET($fd, EVFILT_READ, EV_DISABLE);
+        eval { $kq->EV_SET($fd, EVFILT_WRITE, EV_DISABLE) };
         0;
 }
 
-sub epoll_wait {
+sub ep_wait {
         my ($self, $maxevents, $timeout_msec, $events) = @_;
         @$events = eval { $self->{kq}->kevent($timeout_msec) };
         if (my $err = $@) {
diff --git a/lib/PublicInbox/DSPoll.pm b/lib/PublicInbox/DSPoll.pm
index 56a400c2..fc282de0 100644
--- a/lib/PublicInbox/DSPoll.pm
+++ b/lib/PublicInbox/DSPoll.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # Licensed the same as Danga::Socket (and Perl5)
 # License: GPL-1.0+ or Artistic-1.0-Perl
 #  <https://www.gnu.org/licenses/gpl-1.0.txt>
@@ -9,28 +9,13 @@
 # an all encompassing emulation of epoll via IO::Poll, but just to
 # support cases public-inbox-nntpd/httpd care about.
 package PublicInbox::DSPoll;
-use strict;
-use warnings;
-use parent qw(Exporter);
+use v5.12;
 use IO::Poll;
-use PublicInbox::Syscall qw(EPOLLONESHOT EPOLLIN EPOLLOUT EPOLL_CTL_DEL);
-our @EXPORT_OK = qw(epoll_ctl epoll_wait);
+use PublicInbox::Syscall qw(EPOLLONESHOT EPOLLIN EPOLLOUT);
 
-sub new { bless {}, $_[0] } # fd => events
+sub new { bless {}, __PACKAGE__ } # fd => events
 
-sub epoll_ctl {
-        my ($self, $op, $fd, $ev) = @_;
-
-        # not wasting time on error checking
-        if ($op != EPOLL_CTL_DEL) {
-                $self->{$fd} = $ev;
-        } else {
-                delete $self->{$fd};
-        }
-        0;
-}
-
-sub epoll_wait {
+sub ep_wait {
         my ($self, $maxevents, $timeout_msec, $events) = @_;
         my @pset;
         while (my ($fd, $events) = each %$self) {
@@ -54,4 +39,10 @@ sub epoll_wait {
         }
 }
 
+sub ep_del { delete($_[0]->{fileno($_[1])}); 0 }
+sub ep_add { $_[0]->{fileno($_[1])} = $_[2]; 0 }
+
+no warnings 'once';
+*ep_mod = \&ep_add;
+
 1;
diff --git a/lib/PublicInbox/Epoll.pm b/lib/PublicInbox/Epoll.pm
new file mode 100644
index 00000000..d55c8535
--- /dev/null
+++ b/lib/PublicInbox/Epoll.pm
@@ -0,0 +1,23 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# OO API for epoll
+package PublicInbox::Epoll;
+use v5.12;
+use PublicInbox::Syscall qw(epoll_create epoll_ctl epoll_wait
+        EPOLL_CTL_ADD EPOLL_CTL_MOD EPOLL_CTL_DEL);
+use Fcntl qw(F_SETFD FD_CLOEXEC);
+use autodie qw(open fcntl);
+
+sub new {
+        open(my $fh, '+<&=', epoll_create());
+        fcntl($fh, F_SETFD, FD_CLOEXEC);
+        bless \$fh, __PACKAGE__;
+}
+
+sub ep_add { epoll_ctl(fileno(${$_[0]}), EPOLL_CTL_ADD, fileno($_[1]), $_[2]) }
+sub ep_mod { epoll_ctl(fileno(${$_[0]}), EPOLL_CTL_MOD, fileno($_[1]), $_[2]) }
+sub ep_del { epoll_ctl(fileno(${$_[0]}), EPOLL_CTL_DEL, fileno($_[1]), 0) }
+sub ep_wait { epoll_wait(fileno(${$_[0]}), @_[1, 2, 3]) }
+
+1;
diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm
index 14cd1720..0a0912fb 100644
--- a/lib/PublicInbox/Syscall.pm
+++ b/lib/PublicInbox/Syscall.pm
@@ -29,12 +29,6 @@ our @EXPORT_OK = qw(epoll_ctl epoll_create epoll_wait
                   EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
                   EPOLLONESHOT EPOLLEXCLUSIVE
                   signalfd rename_noreplace %SIGNUM);
-our %EXPORT_TAGS = (epoll => [qw(epoll_ctl epoll_create epoll_wait
-                             EPOLLIN EPOLLOUT
-                             EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
-                             EPOLLONESHOT EPOLLEXCLUSIVE)],
-                );
-
 use constant {
         EPOLLIN => 1,
         EPOLLOUT => 4,