httpd: pause forking thread on resource limitations

We shouldn't abort the program due to resource limitations.
Instead, the least intrusive course of action is probably to
sleep and wait for other threads in the process to do cleanup
work.

1 files changed, 42 insertions, 28 deletions

diff --git a/mwrap_httpd.h b/mwrap_httpd.h
index fc096c0..4864e72 100644
--- a/mwrap_httpd.h
+++ b/mwrap_httpd.h
@@ -79,7 +79,8 @@ struct mw_h1 { /* each HTTP/1.x client (heap) */
 
 struct mw_h1d { /* the daemon + listener, a singleton */
         int lfd;
-        unsigned alive;
+        uint8_t alive; /* set by parent */
+        uint8_t running; /* cleared by child */
         struct cds_list_head conn; /* <=> mw_h1.nd */
         /* use open_memstream + fwrite to implement a growing pollfd array */
         struct mw_fbuf pb; /* pollfd vector */
@@ -996,8 +997,7 @@ static struct pollfd *poll_detach(struct mw_h1d *h1d, nfds_t *nfds)
 
 static void non_fatal_pause(const char *fail_fn)
 {
-        fprintf(stderr,
-"%s: %m (non-fatal, pausing mwrap-httpd thread)\n", fail_fn);
+        fprintf(stderr, "%s: %m (non-fatal, pausing mwrap-httpd)\n", fail_fn);
         poll(NULL, 0, 1000);
 }
 
@@ -1037,13 +1037,19 @@ static void h1d_event_step(struct mw_h1d *h1d)
                         case EPERM:
                                 fail_fn = "accept4";
                                 break;
-                        /* hope other cleanup work gets done: */
+                        /*
+                         * EINVAL, EBADF, ENOTSOCK, EOPNOTSUPP are all fatal
+                         * bugs.  The last 3 would be wayward closes in the
+                         * application being traced
+                         */
                         default:
-                                fprintf(stderr, "accept4: %m (fatal)\n");
+                                fprintf(stderr,
+                                        "accept4: %m (fatal in mwrap-httpd)\n");
                                 abort();
                         }
                 }
         }
+        /* hope other cleanup work gets done by other threads: */
         non_fatal_pause(fail_fn);
 }
 
@@ -1094,12 +1100,6 @@ static int h1d_init(struct mw_h1d *h1d, const char *menv)
         memcpy(h1d->pid_str, p, h1d->pid_len);
         if (unlink(sa.un.sun_path) < 0 && errno != ENOENT)
                 return fprintf(stderr, "unlink(%s): %m\n", sa.un.sun_path);
-        /*
-         * lfd may be >=0 if h1d_stop_join failed in parent and we're now
-         * running in a forked child
-         */
-        if (h1d->lfd >= 0)
-                (void)close(h1d->lfd);
         h1d->lfd = socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0);
         if (h1d->lfd < 0)
                 return fprintf(stderr, "socket: %m\n");
@@ -1112,6 +1112,7 @@ static int h1d_init(struct mw_h1d *h1d, const char *menv)
                 goto close_fail;
         }
         h1d->alive = 1; /* runs in parent, before pthread_create */
+        h1d->running = 1;
         CDS_INIT_LIST_HEAD(&h1d->conn);
         return 0;
 close_fail:
@@ -1145,8 +1146,8 @@ static void *h1d_run(void *x) /* pthread_create cb */
         locating = 1; /* don't report our own memory use */
 
         for (; uatomic_read(&h1d->alive); ) {
-                if (poll_add(h1d, h1d->lfd, POLLIN))
-                        exit(EXIT_FAILURE);
+                while (poll_add(h1d, h1d->lfd, POLLIN))
+                        non_fatal_pause("poll_add(lfd)");
                 cds_list_for_each_entry_safe(h1, nxt, &h1d->conn, nd)
                         if (poll_add(h1d, h1->fd, h1->events))
                                 h1_close(h1);
@@ -1155,13 +1156,14 @@ static void *h1d_run(void *x) /* pthread_create cb */
 
                 if (rc < 0) {
                         switch (errno) {
-                        case EINTR: break;
+                        case EINTR: break; /* shouldn't happen, actually */
                         case ENOMEM: /* may be common */
                         case EINVAL: /* RLIMIT_NOFILE hit */
                                 non_fatal_pause("poll");
                                 break; /* to forloop where rc<0 */
-                        default:
-                                fprintf(stderr, "poll: %m (fatal)\n");
+                        default: /* EFAULT is a fatal bug */
+                                fprintf(stderr,
+                                        "poll: %m (fatal in mwrap-httpd)\n");
                                 abort();
                         }
                 } else {
@@ -1181,6 +1183,7 @@ static void *h1d_run(void *x) /* pthread_create cb */
                         }
                 }
         }
+        uatomic_set(&h1d->running, 0);
         free(poll_detach(h1d, &nfds));
         cds_list_for_each_entry_safe(h1, nxt, &h1d->conn, nd)
                 h1_close(h1);
@@ -1210,27 +1213,34 @@ static void h1d_stop_join(struct mw_h1d *h1d)
         socklen_t len = (socklen_t)sizeof(sa);
         int e, sfd;
         void *ret;
-#define ERR ": %m (can't stop mwrap-httpd before fork)\n"
+#define ERR ": (stopping mwrap-httpd before fork): "
 
         mwrap_assert(uatomic_read(&h1d->alive) == 0);
-        if (getsockname(h1d->lfd, &sa.any, &len) < 0) {
-                fprintf(stderr, "getsockname"ERR);
-                return;
+        while (getsockname(h1d->lfd, &sa.any, &len) < 0) {
+                non_fatal_pause("getsockname"ERR);
+                if (!uatomic_read(&h1d->running))
+                        goto join_thread;
         }
-        sfd = socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0);
-        if (sfd < 0) {
-                fprintf(stderr, "socket"ERR);
-                return;
+retry_socket:
+        while ((sfd = socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0)) < 0) {
+                non_fatal_pause("socket"ERR);
+                if (!uatomic_read(&h1d->running))
+                        goto join_thread;
         }
         if (connect(sfd, &sa.any, len) < 0) {
-                fprintf(stderr, "connect"ERR);
+                int e = errno;
                 close(sfd);
-                return;
+                errno = e;
+                non_fatal_pause("connect"ERR);
+                if (!uatomic_read(&h1d->running))
+                        goto join_thread;
+                goto retry_socket;
         }
 #undef ERR
         (void)close(sfd);
+join_thread:
         e = pthread_join(h1d->tid, &ret);
-        if (e) {
+        if (e) { /* EDEADLK, EINVAL, ESRCH are all fatal bugs */
                 fprintf(stderr, "BUG? pthread_join: %s\n", strerror(e));
                 abort();
         }
@@ -1248,8 +1258,12 @@ static void h1d_start(void) /* may be called as pthread_atfork child cb */
 {
         if (mwrap_env && !h1d_init(&g_h1d, mwrap_env) && g_h1d.alive) {
                 int rc = pthread_create(&g_h1d.tid, NULL, h1d_run, &g_h1d);
-                if (rc) /* non-fatal */
+                if (rc) { /* non-fatal */
                         fprintf(stderr, "pthread_create: %s\n", strerror(rc));
+                        (void)close(g_h1d.lfd);
+                        g_h1d.lfd = -1;
+                        g_h1d.alive = 0;
+                }
         }
 }