[PATCH 15/19] httpd: pause forking thread on resource limitations

[Eric Wong <e@80x24.org>]

We shouldn't abort the program due to resource limitations.
Instead, the least intrusive course of action is probably to
sleep and wait for other threads in the process to do cleanup
work.
---
 mwrap_httpd.h | 70 ++++++++++++++++++++++++++++++---------------------
 1 file changed, 42 insertions(+), 28 deletions(-)

diff --git a/mwrap_httpd.h b/mwrap_httpd.h
index fc096c0..4864e72 100644
--- a/mwrap_httpd.h
+++ b/mwrap_httpd.h
@@ -79,7 +79,8 @@ struct mw_h1 { /* each HTTP/1.x client (heap) */
 
 struct mw_h1d { /* the daemon + listener, a singleton */
 	int lfd;
-	unsigned alive;
+	uint8_t alive; /* set by parent */
+	uint8_t running; /* cleared by child */
 	struct cds_list_head conn; /* <=> mw_h1.nd */
 	/* use open_memstream + fwrite to implement a growing pollfd array */
 	struct mw_fbuf pb; /* pollfd vector */
@@ -996,8 +997,7 @@ static struct pollfd *poll_detach(struct mw_h1d *h1d, nfds_t *nfds)
 
 static void non_fatal_pause(const char *fail_fn)
 {
-	fprintf(stderr,
-"%s: %m (non-fatal, pausing mwrap-httpd thread)\n", fail_fn);
+	fprintf(stderr, "%s: %m (non-fatal, pausing mwrap-httpd)\n", fail_fn);
 	poll(NULL, 0, 1000);
 }
 
@@ -1037,13 +1037,19 @@ static void h1d_event_step(struct mw_h1d *h1d)
 			case EPERM:
 				fail_fn = "accept4";
 				break;
-			/* hope other cleanup work gets done: */
+			/*
+			 * EINVAL, EBADF, ENOTSOCK, EOPNOTSUPP are all fatal
+			 * bugs.  The last 3 would be wayward closes in the
+			 * application being traced
+			 */
 			default:
-				fprintf(stderr, "accept4: %m (fatal)\n");
+				fprintf(stderr,
+					"accept4: %m (fatal in mwrap-httpd)\n");
 				abort();
 			}
 		}
 	}
+	/* hope other cleanup work gets done by other threads: */
 	non_fatal_pause(fail_fn);
 }
 
@@ -1094,12 +1100,6 @@ static int h1d_init(struct mw_h1d *h1d, const char *menv)
 	memcpy(h1d->pid_str, p, h1d->pid_len);
 	if (unlink(sa.un.sun_path) < 0 && errno != ENOENT)
 		return fprintf(stderr, "unlink(%s): %m\n", sa.un.sun_path);
-	/*
-	 * lfd may be >=0 if h1d_stop_join failed in parent and we're now
-	 * running in a forked child
-	 */
-	if (h1d->lfd >= 0)
-		(void)close(h1d->lfd);
 	h1d->lfd = socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0);
 	if (h1d->lfd < 0)
 		return fprintf(stderr, "socket: %m\n");
@@ -1112,6 +1112,7 @@ static int h1d_init(struct mw_h1d *h1d, const char *menv)
 		goto close_fail;
 	}
 	h1d->alive = 1; /* runs in parent, before pthread_create */
+	h1d->running = 1;
 	CDS_INIT_LIST_HEAD(&h1d->conn);
 	return 0;
 close_fail:
@@ -1145,8 +1146,8 @@ static void *h1d_run(void *x) /* pthread_create cb */
 	locating = 1; /* don't report our own memory use */
 
 	for (; uatomic_read(&h1d->alive); ) {
-		if (poll_add(h1d, h1d->lfd, POLLIN))
-			exit(EXIT_FAILURE);
+		while (poll_add(h1d, h1d->lfd, POLLIN))
+			non_fatal_pause("poll_add(lfd)");
 		cds_list_for_each_entry_safe(h1, nxt, &h1d->conn, nd)
 			if (poll_add(h1d, h1->fd, h1->events))
 				h1_close(h1);
@@ -1155,13 +1156,14 @@ static void *h1d_run(void *x) /* pthread_create cb */
 
 		if (rc < 0) {
 			switch (errno) {
-			case EINTR: break;
+			case EINTR: break; /* shouldn't happen, actually */
 			case ENOMEM: /* may be common */
 			case EINVAL: /* RLIMIT_NOFILE hit */
 				non_fatal_pause("poll");
 				break; /* to forloop where rc<0 */
-			default:
-				fprintf(stderr, "poll: %m (fatal)\n");
+			default: /* EFAULT is a fatal bug */
+				fprintf(stderr,
+					"poll: %m (fatal in mwrap-httpd)\n");
 				abort();
 			}
 		} else {
@@ -1181,6 +1183,7 @@ static void *h1d_run(void *x) /* pthread_create cb */
 			}
 		}
 	}
+	uatomic_set(&h1d->running, 0);
 	free(poll_detach(h1d, &nfds));
 	cds_list_for_each_entry_safe(h1, nxt, &h1d->conn, nd)
 		h1_close(h1);
@@ -1210,27 +1213,34 @@ static void h1d_stop_join(struct mw_h1d *h1d)
 	socklen_t len = (socklen_t)sizeof(sa);
 	int e, sfd;
 	void *ret;
-#define ERR ": %m (can't stop mwrap-httpd before fork)\n"
+#define ERR ": (stopping mwrap-httpd before fork): "
 
 	mwrap_assert(uatomic_read(&h1d->alive) == 0);
-	if (getsockname(h1d->lfd, &sa.any, &len) < 0) {
-		fprintf(stderr, "getsockname"ERR);
-		return;
+	while (getsockname(h1d->lfd, &sa.any, &len) < 0) {
+		non_fatal_pause("getsockname"ERR);
+		if (!uatomic_read(&h1d->running))
+			goto join_thread;
 	}
-	sfd = socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0);
-	if (sfd < 0) {
-		fprintf(stderr, "socket"ERR);
-		return;
+retry_socket:
+	while ((sfd = socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0)) < 0) {
+		non_fatal_pause("socket"ERR);
+		if (!uatomic_read(&h1d->running))
+			goto join_thread;
 	}
 	if (connect(sfd, &sa.any, len) < 0) {
-		fprintf(stderr, "connect"ERR);
+		int e = errno;
 		close(sfd);
-		return;
+		errno = e;
+		non_fatal_pause("connect"ERR);
+		if (!uatomic_read(&h1d->running))
+			goto join_thread;
+		goto retry_socket;
 	}
 #undef ERR
 	(void)close(sfd);
+join_thread:
 	e = pthread_join(h1d->tid, &ret);
-	if (e) {
+	if (e) { /* EDEADLK, EINVAL, ESRCH are all fatal bugs */
 		fprintf(stderr, "BUG? pthread_join: %s\n", strerror(e));
 		abort();
 	}
@@ -1248,8 +1258,12 @@ static void h1d_start(void) /* may be called as pthread_atfork child cb */
 {
 	if (mwrap_env && !h1d_init(&g_h1d, mwrap_env) && g_h1d.alive) {
 		int rc = pthread_create(&g_h1d.tid, NULL, h1d_run, &g_h1d);
-		if (rc) /* non-fatal */
+		if (rc) { /* non-fatal */
 			fprintf(stderr, "pthread_create: %s\n", strerror(rc));
+			(void)close(g_h1d.lfd);
+			g_h1d.lfd = -1;
+			g_h1d.alive = 0;
+		}
 	}
 }