[PATCH 3/3] httpd: do not waste TSD space

[Eric Wong <e@80x24.org>]

mwrap-httpd is inherently single-threaded, and the TSD pointer
needs to be alotted to every thread.  So tie the request pointer
to the daemon itself instead of per-thread space.

If this were a dedicated multi-threaded httpd, I would be using
TSD for this.  But this is only a single-threaded embedded httpd
with many threads doing completely unrelated things.
---
 httpd.h | 46 +++++++++++++++++++++++++---------------------
 1 file changed, 25 insertions(+), 21 deletions(-)

diff --git a/httpd.h b/httpd.h
index 128836c..e528d2b 100644
--- a/httpd.h
+++ b/httpd.h
@@ -94,6 +94,7 @@ struct mw_h1d { /* the daemon + listener, a singleton */
 	/* use open_memstream + fwrite to implement a growing pollfd array */
 	struct mw_fbuf pb; /* pollfd vector */
 	pthread_t tid;
+	struct mw_h1req *shared_h1r; /* shared by all fast clients */
 	size_t pid_len;
 	char pid_str[10];
 };
@@ -104,7 +105,6 @@ union mw_sockaddr { /* cast-avoiding convenience :> */
 };
 
 static struct mw_h1d g_h1d = { .lfd = -1 };
-static MWRAP_TSD struct mw_h1req *tsd_h1r;
 
 /* sortable snapshot version of struct src_loc */
 struct h1_src_loc {
@@ -787,12 +787,22 @@ static enum mw_qev h1_dispatch(struct mw_h1 *h1, struct mw_h1req *h1r)
 	return h1_404(h1);
 }
 
+static void
+prep_trickle(struct mw_h1 *h1, struct mw_h1req *h1r, struct mw_h1d *h1d)
+{
+	if (h1->h1r) return; /* already trickling */
+	h1->h1r = h1r;
+	mwrap_assert(h1d->shared_h1r == h1r);
+	h1d->shared_h1r = NULL;
+}
+
 /*
  * nothing in the PSGI app actually reads input, but clients tend
  * to send something in the body of POST requests anyways, so we
  * just drain it
  */
-static enum mw_qev h1_drain_input(struct mw_h1 *h1, struct mw_h1req *h1r)
+static enum mw_qev h1_drain_input(struct mw_h1 *h1, struct mw_h1req *h1r,
+	struct mw_h1d *h1d)
 {
 	if (h1r) { /* initial */
 		ssize_t overread = h1r->rbuf_len - h1r->pret;
@@ -822,11 +832,7 @@ static enum mw_qev h1_drain_input(struct mw_h1 *h1, struct mw_h1req *h1r)
 		} else {
 			switch (errno) {
 			case EAGAIN:
-				if (!h1->h1r) {
-					h1->h1r = h1r;
-					mwrap_assert(tsd_h1r == h1r);
-					tsd_h1r = NULL;
-				}
+				prep_trickle(h1, h1r, h1d);
 				return MW_QEV_RD;
 			case ECONNRESET: /* common */
 			case ENOTCONN:
@@ -860,7 +866,8 @@ static void ctl_set(struct mw_h1 *h1, long n)
 	}
 }
 
-static enum mw_qev h1_parse_harder(struct mw_h1 *h1, struct mw_h1req *h1r)
+static enum mw_qev h1_parse_harder(struct mw_h1 *h1, struct mw_h1req *h1r,
+	struct mw_h1d *h1d)
 {
 	enum { HDR_IGN, HDR_XENC, HDR_CLEN } cur = HDR_IGN;
 	char *end;
@@ -940,10 +947,10 @@ static enum mw_qev h1_parse_harder(struct mw_h1 *h1, struct mw_h1req *h1r)
 		h1r->qlen = h1r->path + h1r->path_len - h1r->qstr;
 		h1r->path_len -= (h1r->qlen + 1);
 	}
-	return h1_drain_input(h1, h1r);
+	return h1_drain_input(h1, h1r, h1d);
 }
 
-static enum mw_qev h1_event_step(struct mw_h1 *h1)
+static enum mw_qev h1_event_step(struct mw_h1 *h1, struct mw_h1d *h1d)
 {
 	struct mw_h1req *h1r;
 
@@ -956,14 +963,14 @@ static enum mw_qev h1_event_step(struct mw_h1 *h1)
 		return h1_send_flush(h1);
 
 	if (h1->has_input)
-		return h1_drain_input(h1, NULL);
+		return h1_drain_input(h1, NULL, h1d);
 	/*
-	 * The majority of requests can be served using TSD rbuf,
+	 * The majority of requests can be served using per-daemon rbuf,
 	 * no need for per-client allocations unless a client trickles
 	 */
-	h1r = h1->h1r ? h1->h1r : tsd_h1r;
+	h1r = h1->h1r ? h1->h1r : h1d->shared_h1r;
 	if (!h1r) {
-		h1r = tsd_h1r = malloc(sizeof(*h1r));
+		h1r = h1d->shared_h1r = malloc(sizeof(*h1r));
 		if (!h1r) {
 			fprintf(stderr, "h1r malloc: %m\n");
 			return h1_close(h1);
@@ -982,7 +989,7 @@ static enum mw_qev h1_event_step(struct mw_h1 *h1)
 						&h1r->minor_ver, h1r->hdr,
 						&h1r->nr_hdr, h1->prev_len);
 			if (h1r->pret > 0)
-				return h1_parse_harder(h1, h1r);
+				return h1_parse_harder(h1, h1r, h1d);
 			if (h1r->pret == -1)
 				return h1_400(h1); /* parser error */
 
@@ -1001,11 +1008,8 @@ static enum mw_qev h1_event_step(struct mw_h1 *h1)
 		} else { /* r < 0 */
 			switch (errno) {
 			case EAGAIN: /* likely, detach to per-client buffer */
-				if (h1->prev_len && !h1->h1r) {
-					h1->h1r = h1r;
-					mwrap_assert(tsd_h1r == h1r);
-					tsd_h1r = NULL;
-				}
+				if (h1->prev_len)
+					prep_trickle(h1, h1r, h1d);
 				return MW_QEV_RD;
 			case ECONNRESET: /* common */
 			case ENOTCONN:
@@ -1258,7 +1262,7 @@ static void *h1d_run(void *x) /* pthread_create cb */
 					h1d_event_step(h1d);
 				} else {
 					h1 = h1_lookup(h1d, pfd[i].fd);
-					ev = h1_event_step(h1);
+					ev = h1_event_step(h1, h1d);
 					if (ev == MW_QEV_IGNORE)
 						continue;
 					h1->events = ev;