httpd: do not waste TSD space

mwrap-httpd is inherently single-threaded, and the TSD pointer
needs to be alotted to every thread.  So tie the request pointer
to the daemon itself instead of per-thread space.

If this were a dedicated multi-threaded httpd, I would be using
TSD for this.  But this is only a single-threaded embedded httpd
with many threads doing completely unrelated things.

1 files changed, 25 insertions, 21 deletions

diff --git a/httpd.h b/httpd.h
index 128836c..e528d2b 100644
--- a/httpd.h
+++ b/httpd.h
@@ -94,6 +94,7 @@ struct mw_h1d { /* the daemon + listener, a singleton */
         /* use open_memstream + fwrite to implement a growing pollfd array */
         struct mw_fbuf pb; /* pollfd vector */
         pthread_t tid;
+        struct mw_h1req *shared_h1r; /* shared by all fast clients */
         size_t pid_len;
         char pid_str[10];
 };
@@ -104,7 +105,6 @@ union mw_sockaddr { /* cast-avoiding convenience :> */
 };
 
 static struct mw_h1d g_h1d = { .lfd = -1 };
-static MWRAP_TSD struct mw_h1req *tsd_h1r;
 
 /* sortable snapshot version of struct src_loc */
 struct h1_src_loc {
@@ -787,12 +787,22 @@ static enum mw_qev h1_dispatch(struct mw_h1 *h1, struct mw_h1req *h1r)
         return h1_404(h1);
 }
 
+static void
+prep_trickle(struct mw_h1 *h1, struct mw_h1req *h1r, struct mw_h1d *h1d)
+{
+        if (h1->h1r) return; /* already trickling */
+        h1->h1r = h1r;
+        mwrap_assert(h1d->shared_h1r == h1r);
+        h1d->shared_h1r = NULL;
+}
+
 /*
  * nothing in the PSGI app actually reads input, but clients tend
  * to send something in the body of POST requests anyways, so we
  * just drain it
  */
-static enum mw_qev h1_drain_input(struct mw_h1 *h1, struct mw_h1req *h1r)
+static enum mw_qev h1_drain_input(struct mw_h1 *h1, struct mw_h1req *h1r,
+        struct mw_h1d *h1d)
 {
         if (h1r) { /* initial */
                 ssize_t overread = h1r->rbuf_len - h1r->pret;
@@ -822,11 +832,7 @@ static enum mw_qev h1_drain_input(struct mw_h1 *h1, struct mw_h1req *h1r)
                 } else {
                         switch (errno) {
                         case EAGAIN:
-                                if (!h1->h1r) {
-                                        h1->h1r = h1r;
-                                        mwrap_assert(tsd_h1r == h1r);
-                                        tsd_h1r = NULL;
-                                }
+                                prep_trickle(h1, h1r, h1d);
                                 return MW_QEV_RD;
                         case ECONNRESET: /* common */
                         case ENOTCONN:
@@ -860,7 +866,8 @@ static void ctl_set(struct mw_h1 *h1, long n)
         }
 }
 
-static enum mw_qev h1_parse_harder(struct mw_h1 *h1, struct mw_h1req *h1r)
+static enum mw_qev h1_parse_harder(struct mw_h1 *h1, struct mw_h1req *h1r,
+        struct mw_h1d *h1d)
 {
         enum { HDR_IGN, HDR_XENC, HDR_CLEN } cur = HDR_IGN;
         char *end;
@@ -940,10 +947,10 @@ static enum mw_qev h1_parse_harder(struct mw_h1 *h1, struct mw_h1req *h1r)
                 h1r->qlen = h1r->path + h1r->path_len - h1r->qstr;
                 h1r->path_len -= (h1r->qlen + 1);
         }
-        return h1_drain_input(h1, h1r);
+        return h1_drain_input(h1, h1r, h1d);
 }
 
-static enum mw_qev h1_event_step(struct mw_h1 *h1)
+static enum mw_qev h1_event_step(struct mw_h1 *h1, struct mw_h1d *h1d)
 {
         struct mw_h1req *h1r;
 
@@ -956,14 +963,14 @@ static enum mw_qev h1_event_step(struct mw_h1 *h1)
                 return h1_send_flush(h1);
 
         if (h1->has_input)
-                return h1_drain_input(h1, NULL);
+                return h1_drain_input(h1, NULL, h1d);
         /*
-         * The majority of requests can be served using TSD rbuf,
+         * The majority of requests can be served using per-daemon rbuf,
          * no need for per-client allocations unless a client trickles
          */
-        h1r = h1->h1r ? h1->h1r : tsd_h1r;
+        h1r = h1->h1r ? h1->h1r : h1d->shared_h1r;
         if (!h1r) {
-                h1r = tsd_h1r = malloc(sizeof(*h1r));
+                h1r = h1d->shared_h1r = malloc(sizeof(*h1r));
                 if (!h1r) {
                         fprintf(stderr, "h1r malloc: %m\n");
                         return h1_close(h1);
@@ -982,7 +989,7 @@ static enum mw_qev h1_event_step(struct mw_h1 *h1)
                                                 &h1r->minor_ver, h1r->hdr,
                                                 &h1r->nr_hdr, h1->prev_len);
                         if (h1r->pret > 0)
-                                return h1_parse_harder(h1, h1r);
+                                return h1_parse_harder(h1, h1r, h1d);
                         if (h1r->pret == -1)
                                 return h1_400(h1); /* parser error */
 
@@ -1001,11 +1008,8 @@ static enum mw_qev h1_event_step(struct mw_h1 *h1)
                 } else { /* r < 0 */
                         switch (errno) {
                         case EAGAIN: /* likely, detach to per-client buffer */
-                                if (h1->prev_len && !h1->h1r) {
-                                        h1->h1r = h1r;
-                                        mwrap_assert(tsd_h1r == h1r);
-                                        tsd_h1r = NULL;
-                                }
+                                if (h1->prev_len)
+                                        prep_trickle(h1, h1r, h1d);
                                 return MW_QEV_RD;
                         case ECONNRESET: /* common */
                         case ENOTCONN:
@@ -1258,7 +1262,7 @@ static void *h1d_run(void *x) /* pthread_create cb */
                                         h1d_event_step(h1d);
                                 } else {
                                         h1 = h1_lookup(h1d, pfd[i].fd);
-                                        ev = h1_event_step(h1);
+                                        ev = h1_event_step(h1, h1d);
                                         if (ev == MW_QEV_IGNORE)
                                                 continue;
                                         h1->events = ev;