From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <e@80x24.org>
X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net
X-Spam-Level: 
X-Spam-ASN:  
X-Spam-Status: No, score=-3.5 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00,
	DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,NORMAL_HTTP_TO_IP,
	NUMERIC_HTTP_ADDR shortcircuit=no autolearn=ham autolearn_force=no
	version=3.4.2
Received: from localhost (dcvr.yhbt.net [127.0.0.1])
	by dcvr.yhbt.net (Postfix) with ESMTP id 5BDE11FAFB
	for <mwrap-perl@80x24.org>; Thu, 15 Dec 2022 20:52:58 +0000 (UTC)
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org;
	s=selector1; t=1671137578;
	bh=pOHcCNUXtZ3eTDeqvsitmJMUDLEdKU/0t7B9FpfI6+Y=;
	h=From:To:Subject:Date:In-Reply-To:References:From;
	b=iLxF7UcvEngg/RviEpz9JJL79r4xVMBI3yG4DllHWpYsB+P+wINEK4N87mS3amFMW
	 dUcf+WeTAjykOm3NfpnDo4O9V++QVsVCCo1i2Zk78ZDM1EGVcopx5mTPFkj/IDGBAF
	 RzfB0tVv+EEv5Cj/6/VA23hWkU+mXEAcmqzVwLys=
From: Eric Wong <e@80x24.org>
To: mwrap-perl@80x24.org
Subject: [PATCH 12/19] httpd: support CSV output
Date: Thu, 15 Dec 2022 20:52:48 +0000
Message-Id: <20221215205255.27840-13-e@80x24.org>
In-Reply-To: <20221215205255.27840-1-e@80x24.org>
References: <20221215205255.27840-1-e@80x24.org>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
List-Id: <mwrap-perl.yhbt.net>

CSV is well-supported by SQLite (and many other tools) so it can
be useful for offline analysis.
---
 mwrap_httpd.h   | 171 ++++++++++++++++++++++++++++++++----------------
 t/mwrap-httpd.t |  18 +++++
 2 files changed, 134 insertions(+), 55 deletions(-)

diff --git a/mwrap_httpd.h b/mwrap_httpd.h
index 3d2bf99..f484bdd 100644
--- a/mwrap_httpd.h
+++ b/mwrap_httpd.h
@@ -30,6 +30,8 @@
 #include <pthread.h>
 #include <stdbool.h>
 #define URL "https://80x24.org/mwrap-perl.git/about"
+#define TYPE_HTML "text/html; charset=UTF-8"
+#define TYPE_CSV "text/csv"
 
 enum mw_qev {
 	MW_QEV_IGNORE = 0,
@@ -128,7 +130,7 @@ static int cmp_location(const void *x, const void *y)
 	return strcmp(a->loc_name, b->loc_name);
 }
 
-/* fields for /each/$MIN/ endpoint */
+/* fields for /each/$MIN{,.csv} endpoints */
 struct h1_tbl {
 	const char *fname;
 	size_t flen;
@@ -257,7 +259,7 @@ static enum mw_qev h1_res_oneshot(struct mw_h1 *h1, const char *buf, size_t len)
 }
 
 #define FPUTS(STR, fp) fwrite(STR, sizeof(STR) - 1, 1, fp)
-static enum mw_qev h1_200(struct mw_h1 *h1, struct mw_fbuf *fb)
+static enum mw_qev h1_200(struct mw_h1 *h1, struct mw_fbuf *fb, const char *ct)
 {
 	/*
 	 * the HTTP header goes at the END of the body buffer,
@@ -275,10 +277,8 @@ static enum mw_qev h1_200(struct mw_h1 *h1, struct mw_fbuf *fb)
 		"Expires: Fri, 01 Jan 1980 00:00:00 GMT\r\n"
 		"Pragma: no-cache\r\n"
 		"Cache-Control: no-cache, max-age=0, must-revalidate\r\n"
-		"Content-Type: text/html; charset=UTF-8\r\n"
-		"Content-Length: ", fb->fp);
-	fprintf(fb->fp, "%zu", (size_t)clen);
-	FPUTS("\r\n\r\n", fb->fp);
+		"Content-Type: ", fb->fp);
+	fprintf(fb->fp, "%s\r\nContent-Length: %zu\r\n\r\n", ct, (size_t)clen);
 
 	if (fbuf_close(fb))
 		return h1_close(h1);
@@ -354,6 +354,25 @@ static void write_html(FILE *fp, const char *s, size_t len)
 	}
 }
 
+/*
+ * quotes multi-line backtraces for CSV (and `\' and `"' in case
+ * we encounter nasty file names).
+ */
+static void write_q_csv(FILE *fp, const char *s, size_t len)
+{
+	fputc('"', fp);
+	for (; len--; ++s) {
+		switch (*s) {
+		case '\n': fputs("\\n", fp); break;
+		case '\\': fputs("\\\\", fp); break;
+		case '"': fputs("\\\"", fp); break;
+		default: fputc(*s, fp);
+		}
+	}
+	fputc('"', fp);
+}
+
+
 /* URI-safe base-64 (RFC 4648) */
 static void write_b64_url(FILE *fp, const uint8_t *in, size_t len)
 {
@@ -559,12 +578,12 @@ static enum mw_qev each_at(struct mw_h1 *h1, struct mw_h1req *h1r)
 	}
 	rcu_read_unlock();
 	FPUTS("</table></body></html>", fp);
-	return h1_200(h1, &html);
+	return h1_200(h1, &html, TYPE_HTML);
 }
 
 /* /$PID/each/$MIN endpoint */
 static enum mw_qev each_gt(struct mw_h1 *h1, struct mw_h1req *h1r,
-				unsigned long min)
+				unsigned long min, bool csv)
 {
 	static const char default_sort[] = "bytes";
 	const char *sort = default_sort;
@@ -593,58 +612,86 @@ static enum mw_qev each_gt(struct mw_h1 *h1, struct mw_h1req *h1r,
 			return h1_close(h1);
 	}
 
-	struct mw_fbuf html;
-	FILE *fp = wbuf_init(&html);
+	struct mw_fbuf bdy;
+	FILE *fp = wbuf_init(&bdy);
 	if (!fp) return h1_close(h1);
-	fprintf(fp, "<html><head><title>mwrap each &gt;%lu"
-		"</title></head><body><p>mwrap each &gt;%lu "
-		"(change `%lu' in URL to adjust filtering) - MWRAP=bt:%u",
-		min, min, min, (unsigned)bt_req_depth);
 
-	show_stats(fp);
-	if (bt_req_depth) /* need borders to distinguish multi-level traces */
-		FPUTS("<table\nborder=1><tr>", fp);
-	else /* save screen space if only tracing one line */
-		FPUTS("<table><tr>", fp);
+	if (!csv) {
+		fprintf(fp, "<html><head><title>mwrap each &gt;%lu"
+			"</title></head><body><p>mwrap each &gt;%lu "
+			"(change `%lu' in URL to adjust filtering) - "
+			"MWRAP=bt:%u", min, min, min, (unsigned)bt_req_depth);
+		show_stats(fp);
+		/* need borders to distinguish multi-level traces */
+		if (bt_req_depth)
+			FPUTS("<table\nborder=1><tr>", fp);
+		else /* save screen space if only tracing one line */
+			FPUTS("<table><tr>", fp);
+	}
 
 	int (*cmp)(const void *, const void *) = NULL;
-	for (size_t i = 0; i < CAA_ARRAY_SIZE(fields); i++) {
-		FPUTS("<th>", fp);
-		if (fields[i].flen == sort_len &&
-				!memcmp(fields[i].fname, sort, sort_len)) {
-			cmp = fields[i].cmp;
-			fprintf(fp, "<b>%s</b>", fields[i].fname);
-		} else {
-			fprintf(fp,
-				"<a\nhref=\"./%lu?sort=%s\">%s</a>",
-				min, fields[i].fname, fields[i].fname);
+	if (csv) {
+		for (size_t i = 0; i < CAA_ARRAY_SIZE(fields); i++) {
+			const char *fn = fields[i].fname;
+			if (i)
+				fputc(',', fp);
+			fputs(fn, fp);
+			if (fields[i].flen == sort_len &&
+					!memcmp(fn, sort, sort_len))
+				cmp = fields[i].cmp;
+		}
+		fputc('\n', fp);
+	} else {
+		for (size_t i = 0; i < CAA_ARRAY_SIZE(fields); i++) {
+			const char *fn = fields[i].fname;
+			FPUTS("<th>", fp);
+			if (fields[i].flen == sort_len &&
+					!memcmp(fn, sort, sort_len)) {
+				cmp = fields[i].cmp;
+				fprintf(fp, "<b>%s</b>", fields[i].fname);
+			} else {
+				fprintf(fp, "<a\nhref=\"./%lu?sort=%s\">%s</a>",
+					min, fn, fn);
+			}
+			FPUTS("</th>", fp);
 		}
-		FPUTS("</th>", fp);
 	}
-	FPUTS("</tr>", fp);
+	if (!csv)
+		FPUTS("</tr>", fp);
 	if (cmp)
 		qsort(hslv, hslc, sizeof(*hslv), cmp);
-	else
+	else if (!csv)
 		FPUTS("<tr><td>sort= not understood</td></tr>", fp);
-	for (size_t i = 0; i < hslc; i++) {
-		struct h1_src_loc *hsl = &hslv[i];
-
-		fprintf(fp, "<tr><td>%zu</td><td>%zu</td><td>%zu</td>"
-			"<td>%zu</td><td>%0.3f</td><td>%zu</td>",
-			hsl->bytes, hsl->allocations, hsl->frees,
-			hsl->live, hsl->mean_life, hsl->max_life);
-		FPUTS("<td><a\nhref=\"../at/", fp);
-
-		/* yes, we're writing our memory addresses into the URI */
-		write_b64_url(fp, (const void *)&hsl->sl->f,
-				src_loc_hash_len(hsl->sl));
-
-		FPUTS("\">", fp);
-		write_html(fp, hsl->loc_name, hsl->lname_len);
-		FPUTS("</a></td></tr>", fp);
+	if (csv) {
+		for (size_t i = 0; i < hslc; i++) {
+			struct h1_src_loc *hsl = &hslv[i];
+
+			fprintf(fp, "%zu,%zu,%zu,%zu,%0.3f,%zu,",
+				hsl->bytes, hsl->allocations, hsl->frees,
+				hsl->live, hsl->mean_life, hsl->max_life);
+			write_q_csv(fp, hsl->loc_name, hsl->lname_len);
+			fputc('\n', fp);
+		}
+	} else {
+		for (size_t i = 0; i < hslc; i++) {
+			struct h1_src_loc *hsl = &hslv[i];
+
+			fprintf(fp, "<tr><td>%zu</td><td>%zu</td><td>%zu</td>"
+				"<td>%zu</td><td>%0.3f</td><td>%zu</td>",
+				hsl->bytes, hsl->allocations, hsl->frees,
+				hsl->live, hsl->mean_life, hsl->max_life);
+			FPUTS("<td><a\nhref=\"../at/", fp);
+
+			write_b64_url(fp, (const void *)&hsl->sl->f,
+					src_loc_hash_len(hsl->sl));
+
+			FPUTS("\">", fp);
+			write_html(fp, hsl->loc_name, hsl->lname_len);
+			FPUTS("</a></td></tr>", fp);
+		}
+		FPUTS("</table></body></html>", fp);
 	}
-	FPUTS("</table></body></html>", fp);
-	return h1_200(h1, &html);
+	return h1_200(h1, &bdy, csv ? TYPE_CSV : TYPE_HTML);
 }
 
 /* /$PID/ root endpoint */
@@ -661,10 +708,19 @@ static enum mw_qev pid_root(struct mw_h1 *h1, struct mw_h1req *h1r)
 	FPUTS("<p><a\nhref=\"each/" default_min "\">allocations &gt;"
 		default_min " bytes</a>"
 		"<p><a\nhref=\"" URL "\">" URL "</a></body></html>", fp);
-	return h1_200(h1, &html);
+	return h1_200(h1, &html, TYPE_HTML);
 #undef default_min
 }
 
+/* @e is not NUL-terminated */
+static bool sfx_eq(const char *e, const char *sfx)
+{
+	for (const char *m = sfx; *m; m++, e++)
+		if (*e != *m)
+			return false;
+	return true;
+}
+
 static enum mw_qev h1_dispatch(struct mw_h1 *h1, struct mw_h1req *h1r)
 {
 	if (h1r->method_len == 3 && !memcmp(h1r->method, "GET", 3)) {
@@ -672,10 +728,15 @@ static enum mw_qev h1_dispatch(struct mw_h1 *h1, struct mw_h1req *h1r)
 
 		if ((c = PATH_SKIP(h1r, "/each/"))) {
 			errno = 0;
-			char *end;
-			unsigned long min = strtoul(c, &end, 10);
-			if ((*end == ' ' || *end == '?') && !errno)
-				return each_gt(h1, h1r, min);
+			char *e;
+			unsigned long min = strtoul(c, &e, 10);
+			if (!errno) {
+				if (*e == ' ' || *e == '?')
+					return each_gt(h1, h1r, min, false);
+				if (sfx_eq(e, ".csv") &&
+						(e[4] == ' ' || e[4] == '?'))
+					return each_gt(h1, h1r, min, true);
+			}
 		} else if ((PATH_SKIP(h1r, "/at/"))) {
 			return each_at(h1, h1r);
 		} else if (h1r->path_len == 1 && h1r->path[0] == '/') {
diff --git a/t/mwrap-httpd.t b/t/mwrap-httpd.t
index f300eae..ca90cf0 100644
--- a/t/mwrap-httpd.t
+++ b/t/mwrap-httpd.t
@@ -134,6 +134,24 @@ SKIP: {
 
 SKIP: {
 	skip 'no reset w/o curl --unix-socket', 1 if !$curl_unix;
+
+	$rc = system(qw(curl -vsSf --unix-socket), $sock, '-o', $cout,
+		"http://0/$pid/each/100.csv");
+	is($rc, 0, '.csv retrieved') or skip 'CSV failed', 1;
+	my $db = "$mwrap_tmp/t.sqlite3";
+	$rc = system(qw(sqlite3), $db, ".import --csv $cout mwrap_each");
+	if ($rc == -1) {
+		diag 'sqlite3 missing';
+	} else {
+		is($rc, 0, 'sqlite3 import');
+		my $n = `sqlite3 $db 'SELECT COUNT(*) FROM mwrap_each'`;
+		is($?, 0, 'sqlite3 count');
+		my $exp = split(/\n/, slurp($cout));
+		is($n + 1, $exp, 'imported all rows into sqlite');
+		my $n = `sqlite3 $db 'SELECT COUNT(*) FROM mwrap_each'`;
+		# diag `sqlite3 $db .schema`;
+	}
+
 	$rc = system(qw(curl -vsSf --unix-socket), $sock, '-o', $cout,
 		'-d', 'x=y', "http://0/$pid/reset");
 	is($rc, 0, 'curl /reset');