[PATCH 1/5] xap_helper.h: use xcalloc to simplify error checking

[PATCH 1/5] xap_helper.h: use xcalloc to simplify error checking

[Eric Wong]

Since we introduced xcalloc for khashl.h usage, we might as well
use it elsewhere.
---
 lib/PublicInbox/xap_helper.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/PublicInbox/xap_helper.h b/lib/PublicInbox/xap_helper.h
index 38575f74..50de4199 100644
--- a/lib/PublicInbox/xap_helper.h
+++ b/lib/PublicInbox/xap_helper.h
@@ -1100,8 +1100,7 @@ int main(int argc, char *argv[])
 	CHECK(int, 0, sigdelset(&workerset, SIGTERM));
 	CHECK(int, 0, sigdelset(&workerset, SIGCHLD));
 	nworker_hwm = nworker;
-	worker_pids = (pid_t *)calloc(nworker, sizeof(pid_t));
-	if (!worker_pids) err(EXIT_FAILURE, "calloc");
+	worker_pids = (pid_t *)xcalloc(nworker, sizeof(pid_t));
 
 	if (pipe(pipefds)) err(EXIT_FAILURE, "pipe");
 	int fl = fcntl(pipefds[1], F_GETFL);

[PATCH 2/5] xh_cidx: comment

[Eric Wong]

---
 lib/PublicInbox/xh_cidx.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/PublicInbox/xh_cidx.h b/lib/PublicInbox/xh_cidx.h
index 94420074..8cc6a845 100644
--- a/lib/PublicInbox/xh_cidx.h
+++ b/lib/PublicInbox/xh_cidx.h
@@ -3,6 +3,10 @@
 // This file is only intended to be included by xap_helper.h
 // it implements pieces used by CodeSearchIdx.pm
 
+// TODO: consider making PublicInbox::CodeSearchIdx emit binary
+// (20 or 32-bit) OIDs instead of ASCII hex.  It would require
+// more code in both Perl and C++, though...
+
 // assumes trusted data from same host
 static inline unsigned int hex2uint(char c)
 {

[PATCH 3/5] WIP

[Eric Wong]

---
 lib/PublicInbox/xap_helper.h | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/PublicInbox/xap_helper.h b/lib/PublicInbox/xap_helper.h
index 50de4199..59dd96cb 100644
--- a/lib/PublicInbox/xap_helper.h
+++ b/lib/PublicInbox/xap_helper.h
@@ -124,7 +124,7 @@ static bool alive = true;
 static FILE *orig_err = stderr;
 #endif
 static int orig_err_fd = -1;
-static void *srch_tree; // tsearch + tdelete + twalk
+static srch_tbl *srch; // tsearch + tdelete + twalk
 static pid_t *worker_pids; // nr => pid
 #define WORKER_MAX USHRT_MAX
 static unsigned long nworker, nworker_hwm;
@@ -526,13 +526,10 @@ again:
 	return false;
 }
 
-static int srch_cmp(const void *pa, const void *pb) // for tfind|tsearch
+static int srch_eq(const struct srch *a, const struct srch *b)
 {
-	const struct srch *a = (const struct srch *)pa;
-	const struct srch *b = (const struct srch *)pb;
-	int diff = a->ckey_len - b->ckey_len;
-
-	return diff ? diff : memcmp(a->ckey, b->ckey, (size_t)a->ckey_len);
+	a->ckey_len != b->ckey_len ? 0 :
+		!memcmp(a->ckey, b->ckey, (size_t)a->ckey_len);
 }
 
 static bool is_chert(const char *dir)
@@ -725,6 +722,7 @@ static void dispatch(struct req *req)
 	kbuf.srch->ckey_len = size - offsetof(struct srch, ckey);
 	if (kbuf.srch->ckey_len <= 0 || !req->dirc)
 		ABORT("no -d args (or too many)");
+	khint_t ki;
 	s = (struct srch **)tsearch(kbuf.srch, &srch_tree, srch_cmp);
 	if (!s) err(EXIT_FAILURE, "tsearch"); // likely ENOMEM
 	req->srch = *s;
@@ -732,6 +730,9 @@ static void dispatch(struct req *req)
 		free_srch(kbuf.srch);
 		req->srch->db->reopen();
 	} else if (!srch_init(req)) {
+		int absent;
+		ki = srch_map_put(srch_map, kbuf.srch, srch_cmp);
+		assert(
 		assert(kbuf.srch == *((struct srch **)tfind(
 					kbuf.srch, &srch_tree, srch_cmp)));
 		void *del = tdelete(kbuf.srch, &srch_tree, srch_cmp);
@@ -895,10 +896,9 @@ static void start_workers(void)
 static void cleanup_all(void)
 {
 	cleanup_pids();
-#ifdef __GLIBC__
-	tdestroy(srch_tree, free_srch);
-	srch_tree = NULL;
-#endif
+	if (srch_cache)
+		srch_cache
+	srch_xdb = NULL;
 }
 
 static void parent_reopen_logs(void)

[PATCH 4/5] xap_helper: havereallocarray typo

[Eric Wong]

---
 lib/PublicInbox/xap_helper.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/xap_helper.h b/lib/PublicInbox/xap_helper.h
index 59dd96cb..095dd0ee 100644
--- a/lib/PublicInbox/xap_helper.h
+++ b/lib/PublicInbox/xap_helper.h
@@ -95,10 +95,10 @@ static void *xcalloc(size_t nmemb, size_t size)
 
 #if defined(__GLIBC__) && defined(__GLIBC_MINOR__) && \
 		MY_VER(__GLIBC__, __GLIBC_MINOR__, 0) >= MY_VER(2, 28, 0)
-#	define HAVE_REALLOC_ARRAY 1
+#	define HAVE_REALLOCARRAY 1
 #elif (defined(__OpenBSD__) || defined(__DragonFly__) || \
 		defined(__FreeBSD__) || defined(__NetBSD__)
-#	define HAVE_REALLOC_ARRAY 1
+#	define HAVE_REALLOCARRAY 1
 #endif
 
 static void *xreallocarray(void *ptr, size_t nmemb, size_t size)

[PATCH 5/5] xap_helper: srch cache is khash cached set

[Eric Wong]

---
 lib/PublicInbox/xap_helper.h | 85 ++++++++++++++++++++----------------
 1 file changed, 47 insertions(+), 38 deletions(-)

diff --git a/lib/PublicInbox/xap_helper.h b/lib/PublicInbox/xap_helper.h
index 095dd0ee..45cb0db9 100644
--- a/lib/PublicInbox/xap_helper.h
+++ b/lib/PublicInbox/xap_helper.h
@@ -37,7 +37,6 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <limits.h>
-#include <search.h>
 #include <signal.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -116,6 +115,30 @@ static void *xreallocarray(void *ptr, size_t nmemb, size_t size)
 	return ret;
 }
 
+#include "khashl.h"
+
+struct srch {
+	int ckey_len; // int for comparisons
+	unsigned qp_flags;
+	bool qp_extra_done;
+	Xapian::Database *db;
+	Xapian::QueryParser *qp;
+	unsigned char ckey[]; // $shard_path0\0$shard_path1\0...
+};
+
+static khint_t srch_hash(const struct srch *srch)
+{
+	return kh_hash_bytes(srch->ckey_len, srch->ckey);
+}
+
+static int srch_eq(const struct srch *a, const struct srch *b)
+{
+	return a->ckey_len == b->ckey_len ?
+		!memcmp(a->ckey, b->ckey, (size_t)a->ckey_len) : 0;
+}
+
+KHASHL_CSET_INIT(KH_LOCAL, srch_set, srch_set, struct srch *, srch_hash, srch_eq)
+static srch_set *srch_cache;
 // sock_fd is modified in signal handler, yes, it's SOCK_SEQPACKET
 static volatile int sock_fd = STDIN_FILENO;
 static sigset_t fullset, workerset;
@@ -124,7 +147,6 @@ static bool alive = true;
 static FILE *orig_err = stderr;
 #endif
 static int orig_err_fd = -1;
-static srch_tbl *srch; // tsearch + tdelete + twalk
 static pid_t *worker_pids; // nr => pid
 #define WORKER_MAX USHRT_MAX
 static unsigned long nworker, nworker_hwm;
@@ -144,15 +166,6 @@ enum exc_iter {
 	ITER_ABORT
 };
 
-struct srch {
-	int ckey_len; // int for comparisons
-	unsigned qp_flags;
-	bool qp_extra_done;
-	Xapian::Database *db;
-	Xapian::QueryParser *qp;
-	char ckey[]; // $shard_path0\0$shard_path1\0...
-};
-
 #define MY_ARG_MAX 256
 typedef bool (*cmd)(struct req *);
 
@@ -435,7 +448,6 @@ static bool cmd_test_sleep(struct req *req)
 	for (;;) poll(NULL, 0, 10);
 	return false;
 }
-#include "khashl.h"
 #include "xh_mset.h" // read-only (WWW, IMAP, lei) stuff
 #include "xh_cidx.h" // CodeSearchIdx.pm stuff
 
@@ -526,12 +538,6 @@ again:
 	return false;
 }
 
-static int srch_eq(const struct srch *a, const struct srch *b)
-{
-	a->ckey_len != b->ckey_len ? 0 :
-		!memcmp(a->ckey, b->ckey, (size_t)a->ckey_len);
-}
-
 static bool is_chert(const char *dir)
 {
 	char iamchert[PATH_MAX];
@@ -622,9 +628,8 @@ static void srch_init_extra(struct req *req)
 	req->srch->qp_extra_done = true;
 }
 
-static void free_srch(void *p) // tdestroy
+static void srch_free(struct srch *srch)
 {
-	struct srch *srch = (struct srch *)p;
 	delete srch->qp;
 	delete srch->db;
 	free(srch);
@@ -640,7 +645,6 @@ static void dispatch(struct req *req)
 	} kbuf;
 	char *end;
 	FILE *kfp;
-	struct srch **s;
 	req->threadid = ULLONG_MAX;
 	for (c = 0; c < (int)MY_ARRAY_SIZE(cmds); c++) {
 		if (cmds[c].fn_len == size &&
@@ -722,22 +726,19 @@ static void dispatch(struct req *req)
 	kbuf.srch->ckey_len = size - offsetof(struct srch, ckey);
 	if (kbuf.srch->ckey_len <= 0 || !req->dirc)
 		ABORT("no -d args (or too many)");
-	khint_t ki;
-	s = (struct srch **)tsearch(kbuf.srch, &srch_tree, srch_cmp);
-	if (!s) err(EXIT_FAILURE, "tsearch"); // likely ENOMEM
-	req->srch = *s;
-	if (req->srch != kbuf.srch) { // reuse existing
-		free_srch(kbuf.srch);
+
+	int absent;
+	khint_t ki = srch_set_put(srch_cache, kbuf.srch, &absent);
+	assert(ki < kh_end(srch_cache));
+	req->srch = kh_key(srch_cache, ki);
+	if (!absent) { // reuse existing
+		assert(req->srch != kbuf.srch);
+		srch_free(kbuf.srch);
 		req->srch->db->reopen();
 	} else if (!srch_init(req)) {
-		int absent;
-		ki = srch_map_put(srch_map, kbuf.srch, srch_cmp);
-		assert(
-		assert(kbuf.srch == *((struct srch **)tfind(
-					kbuf.srch, &srch_tree, srch_cmp)));
-		void *del = tdelete(kbuf.srch, &srch_tree, srch_cmp);
-		assert(del);
-		free_srch(kbuf.srch);
+		int gone = srch_set_del(srch_cache, ki);
+		assert(gone);
+		srch_free(kbuf.srch);
 		goto cmd_err; // srch_init already warned
 	}
 	if (req->qpfxc && !req->srch->qp_extra_done)
@@ -896,9 +897,16 @@ static void start_workers(void)
 static void cleanup_all(void)
 {
 	cleanup_pids();
-	if (srch_cache)
-		srch_cache
-	srch_xdb = NULL;
+	if (!srch_cache)
+		return;
+
+	khint_t k;
+	for (k = kh_begin(srch_cache); k != kh_end(srch_cache); k++) {
+		if (kh_exist(srch_cache, k))
+			srch_free(kh_key(srch_cache, k));
+	}
+	srch_set_destroy(srch_cache);
+	srch_cache = NULL;
 }
 
 static void parent_reopen_logs(void)
@@ -1040,6 +1048,7 @@ int main(int argc, char *argv[])
 
 	mail_nrp_init();
 	code_nrp_init();
+	srch_cache = srch_set_init();
 	atexit(cleanup_all);
 
 	if (!STDERR_ASSIGNABLE) {