* [PATCH 1/5] xap_helper.h: use xcalloc to simplify error checking
@ 2024-05-13 18:24 Eric Wong
2024-05-13 18:24 ` [PATCH 2/5] xh_cidx: comment Eric Wong
` (3 more replies)
0 siblings, 4 replies; 5+ messages in thread
From: Eric Wong @ 2024-05-13 18:24 UTC (permalink / raw)
To: spew
Since we introduced xcalloc for khashl.h usage, we might as well
use it elsewhere.
---
lib/PublicInbox/xap_helper.h | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/lib/PublicInbox/xap_helper.h b/lib/PublicInbox/xap_helper.h
index 38575f74..50de4199 100644
--- a/lib/PublicInbox/xap_helper.h
+++ b/lib/PublicInbox/xap_helper.h
@@ -1100,8 +1100,7 @@ int main(int argc, char *argv[])
CHECK(int, 0, sigdelset(&workerset, SIGTERM));
CHECK(int, 0, sigdelset(&workerset, SIGCHLD));
nworker_hwm = nworker;
- worker_pids = (pid_t *)calloc(nworker, sizeof(pid_t));
- if (!worker_pids) err(EXIT_FAILURE, "calloc");
+ worker_pids = (pid_t *)xcalloc(nworker, sizeof(pid_t));
if (pipe(pipefds)) err(EXIT_FAILURE, "pipe");
int fl = fcntl(pipefds[1], F_GETFL);
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 2/5] xh_cidx: comment
2024-05-13 18:24 [PATCH 1/5] xap_helper.h: use xcalloc to simplify error checking Eric Wong
@ 2024-05-13 18:24 ` Eric Wong
2024-05-13 18:24 ` [PATCH 3/5] WIP Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Eric Wong @ 2024-05-13 18:24 UTC (permalink / raw)
To: spew
---
lib/PublicInbox/xh_cidx.h | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lib/PublicInbox/xh_cidx.h b/lib/PublicInbox/xh_cidx.h
index 94420074..8cc6a845 100644
--- a/lib/PublicInbox/xh_cidx.h
+++ b/lib/PublicInbox/xh_cidx.h
@@ -3,6 +3,10 @@
// This file is only intended to be included by xap_helper.h
// it implements pieces used by CodeSearchIdx.pm
+// TODO: consider making PublicInbox::CodeSearchIdx emit binary
+// (20 or 32-bit) OIDs instead of ASCII hex. It would require
+// more code in both Perl and C++, though...
+
// assumes trusted data from same host
static inline unsigned int hex2uint(char c)
{
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 3/5] WIP
2024-05-13 18:24 [PATCH 1/5] xap_helper.h: use xcalloc to simplify error checking Eric Wong
2024-05-13 18:24 ` [PATCH 2/5] xh_cidx: comment Eric Wong
@ 2024-05-13 18:24 ` Eric Wong
2024-05-13 18:24 ` [PATCH 4/5] xap_helper: havereallocarray typo Eric Wong
2024-05-13 18:24 ` [PATCH 5/5] xap_helper: srch cache is khash cached set Eric Wong
3 siblings, 0 replies; 5+ messages in thread
From: Eric Wong @ 2024-05-13 18:24 UTC (permalink / raw)
To: spew
---
lib/PublicInbox/xap_helper.h | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/lib/PublicInbox/xap_helper.h b/lib/PublicInbox/xap_helper.h
index 50de4199..59dd96cb 100644
--- a/lib/PublicInbox/xap_helper.h
+++ b/lib/PublicInbox/xap_helper.h
@@ -124,7 +124,7 @@ static bool alive = true;
static FILE *orig_err = stderr;
#endif
static int orig_err_fd = -1;
-static void *srch_tree; // tsearch + tdelete + twalk
+static srch_tbl *srch; // tsearch + tdelete + twalk
static pid_t *worker_pids; // nr => pid
#define WORKER_MAX USHRT_MAX
static unsigned long nworker, nworker_hwm;
@@ -526,13 +526,10 @@ again:
return false;
}
-static int srch_cmp(const void *pa, const void *pb) // for tfind|tsearch
+static int srch_eq(const struct srch *a, const struct srch *b)
{
- const struct srch *a = (const struct srch *)pa;
- const struct srch *b = (const struct srch *)pb;
- int diff = a->ckey_len - b->ckey_len;
-
- return diff ? diff : memcmp(a->ckey, b->ckey, (size_t)a->ckey_len);
+ a->ckey_len != b->ckey_len ? 0 :
+ !memcmp(a->ckey, b->ckey, (size_t)a->ckey_len);
}
static bool is_chert(const char *dir)
@@ -725,6 +722,7 @@ static void dispatch(struct req *req)
kbuf.srch->ckey_len = size - offsetof(struct srch, ckey);
if (kbuf.srch->ckey_len <= 0 || !req->dirc)
ABORT("no -d args (or too many)");
+ khint_t ki;
s = (struct srch **)tsearch(kbuf.srch, &srch_tree, srch_cmp);
if (!s) err(EXIT_FAILURE, "tsearch"); // likely ENOMEM
req->srch = *s;
@@ -732,6 +730,9 @@ static void dispatch(struct req *req)
free_srch(kbuf.srch);
req->srch->db->reopen();
} else if (!srch_init(req)) {
+ int absent;
+ ki = srch_map_put(srch_map, kbuf.srch, srch_cmp);
+ assert(
assert(kbuf.srch == *((struct srch **)tfind(
kbuf.srch, &srch_tree, srch_cmp)));
void *del = tdelete(kbuf.srch, &srch_tree, srch_cmp);
@@ -895,10 +896,9 @@ static void start_workers(void)
static void cleanup_all(void)
{
cleanup_pids();
-#ifdef __GLIBC__
- tdestroy(srch_tree, free_srch);
- srch_tree = NULL;
-#endif
+ if (srch_cache)
+ srch_cache
+ srch_xdb = NULL;
}
static void parent_reopen_logs(void)
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 4/5] xap_helper: havereallocarray typo
2024-05-13 18:24 [PATCH 1/5] xap_helper.h: use xcalloc to simplify error checking Eric Wong
2024-05-13 18:24 ` [PATCH 2/5] xh_cidx: comment Eric Wong
2024-05-13 18:24 ` [PATCH 3/5] WIP Eric Wong
@ 2024-05-13 18:24 ` Eric Wong
2024-05-13 18:24 ` [PATCH 5/5] xap_helper: srch cache is khash cached set Eric Wong
3 siblings, 0 replies; 5+ messages in thread
From: Eric Wong @ 2024-05-13 18:24 UTC (permalink / raw)
To: spew
---
lib/PublicInbox/xap_helper.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/xap_helper.h b/lib/PublicInbox/xap_helper.h
index 59dd96cb..095dd0ee 100644
--- a/lib/PublicInbox/xap_helper.h
+++ b/lib/PublicInbox/xap_helper.h
@@ -95,10 +95,10 @@ static void *xcalloc(size_t nmemb, size_t size)
#if defined(__GLIBC__) && defined(__GLIBC_MINOR__) && \
MY_VER(__GLIBC__, __GLIBC_MINOR__, 0) >= MY_VER(2, 28, 0)
-# define HAVE_REALLOC_ARRAY 1
+# define HAVE_REALLOCARRAY 1
#elif (defined(__OpenBSD__) || defined(__DragonFly__) || \
defined(__FreeBSD__) || defined(__NetBSD__)
-# define HAVE_REALLOC_ARRAY 1
+# define HAVE_REALLOCARRAY 1
#endif
static void *xreallocarray(void *ptr, size_t nmemb, size_t size)
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 5/5] xap_helper: srch cache is khash cached set
2024-05-13 18:24 [PATCH 1/5] xap_helper.h: use xcalloc to simplify error checking Eric Wong
` (2 preceding siblings ...)
2024-05-13 18:24 ` [PATCH 4/5] xap_helper: havereallocarray typo Eric Wong
@ 2024-05-13 18:24 ` Eric Wong
3 siblings, 0 replies; 5+ messages in thread
From: Eric Wong @ 2024-05-13 18:24 UTC (permalink / raw)
To: spew
---
lib/PublicInbox/xap_helper.h | 85 ++++++++++++++++++++----------------
1 file changed, 47 insertions(+), 38 deletions(-)
diff --git a/lib/PublicInbox/xap_helper.h b/lib/PublicInbox/xap_helper.h
index 095dd0ee..45cb0db9 100644
--- a/lib/PublicInbox/xap_helper.h
+++ b/lib/PublicInbox/xap_helper.h
@@ -37,7 +37,6 @@
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
-#include <search.h>
#include <signal.h>
#include <stddef.h>
#include <stdint.h>
@@ -116,6 +115,30 @@ static void *xreallocarray(void *ptr, size_t nmemb, size_t size)
return ret;
}
+#include "khashl.h"
+
+struct srch {
+ int ckey_len; // int for comparisons
+ unsigned qp_flags;
+ bool qp_extra_done;
+ Xapian::Database *db;
+ Xapian::QueryParser *qp;
+ unsigned char ckey[]; // $shard_path0\0$shard_path1\0...
+};
+
+static khint_t srch_hash(const struct srch *srch)
+{
+ return kh_hash_bytes(srch->ckey_len, srch->ckey);
+}
+
+static int srch_eq(const struct srch *a, const struct srch *b)
+{
+ return a->ckey_len == b->ckey_len ?
+ !memcmp(a->ckey, b->ckey, (size_t)a->ckey_len) : 0;
+}
+
+KHASHL_CSET_INIT(KH_LOCAL, srch_set, srch_set, struct srch *, srch_hash, srch_eq)
+static srch_set *srch_cache;
// sock_fd is modified in signal handler, yes, it's SOCK_SEQPACKET
static volatile int sock_fd = STDIN_FILENO;
static sigset_t fullset, workerset;
@@ -124,7 +147,6 @@ static bool alive = true;
static FILE *orig_err = stderr;
#endif
static int orig_err_fd = -1;
-static srch_tbl *srch; // tsearch + tdelete + twalk
static pid_t *worker_pids; // nr => pid
#define WORKER_MAX USHRT_MAX
static unsigned long nworker, nworker_hwm;
@@ -144,15 +166,6 @@ enum exc_iter {
ITER_ABORT
};
-struct srch {
- int ckey_len; // int for comparisons
- unsigned qp_flags;
- bool qp_extra_done;
- Xapian::Database *db;
- Xapian::QueryParser *qp;
- char ckey[]; // $shard_path0\0$shard_path1\0...
-};
-
#define MY_ARG_MAX 256
typedef bool (*cmd)(struct req *);
@@ -435,7 +448,6 @@ static bool cmd_test_sleep(struct req *req)
for (;;) poll(NULL, 0, 10);
return false;
}
-#include "khashl.h"
#include "xh_mset.h" // read-only (WWW, IMAP, lei) stuff
#include "xh_cidx.h" // CodeSearchIdx.pm stuff
@@ -526,12 +538,6 @@ again:
return false;
}
-static int srch_eq(const struct srch *a, const struct srch *b)
-{
- a->ckey_len != b->ckey_len ? 0 :
- !memcmp(a->ckey, b->ckey, (size_t)a->ckey_len);
-}
-
static bool is_chert(const char *dir)
{
char iamchert[PATH_MAX];
@@ -622,9 +628,8 @@ static void srch_init_extra(struct req *req)
req->srch->qp_extra_done = true;
}
-static void free_srch(void *p) // tdestroy
+static void srch_free(struct srch *srch)
{
- struct srch *srch = (struct srch *)p;
delete srch->qp;
delete srch->db;
free(srch);
@@ -640,7 +645,6 @@ static void dispatch(struct req *req)
} kbuf;
char *end;
FILE *kfp;
- struct srch **s;
req->threadid = ULLONG_MAX;
for (c = 0; c < (int)MY_ARRAY_SIZE(cmds); c++) {
if (cmds[c].fn_len == size &&
@@ -722,22 +726,19 @@ static void dispatch(struct req *req)
kbuf.srch->ckey_len = size - offsetof(struct srch, ckey);
if (kbuf.srch->ckey_len <= 0 || !req->dirc)
ABORT("no -d args (or too many)");
- khint_t ki;
- s = (struct srch **)tsearch(kbuf.srch, &srch_tree, srch_cmp);
- if (!s) err(EXIT_FAILURE, "tsearch"); // likely ENOMEM
- req->srch = *s;
- if (req->srch != kbuf.srch) { // reuse existing
- free_srch(kbuf.srch);
+
+ int absent;
+ khint_t ki = srch_set_put(srch_cache, kbuf.srch, &absent);
+ assert(ki < kh_end(srch_cache));
+ req->srch = kh_key(srch_cache, ki);
+ if (!absent) { // reuse existing
+ assert(req->srch != kbuf.srch);
+ srch_free(kbuf.srch);
req->srch->db->reopen();
} else if (!srch_init(req)) {
- int absent;
- ki = srch_map_put(srch_map, kbuf.srch, srch_cmp);
- assert(
- assert(kbuf.srch == *((struct srch **)tfind(
- kbuf.srch, &srch_tree, srch_cmp)));
- void *del = tdelete(kbuf.srch, &srch_tree, srch_cmp);
- assert(del);
- free_srch(kbuf.srch);
+ int gone = srch_set_del(srch_cache, ki);
+ assert(gone);
+ srch_free(kbuf.srch);
goto cmd_err; // srch_init already warned
}
if (req->qpfxc && !req->srch->qp_extra_done)
@@ -896,9 +897,16 @@ static void start_workers(void)
static void cleanup_all(void)
{
cleanup_pids();
- if (srch_cache)
- srch_cache
- srch_xdb = NULL;
+ if (!srch_cache)
+ return;
+
+ khint_t k;
+ for (k = kh_begin(srch_cache); k != kh_end(srch_cache); k++) {
+ if (kh_exist(srch_cache, k))
+ srch_free(kh_key(srch_cache, k));
+ }
+ srch_set_destroy(srch_cache);
+ srch_cache = NULL;
}
static void parent_reopen_logs(void)
@@ -1040,6 +1048,7 @@ int main(int argc, char *argv[])
mail_nrp_init();
code_nrp_init();
+ srch_cache = srch_set_init();
atexit(cleanup_all);
if (!STDERR_ASSIGNABLE) {
^ permalink raw reply related [flat|nested] 5+ messages in thread
end of thread, other threads:[~2024-05-13 18:24 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-05-13 18:24 [PATCH 1/5] xap_helper.h: use xcalloc to simplify error checking Eric Wong
2024-05-13 18:24 ` [PATCH 2/5] xh_cidx: comment Eric Wong
2024-05-13 18:24 ` [PATCH 3/5] WIP Eric Wong
2024-05-13 18:24 ` [PATCH 4/5] xap_helper: havereallocarray typo Eric Wong
2024-05-13 18:24 ` [PATCH 5/5] xap_helper: srch cache is khash cached set Eric Wong
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).