[PATCH 3/3] cleanup some FreeBSD-related workarounds

[Eric Wong <e@80x24.org>]

While FreeBSD 12.3 + Ruby 3.0.4p208 remains broken with mwrap
(likely due to threading bugs in Ruby) at least get the code
more consistently closer to a working state.

Thus we'll remember to account for PTHREAD_MUTEX_INITIALIZER
requiring malloc for HeapPageBody tracking, as we do with other
mutexes.

For reference, the Perl5 port of mwrap seems to have no problems
on FreeBSD 12.3, so it seems down to misbehavior w.r.t. pthreads
usage within Ruby itself.  Perl5 on FreeBSD is configured with
threads support, but Perl5 doesn't spawn background threads by
default like Ruby can.
---
 README            |  2 +-
 ext/mwrap/mwrap.c | 93 +++++++++++++++++++++++++++--------------------
 2 files changed, 54 insertions(+), 41 deletions(-)

diff --git a/README b/README
index f387bc4..539073e 100644
--- a/README
+++ b/README
@@ -23,7 +23,7 @@ It does not require recompiling or rebuilding Ruby, but only
 supports Ruby trunk (2.6.0dev+) on a few platforms:
 
 * GNU/Linux
-* FreeBSD (tested 11.1)
+* FreeBSD (tested 11.1 on Ruby 2.6, currently broken with Ruby 3.x)
 
 It may work on NetBSD, OpenBSD and DragonFly BSD.
 
diff --git a/ext/mwrap/mwrap.c b/ext/mwrap/mwrap.c
index 9d90298..1d6baec 100644
--- a/ext/mwrap/mwrap.c
+++ b/ext/mwrap/mwrap.c
@@ -99,9 +99,43 @@ union padded_mutex {
 /* a round-robin pool of mutexes */
 #define MUTEX_NR   (1 << 6)
 #define MUTEX_MASK (MUTEX_NR - 1)
+#ifdef __FreeBSD__
+#  define STATIC_MTX_INIT_OK (0)
+#else /* only tested on Linux + glibc */
+#  define STATIC_MTX_INIT_OK (1)
+#endif
 static size_t mutex_i;
 static union padded_mutex mutexes[MUTEX_NR] = {
+#if STATIC_MTX_INIT_OK
 	[0 ... (MUTEX_NR-1)].mtx = PTHREAD_MUTEX_INITIALIZER
+#endif
+};
+
+#define ACC_INIT(name) { .nr=0, .min=INT64_MAX, .max=-1, .m2=0, .mean=0 }
+struct acc {
+	uint64_t nr;
+	int64_t min;
+	int64_t max;
+	double m2;
+	double mean;
+};
+
+/* for tracking 16K-aligned heap page bodies (protected by GVL) */
+struct {
+	pthread_mutex_t lock;
+	struct cds_list_head bodies;
+	struct cds_list_head freed;
+
+	struct acc alive;
+	struct acc reborn;
+} hpb_stats = {
+#if STATIC_MTX_INIT_OK
+	.lock = PTHREAD_MUTEX_INITIALIZER,
+#endif
+	.bodies = CDS_LIST_HEAD_INIT(hpb_stats.bodies),
+	.freed = CDS_LIST_HEAD_INIT(hpb_stats.freed),
+	.alive = ACC_INIT(hpb_stats.alive),
+	.reborn = ACC_INIT(hpb_stats.reborn)
 };
 
 static pthread_mutex_t *mutex_assign(void)
@@ -120,12 +154,11 @@ __attribute__((constructor)) static void resolve_malloc(void)
 	int err;
 	++locating;
 
-#ifdef __FreeBSD__
 	/*
 	 * PTHREAD_MUTEX_INITIALIZER on FreeBSD means lazy initialization,
 	 * which happens at pthread_mutex_lock, and that calls calloc
 	 */
-	{
+	if (!STATIC_MTX_INIT_OK) {
 		size_t i;
 
 		for (i = 0; i < MUTEX_NR; i++) {
@@ -135,22 +168,28 @@ __attribute__((constructor)) static void resolve_malloc(void)
 				_exit(1);
 			}
 		}
+		err = pthread_mutex_init(&hpb_stats.lock, 0);
+		if (err) {
+			fprintf(stderr, "error: %s\n", strerror(err));
+			_exit(1);
+		}
 		/* initialize mutexes used by urcu-bp */
 		rcu_read_lock();
 		rcu_read_unlock();
+#ifndef __FreeBSD__
+	} else {
+		if (!real_malloc) {
+			resolving_malloc = 1;
+			real_malloc = dlsym(RTLD_NEXT, "malloc");
+		}
+		real_free = dlsym(RTLD_NEXT, "free");
+		if (!real_malloc || !real_free) {
+			fprintf(stderr, "missing malloc/aligned_alloc/free\n"
+				"\t%p %p\n", real_malloc, real_free);
+			_exit(1);
+		}
+#endif /* !__FreeBSD__ */
 	}
-#else /* !FreeBSD (tested on GNU/Linux) */
-	if (!real_malloc) {
-		resolving_malloc = 1;
-		real_malloc = dlsym(RTLD_NEXT, "malloc");
-	}
-	real_free = dlsym(RTLD_NEXT, "free");
-	if (!real_malloc || !real_free) {
-		fprintf(stderr, "missing malloc/aligned_alloc/free\n"
-			"\t%p %p\n", real_malloc, real_free);
-		_exit(1);
-	}
-#endif /* !FreeBSD */
 	CMM_STORE_SHARED(totals, lfht_new());
 	if (!CMM_LOAD_SHARED(totals))
 		fprintf(stderr, "failed to allocate totals table\n");
@@ -237,32 +276,6 @@ static int has_ec_p(void)
 		ruby_current_vm_ptr && ruby_current_ec;
 }
 
-struct acc {
-	uint64_t nr;
-	int64_t min;
-	int64_t max;
-	double m2;
-	double mean;
-};
-
-#define ACC_INIT(name) { .nr=0, .min=INT64_MAX, .max=-1, .m2=0, .mean=0 }
-
-/* for tracking 16K-aligned heap page bodies (protected by GVL) */
-struct {
-	pthread_mutex_t lock;
-	struct cds_list_head bodies;
-	struct cds_list_head freed;
-
-	struct acc alive;
-	struct acc reborn;
-} hpb_stats = {
-	.lock = PTHREAD_MUTEX_INITIALIZER,
-	.bodies = CDS_LIST_HEAD_INIT(hpb_stats.bodies),
-	.freed = CDS_LIST_HEAD_INIT(hpb_stats.freed),
-	.alive = ACC_INIT(hpb_stats.alive),
-	.reborn = ACC_INIT(hpb_stats.reborn)
-};
-
 /* allocated via real_malloc/real_free */
 struct src_loc {
 	pthread_mutex_t *mtx;