[PATCH 0/3] Ruby 3.x support

[PATCH 0/3] Ruby 3.x support

[Eric Wong]

1st patch is a cleanup patch I noticed along the way,
2/3 fixes things for Ruby 3.0.x, and 3/3 disables a test
for Ruby 3.1.

3.1 support of heap page bodies will require wrapping mmap,
which may happen at some point

Haven't tested on current ruby.git nor 3.2 previews,
nor anything aside from x86-64.

Eric Wong (3):
  constify arg for totals_add_rcu
  support Ruby 3.0.x
  disable HeapPageBody count test for Ruby 3.1

 ext/mwrap/extconf.rb |  7 +++++++
 ext/mwrap/mwrap.c    | 27 ++++++++++++++++++++++-----
 test/test_mwrap.rb   | 29 ++++++++++++++++++-----------
 3 files changed, 47 insertions(+), 16 deletions(-)

[PATCH 1/3] constify arg for totals_add_rcu

[Eric Wong]

It's not modified by that function, so constify it for
ease-of-reading and review.
---
 ext/mwrap/mwrap.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ext/mwrap/mwrap.c b/ext/mwrap/mwrap.c
index 477b1cb..8592ea7 100644
--- a/ext/mwrap/mwrap.c
+++ b/ext/mwrap/mwrap.c
@@ -367,7 +367,7 @@ acc_stddev(const struct acc *acc)
 	return DBL2NUM(acc_stddev_dbl(acc));
 }
 
-static struct src_loc *totals_add_rcu(struct src_loc *k)
+static struct src_loc *totals_add_rcu(const struct src_loc *k)
 {
 	struct cds_lfht_iter iter;
 	struct cds_lfht_node *cur;

[PATCH 2/3] support Ruby 3.0.x

[Eric Wong]

HEAP_PAGE_SIZE no longer estimates malloc overhead in Ruby 3.0.x,
but we can now rely on the GC::INTERNAL_CONSTANTS hash to access
the true value.

We also need to account for Ractors in 3.0+, and thus we need to
rely on thread-specific `ruby_current_ec' instead of process-wide
`ruby_current_execution_context_ptr' (which no longer exists in
3.0+).

Finally, the VM seems prone to making some small immortal
allocations in a few places we were not expecting in 2.7 and
earlier.  Account for that and loosen some exact checks to
account for it.

Tested on Ruby 3.0.3 and 3.0.4
---
 ext/mwrap/extconf.rb |  7 +++++++
 ext/mwrap/mwrap.c    | 25 +++++++++++++++++++++----
 test/test_mwrap.rb   | 27 ++++++++++++++++-----------
 3 files changed, 44 insertions(+), 15 deletions(-)

diff --git a/ext/mwrap/extconf.rb b/ext/mwrap/extconf.rb
index e9dbb1e..254a3bb 100644
--- a/ext/mwrap/extconf.rb
+++ b/ext/mwrap/extconf.rb
@@ -25,4 +25,11 @@ else
   abort 'missing __builtin_add_overflow'
 end
 
+begin
+  if n = GC::INTERNAL_CONSTANTS[:HEAP_PAGE_SIZE]
+    $defs << "-DHEAP_PAGE_SIZE=#{n}"
+  end
+rescue NameError
+end
+
 create_makefile 'mwrap'
diff --git a/ext/mwrap/mwrap.c b/ext/mwrap/mwrap.c
index 8592ea7..cd68eed 100644
--- a/ext/mwrap/mwrap.c
+++ b/ext/mwrap/mwrap.c
@@ -3,7 +3,7 @@
  * License: GPL-2.0+ <https://www.gnu.org/licenses/gpl-2.0.txt>
  */
 #define _LGPL_SOURCE /* allows URCU to inline some stuff */
-#include <ruby/ruby.h>
+#include <ruby.h> /* defines HAVE_RUBY_RACTOR_H on 3.0+ */
 #include <ruby/thread.h>
 #include <ruby/io.h>
 #include <execinfo.h>
@@ -22,10 +22,24 @@
 #include <urcu/rculist.h>
 #include "jhash.h"
 
+#if __STDC_VERSION__ >= 201112
+#	define MWRAP_TSD _Thread_local
+#elif defined(__GNUC__)
+#	define MWRAP_TSD __thread
+#else
+#	error _Thread_local nor __thread supported
+#endif
+
 static ID id_uminus;
 const char *rb_source_location_cstr(int *line); /* requires 2.6.0dev */
 extern int __attribute__((weak)) ruby_thread_has_gvl_p(void);
+
+#ifdef HAVE_RUBY_RACTOR_H /* Ruby 3.0+ */
+extern MWRAP_TSD void * __attribute__((weak)) ruby_current_ec;
+#else /* Ruby 2.6-2.7 */
 extern void * __attribute__((weak)) ruby_current_execution_context_ptr;
+#	define ruby_current_ec ruby_current_execution_context_ptr
+#endif
 extern void * __attribute__((weak)) ruby_current_vm_ptr; /* for rb_gc_count */
 extern size_t __attribute__((weak)) rb_gc_count(void);
 extern VALUE __attribute__((weak)) rb_cObject;
@@ -40,9 +54,12 @@ static size_t total_bytes_inc, total_bytes_dec;
 /* match values in Ruby gc.c */
 #define HEAP_PAGE_ALIGN_LOG 14
 enum {
-	HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
+	HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG)
+#ifndef HEAP_PAGE_SIZE /* Ruby 2.6-2.7 only */
+	,
 	REQUIRED_SIZE_BY_MALLOC = (sizeof(size_t) * 5),
 	HEAP_PAGE_SIZE = (HEAP_PAGE_ALIGN - REQUIRED_SIZE_BY_MALLOC)
+#endif
 };
 
 #define IS_HEAP_PAGE_BODY ((struct src_loc *)-1)
@@ -75,7 +92,7 @@ static int resolving_malloc;
 	} \
 } while (0)
 
-static __thread size_t locating;
+static MWRAP_TSD size_t locating;
 static size_t generation;
 static size_t page_size;
 static struct cds_lfht *totals;
@@ -214,7 +231,7 @@ static char *int2str(int num, char *dst, size_t * size)
 static int has_ec_p(void)
 {
 	return (ruby_thread_has_gvl_p() && ruby_current_vm_ptr &&
-		ruby_current_execution_context_ptr);
+		ruby_current_ec);
 }
 
 struct acc {
diff --git a/test/test_mwrap.rb b/test/test_mwrap.rb
index 48fba23..fadaef6 100644
--- a/test/test_mwrap.rb
+++ b/test/test_mwrap.rb
@@ -29,7 +29,8 @@ class TestMwrap < Test::Unit::TestCase
       tmp.rewind
       lines = tmp.readlines
       line_1 = lines.grep(/\s-e:1\b/)[0].strip
-      assert_equal '10001', line_1.split(/\s+/)[0]
+      bytes = line_1.split(/\s+/)[0].to_i
+      assert_operator bytes, :>=, 10001
     end
   end
 
@@ -42,7 +43,7 @@ class TestMwrap < Test::Unit::TestCase
       res = system(env, *cmd, { 5 => tmp })
       assert res, $?.inspect
       tmp.rewind
-      assert_match(/\b10001\s+1\s+-e:1$/, tmp.read)
+      assert_match(/\b1\d{4}\s+[1-9]\d*\s+-e:1$/, tmp.read)
 
       env['MWRAP'] = 'dump_fd:1,dump_min:10000'
       tmp.rewind
@@ -50,14 +51,14 @@ class TestMwrap < Test::Unit::TestCase
       res = system(env, *cmd, { 1 => tmp })
       assert res, $?.inspect
       tmp.rewind
-      assert_match(/\b10001\s+1\s+-e:1$/, tmp.read)
+      assert_match(/\b1\d{4}\s+[1-9]\d*\s+-e:1$/, tmp.read)
 
       tmp.rewind
       tmp.truncate(0)
       env['MWRAP'] = "dump_path:#{tmp.path},dump_min:10000"
       res = system(env, *cmd)
       assert res, $?.inspect
-      assert_match(/\b10001\s+1\s+-e:1$/, tmp.read)
+      assert_match(/\b1\d{4}\s+[1-9]\d*\s+-e:1$/, tmp.read)
 
       tmp.rewind
       tmp.truncate(0)
@@ -98,7 +99,7 @@ class TestMwrap < Test::Unit::TestCase
       tmp.rewind
       buf = tmp.read
       assert_not_match(/\s+-e:1$/, buf)
-      assert_match(/\b20001\s+1\s+-e:3$/, buf)
+      assert_match(/\b2\d{4}\s+[0-9]\d*\s+-e:3$/, buf)
     end
   end
 
@@ -176,8 +177,8 @@ class TestMwrap < Test::Unit::TestCase
       -e GC.disable
       -e keep=("0"*10000)
       -e loc=Mwrap["-e:3"]
-      -e loc.each{|size,gen|p([size,gen,count])}
-    )
+      -e
+    ) + [ 'loc.each{|size,gen|p([size,gen,count]) if size > 10000}' ]
     buf = IO.popen(@@env, cmd, &:read)
     assert_predicate $?, :success?
     assert_match(/\A\[\s*\d+,\s*\d+,\s*\d+\]\s*\z/s, buf)
@@ -230,7 +231,8 @@ class TestMwrap < Test::Unit::TestCase
       loc.name == k or abort 'SourceLocation#name broken'
       loc.total >= 10000 or abort 'SourceLocation#total broken'
       loc.frees == 0 or abort 'SourceLocation#frees broken'
-      loc.allocations == 1 or abort 'SourceLocation#allocations broken'
+      loc.allocations >= 1 or
+        abort "SourceLocation#allocations broken: #{loc.allocations}"
       seen = false
       loc.each do |*x| seen = x end
       seen[1] == loc.total or 'SourceLocation#each broken'
@@ -240,7 +242,9 @@ class TestMwrap < Test::Unit::TestCase
       freed = false
       until freed
         freed = true
-        loc.each do freed = false end
+        loc.each do |size, gen|
+          freed = false if size >= 10000
+        end
       end
       loc.frees == 1 or abort 'SourceLocation#frees broken (after free)'
       Float === loc.mean_lifespan or abort 'mean_lifespan broken'
@@ -264,8 +268,9 @@ class TestMwrap < Test::Unit::TestCase
     assert_separately(+"#{<<~"begin;"}\n#{<<~'end;'}")
     begin;
       require 'mwrap'
-      before = __LINE__
+      before = nil
       res = Mwrap.quiet do |depth|
+        before = __LINE__
         depth == 1 or abort 'depth is not 1'
         ('a' * 10000).clear
         Mwrap.quiet { |d| d == 2 or abort 'depth is not 2' }
@@ -304,7 +309,7 @@ class TestMwrap < Test::Unit::TestCase
         gen <= GC.count && gen >= 0 or abort "bad generation: #{gen}"
         (0 == (addr & 16383)) or abort "addr not aligned: #{'%x' % addr}"
       end
-      nr == ap or abort 'HeapPageBody.each missed page'
+      nr == ap or abort "HeapPageBody.each missed page #{nr} != #{ap}"
       10.times { (1..20000).to_a.map(&:to_s) }
       3.times { GC.start }
       Mwrap::HeapPageBody.stat(h)

[PATCH 3/3] disable HeapPageBody count test for Ruby 3.1

[Eric Wong]

Ruby 3.1+ uses mmap on platforms relevant to us, so we currently
can't account for it with various malloc wrappers.

Tested on Ruby 3.1.2, this is the only change necessary to
support 3.1.x so far; but the functionality is gone unless
we decide to wrap mmap, as well.
---
 test/test_mwrap.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_mwrap.rb b/test/test_mwrap.rb
index fadaef6..c506554 100644
--- a/test/test_mwrap.rb
+++ b/test/test_mwrap.rb
@@ -309,7 +309,9 @@ class TestMwrap < Test::Unit::TestCase
         gen <= GC.count && gen >= 0 or abort "bad generation: #{gen}"
         (0 == (addr & 16383)) or abort "addr not aligned: #{'%x' % addr}"
       end
-      nr == ap or abort "HeapPageBody.each missed page #{nr} != #{ap}"
+      if RUBY_VERSION.to_f < 3.1 # 3.1+ uses mmap on platforms we care about
+        nr == ap or abort "HeapPageBody.each missed page #{nr} != #{ap}"
+      end
       10.times { (1..20000).to_a.map(&:to_s) }
       3.times { GC.start }
       Mwrap::HeapPageBody.stat(h)

Re: [PATCH 3/3] disable HeapPageBody count test for Ruby 3.1

[Eric Wong]

Eric Wong <e@80x24.org> wrote:
> Ruby 3.1+ uses mmap on platforms relevant to us, so we currently
> can't account for it with various malloc wrappers.
> 
> Tested on Ruby 3.1.2, this is the only change necessary to
> support 3.1.x so far; but the functionality is gone unless
> we decide to wrap mmap, as well.

I should note that wrapping mmap probably isn't worth it.
My original reason for tracking heap_page_body was specifically
to track problems related to memalign itself (at least as far as
glibc goes).  Using mmap neatly sidesteps the problems of (glibc)
memalign:

  https://80x24.org/mwrap-public/20180810064943.24287-1-e@80x24.org/