* [PATCH] gc: per-thread malloc accounting [WIP]
@ 2018-06-01 8:33 Eric Wong
0 siblings, 0 replies; only message in thread
From: Eric Wong @ 2018-06-01 8:33 UTC (permalink / raw)
To: spew
---
gc.c | 183 +++++++++++++++++++++++++++++++++--------------
gc.h | 1 +
internal.h | 2 +
thread.c | 2 +
thread_pthread.c | 2 +-
thread_win32.c | 2 +-
vm_core.h | 6 ++
7 files changed, 143 insertions(+), 55 deletions(-)
diff --git a/gc.c b/gc.c
index 9f44381fd5e..f0869cd7530 100644
--- a/gc.c
+++ b/gc.c
@@ -516,7 +516,7 @@ enum gc_mode {
typedef struct rb_objspace {
struct {
size_t limit;
- size_t increase;
+ size_t increase; /* only used for dead threads, now */
#if MALLOC_ALLOCATED_SIZE
size_t allocated_size;
size_t allocations;
@@ -739,7 +739,6 @@ static rb_objspace_t rb_objspace = {{GC_MALLOC_LIMIT_MIN}};
VALUE *ruby_initial_gc_stress_ptr = &ruby_initial_gc_stress;
#define malloc_limit objspace->malloc_params.limit
-#define malloc_increase objspace->malloc_params.increase
#define malloc_allocated_size objspace->malloc_params.allocated_size
#define heap_pages_sorted objspace->heap_pages.sorted
#define heap_allocated_pages objspace->heap_pages.allocated_pages
@@ -6328,14 +6327,43 @@ ready_to_gc(rb_objspace_t *objspace)
}
}
+static size_t
+malloc_increase_all(void) /* cold function only for statistics */
+{
+ rb_objspace_t *objspace = &rb_objspace;
+ size_t inc = objspace->malloc_params.increase;
+ size_t dec = 0;
+ rb_vm_t *vm = GET_VM();
+ rb_thread_t *th = 0;
+
+ list_for_each(&vm->living_threads, th, vmlt_node) {
+ inc += th->gc.malloc_inc;
+ dec += th->gc.malloc_dec;
+ }
+ return inc > dec ? inc - dec : 0;
+}
+
static void
gc_reset_malloc_info(rb_objspace_t *objspace)
{
gc_prof_set_malloc_info(objspace);
{
- size_t inc = ATOMIC_SIZE_EXCHANGE(malloc_increase, 0);
- size_t old_limit = malloc_limit;
-
+ size_t dec = 0;
+ rb_vm_t *vm = GET_VM();
+ rb_thread_t *th = 0;
+ size_t old_limit = malloc_limit;
+ size_t inc = objspace->malloc_params.increase;
+
+ objspace->malloc_params.increase = 0;
+ list_for_each(&vm->living_threads, th, vmlt_node) {
+ inc += th->gc.malloc_inc;
+ dec += th->gc.malloc_dec;
+ th->gc.malloc_inc = th->gc.malloc_dec = 0;
+ }
+ inc = inc > dec ? inc - dec : 0;
+#if RGENGC_ESTIMATE_OLDMALLOC
+ objspace->rgengc.oldmalloc_increase += inc;
+#endif
if (inc > malloc_limit) {
malloc_limit = (size_t)(inc * gc_params.malloc_limit_growth_factor);
if (gc_params.malloc_limit_max > 0 && /* ignore max-check if 0 */
@@ -6641,15 +6669,22 @@ gc_with_gvl(void *ptr)
return (void *)(VALUE)garbage_collect(oar->objspace, oar->reason);
}
+static inline int
+thread_has_gvl_p(const rb_thread_t *th)
+{
+ return (th && th->blocking_region_buffer == 0);
+}
+
static int
-garbage_collect_with_gvl(rb_objspace_t *objspace, int reason)
+garbage_collect_with_gvl(rb_objspace_t *objspace, const rb_thread_t *th,
+ int reason)
{
if (dont_gc) return TRUE;
- if (ruby_thread_has_gvl_p()) {
+ if (thread_has_gvl_p(th)) {
return garbage_collect(objspace, reason);
}
else {
- if (ruby_native_thread_p()) {
+ if (th) {
struct objspace_and_reason oar;
oar.objspace = objspace;
oar.reason = reason;
@@ -7188,7 +7223,7 @@ gc_stat_internal(VALUE hash_or_sym)
SET(total_freed_pages, objspace->profile.total_freed_pages);
SET(total_allocated_objects, objspace->total_allocated_objects);
SET(total_freed_objects, objspace->profile.total_freed_objects);
- SET(malloc_increase_bytes, malloc_increase);
+ SET(malloc_increase_bytes, malloc_increase_all());
SET(malloc_increase_bytes_limit, malloc_limit);
#if USE_RGENGC
SET(minor_gc_count, objspace->profile.minor_gc_count);
@@ -7811,44 +7846,50 @@ atomic_sub_nounderflow(size_t *var, size_t sub)
}
static void
-objspace_malloc_gc_stress(rb_objspace_t *objspace)
+objspace_malloc_gc_stress(rb_objspace_t *objspace, const rb_thread_t *th)
{
- if (ruby_gc_stressful && ruby_native_thread_p()) {
+ if (ruby_gc_stressful && th) {
int reason = GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP |
GPR_FLAG_STRESS | GPR_FLAG_MALLOC;
if (gc_stress_full_mark_after_malloc_p()) {
reason |= GPR_FLAG_FULL_MARK;
}
- garbage_collect_with_gvl(objspace, reason);
+ garbage_collect_with_gvl(objspace, th, reason);
}
}
-static void
-objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type)
+static size_t
+malloc_diff(const rb_thread_t *th)
{
- if (new_size > old_size) {
- ATOMIC_SIZE_ADD(malloc_increase, new_size - old_size);
-#if RGENGC_ESTIMATE_OLDMALLOC
- ATOMIC_SIZE_ADD(objspace->rgengc.oldmalloc_increase, new_size - old_size);
-#endif
- }
- else {
- atomic_sub_nounderflow(&malloc_increase, old_size - new_size);
-#if RGENGC_ESTIMATE_OLDMALLOC
- atomic_sub_nounderflow(&objspace->rgengc.oldmalloc_increase, old_size - new_size);
-#endif
+ return th->gc.malloc_inc;
+ if (th->gc.malloc_inc > th->gc.malloc_dec) {
+ return th->gc.malloc_inc - th->gc.malloc_dec;
}
+ return 0;
+}
- if (type == MEMOP_TYPE_MALLOC) {
- retry:
- if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc) {
- if (ruby_thread_has_gvl_p() && is_lazy_sweeping(heap_eden)) {
- gc_rest(objspace); /* gc_rest can reduce malloc_increase */
- goto retry;
- }
- garbage_collect_with_gvl(objspace, GPR_FLAG_MALLOC);
- }
+static void
+objspace_malloc_increase(rb_objspace_t *objspace, rb_thread_t *th,
+ void *mem, size_t new_size, size_t old_size, enum memop_type type)
+{
+ if (th) {
+ if (new_size) {
+ th->gc.malloc_inc += new_size;
+ }
+ if (old_size) {
+ th->gc.malloc_dec += old_size;
+ }
+ if (type == MEMOP_TYPE_MALLOC) {
+ retry:
+ if (th && malloc_diff(th) > malloc_limit && !dont_gc) {
+ if (thread_has_gvl_p(th) && is_lazy_sweeping(heap_eden)) {
+ gc_rest(objspace); /* gc_rest can reduce malloc_diff */
+ goto retry;
+ }
+ garbage_collect_with_gvl(objspace, th, GPR_FLAG_MALLOC);
+ }
+ }
}
#if MALLOC_ALLOCATED_SIZE
@@ -7896,23 +7937,36 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si
#endif
}
-static inline size_t
-objspace_malloc_prepare(rb_objspace_t *objspace, size_t size)
+static size_t
+objspace_malloc_prepare(rb_objspace_t *objspace, const rb_thread_t *th,
+ size_t size)
{
if (size == 0) size = 1;
#if CALC_EXACT_MALLOC_SIZE
size += sizeof(size_t);
#endif
+ if (thread_has_gvl_p(th) && !dont_gc) {
+ if (is_lazy_sweeping(heap_eden)) {
+ gc_sweep_continue(objspace, heap_eden);
+ }
+ else if ((malloc_diff(th) + size + sizeof(size_t) * 2) > malloc_limit) {
+ gc_rest(objspace); /* gc_rest can reduce malloc_diff */
+ if ((malloc_diff(th) + size + sizeof(size_t) * 2) > malloc_limit) {
+ garbage_collect_with_gvl(objspace, th, GPR_FLAG_MALLOC);
+ }
+ }
+ }
return size;
}
static inline void *
-objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size)
+objspace_malloc_fixup(rb_objspace_t *objspace, rb_thread_t *th,
+ void *mem, size_t size)
{
size = objspace_malloc_size(objspace, mem, size);
- objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC);
+ objspace_malloc_increase(objspace, th, mem, size, 0, MEMOP_TYPE_MALLOC);
#if CALC_EXACT_MALLOC_SIZE
((size_t *)mem)[0] = size;
@@ -7922,10 +7976,10 @@ objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size)
return mem;
}
-#define TRY_WITH_GC(alloc) do { \
- objspace_malloc_gc_stress(objspace); \
+#define TRY_WITH_GC(th, alloc) do { \
+ objspace_malloc_gc_stress(objspace, th); \
if (!(alloc) && \
- (!garbage_collect_with_gvl(objspace, GPR_FLAG_FULL_MARK | \
+ (!garbage_collect_with_gvl(objspace, th, GPR_FLAG_FULL_MARK | \
GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP | \
GPR_FLAG_MALLOC) || \
!(alloc))) { \
@@ -7940,10 +7994,11 @@ static void *
objspace_xmalloc0(rb_objspace_t *objspace, size_t size)
{
void *mem;
+ rb_thread_t *th = ruby_thread_from_native();
- size = objspace_malloc_prepare(objspace, size);
- TRY_WITH_GC(mem = malloc(size));
- return objspace_malloc_fixup(objspace, mem, size);
+ size = objspace_malloc_prepare(objspace, th, size);
+ TRY_WITH_GC(th, mem = malloc(size));
+ return objspace_malloc_fixup(objspace, th, mem, size);
}
static inline size_t
@@ -7960,6 +8015,7 @@ static void *
objspace_xrealloc(rb_objspace_t *objspace, void *ptr, size_t new_size, size_t old_size)
{
void *mem;
+ rb_thread_t *th;
if (!ptr) return objspace_xmalloc0(objspace, new_size);
@@ -7979,8 +8035,9 @@ objspace_xrealloc(rb_objspace_t *objspace, void *ptr, size_t new_size, size_t ol
old_size = ((size_t *)ptr)[0];
#endif
+ th = ruby_thread_from_native();
old_size = objspace_malloc_size(objspace, ptr, old_size);
- TRY_WITH_GC(mem = realloc(ptr, new_size));
+ TRY_WITH_GC(th, mem = realloc(ptr, new_size));
new_size = objspace_malloc_size(objspace, mem, new_size);
#if CALC_EXACT_MALLOC_SIZE
@@ -7988,7 +8045,8 @@ objspace_xrealloc(rb_objspace_t *objspace, void *ptr, size_t new_size, size_t ol
mem = (size_t *)mem + 1;
#endif
- objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC);
+ objspace_malloc_increase(objspace, th, mem, new_size, old_size,
+ MEMOP_TYPE_REALLOC);
return mem;
}
@@ -7996,6 +8054,7 @@ objspace_xrealloc(rb_objspace_t *objspace, void *ptr, size_t new_size, size_t ol
static void
objspace_xfree(rb_objspace_t *objspace, void *ptr, size_t old_size)
{
+ rb_thread_t *th = ruby_thread_from_native();
#if CALC_EXACT_MALLOC_SIZE
ptr = ((size_t *)ptr) - 1;
old_size = ((size_t*)ptr)[0];
@@ -8004,7 +8063,7 @@ objspace_xfree(rb_objspace_t *objspace, void *ptr, size_t old_size)
free(ptr);
- objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE);
+ objspace_malloc_increase(objspace, th, ptr, 0, old_size, MEMOP_TYPE_FREE);
}
static void *
@@ -8040,10 +8099,11 @@ static void *
objspace_xcalloc(rb_objspace_t *objspace, size_t size)
{
void *mem;
+ rb_thread_t *th = ruby_thread_from_native();
- size = objspace_malloc_prepare(objspace, size);
- TRY_WITH_GC(mem = calloc(1, size));
- return objspace_malloc_fixup(objspace, mem, size);
+ size = objspace_malloc_prepare(objspace, th, size);
+ TRY_WITH_GC(th, mem = calloc(1, size));
+ return objspace_malloc_fixup(objspace, th, mem, size);
}
void *
@@ -8213,11 +8273,28 @@ void
rb_gc_adjust_memory_usage(ssize_t diff)
{
rb_objspace_t *objspace = &rb_objspace;
+ rb_thread_t *th = ruby_thread_from_native();
+
if (diff > 0) {
- objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC);
+ objspace_malloc_increase(objspace, th, 0, diff, 0, MEMOP_TYPE_REALLOC);
}
else if (diff < 0) {
- objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC);
+ objspace_malloc_increase(objspace, th, 0, 0, -diff, MEMOP_TYPE_REALLOC);
+ }
+}
+
+void
+rb_gc_thread_at_exit(const rb_thread_t *th)
+{
+ rb_objspace_t *objspace = &rb_objspace;
+ objspace->malloc_params.increase += th->gc.malloc_inc;
+ if (objspace->malloc_params.increase > malloc_limit && !dont_gc) {
+ if (is_lazy_sweeping(heap_eden)) {
+ gc_rest(objspace);
+ }
+ else {
+ garbage_collect_with_gvl(objspace, th, GPR_FLAG_MALLOC);
+ }
}
}
@@ -8826,7 +8903,7 @@ gc_prof_set_malloc_info(rb_objspace_t *objspace)
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
- record->allocate_increase = malloc_increase;
+ record->allocate_increase = malloc_increase_all();
record->allocate_limit = malloc_limit;
}
#endif
diff --git a/gc.h b/gc.h
index 2c91e06620a..324a1dcff54 100644
--- a/gc.h
+++ b/gc.h
@@ -91,6 +91,7 @@ const char *rb_raw_obj_info(char *buff, const int buff_size, VALUE obj);
void rb_obj_info_dump(VALUE obj);
struct rb_thread_struct;
+void rb_gc_thread_at_exit(const struct rb_thread_struct *);
RUBY_SYMBOL_EXPORT_BEGIN
diff --git a/internal.h b/internal.h
index a072fa30dd7..c2c0679fdd3 100644
--- a/internal.h
+++ b/internal.h
@@ -2047,6 +2047,8 @@ VALUE rb_str_upto_endless_each(VALUE, int (*each)(VALUE, VALUE), VALUE);
/* thread.c (export) */
int ruby_thread_has_gvl_p(void); /* for ext/fiddle/closure.c */
+struct rb_thread_struct;
+struct rb_thread_struct *ruby_thread_from_native(void); /* for gc.c */
/* util.c (export) */
extern const signed char ruby_digit36_to_number_table[];
diff --git a/thread.c b/thread.c
index 214df5b0dff..040aa49cde1 100644
--- a/thread.c
+++ b/thread.c
@@ -725,6 +725,8 @@ thread_start_func_2(rb_thread_t *th, VALUE *stack_start, VALUE *register_stack_s
(void *)th, th->locking_mutex);
}
+ rb_gc_thread_at_exit(th);
+ ruby_thread_set_native(0);
/* delete self other than main thread from living_threads */
rb_vm_living_threads_remove(th->vm, th);
if (main_th->status == THREAD_KILLED && rb_thread_alone()) {
diff --git a/thread_pthread.c b/thread_pthread.c
index e17ca36819e..32c607e3c42 100644
--- a/thread_pthread.c
+++ b/thread_pthread.c
@@ -380,7 +380,7 @@ null_func(int i)
/* null */
}
-static rb_thread_t *
+rb_thread_t *
ruby_thread_from_native(void)
{
return pthread_getspecific(ruby_native_thread_key);
diff --git a/thread_win32.c b/thread_win32.c
index 3c6d0e73693..3e25969c24a 100644
--- a/thread_win32.c
+++ b/thread_win32.c
@@ -127,7 +127,7 @@ gvl_destroy(rb_vm_t *vm)
CloseHandle(vm->gvl.lock);
}
-static rb_thread_t *
+rb_thread_t *
ruby_thread_from_native(void)
{
return TlsGetValue(ruby_native_thread_key);
diff --git a/vm_core.h b/vm_core.h
index 8e34ff14a57..d56cca13ef6 100644
--- a/vm_core.h
+++ b/vm_core.h
@@ -892,6 +892,12 @@ typedef struct rb_thread_struct {
unsigned int report_on_exception: 1;
uint32_t running_time_us; /* 12500..800000 */
VALUE name;
+
+ /* reset at each GC */
+ struct {
+ size_t malloc_inc;
+ size_t malloc_dec;
+ } gc;
} rb_thread_t;
typedef enum {
--
EW
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2018-06-01 8:33 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2018-06-01 8:33 [PATCH] gc: per-thread malloc accounting [WIP] Eric Wong
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).