From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on dcvr.yhbt.net X-Spam-Flag: YES X-Spam-Level: *** X-Spam-ASN: AS6939 65.19.128.0/18 X-Spam-Status: Yes, score=3.7 required=3.0 tests=BAYES_00,RCVD_IN_MSPIKE_BL, RCVD_IN_MSPIKE_ZBI,RCVD_IN_SORBS_WEB,RCVD_IN_XBL,RDNS_NONE,SPF_FAIL, SPF_HELO_FAIL,TO_EQ_FM_DOM_SPF_FAIL shortcircuit=no autolearn=no autolearn_force=no version=3.4.0 X-Spam-Report: * 0.4 RCVD_IN_XBL RBL: Received via a relay in Spamhaus XBL * [65.19.167.132 listed in zen.spamhaus.org] * 1.5 RCVD_IN_SORBS_WEB RBL: SORBS: sender is an abusable web server * [65.19.167.132 listed in dnsbl.sorbs.net] * 0.0 SPF_FAIL SPF: sender does not match SPF record (fail) * [SPF failed: Please see http://www.openspf.org/Why?s=mfrom;id=e%4080x24.org;ip=65.19.167.132;r=dcvr.yhbt.net] * 0.0 SPF_HELO_FAIL SPF: HELO does not match SPF record (fail) * [SPF failed: Please see http://www.openspf.org/Why?s=helo;id=80x24.org;ip=65.19.167.132;r=dcvr.yhbt.net] * -3.0 BAYES_00 BODY: Bayes spam probability is 0 to 1% * [score: 0.0000] * 0.8 RDNS_NONE Delivered to internal network by a host with no rDNS * 0.0 RCVD_IN_MSPIKE_BL Mailspike blacklisted * 4.0 RCVD_IN_MSPIKE_ZBI No description available. * 0.0 TO_EQ_FM_DOM_SPF_FAIL To domain == From domain and external SPF * failed Received: from 80x24.org (unknown [65.19.167.132]) by dcvr.yhbt.net (Postfix) with ESMTP id DE02E1F406 for ; Thu, 17 May 2018 02:42:20 +0000 (UTC) From: Eric Wong To: spew@80x24.org Subject: [PATCH] gc.c: use monotonic counters for objspace_malloc_increase Date: Thu, 17 May 2018 02:42:18 +0000 Message-Id: <20180517024218.30730-1-e@80x24.org> List-Id: atomic_sub_nounderflow is expensive and objspace_malloc_increase was showing up near the top of some `perf` profiles. The new implementation allows the compiler to inline and eliminate some branches from objspace_malloc_increase. This consistently improves bm_so_count_words benchmark by around 10% on my hardware. name built so_count_words 1.107 --- gc.c | 87 +++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 25 deletions(-) diff --git a/gc.c b/gc.c index 6ab41e69435..7264a62fced 100644 --- a/gc.c +++ b/gc.c @@ -511,10 +511,15 @@ enum gc_mode { gc_mode_sweeping }; +struct monoctr { + size_t add; + size_t sub; +}; + typedef struct rb_objspace { struct { size_t limit; - size_t increase; + struct monoctr m; #if MALLOC_ALLOCATED_SIZE size_t allocated_size; size_t allocations; @@ -631,7 +636,7 @@ typedef struct rb_objspace { size_t old_objects_limit; #if RGENGC_ESTIMATE_OLDMALLOC - size_t oldmalloc_increase; + struct monoctr oldmalloc; size_t oldmalloc_increase_limit; #endif @@ -737,7 +742,6 @@ static rb_objspace_t rb_objspace = {{GC_MALLOC_LIMIT_MIN}}; VALUE *ruby_initial_gc_stress_ptr = &ruby_initial_gc_stress; #define malloc_limit objspace->malloc_params.limit -#define malloc_increase objspace->malloc_params.increase #define malloc_allocated_size objspace->malloc_params.allocated_size #define heap_pages_sorted objspace->heap_pages.sorted #define heap_allocated_pages objspace->heap_pages.allocated_pages @@ -5094,9 +5098,9 @@ gc_check_after_marks_i(st_data_t k, st_data_t v, void *ptr) static void gc_marks_check(rb_objspace_t *objspace, int (*checker_func)(ANYARGS), const char *checker_name) { - size_t saved_malloc_increase = objspace->malloc_params.increase; + struct monoctr saved_malloc = objspace->malloc_params.m; #if RGENGC_ESTIMATE_OLDMALLOC - size_t saved_oldmalloc_increase = objspace->rgengc.oldmalloc_increase; + struct monoctr saved_oldmalloc = objspace->rgengc.oldmalloc; #endif VALUE already_disabled = rb_gc_disable(); @@ -5117,9 +5121,9 @@ gc_marks_check(rb_objspace_t *objspace, int (*checker_func)(ANYARGS), const char objspace->rgengc.allrefs_table = 0; if (already_disabled == Qfalse) rb_gc_enable(); - objspace->malloc_params.increase = saved_malloc_increase; + objspace->malloc_params.m = saved_malloc; #if RGENGC_ESTIMATE_OLDMALLOC - objspace->rgengc.oldmalloc_increase = saved_oldmalloc_increase; + objspace->rgengc.oldmalloc = saved_oldmalloc; #endif } #endif /* RGENGC_CHECK_MODE >= 4 */ @@ -6326,12 +6330,46 @@ ready_to_gc(rb_objspace_t *objspace) } } +static size_t +monoctr_read(const struct monoctr *mc) +{ + size_t add = mc->add; + size_t sub = mc->sub; + size_t diff = add - sub; + + return (diff <= add) ? diff : 0; +} + +static size_t +monoctr_xchg0(struct monoctr *mc) +{ + size_t add = ATOMIC_SIZE_EXCHANGE(mc->add, 0); + size_t sub = ATOMIC_SIZE_EXCHANGE(mc->sub, 0); + size_t diff = add - sub; + + return (diff <= add) ? diff : 0; +} + +static size_t +malloc_increase(const rb_objspace_t *objspace) +{ + return monoctr_read(&objspace->malloc_params.m); +} + +#if RGENGC_ESTIMATE_OLDMALLOC +static size_t +oldmalloc_increase(const rb_objspace_t *objspace) +{ + return monoctr_read(&objspace->rgengc.oldmalloc); +} +#endif + static void gc_reset_malloc_info(rb_objspace_t *objspace) { gc_prof_set_malloc_info(objspace); { - size_t inc = ATOMIC_SIZE_EXCHANGE(malloc_increase, 0); + size_t inc = monoctr_xchg0(&objspace->malloc_params.m); size_t old_limit = malloc_limit; if (inc > malloc_limit) { @@ -6363,7 +6401,7 @@ gc_reset_malloc_info(rb_objspace_t *objspace) /* reset oldmalloc info */ #if RGENGC_ESTIMATE_OLDMALLOC if (!is_full_marking(objspace)) { - if (objspace->rgengc.oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) { + if (oldmalloc_increase(objspace) > objspace->rgengc.oldmalloc_increase_limit) { objspace->rgengc.need_major_gc |= GPR_FLAG_MAJOR_BY_OLDMALLOC; objspace->rgengc.oldmalloc_increase_limit = (size_t)(objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor); @@ -6376,13 +6414,13 @@ gc_reset_malloc_info(rb_objspace_t *objspace) if (0) fprintf(stderr, "%d\t%d\t%u\t%u\t%d\n", (int)rb_gc_count(), (int)objspace->rgengc.need_major_gc, - (unsigned int)objspace->rgengc.oldmalloc_increase, + (unsigned int)oldmalloc_increase(objspace), (unsigned int)objspace->rgengc.oldmalloc_increase_limit, (unsigned int)gc_params.oldmalloc_limit_max); } else { /* major GC */ - objspace->rgengc.oldmalloc_increase = 0; + MEMZERO(&objspace->rgengc.oldmalloc, struct monoctr, 1); if ((objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) { objspace->rgengc.oldmalloc_increase_limit = @@ -7180,7 +7218,7 @@ gc_stat_internal(VALUE hash_or_sym) SET(total_freed_pages, objspace->profile.total_freed_pages); SET(total_allocated_objects, objspace->total_allocated_objects); SET(total_freed_objects, objspace->profile.total_freed_objects); - SET(malloc_increase_bytes, malloc_increase); + SET(malloc_increase_bytes, malloc_increase(objspace)); SET(malloc_increase_bytes_limit, malloc_limit); #if USE_RGENGC SET(minor_gc_count, objspace->profile.minor_gc_count); @@ -7190,7 +7228,7 @@ gc_stat_internal(VALUE hash_or_sym) SET(old_objects, objspace->rgengc.old_objects); SET(old_objects_limit, objspace->rgengc.old_objects_limit); #if RGENGC_ESTIMATE_OLDMALLOC - SET(oldmalloc_increase_bytes, objspace->rgengc.oldmalloc_increase); + SET(oldmalloc_increase_bytes, oldmalloc_increase(objspace)); SET(oldmalloc_increase_bytes_limit, objspace->rgengc.oldmalloc_increase_limit); #endif @@ -7790,6 +7828,7 @@ enum memop_type { MEMOP_TYPE_REALLOC = 3 }; +#if MALLOC_ALLOCATED_SIZE static inline void atomic_sub_nounderflow(size_t *var, size_t sub) { @@ -7801,6 +7840,7 @@ atomic_sub_nounderflow(size_t *var, size_t sub) if (ATOMIC_SIZE_CAS(*var, val, val-sub) == val) break; } } +#endif static void objspace_malloc_gc_stress(rb_objspace_t *objspace) @@ -7813,22 +7853,19 @@ objspace_malloc_gc_stress(rb_objspace_t *objspace) static void objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type) { - if (new_size > old_size) { - ATOMIC_SIZE_ADD(malloc_increase, new_size - old_size); -#if RGENGC_ESTIMATE_OLDMALLOC - ATOMIC_SIZE_ADD(objspace->rgengc.oldmalloc_increase, new_size - old_size); -#endif + /* n.b. these checks for non-zero get inlined */ + if (new_size) { + ATOMIC_SIZE_ADD(objspace->malloc_params.m.add, new_size); + ATOMIC_SIZE_ADD(objspace->rgengc.oldmalloc.add, new_size); } - else { - atomic_sub_nounderflow(&malloc_increase, old_size - new_size); -#if RGENGC_ESTIMATE_OLDMALLOC - atomic_sub_nounderflow(&objspace->rgengc.oldmalloc_increase, old_size - new_size); -#endif + if (old_size) { + ATOMIC_SIZE_ADD(objspace->malloc_params.m.sub, old_size); + ATOMIC_SIZE_ADD(objspace->rgengc.oldmalloc.sub, old_size); } if (type == MEMOP_TYPE_MALLOC) { retry: - if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc) { + if (malloc_increase(objspace) > malloc_limit && ruby_native_thread_p() && !dont_gc) { if (ruby_thread_has_gvl_p() && is_lazy_sweeping(heap_eden)) { gc_rest(objspace); /* gc_rest can reduce malloc_increase */ goto retry; @@ -8810,7 +8847,7 @@ gc_prof_set_malloc_info(rb_objspace_t *objspace) #if GC_PROFILE_MORE_DETAIL if (gc_prof_enabled(objspace)) { gc_profile_record *record = gc_prof_record(objspace); - record->allocate_increase = malloc_increase; + record->allocate_increase = malloc_increase(objspace); record->allocate_limit = malloc_limit; } #endif -- EW