diff options
Diffstat (limited to 'mwrap_core.h')
-rw-r--r-- | mwrap_core.h | 263 |
1 files changed, 157 insertions, 106 deletions
diff --git a/mwrap_core.h b/mwrap_core.h index 179f136..ec08ee1 100644 --- a/mwrap_core.h +++ b/mwrap_core.h @@ -67,6 +67,7 @@ typedef void COP; * hardly anybody still uses it). */ static size_t total_bytes_inc, total_bytes_dec; +static uint32_t bt_req_depth; #if MWRAP_PERL extern pthread_key_t __attribute__((weak)) PL_thr_key; @@ -131,7 +132,22 @@ static void *my_mempcpy(void *dest, const void *src, size_t n) /* stolen from glibc: */ #define RETURN_ADDRESS(nr) \ - (uintptr_t)(__builtin_extract_return_addr(__builtin_return_address(nr))) + __builtin_extract_return_addr(__builtin_return_address(nr)) + +#define SRC_LOC_BT(bt) union stk_bt bt; do { \ + uint32_t depth = locating ? 1 : bt_req_depth; \ + switch (depth) { \ + case 0: \ + case 1: bt.sl.bt_len = 1; bt.sl.bt[0] = RETURN_ADDRESS(0); break; \ + default: /* skip 1st level of BT since thats our function */ \ + mwrap_assert(bt_req_depth <= MWRAP_BT_MAX); \ + ++locating; \ + long n = (long)backtrace(&bt.sl.bt[-1], bt_req_depth); \ + --locating; \ + bt.sl.bt_len = n <= 1 ? 0 : (uint32_t)n - 1; \ + if (n > 1) mwrap_assert(bt.sl.bt[0] == RETURN_ADDRESS(0)); \ + } \ +} while (0) /* * only for interpreted sources (Perl/Ruby/etc), not backtrace_symbols* files @@ -163,6 +179,11 @@ struct src_loc { void *bt[]; }; +#ifdef static_assert +static_assert(sizeof(struct src_file *) == sizeof(size_t), + "size_t is the same size as a pointer"); +#endif + /* * Every allocation has this in the header, maintain alignment with malloc * Do not expose this to Perl code because of use-after-free concerns. @@ -180,12 +201,16 @@ struct alloc_hdr { size_t size; }; -static MWRAP_TSD union { - char kbuf[sizeof(struct src_file) + PATH_MAX]; - char btbuf[sizeof(struct src_loc) + sizeof(uintptr_t) * MWRAP_BT_MAX]; - struct src_file src_file; - struct src_loc src_loc; -} tsd; +/* on-stack structures */ +union stk_sf { + struct src_file sf; + char buf_[sizeof(struct src_file) + PATH_MAX]; +}; + +union stk_bt { + struct src_loc sl; + char buf_[sizeof(struct src_loc) + sizeof(void *) * MWRAP_BT_MAX]; +}; static struct alloc_hdr *ptr2hdr(void *p) { @@ -207,6 +232,11 @@ static size_t bt_bytelen(const struct src_loc *l) return sizeof(l->bt[0]) * l->bt_len; } +static size_t src_loc_hash_len(const struct src_loc *l) +{ + return sizeof(l->f) + sizeof(l->lineno) + + bt_bytelen(l); +} + static int loc_eq(struct cds_lfht_node *node, const void *key) { const struct src_loc *existing; @@ -214,10 +244,8 @@ static int loc_eq(struct cds_lfht_node *node, const void *key) existing = caa_container_of(node, struct src_loc, hnode); - return (k->f == existing->f && - k->lineno == existing->lineno && - k->bt_len == existing->bt_len && - !memcmp(k->bt, existing->bt, bt_bytelen(k))); + return (k->bt_len == existing->bt_len && + !memcmp(&k->f, &existing->f, src_loc_hash_len(k))); } static int fn_eq(struct cds_lfht_node *node, const void *key) @@ -314,8 +342,7 @@ static uint32_t do_hash(const void *p, size_t len) static void hash_src_loc(struct src_loc *l) { - l->loc_hash = do_hash(&l->f, sizeof(l->f) + sizeof(l->lineno) + - + bt_bytelen(l)); + l->loc_hash = do_hash(&l->f, src_loc_hash_len(l)); } static struct src_file *src_file_get(struct cds_lfht *t, struct src_file *k, @@ -338,29 +365,34 @@ static struct src_file *src_file_get(struct cds_lfht *t, struct src_file *k, return cur ? caa_container_of(cur, struct src_file, nd) : NULL; } -static struct src_loc *assign_line(size_t size, const char *fn, unsigned lineno) +#if !MWRAP_PERL +# define CopFILE(cop) NULL +# define CopLINE(cop) 0 +#endif +static struct src_loc *assign_line(size_t size, const COP *cop, + struct src_loc *sl) { /* avoid vsnprintf or anything which could call malloc here: */ - size_t len; + if (!cop) return NULL; + const char *fn = CopFILE(cop); + if (!fn) return NULL; + unsigned lineno = CopLINE(cop); struct src_file *f; - struct src_file *k = &tsd.src_file; - struct src_loc *l; + union stk_sf sf; struct cds_lfht_node *cur; struct cds_lfht *t = CMM_LOAD_SHARED(files); mwrap_assert(t); - if (!fn) - return NULL; - len = strlen(fn); + size_t len = strlen(fn); if (len >= PATH_MAX) len = PATH_MAX - 1; again: - f = src_file_get(t, k, fn, len); + f = src_file_get(t, &sf.sf, fn, len); if (!f) { /* doesn't exist, add a new one */ f = real_malloc(sizeof(*f) + len + 1); if (!f) return NULL; - memcpy(f, k, sizeof(*f) + len + 1); + memcpy(f, &sf.sf, sizeof(*f) + len + 1); cur = cds_lfht_add_unique(t, f->fn_hash, fn_eq, f, &f->nd); if (cur != &f->nd) { /* lost race */ rcu_read_unlock(); @@ -369,41 +401,34 @@ again: goto again; } } - l = &tsd.src_loc; - l->total = size; - l->f = f; - l->lineno = lineno; - l->bt_len = 0; - hash_src_loc(l); - return totals_add_rcu(l); + sl->total = size; + sl->f = f; + sl->lineno = lineno; + if (f && !bt_req_depth) + sl->bt_len = 0; + hash_src_loc(sl); + return totals_add_rcu(sl); } static struct src_loc * -update_stats_rcu_lock(size_t *generation, size_t size, uintptr_t caller) +update_stats_rcu_lock(size_t *generation, size_t size, struct src_loc *sl) { - struct src_loc *k, *ret = 0; struct cds_lfht *t = CMM_LOAD_SHARED(totals); - const COP *cop = NULL; + struct src_loc *ret = NULL; if (caa_unlikely(!t)) return 0; /* not initialized */ if (locating++) goto out; /* do not recurse into another *alloc */ *generation = uatomic_add_return(&total_bytes_inc, size); - cop = mwp_curcop(); + const COP *cop = mwp_curcop(); rcu_read_lock(); -#if MWRAP_PERL - if (cop) - ret = assign_line(size, OutCopFILE(cop), CopLINE(cop)); -#endif /* MWRAP_PERL */ + ret = assign_line(size, cop, sl); if (!ret) { /* no associated Perl code, just C/C++ */ - k = &tsd.src_loc; - k->total = size; - k->f = NULL; - k->lineno = 0; - k->bt[0] = (void *)caller; - k->bt_len = 1; - hash_src_loc(k); - ret = totals_add_rcu(k); + sl->total = size; + sl->f = NULL; + sl->lineno = 0; + hash_src_loc(sl); + ret = totals_add_rcu(sl); } out: --locating; @@ -482,7 +507,7 @@ static bool is_power_of_two(size_t n) } static int -mwrap_memalign(void **pp, size_t alignment, size_t size, uintptr_t caller) +mwrap_memalign(void **pp, size_t alignment, size_t size, struct src_loc *sl) { struct src_loc *l; struct alloc_hdr *h; @@ -507,7 +532,7 @@ mwrap_memalign(void **pp, size_t alignment, size_t size, uintptr_t caller) __builtin_add_overflow(asize, sizeof(struct alloc_hdr), &asize)) return ENOMEM; - l = update_stats_rcu_lock(&generation, size, caller); + l = update_stats_rcu_lock(&generation, size, sl); real = real_malloc(asize); if (real) { @@ -533,13 +558,15 @@ static void *memalign_result(int err, void *p) void *memalign(size_t alignment, size_t size) { void *p = NULL; - int err = mwrap_memalign(&p, alignment, size, RETURN_ADDRESS(0)); + SRC_LOC_BT(bt); + int err = mwrap_memalign(&p, alignment, size, &bt.sl); return memalign_result(err, p); } int posix_memalign(void **p, size_t alignment, size_t size) { - return mwrap_memalign(p, alignment, size, RETURN_ADDRESS(0)); + SRC_LOC_BT(bt); + return mwrap_memalign(p, alignment, size, &bt.sl); } /* these aliases aren't needed for glibc, not sure about other libcs... */ @@ -549,12 +576,10 @@ void cfree(void *) __attribute__((__nothrow__)) void *valloc(size_t size) { - void *p = NULL; - int err; - ensure_initialization(); - err = mwrap_memalign(&p, mparams.page_size, - size, RETURN_ADDRESS(0)); + SRC_LOC_BT(bt); + void *p = NULL; + int err = mwrap_memalign(&p, mparams.page_size, size, &bt.sl); return memalign_result(err, p); } @@ -576,7 +601,6 @@ static size_t size_align(size_t size, size_t alignment) void *pvalloc(size_t size) { void *p = NULL; - int err; ensure_initialization(); @@ -585,24 +609,23 @@ void *pvalloc(size_t size) return 0; } size = size_align(size, mparams.page_size); - err = mwrap_memalign(&p, mparams.page_size, - size, RETURN_ADDRESS(0)); + SRC_LOC_BT(bt); + int err = mwrap_memalign(&p, mparams.page_size, size, &bt.sl); return memalign_result(err, p); } void *malloc(size_t size) { - struct src_loc *l; - struct alloc_hdr *h; size_t asize; - void *p; - size_t generation = 0; if (__builtin_add_overflow(size, sizeof(struct alloc_hdr), &asize)) goto enomem; - l = update_stats_rcu_lock(&generation, size, RETURN_ADDRESS(0)); - p = h = real_malloc(asize); + size_t generation = 0; + SRC_LOC_BT(bt); + struct src_loc *l = update_stats_rcu_lock(&generation, size, &bt.sl); + struct alloc_hdr *h; + void *p = h = real_malloc(asize); if (h) { alloc_insert_rcu(l, h, size, h, generation); p = hdr2ptr(h); @@ -617,9 +640,6 @@ enomem: void *calloc(size_t nmemb, size_t size) { - void *p; - struct src_loc *l; - struct alloc_hdr *h; size_t asize; size_t generation = 0; @@ -631,8 +651,10 @@ void *calloc(size_t nmemb, size_t size) errno = ENOMEM; return 0; } - l = update_stats_rcu_lock(&generation, size, RETURN_ADDRESS(0)); - p = h = real_malloc(asize); + struct alloc_hdr *h; + SRC_LOC_BT(bt); + struct src_loc *l = update_stats_rcu_lock(&generation, size, &bt.sl); + void *p = h = real_malloc(asize); if (p) { alloc_insert_rcu(l, h, size, h, generation); p = hdr2ptr(h); @@ -645,11 +667,7 @@ void *calloc(size_t nmemb, size_t size) void *realloc(void *ptr, size_t size) { - void *p; - struct src_loc *l; - struct alloc_hdr *h; size_t asize; - size_t generation = 0; if (!size) { free(ptr); @@ -659,8 +677,11 @@ void *realloc(void *ptr, size_t size) errno = ENOMEM; return 0; } - l = update_stats_rcu_lock(&generation, size, RETURN_ADDRESS(0)); - p = h = real_malloc(asize); + struct alloc_hdr *h; + size_t generation = 0; + SRC_LOC_BT(bt); + struct src_loc *l = update_stats_rcu_lock(&generation, size, &bt.sl); + void *p = h = real_malloc(asize); if (p) { alloc_insert_rcu(l, h, size, h, generation); p = hdr2ptr(h); @@ -752,15 +773,14 @@ static struct src_loc *src_loc_lookup(const char *str, size_t len) char *c = memrchr(str, ':', len); const char *end = str + len; unsigned lineno; - size_t fn_len; - struct src_file *f; struct src_loc *l = NULL; struct cds_lfht *t = CMM_LOAD_SHARED(files); + union stk_sf sf; if (!c || c == end || !t) return NULL; - fn_len = c - str; + size_t fn_len = c - str; c++; if (*c == '-') { lineno = UINT_MAX; @@ -774,16 +794,16 @@ static struct src_loc *src_loc_lookup(const char *str, size_t len) } } rcu_read_lock(); - f = src_file_get(t, &tsd.src_file, str, fn_len); + struct src_file *f = src_file_get(t, &sf.sf, str, fn_len); t = CMM_LOAD_SHARED(totals); if (f && t) { - struct src_loc *k = &tsd.src_loc; + struct src_loc k; - k->f = f; - k->lineno = lineno; - k->bt_len = 0; - hash_src_loc(k); - l = src_loc_get(t, k); + k.f = f; + k.lineno = lineno; + k.bt_len = 0; + hash_src_loc(&k); + l = src_loc_get(t, &k); } rcu_read_unlock(); return l; @@ -815,13 +835,12 @@ static void dump_destructor(void) (dump_path += sizeof("dump_path")) && *dump_path) { char *end = strchr(dump_path, ','); + char buf[PATH_MAX]; if (end) { - char *tmp = tsd.kbuf; - mwrap_assert((end - dump_path) < - (intptr_t)sizeof(tsd.kbuf)); - end = mempcpy(tmp, dump_path, end - dump_path); + mwrap_assert((end - dump_path) < (intptr_t)sizeof(buf)); + end = mempcpy(buf, dump_path, end - dump_path); *end = 0; - dump_path = tmp; + dump_path = buf; } dump_fd = open(dump_path, O_CLOEXEC|O_WRONLY|O_APPEND|O_CREAT, 0666); @@ -884,31 +903,48 @@ static void mwrap_reset(void) static struct src_loc *mwrap_get(const char *str, size_t len) { - struct src_loc *l = NULL; void *p; - if (len >= PATH_MAX) - return l; - if (extract_addr(str, len, &p)) { + if (!extract_addr(str, len, &p)) + return src_loc_lookup(str, len); + + union stk_bt k; + struct cds_lfht *t = CMM_LOAD_SHARED(totals); + + if (!t) return NULL; + k.sl.f = NULL; + k.sl.lineno = 0; + k.sl.bt[0] = p; + k.sl.bt_len = 1; + hash_src_loc(&k.sl); + rcu_read_lock(); + struct src_loc *l = src_loc_get(t, &k.sl); + rcu_read_unlock(); + return l; +} + +static struct src_loc *mwrap_get_bin(const char *buf, size_t len) +{ + if ((len % sizeof(void *)) == 0 && len >= (2 * sizeof(void *))) { + union stk_bt k; struct cds_lfht *t = CMM_LOAD_SHARED(totals); - struct src_loc *k; + if (!t) return NULL; - if (!t) - return l; - k = &tsd.src_loc; + k.sl.bt_len = len / sizeof(void *); + k.sl.bt_len -= 2; /* lineno + src_file *f */ - k->f = NULL; - k->lineno = 0; - k->bt[0] = p; - k->bt_len = 1; - hash_src_loc(k); + if (k.sl.bt_len > MWRAP_BT_MAX) + return NULL; + + memcpy(&k.sl.f, buf, len); + + hash_src_loc(&k.sl); rcu_read_lock(); - l = src_loc_get(t, k); + struct src_loc *l = src_loc_get(t, &k.sl); rcu_read_unlock(); - } else { - l = src_loc_lookup(str, len); + return l; } - return l; + return NULL; } static const char *mwrap_env; @@ -952,5 +988,20 @@ __attribute__((constructor)) static void mwrap_ctor(void) CHECK(int, 0, pthread_sigmask(SIG_SETMASK, &old, NULL)); CHECK(int, 0, pthread_atfork(atfork_prepare, atfork_parent, atfork_child)); + + if (mwrap_env) { + const char *bt = strstr(mwrap_env, "bt:"); + if (bt) { + bt += sizeof("bt"); + errno = 0; + char *end; + unsigned long n = strtoul(bt, &end, 10); + if (n && !errno && (*end == ',' || *end == 0)) { + if (n >= MWRAP_BT_MAX) + n = MWRAP_BT_MAX; + bt_req_depth = (uint32_t)n; + } + } + } --locating; } |