From df5335f9b3f324758e21871f8ac9086b2c33ab4a Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Fri, 13 Jan 2023 10:18:20 +0000 Subject: resync docs, dump_csv: support from the Perl version I somehow forgot about the existence of the perror(3) function :x --- Documentation/mwrap.pod | 21 ++++- README | 31 +++++-- ext/mwrap/httpd.h | 241 +++++++++++++++++++++++++----------------------- ext/mwrap/mwrap_core.h | 72 +++++++++++---- 4 files changed, 222 insertions(+), 143 deletions(-) diff --git a/Documentation/mwrap.pod b/Documentation/mwrap.pod index 6832430..a31bc1f 100644 --- a/Documentation/mwrap.pod +++ b/Documentation/mwrap.pod @@ -7,8 +7,8 @@ mwrap - run any command under mwrap # to trace a long-running program and access it via $DIR/$PID.sock: MWRAP=socket_dir:$DIR mwrap COMMAND - # to trace a short-lived command and dump its output to a log: - MWRAP=dump_path:$FILENAME mwrap COMMAND + # to trace a short-lived command and dump its output to a CSV: + MWRAP=dump_csv:$FILENAME mwrap COMMAND =head1 DESCRIPTION @@ -46,13 +46,28 @@ This may be changed via POST request (see below). Default: 0 +=item dump_csv:$FILENAME + +Dump CSV to the given filename. + +This output matches the HTTP server output and includes column headers, +but is subject to change in future releases. + +C without the C<:> may also be used in conjunction with +C, such as C. + +As of mwrap 3.0, +C<$FILENAME> may contain C<%p> where C<%p> is a placeholder for +the PID being dumped. No other use of C<%> is accepted, and +multiple C<%> means all C<%> (including C<%p>) are handled as-is. + =item dump_path:$FILENAME Dumps the output at exit to a given filename: total_bytes call_count location -In the future, dumping to a self-describing CSV will be supported. +Expands C<%p> to the PID in C<$FILENAME> as described for C =item dump_fd:$DESCRIPTOR diff --git a/README b/README index 761f87e..f969c14 100644 --- a/README +++ b/README @@ -16,20 +16,24 @@ numeric caller addresses for allocations made without GVL so you can get an idea of how much memory usage certain extensions and native libraries use. +As of 3.0, it also gives configurable C backtraces of all +dynamically-linked malloc callsites for any program where backtrace(3) +works, including programs not linked to Ruby. + It requires the concurrent lock-free hash table from the Userspace RCU project: https://liburcu.org/ It does not require recompiling or rebuilding Ruby, but only supports Ruby 2.7.0 or later on a few platforms: -* GNU/Linux (only tested --without-jemalloc, mwrap 3.x provides its own) +* GNU/Linux (only tested --without-jemalloc, mwrap 3.x provides its own malloc) It may work on FreeBSD, NetBSD, OpenBSD and DragonFly BSD if given appropriate build options. == Install - # Debian-based systems: apt-get liburcu-dev + # Debian-based systems: apt-get install liburcu-dev # Install mwrap via RubyGems.org gem install mwrap @@ -37,13 +41,21 @@ appropriate build options. == Usage mwrap works as an LD_PRELOAD and supplies a mwrap RubyGem executable to -improve ease-of-use. You can set dump_path: in the MWRAP environment -variable to append the results to a log file: +improve ease-of-use. You can set `dump_csv:' in the MWRAP environment +variable to append the results to a CSV file: + + MWRAP=dump_csv:/path/to/log mwrap RUBY_COMMAND + +(`dump_csv:' is new in mwrap 3.x, `dump_file:' from earlier versions is +still supported). - MWRAP=dump_path:/path/to/log mwrap RUBY_COMMAND +For long running processes, you can see the AF_UNIX HTTP interface: - # And to display the locations with the most allocations: - sort -k1,1rn +for more info. You may also `require "mwrap"' in your Ruby code and use Mwrap.dump, Mwrap.reset, Mwrap.each, etc. @@ -53,7 +65,10 @@ effect in tracking malloc use. However, it is safe to keep "require 'mwrap'" in performance-critical deployments, as overhead is only incurred when used as an LD_PRELOAD. -The output of the mwrap dump is a text file with 3 columns: +The output of `dump_csv:' is has self-describing columns and is +subject to change. SQLite 3.32+ can load it with: `.import --csv'. + +The output of the `dump_file:' output is a text file with 3 columns: total_bytes call_count location diff --git a/ext/mwrap/httpd.h b/ext/mwrap/httpd.h index ef4d83c..8a105aa 100644 --- a/ext/mwrap/httpd.h +++ b/ext/mwrap/httpd.h @@ -221,7 +221,7 @@ static FILE *fbuf_init(struct mw_fbuf *fb) { fb->ptr = NULL; fb->fp = open_memstream(&fb->ptr, &fb->len); - if (!fb->fp) fprintf(stderr, "open_memstream: %m\n"); + if (!fb->fp) perror("open_memstream"); return fb->fp; } @@ -237,7 +237,7 @@ static int fbuf_close(struct mw_fbuf *fb) { int e = ferror(fb->fp) | fclose(fb->fp); fb->fp = NULL; - if (e) fprintf(stderr, "ferror|fclose: %m\n"); + if (e) perror("ferror|fclose"); return e; } @@ -279,7 +279,7 @@ static enum mw_qev h1_200(struct mw_h1 *h1, struct mw_fbuf *fb, const char *ct) */ off_t clen = ftello(fb->fp); if (clen < 0) { - fprintf(stderr, "ftello: %m\n"); + perror("ftello"); fbuf_close(fb); return h1_close(h1); } @@ -468,7 +468,7 @@ static off_t write_loc_name(FILE *fp, const struct src_loc *l) off_t beg = ftello(fp); if (beg < 0) { - fprintf(stderr, "ftello: %m\n"); + perror("ftello"); return beg; } if (l->f) { @@ -498,15 +498,17 @@ static off_t write_loc_name(FILE *fp, const struct src_loc *l) } off_t end = ftello(fp); if (end < 0) { - fprintf(stderr, "ftello: %m\n"); + perror("ftello"); return end; } return end - beg; } -static struct h1_src_loc *accumulate(unsigned long min, size_t *hslc, FILE *lp) +static struct h1_src_loc * +accumulate(struct mw_fbuf *lb, unsigned long min, size_t *hslc) { struct mw_fbuf fb; + if (!fbuf_init(lb)) return NULL; if (!fbuf_init(&fb)) return NULL; rcu_read_lock(); struct cds_lfht *t = CMM_LOAD_SHARED(totals); @@ -528,18 +530,23 @@ static struct h1_src_loc *accumulate(unsigned long min, size_t *hslc, FILE *lp) HUGE_VAL; hsl.max_life = uatomic_read(&l->max_lifespan); hsl.sl = l; - hsl.lname_len = write_loc_name(lp, l); + hsl.lname_len = write_loc_name(lb->fp, l); fwrite(&hsl, sizeof(hsl), 1, fb.fp); } rcu_read_unlock(); - struct h1_src_loc *hslv; - if (fbuf_close(&fb)) { - hslv = NULL; - } else { - *hslc = fb.len / sizeof(*hslv); - mwrap_assert((fb.len % sizeof(*hslv)) == 0); - hslv = (struct h1_src_loc *)fb.ptr; + if (fbuf_close(&fb) || fbuf_close(lb)) + return NULL; + + struct h1_src_loc *hslv = (struct h1_src_loc *)fb.ptr; + *hslc = fb.len / sizeof(*hslv); + mwrap_assert((fb.len % sizeof(*hslv)) == 0); + char *n = lb->ptr; + for (size_t i = 0; i < *hslc; ++i) { + hslv[i].loc_name = n; + n += hslv[i].lname_len; + if (hslv[i].lname_len < 0) + return NULL; } return hslv; } @@ -609,124 +616,128 @@ static enum mw_qev each_at(struct mw_h1 *h1, struct mw_h1req *h1r) return h1_200(h1, &html, TYPE_HTML); } -/* /$PID/each/$MIN endpoint */ -static enum mw_qev each_gt(struct mw_h1 *h1, struct mw_h1req *h1r, - unsigned long min, bool csv) -{ - static const char default_sort[] = "bytes"; - const char *sort; - size_t sort_len = 0; +typedef int (*cmp_fn)(const void *, const void *); - if (!csv) { - sort = default_sort; - sort_len = sizeof(default_sort) - 1; +static cmp_fn write_csv_header(FILE *fp, const char *sort, size_t sort_len) +{ + cmp_fn cmp = NULL; + for (size_t i = 0; i < CAA_ARRAY_SIZE(fields); i++) { + const char *fn = fields[i].fname; + if (i) + fputc(',', fp); + fputs(fn, fp); + if (fields[i].flen == sort_len && !memcmp(fn, sort, sort_len)) + cmp = fields[i].cmp; } + fputc('\n', fp); + return cmp; +} - if (h1r->qstr && h1r->qlen > 5 && !memcmp(h1r->qstr, "sort=", 5)) { - sort = h1r->qstr + 5; - sort_len = h1r->qlen - 5; +static void write_csv_data(FILE *fp, struct h1_src_loc *hslv, size_t hslc) +{ + for (size_t i = 0; i < hslc; i++) { + struct h1_src_loc *hsl = &hslv[i]; + + fprintf(fp, "%zu,%zu,%zu,%zu,%0.3f,%zu,", + hsl->bytes, hsl->allocations, hsl->frees, + hsl->live, hsl->mean_life, hsl->max_life); + write_q_csv(fp, hsl->loc_name, hsl->lname_len); + fputc('\n', fp); } +} - size_t hslc; +static void *write_csv(FILE *fp, size_t min, const char *sort, size_t sort_len) +{ AUTO_CLOFREE struct mw_fbuf lb; - if (!fbuf_init(&lb)) return h1_close(h1); - AUTO_FREE struct h1_src_loc *hslv = accumulate(min, &hslc, lb.fp); - if (!hslv) - return h1_close(h1); + size_t hslc; + AUTO_FREE struct h1_src_loc *hslv = accumulate(&lb, min, &hslc); + if (!hslv) return NULL; - if (fbuf_close(&lb)) - return h1_close(h1); + cmp_fn cmp = write_csv_header(fp, sort, sort_len); + if (cmp) + qsort(hslv, hslc, sizeof(*hslv), cmp); + write_csv_data(fp, hslv, hslc); + return fp; +} - char *n = lb.ptr; - for (size_t i = 0; i < hslc; ++i) { - hslv[i].loc_name = n; - n += hslv[i].lname_len; - if (hslv[i].lname_len < 0) - return h1_close(h1); +/* /$PID/each/$MIN endpoint */ +static enum mw_qev each_gt(struct mw_h1 *h1, struct mw_h1req *h1r, + size_t min, bool csv) +{ + static const char default_sort[] = "bytes"; + const char *sort = csv ? NULL : default_sort; + size_t sort_len = csv ? 0 : (sizeof(default_sort) - 1); + + if (h1r->qstr && h1r->qlen > 5 && !memcmp(h1r->qstr, "sort=", 5)) { + sort = h1r->qstr + 5; + sort_len = h1r->qlen - 5; } struct mw_fbuf bdy; FILE *fp = wbuf_init(&bdy); if (!fp) return h1_close(h1); - - if (!csv) { - unsigned depth = (unsigned)CMM_LOAD_SHARED(bt_req_depth); - fprintf(fp, "mwrap each >%lu" - "

mwrap each >%lu " - "(change `%lu' in URL to adjust filtering) - " - "MWRAP=bt:%u .csv", - min, min, min, depth, min); - show_stats(fp); - /* need borders to distinguish multi-level traces */ - if (depth) - FPUTS("", fp); - else /* save screen space if only tracing one line */ - FPUTS("", fp); + if (csv) { + if (write_csv(fp, min, sort, sort_len)) + return h1_200(h1, &bdy, TYPE_CSV); + return h1_close(h1); } - int (*cmp)(const void *, const void *) = NULL; - if (csv) { - for (size_t i = 0; i < CAA_ARRAY_SIZE(fields); i++) { - const char *fn = fields[i].fname; - if (i) - fputc(',', fp); - fputs(fn, fp); - if (fields[i].flen == sort_len && - !memcmp(fn, sort, sort_len)) - cmp = fields[i].cmp; - } - fputc('\n', fp); - } else { - for (size_t i = 0; i < CAA_ARRAY_SIZE(fields); i++) { - const char *fn = fields[i].fname; - FPUTS("", fp); + size_t hslc; + AUTO_CLOFREE struct mw_fbuf lb; + AUTO_FREE struct h1_src_loc *hslv = accumulate(&lb, min, &hslc); + if (!hslv) + return h1_close(h1); + + unsigned depth = (unsigned)CMM_LOAD_SHARED(bt_req_depth); + fprintf(fp, "mwrap each >%lu" + "

mwrap each >%lu " + "(change `%lu' in URL to adjust filtering) - " + "MWRAP=bt:%u .csv", + min, min, min, depth, min); + show_stats(fp); + /* need borders to distinguish multi-level traces */ + if (depth) + FPUTS("

", fp); + else /* save screen space if only tracing one line */ + FPUTS("
", fp); - if (fields[i].flen == sort_len && - !memcmp(fn, sort, sort_len)) { - cmp = fields[i].cmp; - fprintf(fp, "%s", fields[i].fname); - } else { - fprintf(fp, "%s", - min, fn, fn); - } - FPUTS("
", fp); + cmp_fn cmp = NULL; + for (size_t i = 0; i < CAA_ARRAY_SIZE(fields); i++) { + const char *fn = fields[i].fname; + FPUTS("", fp); } - if (!csv) - FPUTS("", fp); + FPUTS("", fp); if (cmp) qsort(hslv, hslc, sizeof(*hslv), cmp); - else if (!csv) + else FPUTS("", fp); - if (csv) { - for (size_t i = 0; i < hslc; i++) { - struct h1_src_loc *hsl = &hslv[i]; - fprintf(fp, "%zu,%zu,%zu,%zu,%0.3f,%zu,", - hsl->bytes, hsl->allocations, hsl->frees, - hsl->live, hsl->mean_life, hsl->max_life); - write_q_csv(fp, hsl->loc_name, hsl->lname_len); - fputc('\n', fp); - } - } else { - for (size_t i = 0; i < hslc; i++) { - struct h1_src_loc *hsl = &hslv[i]; + for (size_t i = 0; i < hslc; i++) { + struct h1_src_loc *hsl = &hslv[i]; - fprintf(fp, "" - "", - hsl->bytes, hsl->allocations, hsl->frees, - hsl->live, hsl->mean_life, hsl->max_life); - FPUTS("" + "", + hsl->bytes, hsl->allocations, hsl->frees, + hsl->live, hsl->mean_life, hsl->max_life); + FPUTS("", fp); - } - FPUTS("
", fp); + if (fields[i].flen == sort_len && + !memcmp(fn, sort, sort_len)) { + cmp = fields[i].cmp; + fprintf(fp, "%s", fields[i].fname); + } else { + fprintf(fp, "%s", + min, fn, fn); } + FPUTS("
sort= not understood
%zu%zu%zu%zu%0.3f%zu%zu%zu%zu%zu%0.3f%zusl), - src_loc_hash_len(hsl->sl)); + write_b64_url(fp, src_loc_hash_tip(hsl->sl), + src_loc_hash_len(hsl->sl)); - FPUTS("\">", fp); - write_html(fp, hsl->loc_name, hsl->lname_len); - FPUTS("
", fp); + FPUTS("\">", fp); + write_html(fp, hsl->loc_name, hsl->lname_len); + FPUTS("", fp); } - return h1_200(h1, &bdy, csv ? TYPE_CSV : TYPE_HTML); + FPUTS("", fp); + return h1_200(h1, &bdy, TYPE_HTML); } /* /$PID/ root endpoint */ @@ -781,7 +792,7 @@ static enum mw_qev h1_dispatch(struct mw_h1 *h1, struct mw_h1req *h1r) if ((c = PATH_SKIP(h1r, "/each/"))) { errno = 0; char *e; - unsigned long min = strtoul(c, &e, 10); + size_t min = (size_t)strtoul(c, &e, 10); if (!errno) { if (*e == ' ' || *e == '?') return each_gt(h1, h1r, min, false); @@ -857,7 +868,7 @@ static enum mw_qev h1_drain_input(struct mw_h1 *h1, struct mw_h1req *h1r, return h1_close(h1); default: /* ENOMEM, ENOBUFS, ... */ assert(errno != EBADF); - fprintf(stderr, "read: %m\n"); + perror("read"); return h1_close(h1); } } @@ -990,7 +1001,7 @@ static enum mw_qev h1_event_step(struct mw_h1 *h1, struct mw_h1d *h1d) if (!h1r) { h1r = h1d->shared_h1r = malloc(sizeof(*h1r)); if (!h1r) { - fprintf(stderr, "h1r malloc: %m\n"); + perror("h1r malloc"); return h1_close(h1); } } @@ -1034,7 +1045,7 @@ static enum mw_qev h1_event_step(struct mw_h1 *h1, struct mw_h1d *h1d) return h1_close(h1); default: /* ENOMEM, ENOBUFS, ... */ assert(errno != EBADF); - fprintf(stderr, "read: %m\n"); + perror("read"); return h1_close(h1); } } @@ -1142,7 +1153,7 @@ static void h1d_unlink(struct mw_h1d *h1d, bool do_close) if (h1d->lfd < 0 || !h1d->pid_len) return; if (getsockname(h1d->lfd, &sa.any, &len) < 0) { - fprintf(stderr, "getsockname: %m\n"); + perror("getsockname"); return; } if (do_close) { /* only safe to close if thread isn't running */ @@ -1208,13 +1219,13 @@ static int h1d_init(struct mw_h1d *h1d, const char *menv) return fprintf(stderr, "unlink(%s): %m\n", sa.un.sun_path); h1d->lfd = socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0); if (h1d->lfd < 0) - return fprintf(stderr, "socket: %m\n"); + return perror("socket"), 1; if (bind(h1d->lfd, &sa.any, (socklen_t)sizeof(sa)) < 0) { - fprintf(stderr, "bind: %m\n"); + perror("bind"); goto close_fail; } if (listen(h1d->lfd, 1024) < 0) { - fprintf(stderr, "listen: %m\n"); + perror("listen"); goto close_fail; } h1d->alive = 1; /* runs in parent, before pthread_create */ diff --git a/ext/mwrap/mwrap_core.h b/ext/mwrap/mwrap_core.h index 827ee7b..a84cd6d 100644 --- a/ext/mwrap/mwrap_core.h +++ b/ext/mwrap/mwrap_core.h @@ -4,7 +4,9 @@ * Disclaimer: I don't really know my way around XS or Perl internals well */ #define _LGPL_SOURCE /* allows URCU to inline some stuff */ -#define _GNU_SOURCE +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif #include "mymalloc.h" /* includes dlmalloc_c.h */ #ifndef MWRAP_PERL # define MWRAP_PERL 0 @@ -19,9 +21,6 @@ # define MWRAP_BT_MAX 32 #endif -#ifndef _GNU_SOURCE -# define _GNU_SOURCE -#endif #include #include #include @@ -389,11 +388,8 @@ static const COP *mwp_curcop(void) static const char *mw_perl_src_file_cstr(unsigned *lineno) { const COP *cop = mwp_curcop(); - if (!cop) return NULL; - const char *fn = CopFILE(cop); - if (!fn) return NULL; - *lineno = CopLINE(cop); - return fn; + *lineno = cop ? CopLINE(cop) : 0; + return cop ? CopFILE(cop) : NULL; } # define SRC_FILE_CSTR(lineno) mw_perl_src_file_cstr(lineno) #endif /* MWRAP_PERL */ @@ -735,6 +731,7 @@ enomem: struct dump_arg { FILE *fp; size_t min; + bool dump_csv; }; char **bt_syms(void * const *addrlist, uint32_t size) @@ -745,7 +742,7 @@ char **bt_syms(void * const *addrlist, uint32_t size) #else /* make FreeBSD look like glibc output: */ char **s = backtrace_symbols_fmt(addrlist, size, "%f(%n%D) [%a]"); #endif - if (!s) fprintf(stderr, "backtrace_symbols: %m\n"); + if (!s) perror("backtrace_symbols"); return s; } @@ -757,12 +754,16 @@ static void cleanup_free(void *any) free(*p); } +static void *write_csv(FILE *, size_t min, const char *sort, size_t sort_len); static void *dump_to_file(struct dump_arg *a) { struct cds_lfht_iter iter; struct src_loc *l; struct cds_lfht *t; + if (a->dump_csv) + return write_csv(a->fp, a->min, NULL, 0); + ++locating; rcu_read_lock(); t = CMM_LOAD_SHARED(totals); @@ -860,7 +861,7 @@ __attribute__ ((destructor)) static void mwrap_dtor(void) { const char *opt = getenv("MWRAP"); const char *modes[] = { "a", "a+", "w", "w+", "r+" }; - struct dump_arg a = { .min = 0 }; + struct dump_arg a = { .min = 0, .dump_csv = false }; size_t i; int dump_fd; char *dump_path; @@ -873,27 +874,64 @@ __attribute__ ((destructor)) static void mwrap_dtor(void) return; ++locating; - if ((dump_path = strstr(opt, "dump_path:")) && - (dump_path += sizeof("dump_path")) && - *dump_path) { + + /* parse dump_csv:$PATHNAME */ + if ((dump_path = strstr(opt, "dump_csv:"))) { + dump_path += sizeof("dump_csv"); + if (!*dump_path) + dump_path = NULL; + else + a.dump_csv = true; + } + if (!dump_path) { + /* parse dump_path:$PATHNAME */ + if ((dump_path = strstr(opt, "dump_path:"))) { + dump_path += sizeof("dump_path"); + if (!*dump_path) + dump_path = NULL; + } + } + if (dump_path) { char *end = strchr(dump_path, ','); char buf[PATH_MAX]; + AUTO_FREE char *pid_path = NULL; if (end) { mwrap_assert((end - dump_path) < (intptr_t)sizeof(buf)); end = mempcpy(buf, dump_path, end - dump_path); *end = 0; dump_path = buf; } + + /* %p => PID expansion (Linux core_pattern uses %p, too) */ + if ((s = strchr(dump_path, '%')) && s[1] == 'p' && + /* don't allow injecting extra formats: */ + !strchr(s + 2, '%')) { + s[1] = 'd'; /* s/%p/%d/ to make asprintf happy */ + int n = asprintf(&pid_path, dump_path, (int)getpid()); + if (n < 0) + fprintf(stderr, + "asprintf failed: %m, dumping to %s\n", + dump_path); + else + dump_path = pid_path; + } dump_fd = open(dump_path, O_CLOEXEC|O_WRONLY|O_APPEND|O_CREAT, 0666); if (dump_fd < 0) { fprintf(stderr, "open %s failed: %m\n", dump_path); goto out; } + } else { + s = strstr(opt, "dump_fd:"); + if (!s) + goto out; + if (!sscanf(s, "dump_fd:%d", &dump_fd)) + goto out; } - else if (!sscanf(opt, "dump_fd:%d", &dump_fd)) - goto out; + /* allow dump_csv standalone for dump_fd */ + if (!a.dump_csv && strstr(opt, "dump_csv")) + a.dump_csv = true; if ((s = strstr(opt, "dump_min:"))) sscanf(s, "dump_min:%zu", &a.min); @@ -1019,7 +1057,7 @@ __attribute__((constructor)) static void mwrap_ctor(void) h->real = h; call_rcu(&h->as.dead, free_hdr_rcu); } else - fprintf(stderr, "malloc: %m\n"); + perror("malloc"); h1d_start(); CHECK(int, 0, pthread_sigmask(SIG_SETMASK, &old, NULL)); -- cgit v1.2.3-24-ge0c7