diff options
author | Sheena Artrip <sheenobu@fb.com> | 2017-08-16 16:26:08 -0700 |
---|---|---|
committer | Jeff Darcy <jeff@pl.atyp.us> | 2017-09-13 19:32:29 +0000 |
commit | 07b32d43b05379af5e889e0983894a822bd9bbb4 (patch) | |
tree | 81f037b5220e963275f753a4f4db7a6ee3f456c3 | |
parent | 627611998b6992797e2c03f9b263816beb2065a1 (diff) |
Get glusterfs to output p50, p90, and p95 latencies
Summary:
[done] separate p99 dumping into general funcs
[done] add p95, p90, and p50 stats
- add p95, p90, p50 within p99, and generalize
- rename config to dump-percentile-lantencies
Test Plan:
make install glusterfs on dev machine.
gluster volume create $name ...
mount volume on /mnt/$name <brick1, brick2, ...>
dd if=/dev/zero of=/mnt/$name/test
check each brick for pn printing
/var/lib/glusterd/stats/glusterfsd__$brick.dump
Reviewers: sshreyas, kvigor, jdarcy
Reviewed By: jdarcy
Differential Revision: https://phabricator.intern.facebook.com/D5645951
Change-Id: I7bcd7201fc3753316db0ece809491a1cbdbefd32
Signed-off-by: Jeff Darcy <jdarcy@fb.com>
Reviewed-on: https://review.gluster.org/18278
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>
Tested-by: Jeff Darcy <jeff@pl.atyp.us>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
-rw-r--r-- | xlators/debug/io-stats/src/io-stats.c | 162 | ||||
-rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-volume-set.c | 4 |
2 files changed, 114 insertions, 52 deletions
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c index e958601b304..4640c3c37dd 100644 --- a/xlators/debug/io-stats/src/io-stats.c +++ b/xlators/debug/io-stats/src/io-stats.c @@ -308,7 +308,7 @@ struct ios_conf { gf_boolean_t sample_hard_errors; gf_boolean_t sample_all_errors; uint32_t outstanding_req; - gf_boolean_t dump_p99_latencies; + gf_boolean_t dump_percentile_latencies; }; @@ -905,43 +905,55 @@ ios_stats_cleanup (xlator_t *this, inode_t *inode) gf_log (this->name, GF_LOG_TRACE, fmt); \ } while (0) -double -io_stats_dump_p99_latencies (xlator_t *this, FILE* logfp, char * key_prefix, char * str_prefix) -{ - struct ios_conf *conf = NULL; - double *ios_latencies[GF_FOP_MAXVALUE] = {NULL, }; - double *global_ios_latency = NULL; - int num_fop[GF_FOP_MAXVALUE] = {0, }; - ios_sample_t *ios_samples = NULL; - int collected = 0; - int i = 0; - int j = 0; - int fop_idx = 0; - glusterfs_fop_t fop_type = 0; - double p99_val = 0.0; - int p99_idx = 0; - int ret = 0; +void +io_stats_log_px_stat (xlator_t *this, FILE* logfp, float pval, const char *desc, + int *num_fop, double **ios_latencies, + double *global_ios_latency) +{ + int pn_idx = 0; + double pn_val = 0; + int collected = 0; + int i = 0; + int px = 0; + struct ios_conf *conf = NULL; + conf = this->private; collected = conf->ios_sample_buf->collected; + + px = (int)(pval * 100); + pn_idx = (int)(pval * collected); + pn_val = global_ios_latency[pn_idx]; + + ios_log (this, logfp, "\"%s.%s.p%d_latency_usec\":\"%0.4lf\",", desc, + "global", px, pn_val); for (i = 0; i < GF_FOP_MAXVALUE; i++) { - ios_latencies[i] = GF_CALLOC (collected, - sizeof (double), - 0); - if (!ios_latencies[i]) { - ret = -ENOMEM; - goto out; - } - } - global_ios_latency = GF_CALLOC (collected, sizeof (double), - 0); - if (!global_ios_latency) { - ret = -ENOMEM; - goto out; + qsort (ios_latencies[i], num_fop[i], sizeof (double), + gf_compare_double); + pn_idx = (int)(pval*num_fop[i]); + pn_val = ios_latencies[i][pn_idx]; + ios_log (this, logfp, + "\"%s.%s.p%d_latency_usec\":\"%0.4lf\",", desc, gf_fop_list[i], px, pn_val); } +} + + +double +io_stats_prepare_latency_samples (struct ios_conf *conf, int *num_fop, + double **ios_latencies, + double *global_ios_latency) +{ + int ret = 0; + int collected = 0; + int fop_idx = 0; + int i = 0; + ios_sample_t *ios_samples = NULL; + glusterfs_fop_t fop_type = 0; + collected = conf->ios_sample_buf->collected; ios_samples = conf->ios_sample_buf->ios_samples; + for (i = 0; i < collected; i++) { global_ios_latency[i] = ios_samples[i].elapsed; fop_type = ios_samples[i].fop_type; @@ -953,23 +965,72 @@ io_stats_dump_p99_latencies (xlator_t *this, FILE* logfp, char * key_prefix, cha qsort (global_ios_latency, collected, sizeof (double), gf_compare_double); - p99_idx = (int)(0.99 * collected); - p99_val = global_ios_latency[p99_idx]; - ios_log (this, logfp, - "\"%s.%s.fop.%s.p99_latency_usec\":\"%0.4lf\",", - key_prefix, str_prefix, "global", p99_val); + return ret; +} + +double +io_stats_dump_pn_latencies (xlator_t *this, FILE* logfp, char * key_prefix, + char * str_prefix) +{ + double *ios_latencies[GF_FOP_MAXVALUE] = {NULL, }; + double *global_ios_latency = NULL; + int num_fop[GF_FOP_MAXVALUE] = {0, }; + int ret = 0; + int i = 0; + int j = 0; + int collected = 0; + char *prefix = NULL; + struct ios_conf *conf = NULL; + + conf = this->private; + collected = conf->ios_sample_buf->collected; + + /* allocate memory */ + global_ios_latency = GF_CALLOC (collected, sizeof (double), 0); + if (!global_ios_latency) { + ret = -ENOMEM; + goto out; + } for (i = 0; i < GF_FOP_MAXVALUE; i++) { qsort (ios_latencies[i], num_fop[i], sizeof (double), gf_compare_double); - p99_idx = (int)(0.99*num_fop[i]); - p99_val = ios_latencies[i][p99_idx]; - ios_log (this, logfp, - "\"%s.%s.fop.%s.p99_latency_usec\":\"%0.4lf\",", - key_prefix, str_prefix, gf_fop_list[i], p99_val); + ios_latencies[i] = GF_CALLOC (collected, + sizeof (double), + 0); + if (!ios_latencies[i]) { + ret = -ENOMEM; + goto out; + } + } + + /* gather data */ + ret = io_stats_prepare_latency_samples (conf, + num_fop, + ios_latencies, + global_ios_latency); + if (ret) { + goto out; } + /* build prefix and log data */ + ret = gf_asprintf (&prefix, "%s.%s", key_prefix, str_prefix); + if (ret <= 0) { + ret = -ENOMEM; + goto out; + } + + io_stats_log_px_stat (this, logfp, 0.50, prefix, num_fop, + ios_latencies, global_ios_latency); + io_stats_log_px_stat (this, logfp, 0.90, prefix, num_fop, + ios_latencies, global_ios_latency); + io_stats_log_px_stat (this, logfp, 0.95, prefix, num_fop, + ios_latencies, global_ios_latency); + io_stats_log_px_stat (this, logfp, 0.99, prefix, num_fop, + ios_latencies, global_ios_latency); + out: + GF_FREE (prefix); GF_FREE (global_ios_latency); for (j = 0; j < i; j++) { GF_FREE (ios_latencies[j]); @@ -1311,8 +1372,8 @@ io_stats_dump_global_to_json_logfp (xlator_t *this, ios_log (this, logfp, "\"%s.%s.fop.GOT1\":\"%0.4lf\",", key_prefix, str_prefix, fop_ave_usec); - if (conf->dump_p99_latencies) { - io_stats_dump_p99_latencies (this, logfp, key_prefix, + if (conf->dump_percentile_latencies) { + io_stats_dump_pn_latencies (this, logfp, key_prefix, str_prefix); ios_log (this, logfp, "\"%s.%s.fop.GOT2\":\"%0.4lf\",", @@ -4400,8 +4461,8 @@ reconfigure (xlator_t *this, dict_t *options) GF_OPTION_RECONF ("fop-sample-hard-errors", conf->sample_hard_errors, options, bool, out); - GF_OPTION_RECONF ("dump-p99-latencies", conf->dump_p99_latencies, - options, bool, out); + GF_OPTION_RECONF ("dump-percentile-latencies", + conf->dump_percentile_latencies, options, bool, out); ret = 0; out: @@ -4568,9 +4629,9 @@ init (xlator_t *this) GF_OPTION_INIT ("log-flush-timeout", log_flush_timeout, time, out); gf_log_set_log_flush_timeout (log_flush_timeout); - GF_OPTION_INIT ("dump-p99-latencies", conf->dump_p99_latencies, - bool, out); - + GF_OPTION_INIT ("dump-percentile-latencies", + conf->dump_percentile_latencies, bool, out); + this->private = conf; if (conf->ios_dump_interval > 0) { pthread_create (&conf->dump_thread, NULL, @@ -4977,11 +5038,12 @@ struct volume_options options[] = { .description = "This option samples all fops with \"hard errors\"" "including EROFS, ENOSPC, etc." }, - { .key = { "dump-p99-latencies" }, + { .key = { "dump-percentile-latencies" }, .type = GF_OPTION_TYPE_BOOL, .default_value = "off", - .description = "If on the p99 latency of each operation " - "and all types is dumped at each sample interval. " + .description = "If on the p50, p90, p95, and p99 latency of each " + "operation and all types is dumped at each sample " + "interval." }, { .key = {NULL} }, diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c index 996be7f1618..3f4c5bbb686 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c @@ -1360,9 +1360,9 @@ struct volopt_map_entry glusterd_volopt_map[] = { .type = NO_DOC, .op_version = 1 }, - { .key = "diagnostics.dump-p99-latencies", + { .key = "diagnostics.dump-percentile-latencies", .voltype = "debug/io-stats", - .option = "dump-p99-latencies", + .option = "dump-percentile-latencies", .value = "off", .type = NO_DOC, .op_version = 1 |