metrics: provide options to dump metrics from xlators

* Introduce xlator methods to allow dumping of metrics
* Separate options to get the metrics dumped in a path

Updates #168

Change-Id: I7df80df33b71d6f449f03c2332665b4a45f6ddf2
Signed-off-by: Amar Tumballi <amarts@redhat.com>

13 files changed, 419 insertions, 23 deletions

diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index b8ebdefb876..646b4399019 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -32,6 +32,7 @@
 #include "syncop.h"
 #include "xlator.h"
 #include "syscall.h"
+#include "monitoring.h"
 
 static gf_boolean_t is_mgmt_rpc_reconnect = _gf_false;
 int need_emancipate = 0;
@@ -895,6 +896,74 @@ out:
 }
 
 int
+glusterfs_handle_dump_metrics (rpcsvc_request_t *req)
+{
+        int32_t                 ret             = -1;
+        gd1_mgmt_brick_op_req   xlator_req      = {0,};
+        xlator_t                *this           = NULL;
+        glusterfs_ctx_t         *ctx            = NULL;
+        char  *filepath = NULL;
+        int    fd  = -1;
+        struct stat statbuf = {0,};
+        char *msg = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = xdr_to_generic (req->msg[0], &xlator_req,
+                             (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+
+        if (ret < 0) {
+                /*failed to decode msg;*/
+                req->rpc_err = GARBAGE_ARGS;
+                return -1;
+        }
+        ret = -1;
+        ctx = this->ctx;
+
+        /* Infra for monitoring */
+        filepath = gf_monitor_metrics (ctx);
+        if (!filepath)
+                goto out;
+
+        fd = sys_open (filepath, O_RDONLY, 0);
+        if (fd < 0)
+                goto out;
+
+        if (sys_fstat (fd, &statbuf) < 0)
+                goto out;
+
+        if (statbuf.st_size > GF_UNIT_MB) {
+                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
+                        LG_MSG_NO_MEMORY,
+                        "Allocated size exceeds expectation: "
+                        "reconsider logic (%"GF_PRI_SIZET")",
+                        statbuf.st_size);
+        }
+        msg = GF_CALLOC (1, (statbuf.st_size + 1), gf_common_mt_char);
+        if (!msg)
+                goto out;
+
+        ret = sys_read (fd, msg, statbuf.st_size);
+        if (ret < 0)
+                goto out;
+
+        /* Send all the data in errstr, instead of dictionary for now */
+        glusterfs_translator_info_response_send (req, 0, msg, NULL);
+
+        ret = 0;
+out:
+        if (fd >= 0)
+                sys_close (fd);
+
+        GF_FREE (msg);
+        GF_FREE (filepath);
+
+        return ret;
+}
+
+int
 glusterfs_handle_defrag (rpcsvc_request_t *req)
 {
         int32_t                  ret     = -1;
@@ -1695,6 +1764,11 @@ rpcsvc_actor_t glusterfs_actors[GLUSTERD_BRICK_MAXVALUE] = {
                                           GLUSTERD_BRICK_ATTACH,
                                           glusterfs_handle_attach,
                                           NULL, 0, DRC_NA},
+
+        [GLUSTERD_DUMP_METRICS]        = {"DUMP METRICS",
+                                          GLUSTERD_DUMP_METRICS,
+                                          glusterfs_handle_dump_metrics,
+                                          NULL, 0, DRC_NA},
 };
 
 struct rpcsvc_program glusterfs_mop_prog = {
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 69383f87b17..921a86bfd65 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -76,6 +76,7 @@
 #include "client_t.h"
 #include "netgroups.h"
 #include "exports.h"
+#include "monitoring.h"
 
 #include "daemon.h"
 
@@ -1542,6 +1543,9 @@ glusterfs_ctx_defaults_init (glusterfs_ctx_t *ctx)
          */
         ret = -1;
 
+        /* monitoring should be enabled by default */
+        ctx->measure_latency = true;
+
         ctx->process_uuid = generate_glusterfs_ctx_id ();
         if (!ctx->process_uuid) {
                 gf_msg ("", GF_LOG_CRITICAL, 0, glusterfsd_msg_13);
@@ -2175,14 +2179,14 @@ glusterfs_sigwaiter (void *arg)
         sigset_t  set;
         int       ret = 0;
         int       sig = 0;
-
+        char     *file = NULL;
 
         sigemptyset (&set);
         sigaddset (&set, SIGINT);   /* cleanup_and_exit */
         sigaddset (&set, SIGTERM);  /* cleanup_and_exit */
         sigaddset (&set, SIGHUP);   /* reincarnate */
         sigaddset (&set, SIGUSR1);  /* gf_proc_dump_info */
-        sigaddset (&set, SIGUSR2);  /* gf_latency_toggle */
+        sigaddset (&set, SIGUSR2);
 
         for (;;) {
                 ret = sigwait (&set, &sig);
@@ -2202,7 +2206,11 @@ glusterfs_sigwaiter (void *arg)
                         gf_proc_dump_info (sig, glusterfsd_ctx);
                         break;
                 case SIGUSR2:
-                        gf_latency_toggle (sig, glusterfsd_ctx);
+                        file = gf_monitor_metrics (glusterfsd_ctx);
+
+                        /* Nothing needed to be done here */
+                        GF_FREE (file);
+
                         break;
                 default:
 
@@ -2243,7 +2251,7 @@ glusterfs_signals_setup (glusterfs_ctx_t *ctx)
         sigaddset (&set, SIGTERM);  /* cleanup_and_exit */
         sigaddset (&set, SIGHUP);   /* reincarnate */
         sigaddset (&set, SIGUSR1);  /* gf_proc_dump_info */
-        sigaddset (&set, SIGUSR2);  /* gf_latency_toggle */
+        sigaddset (&set, SIGUSR2);
 
         ret = pthread_sigmask (SIG_BLOCK, &set, NULL);
         if (ret) {
diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am
index 094646c5e3f..e4fe6125a3d 100644
--- a/libglusterfs/src/Makefile.am
+++ b/libglusterfs/src/Makefile.am
@@ -35,7 +35,7 @@ libglusterfs_la_SOURCES = dict.c xlator.c logging.c \
 	$(CONTRIBDIR)/timer-wheel/timer-wheel.c \
 	$(CONTRIBDIR)/timer-wheel/find_last_bit.c default-args.c locking.c \
 	$(CONTRIBDIR)/xxhash/xxhash.c \
-	compound-fop-utils.c throttle-tbf.c
+	compound-fop-utils.c throttle-tbf.c monitoring.c
 
 nodist_libglusterfs_la_SOURCES = y.tab.c graph.lex.c defaults.c
 nodist_libglusterfs_la_HEADERS = y.tab.h protocol-common.h
@@ -58,7 +58,7 @@ libglusterfs_la_HEADERS = common-utils.h defaults.h default-args.h \
 	syncop-utils.h parse-utils.h libglusterfs-messages.h \
 	lvm-defaults.h quota-common-utils.h rot-buffs.h \
 	compat-uuid.h upcall-utils.h throttle-tbf.h events.h\
-	compound-fop-utils.h atomic.h
+	compound-fop-utils.h atomic.h monitoring.h
 
 libglusterfs_ladir = $(includedir)/glusterfs
 
diff --git a/libglusterfs/src/globals.c b/libglusterfs/src/globals.c
index 6bed1b546ee..f1eaccc9305 100644
--- a/libglusterfs/src/globals.c
+++ b/libglusterfs/src/globals.c
@@ -157,6 +157,9 @@ global_xl_reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("measure-latency", bool_opt, options, bool, out);
         this->ctx->measure_latency = bool_opt;
 
+        GF_OPTION_RECONF ("metrics-dump-path", this->ctx->config.metrics_dumppath,
+                          options, str, out);
+
         /* TODO: add more things here */
         ret = 0;
 out:
@@ -166,7 +169,19 @@ out:
 static int
 global_xl_init (xlator_t *this)
 {
-        return 0;
+        int ret = -1;
+        gf_boolean_t bool_opt = false;
+
+        GF_OPTION_INIT ("measure-latency", bool_opt, bool, out);
+        this->ctx->measure_latency = bool_opt;
+
+        GF_OPTION_INIT ("metrics-dump-path", this->ctx->config.metrics_dumppath,
+                        str, out);
+
+        ret = 0;
+
+out:
+        return ret;
 }
 
 static void
@@ -184,6 +199,14 @@ struct volume_options global_xl_options[] = {
           .tags = {"global", "context"},
           .description = "Use this option to toggle measuring latency"
         },
+        { .key   = {"metrics-dump-path"},
+          .type  = GF_OPTION_TYPE_STR,
+          .default_value = "{{gluster_workdir}}/metrics",
+          .op_version = {GD_OP_VERSION_4_0_0},
+          .flags = OPT_FLAG_SETTABLE,
+          .tags = {"global", "context"},
+          .description = "Use this option to set the metrics dump path"
+        },
 
         { .key = {NULL},},
 };
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index aa28b80222e..4dee7f00ce6 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -553,6 +553,11 @@ struct _glusterfs_ctx {
 
         gf_lock_t          volfile_lock;
 
+        /* configuration related elements, which gets changed
+           from global xlator */
+        struct {
+                char *metrics_dumppath;
+        } config;
 
         struct {
                 gf_atomic_t max_dict_pairs;
diff --git a/libglusterfs/src/latency.c b/libglusterfs/src/latency.c
index 4c83024e85d..076c019c607 100644
--- a/libglusterfs/src/latency.c
+++ b/libglusterfs/src/latency.c
@@ -93,15 +93,3 @@ gf_proc_dump_latency_info (xlator_t *xl)
         }
 }
 
-
-void
-gf_latency_toggle (int signum, glusterfs_ctx_t *ctx)
-{
-        if (ctx) {
-                ctx->measure_latency = !ctx->measure_latency;
-                gf_msg ("[core]", GF_LOG_INFO, 0,
-                        LG_MSG_LATENCY_MEASUREMENT_STATE,
-                        "Latency measurement turned %s",
-                        ctx->measure_latency ? "on" : "off");
-        }
-}
diff --git a/libglusterfs/src/latency.h b/libglusterfs/src/latency.h
index 132520c1263..73a93242341 100644
--- a/libglusterfs/src/latency.h
+++ b/libglusterfs/src/latency.h
@@ -20,7 +20,4 @@ typedef struct fop_latency {
         uint64_t count;
 } fop_latency_t;
 
-void
-gf_latency_toggle (int signum, glusterfs_ctx_t *ctx);
-
 #endif /* __LATENCY_H__ */
diff --git a/libglusterfs/src/monitoring.c b/libglusterfs/src/monitoring.c
new file mode 100644
index 00000000000..25103867e24
--- /dev/null
+++ b/libglusterfs/src/monitoring.c
@@ -0,0 +1,268 @@
+/*
+  Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "monitoring.h"
+#include "xlator.h"
+#include "syscall.h"
+
+#include <stdlib.h>
+
+static void
+dump_mem_acct_details(xlator_t *xl, int fd)
+{
+        struct mem_acct_rec *mem_rec;
+        int i = 0;
+
+        if (!xl || !xl->mem_acct || (xl->ctx->active != xl->graph))
+                return;
+
+        dprintf (fd, "# %s.%s.total.num_types %d\n", xl->type, xl->name,
+                 xl->mem_acct->num_types);
+
+        dprintf (fd, "# type, in-use-size, in-use-units, max-size, "
+                 "max-units, total-allocs\n");
+
+        for (i = 0; i < xl->mem_acct->num_types; i++) {
+                mem_rec = &xl->mem_acct->rec[i];
+                if (mem_rec->num_allocs == 0)
+                        continue;
+                dprintf (fd, "# %s, %"GF_PRI_SIZET", %u, %"GF_PRI_SIZET", %u,"
+                         " %u\n", mem_rec->typestr, mem_rec->size,
+                         mem_rec->num_allocs, mem_rec->max_size,
+                         mem_rec->max_num_allocs, mem_rec->total_allocs);
+        }
+}
+
+static void
+dump_global_memory_accounting (int fd)
+{
+#if MEMORY_ACCOUNTING_STATS
+        int      i        = 0;
+        uint64_t count    = 0;
+
+        uint64_t tcalloc = GF_ATOMIC_GET (gf_memory_stat_counts.total_calloc);
+        uint64_t tmalloc = GF_ATOMIC_GET (gf_memory_stat_counts.total_malloc);
+        uint64_t tfree   = GF_ATOMIC_GET (gf_memory_stat_counts.total_free);
+
+        dprintf (fd, "memory.total.calloc %lu\n", tcalloc);
+        dprintf (fd, "memory.total.malloc %lu\n", tmalloc);
+        dprintf (fd, "memory.total.realloc %lu\n",
+                 GF_ATOMIC_GET (gf_memory_stat_counts.total_realloc));
+        dprintf (fd, "memory.total.free %lu\n", tfree);
+        dprintf (fd, "memory.total.in-use %lu\n", ((tcalloc + tmalloc) - tfree));
+
+        for (i = 0; i < GF_BLK_MAX_VALUE; i++) {
+                count = GF_ATOMIC_GET (gf_memory_stat_counts.blk_size[i]);
+                dprintf (fd, "memory.total.blk_size.%s %lu\n",
+                         gf_mem_stats_blk[i].blk_size_str, count);
+        }
+
+        dprintf (fd, "#----\n");
+#endif
+
+        /* This is not a metric to be watched in admin guide,
+           but keeping it here till we resolve all leak-issues
+           would be great */
+}
+
+
+static void
+dump_latency_and_count (xlator_t *xl, int fd)
+{
+        int32_t  index = 0;
+        uint64_t fop;
+        uint64_t cbk;
+        uint64_t count;
+
+        dprintf (fd, "%s.total.winds.count %lu\n", xl->name, xl->winds);
+
+        /* Need 'fuse' data, and don't need all the old graph info */
+        if ((xl != xl->ctx->master) && (xl->ctx->active != xl->graph))
+                return;
+
+        count = GF_ATOMIC_GET (xl->stats.total.count);
+        dprintf (fd, "%s.total.fop-count %lu\n", xl->name, count);
+
+        count = GF_ATOMIC_GET (xl->stats.interval.count);
+        dprintf (fd, "%s.interval.fop-count %lu\n", xl->name, count);
+        GF_ATOMIC_INIT (xl->stats.interval.count, 0);
+
+        for (index = 0; index < GF_FOP_MAXVALUE; index++) {
+                fop = GF_ATOMIC_GET (xl->stats.total.metrics[index].fop);
+                if (fop) {
+                        dprintf (fd, "%s.total.%s.count %lu\n",
+                                 xl->name, gf_fop_list[index], fop);
+                }
+                fop = GF_ATOMIC_GET (xl->stats.interval.metrics[index].fop);
+                if (fop) {
+                        dprintf (fd, "%s.interval.%s.count %lu\n",
+                                 xl->name, gf_fop_list[index], fop);
+                }
+                cbk = GF_ATOMIC_GET (xl->stats.interval.metrics[index].cbk);
+                if (cbk) {
+                        dprintf (fd, "%s.interval.%s.fail_count %lu\n",
+                                 xl->name, gf_fop_list[index], cbk);
+                }
+                if (xl->stats.interval.latencies[index].count != 0.0) {
+                        dprintf (fd, "%s.interval.%s.latency %lf\n",
+                                 xl->name, gf_fop_list[index],
+                                 (xl->stats.interval.latencies[index].total /
+                                  xl->stats.interval.latencies[index].count));
+                        dprintf (fd, "%s.interval.%s.max %lf\n",
+                                 xl->name, gf_fop_list[index],
+                                 xl->stats.interval.latencies[index].max);
+                        dprintf (fd, "%s.interval.%s.min %lf\n",
+                                 xl->name, gf_fop_list[index],
+                                 xl->stats.interval.latencies[index].min);
+                }
+                GF_ATOMIC_INIT (xl->stats.interval.metrics[index].cbk, 0);
+                GF_ATOMIC_INIT (xl->stats.interval.metrics[index].fop, 0);
+        }
+        memset (xl->stats.interval.latencies, 0,
+                sizeof (xl->stats.interval.latencies));
+}
+
+static inline void
+dump_call_stack_details (glusterfs_ctx_t *ctx, int fd)
+{
+        dprintf (fd, "total.stack.count %lu\n",
+                 GF_ATOMIC_GET (ctx->pool->total_count));
+        dprintf (fd, "total.stack.in-flight %lu\n",
+                 ctx->pool->cnt);
+}
+
+static inline void
+dump_dict_details (glusterfs_ctx_t *ctx, int fd)
+{
+        uint64_t total_dicts = 0;
+        uint64_t total_pairs = 0;
+
+        total_dicts = GF_ATOMIC_GET (ctx->stats.total_dicts_used);
+        total_pairs = GF_ATOMIC_GET (ctx->stats.total_pairs_used);
+
+        dprintf (fd, "total.dict.max-pairs-per %lu\n",
+                 GF_ATOMIC_GET (ctx->stats.max_dict_pairs));
+        dprintf (fd, "total.dict.pairs-used %lu\n", total_pairs);
+        dprintf (fd, "total.dict.used %lu\n", total_dicts);
+        dprintf (fd, "total.dict.average-pairs %lu\n",
+                            (total_pairs / total_dicts));
+}
+
+static void
+dump_inode_stats (glusterfs_ctx_t *ctx, int fd)
+{
+}
+
+static void
+dump_global_metrics (glusterfs_ctx_t *ctx, int fd)
+{
+        struct timeval tv;
+        time_t nowtime;
+        struct tm *nowtm;
+        char tmbuf[64] = {0,};
+
+        gettimeofday(&tv, NULL);
+        nowtime = tv.tv_sec;
+        nowtm = localtime(&nowtime);
+        strftime(tmbuf, sizeof tmbuf, "%Y-%m-%d %H:%M:%S", nowtm);
+
+        /* Let every file have information on which process dumped info */
+        dprintf (fd, "## %s\n", ctx->cmdlinestr);
+        dprintf (fd, "### %s\n", tmbuf);
+        dprintf (fd, "### BrickName: %s\n", ctx->cmd_args.brick_name);
+        dprintf (fd, "### MountName: %s\n", ctx->cmd_args.mount_point);
+        dprintf (fd, "### VolumeName: %s\n", ctx->cmd_args.volume_name);
+
+        /* Dump memory accounting */
+        dump_global_memory_accounting (fd);
+        dprintf (fd, "# -----\n");
+
+        dump_call_stack_details (ctx, fd);
+        dump_dict_details (ctx, fd);
+        dprintf (fd, "# -----\n");
+
+        dump_inode_stats (ctx, fd);
+        dprintf (fd, "# -----\n");
+}
+
+static void
+dump_xl_metrics (glusterfs_ctx_t *ctx, int fd)
+{
+        xlator_t *xl;
+
+        xl = ctx->active->top;
+
+        while (xl) {
+                dump_latency_and_count (xl, fd);
+                dump_mem_acct_details (xl, fd);
+                if (xl->dump_metrics)
+                        xl->dump_metrics (xl, fd);
+                xl = xl->next;
+        }
+
+        if (ctx->master) {
+                xl = ctx->master;
+
+                dump_latency_and_count (xl, fd);
+                dump_mem_acct_details (xl, fd);
+                if (xl->dump_metrics)
+                        xl->dump_metrics (xl, fd);
+        }
+
+        return;
+}
+
+char *
+gf_monitor_metrics (glusterfs_ctx_t *ctx)
+{
+        int ret = -1;
+        int fd = 0;
+        char *filepath, *dumppath;
+
+        dumppath = ctx->config.metrics_dumppath;
+        if (dumppath == NULL) {
+                dumppath = GLUSTER_METRICS_DIR;
+        }
+
+        ret = gf_asprintf(&filepath, "%s/gmetrics.XXXXXX", dumppath);
+        if (ret < 0) {
+                return NULL;
+        }
+
+        fd = mkstemp (filepath);
+        if (fd < 0) {
+                gf_msg ("monitoring", GF_LOG_ERROR, 0, LG_MSG_STRDUP_ERROR,
+                        "failed to open tmp file %s (%s)",
+                        filepath, strerror (errno));
+                GF_FREE (filepath);
+                return NULL;
+        }
+
+        dump_global_metrics (ctx, fd);
+
+        dump_xl_metrics (ctx, fd);
+
+        /* This below line is used just to capture any errors with dprintf() */
+        ret = dprintf (fd, "\n# End of metrics\n");
+        if (ret < 0) {
+                gf_msg ("monitoring", GF_LOG_WARNING, 0, LG_MSG_STRDUP_ERROR,
+                        "dprintf() failed: %s", strerror (errno));
+        }
+
+        ret = sys_fsync (fd);
+        if (ret < 0) {
+                gf_msg ("monitoring", GF_LOG_WARNING, 0, LG_MSG_STRDUP_ERROR,
+                        "fsync() failed: %s", strerror (errno));
+        }
+        sys_close (fd);
+
+        /* Figure this out, not happy with returning this string */
+        return filepath;
+}
diff --git a/libglusterfs/src/monitoring.h b/libglusterfs/src/monitoring.h
new file mode 100644
index 00000000000..1c08bfc7eb6
--- /dev/null
+++ b/libglusterfs/src/monitoring.h
@@ -0,0 +1,21 @@
+/*
+  Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __MONITORING_H__
+#define __MONITORING_H__
+
+#include "glusterfs.h"
+
+#define GLUSTER_METRICS_DIR "/var/run/gluster/metrics"
+
+char *
+gf_monitor_metrics (glusterfs_ctx_t *ctx);
+
+#endif /* __MONITORING_H__ */
diff --git a/libglusterfs/src/xlator.c b/libglusterfs/src/xlator.c
index 3e108312d60..03495463a53 100644
--- a/libglusterfs/src/xlator.c
+++ b/libglusterfs/src/xlator.c
@@ -373,6 +373,12 @@ int xlator_dynload_newway (xlator_t *xl)
                               xl->name);
         }
 
+        xl->dump_metrics = xlapi->dump_metrics;
+        if (!xl->dump_metrics) {
+                gf_msg_trace ("xlator", 0, "%s: method missing (dump_metrics)",
+                              xl->name);
+        }
+
         vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
                              gf_common_mt_volume_opt_list_t);
         if (!vol_opt) {
diff --git a/libglusterfs/src/xlator.h b/libglusterfs/src/xlator.h
index 81199bd1809..8a5ee819e5c 100644
--- a/libglusterfs/src/xlator.h
+++ b/libglusterfs/src/xlator.h
@@ -982,6 +982,8 @@ struct _xlator {
         int32_t           (*init) (xlator_t *this);
         int32_t           (*reconfigure) (xlator_t *this, dict_t *options);
 	int32_t           (*mem_acct_init) (xlator_t *this);
+	int32_t           (*dump_metrics) (xlator_t *this, int fd);
+
 	event_notify_fn_t notify;
 
         gf_loglevel_t    loglevel;   /* Log level for translator */
@@ -1071,6 +1073,9 @@ typedef struct {
            optional. called during translator initialization */
 	int32_t (*mem_acct_init) (xlator_t *this);
 
+        /* dump_metrics(): used for providing internal metrics. optional */
+	int32_t (*dump_metrics) (xlator_t *this, int fd);
+
         /* notify(): used for handling the notification of events from either
            the parent or child in the graph. optional. */
         event_notify_fn_t notify;
diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h
index ebfb651c96e..aee34302205 100644
--- a/rpc/rpc-lib/src/protocol-common.h
+++ b/rpc/rpc-lib/src/protocol-common.h
@@ -239,6 +239,7 @@ enum glusterd_brick_procnum {
         GLUSTERD_BRICK_BARRIER,
         GLUSTERD_NODE_BITROT,
         GLUSTERD_BRICK_ATTACH,
+        GLUSTERD_DUMP_METRICS,
         GLUSTERD_BRICK_MAXVALUE,
 };
 
diff --git a/tests/bugs/glusterfs/bug-844688.t b/tests/bugs/glusterfs/bug-844688.t
index a1b0b15f5ed..39f04092cf7 100755
--- a/tests/bugs/glusterfs/bug-844688.t
+++ b/tests/bugs/glusterfs/bug-844688.t
@@ -13,7 +13,7 @@ TEST glusterfs -s $H0 --volfile-id $V0 $M0
 
 mount_pid=$(get_mount_process_pid $V0);
 # enable dumping of call stack creation and frame creation times in statedump
-kill -USR2 $mount_pid;
+# monitoring is enabled by default
 
 TEST touch $M0/touchfile;
 (dd if=/dev/urandom of=$M0/file bs=5k 2>/dev/null 1>/dev/null)&