glusterd: run 'volume top read-perf/write-perf' in different thread

Runs the 'volume top read-perf/write-perf' operations in a different
thread without blocking glusterd. Prvents glusterd from being
unresponsive when large values of 'bs' and 'count' are given.

Also increase cli timeout for top/profile commands , from 120s to 300s
to allow large i/o top read-perf and write-perf to return result.

Change-Id: I4b7de1d735f33643d836772db7f25133f112b75a
BUG: 2720
Reviewed-on: http://review.gluster.com/375
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Shishir Gowda <shishirng@gluster.com>

7 files changed, 352 insertions, 228 deletions

diff --git a/cli/src/cli-cmd.c b/cli/src/cli-cmd.c
index 385e2ac2e..17869eb61 100644
--- a/cli/src/cli-cmd.c
+++ b/cli/src/cli-cmd.c
@@ -31,6 +31,7 @@
 #include "cli.h"
 #include "cli-cmd.h"
 #include "cli-mem-types.h"
+#include "protocol-common.h"
 
 #include <fnmatch.h>
 
@@ -281,14 +282,14 @@ seconds_from_now (unsigned secs, struct timespec *ts)
 }
 
 int
-cli_cmd_await_response ()
+cli_cmd_await_response (unsigned time)
 {
         struct  timespec        ts = {0,};
         int                     ret = 0;
 
         cli_op_ret = -1;
 
-        seconds_from_now (CLI_DEFAULT_CMD_TIMEOUT, &ts);
+        seconds_from_now (time, &ts);
         while (!cmd_done && !ret) {
                 ret = pthread_cond_timedwait (&cond, &cond_mutex,
                                         &ts);
@@ -366,7 +367,11 @@ cli_cmd_submit (void *req, call_frame_t *frame,
                 int procnum, struct iobref *iobref,
                 xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int     ret = -1;
+        int             ret = -1;
+        unsigned        timeout = 0;
+
+        timeout = (GLUSTER_CLI_PROFILE_VOLUME == procnum) ?
+                   CLI_TOP_CMD_TIMEOUT : CLI_DEFAULT_CMD_TIMEOUT;
 
         cli_cmd_lock ();
         cmd_sent = 0;
@@ -375,7 +380,7 @@ cli_cmd_submit (void *req, call_frame_t *frame,
 
         if (!ret) {
                 cmd_sent = 1;
-                ret = cli_cmd_await_response ();
+                ret = cli_cmd_await_response (timeout);
         } else
                 cli_cmd_unlock ();
 
diff --git a/cli/src/cli-cmd.h b/cli/src/cli-cmd.h
index 1878571fa..82e0ff113 100644
--- a/cli/src/cli-cmd.h
+++ b/cli/src/cli-cmd.h
@@ -83,7 +83,7 @@ struct cli_cmd_word *cli_cmd_nextword (struct cli_cmd_word *word,
                                        const char *text);
 void cli_cmd_tokens_destroy (char **tokens);
 
-int cli_cmd_await_response ();
+int cli_cmd_await_response (unsigned time);
 
 int cli_cmd_broadcast_response (int32_t status);
 
diff --git a/cli/src/cli.h b/cli/src/cli.h
index 1e0d69cd8..bf3437827 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -33,6 +33,7 @@
 #define CLI_GLUSTERD_PORT                  24007
 #define CLI_DEFAULT_CONN_TIMEOUT             120
 #define CLI_DEFAULT_CMD_TIMEOUT              120
+#define CLI_TOP_CMD_TIMEOUT                  300 //Longer timeout for volume top
 #define DEFAULT_CLI_LOG_FILE_DIRECTORY     DATADIR "/log/glusterfs"
 #define CLI_VOL_STATUS_BRICK_LEN              55
 #define CLI_TAB_LENGTH                         8
diff --git a/glusterfsd/src/glusterfsd-mem-types.h b/glusterfsd/src/glusterfsd-mem-types.h
index 7f953d1ff..a28a7b2e3 100644
--- a/glusterfsd/src/glusterfsd-mem-types.h
+++ b/glusterfsd/src/glusterfsd-mem-types.h
@@ -30,6 +30,7 @@ enum gfd_mem_types_ {
         gfd_mt_xlator_cmdline_option_t,
         gfd_mt_char,
         gfd_mt_call_pool_t,
+        gfd_mt_vol_top_priv_t,
         gfd_mt_end
 
 };
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 330b83fa3..43d49b3f4 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -22,6 +22,7 @@
 #include <sys/wait.h>
 #include <stdlib.h>
 #include <signal.h>
+#include <pthread.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -38,11 +39,12 @@
 #include "protocol-common.h"
 #include "glusterfs3.h"
 #include "portmap-xdr.h"
-#include "glusterd1-xdr.h"
 #include "xdr-generic.h"
 
 #include "glusterfsd.h"
+#include "glusterfsd-mem-types.h"
 #include "rpcsvc.h"
+#include "cli1-xdr.h"
 
 static char is_mgmt_rpc_reconnect;
 
@@ -253,28 +255,83 @@ glusterfs_translator_info_response_send (rpcsvc_request_t *req, int ret,
 }
 
 int
+glusterfs_handle_translator_info_get_cont (gfd_vol_top_priv_t *priv)
+{
+        int                     ret = -1;
+        xlator_t                *any = NULL;
+        xlator_t                *xlator = NULL;
+        glusterfs_graph_t       *active = NULL;
+        glusterfs_ctx_t         *ctx = NULL;
+        char                    msg[2048] = {0,};
+        dict_t                  *output = NULL;
+        dict_t                  *dict = NULL;
+
+        GF_ASSERT (priv);
+
+        dict = dict_new ();
+        ret = dict_unserialize (priv->xlator_req.input.input_val,
+                                priv->xlator_req.input.input_len, &dict);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to unserialize dict");
+                goto cont;
+        }
+        ret = dict_set_double (dict, "time", priv->time);
+        if (ret)
+                goto cont;
+        ret = dict_set_double (dict, "throughput", priv->throughput);
+        if (ret)
+                goto cont;
+
+cont:
+        ctx = glusterfs_ctx_get ();
+        GF_ASSERT (ctx);
+        active = ctx->active;
+        any = active->first;
+
+        xlator = xlator_search_by_name (any, priv->xlator_req.name);
+        if (!xlator) {
+                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
+                          priv->xlator_req.name);
+                goto out;
+        }
+
+        output = dict_new ();
+        ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_INFO, dict, output);
+
+out:
+        ret = glusterfs_translator_info_response_send (priv->req, ret,
+                                                       msg, output);
+
+        if (priv->xlator_req.name)
+                free (priv->xlator_req.name);
+        if (priv->xlator_req.input.input_val)
+                free (priv->xlator_req.input.input_val);
+        if (dict)
+                dict_unref (dict);
+        if (output)
+                dict_unref (output);
+        GF_FREE (priv);
+
+        return ret;
+}
+
+int
 glusterfs_handle_translator_info_get (rpcsvc_request_t *req)
 {
         int32_t                  ret     = -1;
         gd1_mgmt_brick_op_req    xlator_req = {0,};
         dict_t                   *dict    = NULL;
-        xlator_t                 *xlator = NULL;
-        xlator_t                 *any = NULL;
-        dict_t                   *output = NULL;
-        char                     msg[2048] = {0};
-        glusterfs_ctx_t          *ctx = NULL;
-        glusterfs_graph_t        *active = NULL;
         xlator_t                 *this = NULL;
+        gf1_cli_top_op            top_op = 0;
+        int32_t                  blk_size = 0;
+        int32_t                  blk_count = 0;
+        gfd_vol_top_priv_t       *priv = NULL;
+        pthread_t                tid = -1;
 
         GF_ASSERT (req);
         this = THIS;
         GF_ASSERT (this);
 
-        ctx = glusterfs_ctx_get ();
-        GF_ASSERT (ctx);
-
-        active = ctx->active;
-        any = active->first;
         if (!xdr_to_generic (req->msg[0], &xlator_req,
                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req)) {
                 //failed to decode msg;
@@ -293,29 +350,270 @@ glusterfs_handle_translator_info_get (rpcsvc_request_t *req)
                 goto out;
         }
 
-        xlator = xlator_search_by_name (any, xlator_req.name);
-        if (!xlator) {
-                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
-                          xlator_req.name);
-                ret = -1;
+        priv = GF_MALLOC (sizeof (gfd_vol_top_priv_t), gfd_mt_vol_top_priv_t);
+        if (!priv) {
+                gf_log ("glusterd", GF_LOG_ERROR, "failed to allocate memory");
                 goto out;
         }
-
-        output = dict_new ();
-        ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_INFO, dict, output);
+        priv->xlator_req = xlator_req;
+        priv->req = req;
+
+        ret = dict_get_int32 (dict, "top-op", (int32_t *)&top_op);
+        if ((!ret) && (GF_CLI_TOP_READ_PERF == top_op ||
+            GF_CLI_TOP_WRITE_PERF == top_op)) {
+                ret = dict_get_int32 (dict, "blk-size", &blk_size);
+                if (ret)
+                        goto cont;
+                ret = dict_get_int32 (dict, "blk-cnt", &blk_count);
+                if (ret)
+                        goto cont;
+                priv->blk_size = blk_size;
+                priv->blk_count = blk_count;
+                if (GF_CLI_TOP_READ_PERF == top_op) {
+                        ret = pthread_create (&tid, NULL,
+                                              glusterfs_volume_top_read_perf,
+                                              priv);
+                } else if ( GF_CLI_TOP_WRITE_PERF == top_op) {
+                        ret = pthread_create (&tid, NULL,
+                                              glusterfs_volume_top_write_perf,
+                                              priv);
+                }
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Thread create failed");
+                        goto cont;
+                }
+                gf_log ("glusterd", GF_LOG_DEBUG, "Created new thread with "
+                        "tid %u", (unsigned int)tid);
+                goto out;
+        }
+cont:
+        priv->throughput = 0;
+        priv->time = 0;
+        ret = glusterfs_handle_translator_info_get_cont (priv);
 out:
-        ret = glusterfs_translator_info_response_send (req, ret, msg, output);
         if (dict)
                 dict_unref (dict);
-        if (xlator_req.input.input_val)
-                free (xlator_req.input.input_val); // malloced by xdr
-        if (output)
-                dict_unref (output);
-        if (xlator_req.name)
-                free (xlator_req.name); //malloced by xdr
         return ret;
 }
 
+void *
+glusterfs_volume_top_write_perf (void *args)
+{
+        int32_t                 fd = -1;
+        int32_t                 input_fd = -1;
+        char                    export_path[PATH_MAX];
+        char                    *buf = NULL;
+        int32_t                 blk_size = 0;
+        int32_t                 blk_count = 0;
+        int32_t                 iter = 0;
+        int32_t                 ret = -1;
+        int64_t                 total_blks = 0;
+        struct timeval          begin, end = {0,};
+        double                  throughput = 0;
+        double                  time = 0;
+        gfd_vol_top_priv_t      *priv = NULL;
+
+        GF_ASSERT (args);
+        priv = (gfd_vol_top_priv_t *)args;
+
+        blk_size = priv->blk_size;
+        blk_count = priv->blk_count;
+
+        snprintf (export_path, sizeof (export_path), "%s/%s",
+                  priv->xlator_req.name, ".gf-tmp-stats-perf");
+
+        fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
+        if (-1 == fd) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not open tmp file");
+                goto out;
+        }
+
+        buf = GF_MALLOC (blk_size * sizeof(*buf), gf_common_mt_char);
+        if (!buf) {
+                ret = -1;
+                goto out;
+        }
+
+        input_fd = open ("/dev/urandom", O_RDONLY);
+        if (-1 == input_fd) {
+                ret = -1;
+                gf_log ("glusterd",GF_LOG_ERROR, "Unable to open input file");
+                goto out;
+        }
+
+        gettimeofday (&begin, NULL);
+        for (iter = 0; iter < blk_count; iter++) {
+                ret = read (input_fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = write (fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+                total_blks += ret;
+        }
+        ret = 0;
+        if (total_blks != (blk_size * blk_count)) {
+                gf_log ("glusterd", GF_LOG_WARNING, "Error in write");
+                ret = -1;
+                goto out;
+        }
+
+        gettimeofday (&end, NULL);
+        time = (end.tv_sec - begin.tv_sec) * 1e6
+                     + (end.tv_usec - begin.tv_usec);
+        throughput = total_blks / time;
+        gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs "
+                "bytes written %"PRId64, throughput, time, total_blks);
+
+out:
+        priv->throughput = throughput;
+        priv->time = time;
+
+        if (fd >= 0)
+                close (fd);
+        if (input_fd >= 0)
+                close (input_fd);
+        if (buf)
+                GF_FREE (buf);
+        unlink (export_path);
+
+        (void)glusterfs_handle_translator_info_get_cont (priv);
+
+        return NULL;
+}
+
+void *
+glusterfs_volume_top_read_perf (void *args)
+{
+        int32_t                 fd = -1;
+        int32_t                 input_fd = -1;
+        int32_t                 output_fd = -1;
+        char                    export_path[PATH_MAX];
+        char                    *buf = NULL;
+        int32_t                 blk_size = 0;
+        int32_t                 blk_count = 0;
+        int32_t                 iter = 0;
+        int32_t                 ret = -1;
+        int64_t                 total_blks = 0;
+        struct timeval          begin, end = {0,};
+        double                  throughput = 0;
+        double                  time = 0;
+        gfd_vol_top_priv_t      *priv = NULL;
+
+        GF_ASSERT (args);
+        priv = (gfd_vol_top_priv_t *)args;
+
+        blk_size = priv->blk_size;
+        blk_count = priv->blk_count;
+
+        snprintf (export_path, sizeof (export_path), "%s/%s",
+                  priv->xlator_req.name, ".gf-tmp-stats-perf");
+        fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
+        if (-1 == fd) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not open tmp file");
+                goto out;
+        }
+
+        buf = GF_MALLOC (blk_size * sizeof(*buf), gf_common_mt_char);
+        if (!buf) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not allocate memory");
+                goto out;
+        }
+
+        input_fd = open ("/dev/urandom", O_RDONLY);
+        if (-1 == input_fd) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not open input file");
+                goto out;
+        }
+
+        output_fd = open ("/dev/null", O_RDWR);
+        if (-1 == output_fd) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not open output file");
+                goto out;
+        }
+
+        for (iter = 0; iter < blk_count; iter++) {
+                ret = read (input_fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = write (fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = fsync (fd);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "could not flush cache");
+                goto out;
+        }
+        ret = lseek (fd, 0L, 0);
+        if (ret != 0) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "could not seek back to start");
+                ret = -1;
+                goto out;
+        }
+        gettimeofday (&begin, NULL);
+        for (iter = 0; iter < blk_count; iter++) {
+                ret = read (fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = write (output_fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+                total_blks += ret;
+        }
+        ret = 0;
+        if ((blk_size * blk_count) != total_blks) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_WARNING, "Error in write");
+                goto out;
+        }
+
+        gettimeofday (&end, NULL);
+        time = (end.tv_sec - begin.tv_sec) * 1e6
+               + (end.tv_usec - begin.tv_usec);
+        throughput = total_blks / time;
+        gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs "
+                "bytes read %"PRId64, throughput, time, total_blks);
+
+out:
+        priv->throughput = throughput;
+        priv->time = time;
+
+        if (fd >= 0)
+                close (fd);
+        if (input_fd >= 0)
+                close (input_fd);
+        if (output_fd >= 0)
+                close (output_fd);
+        if (buf)
+                GF_FREE (buf);
+        unlink (export_path);
+
+        (void)glusterfs_handle_translator_info_get_cont (priv);
+
+        return NULL;
+}
+
 int
 glusterfs_handle_rpc_msg (rpcsvc_request_t *req)
 {
diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h
index 5106559c1..0e68f7f0e 100644
--- a/glusterfsd/src/glusterfsd.h
+++ b/glusterfsd/src/glusterfsd.h
@@ -24,7 +24,8 @@
 #define _CONFIG_H
 #include "config.h"
 #endif
-
+#include "rpcsvc.h"
+#include "glusterd1-xdr.h"
 
 #define DEFAULT_GLUSTERD_VOLFILE              CONFDIR "/glusterd.vol"
 #define DEFAULT_CLIENT_VOLFILE                CONFDIR "/glusterfs.vol"
@@ -81,9 +82,22 @@ enum argp_option_keys {
         ARGP_USER_MAP_ROOT_KEY            = 156,
 };
 
+struct _gfd_vol_top_priv_t {
+        rpcsvc_request_t        *req;
+        gd1_mgmt_brick_op_req   xlator_req;
+        int32_t                 blk_count;
+        int32_t                 blk_size;
+        double                  throughput;
+        double                  time;
+        int32_t                 ret;
+};
+typedef struct _gfd_vol_top_priv_t gfd_vol_top_priv_t;
+
 int glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx);
 int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx);
 int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx);
 void cleanup_and_exit (int signum);
 
+void *glusterfs_volume_top_read_perf (void *args);
+void *glusterfs_volume_top_write_perf (void *args);
 #endif /* __GLUSTERFSD_H__ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 2b2a1e766..c9d1c99a1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -171,11 +171,6 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
 {
         int                     ret = -1;
         gd1_mgmt_brick_op_req   *brick_req = NULL;
-        gf1_cli_top_op          top_op = 0;
-        double                  throughput = 0;
-        double                  time = 0;
-        int32_t                 blk_size = 0;
-        int32_t                 blk_count = 0;
 
         GF_ASSERT (op < GD_OP_MAX);
         GF_ASSERT (op > GD_OP_NONE);
@@ -206,46 +201,12 @@ glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickin
                 brick_req->op = GF_BRICK_XLATOR_INFO;
                 brick_req->name = brickinfo->path;
 
-                ret = dict_get_int32 (dict, "top-op", (int32_t*)&top_op);
-                if (ret)
-                        goto cont; 
-                if (top_op == GF_CLI_TOP_READ_PERF ||
-                    top_op == GF_CLI_TOP_WRITE_PERF) {
-
-                        ret = dict_get_int32 (dict, "blk-size", &blk_size);
-                        if (ret) {
-                                goto cont;
-                        }
-                        ret = dict_get_int32 (dict, "blk-cnt", &blk_count);
-                        if (ret)
-                                goto out;
-                        if (top_op == GF_CLI_TOP_READ_PERF)
-                                 ret = glusterd_volume_stats_read_perf (
-                                        brickinfo->path, blk_size, blk_count, 
-                                        &throughput, &time);
-                        else if (!ret && top_op == GF_CLI_TOP_WRITE_PERF)
-                                ret = glusterd_volume_stats_write_perf (
-                                        brickinfo->path, blk_size, blk_count, 
-                                        &throughput, &time);
-
-                        if (ret) 
-                                goto out;
-
-                        ret = dict_set_double (dict, "throughput",
-                                              throughput);
-                        if (ret)
-                                goto out;
-                        ret = dict_set_double (dict, "time", time);
-                        if (ret)
-                                goto out;
-                }
                 break;
         default:
                 goto out;
         break;
         }
 
-cont:
         ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
                                            (size_t*)&brick_req->input.input_len);
         if (ret)
@@ -3415,159 +3376,3 @@ glusterd_op_sm_init ()
         return 0;
 }
 
-int32_t
-glusterd_volume_stats_write_perf (char *brick_path, int32_t blk_size,
-                int32_t blk_count, double *throughput, double *time)
-{
-        int32_t          fd = -1;
-        int32_t          input_fd = -1;
-        char             export_path[1024];
-        char            *buf = NULL;
-        int32_t          iter = 0;
-        int32_t          ret = -1;
-        int64_t          total_blks = 0;
-        struct timeval   begin, end = {0, };
-
-
-        GF_VALIDATE_OR_GOTO ("stripe", brick_path, out);
-
-        snprintf (export_path, sizeof(export_path), "%s/%s",
-                 brick_path, ".gf_tmp_stats_perf");
-        fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
-        if (fd == -1)
-                return errno;
-        buf = GF_MALLOC (blk_size * sizeof(*buf), gf_common_mt_char);
-
-        if (!buf)
-                return ret;
-
-        input_fd = open("/dev/zero", O_RDONLY);
-        if (input_fd == -1)
-                return errno;
-        gettimeofday (&begin, NULL);
-        for (iter = 0; iter < blk_count; iter++) {
-                ret = read (input_fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = write (fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-                total_blks += ret;
-        }
-        ret = 0;
-        if (total_blks != (blk_size * blk_count)) {
-                gf_log ("glusterd", GF_LOG_WARNING, "Errors in write");
-                ret = -1;
-                goto out;
-        }
-
-        gettimeofday (&end, NULL);
-        *time = (end.tv_sec - begin.tv_sec) * 1e6
-                + (end.tv_usec - begin.tv_usec);
-
-        *throughput = total_blks / *time;
-        gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f MBps time %.2f secs bytes "
-                "written %"PRId64, *throughput, *time / 1e6, total_blks);
-out:
-        if (fd >= 0)
-                close (fd);
-        if (input_fd >= 0)
-                close (input_fd);
-        if (buf)
-                GF_FREE (buf);
-        unlink (export_path);
-        return ret;
-}
-
-int32_t
-glusterd_volume_stats_read_perf (char *brick_path, int32_t blk_size,
-                int32_t blk_count, double *throughput, double *time)
-{
-        int32_t          fd = -1;
-        int32_t          output_fd = -1;
-        int32_t          input_fd = -1;
-        char             export_path[1024];
-        char            *buf = NULL;
-        int32_t          iter = 0;
-        int32_t          ret = -1;
-        int64_t          total_blks = 0;
-        struct timeval   begin, end = {0, };
-
-
-        GF_VALIDATE_OR_GOTO ("glusterd", brick_path, out);
-
-        snprintf (export_path, sizeof(export_path), "%s/%s",
-                 brick_path, ".gf_tmp_stats_perf");
-        fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
-        if (fd == -1)
-                return errno;
-        buf = GF_MALLOC (blk_size * sizeof(*buf), gf_common_mt_char);
-
-        if (!buf)
-                return ret;
-
-        output_fd = open("/dev/null", O_RDWR);
-        if (output_fd == -1)
-                return errno;
-        input_fd = open("/dev/zero", O_RDONLY);
-        if (input_fd == -1)
-                return errno;
-        for (iter = 0; iter < blk_count; iter++) {
-                ret = read (input_fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = write (fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-
-        lseek (fd, 0L, 0);
-        gettimeofday (&begin, NULL);
-        for (iter = 0; iter < blk_count; iter++) {
-                ret = read (fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = write (output_fd, buf, blk_size);
-                if (ret != blk_size) {
-                        ret = -1;
-                        goto out;
-                }
-                total_blks += ret;
-        }
-        ret = 0;
-        if (total_blks != (blk_size * blk_count)) {
-                gf_log ("glusterd", GF_LOG_WARNING, "Errors in write");
-                ret = -1;
-                goto out;
-        }
-
-        gettimeofday (&end, NULL);
-        *time = (end.tv_sec - begin.tv_sec) * 1e6
-                + (end.tv_usec - begin.tv_usec);
-
-        *throughput = total_blks / *time;
-        gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f MBps time %.2f secs bytes "
-                "read %"PRId64, *throughput, *time / 1e6, total_blks);
-out:
-        if (fd >= 0)
-                close (fd);
-        if (input_fd >= 0)
-                close (input_fd);
-        if (output_fd >= 0)
-                close (output_fd);
-        if (buf)
-                GF_FREE (buf);
-        unlink (export_path);
-        return ret;
-}