glusterd: run 'volume top read-perf/write-perf' in different thread

Runs the 'volume top read-perf/write-perf' operations in a different
thread without blocking glusterd. Prvents glusterd from being
unresponsive when large values of 'bs' and 'count' are given.

Also increase cli timeout for top/profile commands , from 120s to 300s
to allow large i/o top read-perf and write-perf to return result.

Change-Id: I4b7de1d735f33643d836772db7f25133f112b75a
BUG: 2720
Reviewed-on: http://review.gluster.com/375
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Shishir Gowda <shishirng@gluster.com>

3 files changed, 341 insertions, 28 deletions

diff --git a/glusterfsd/src/glusterfsd-mem-types.h b/glusterfsd/src/glusterfsd-mem-types.h
index 7f953d1ff..a28a7b2e3 100644
--- a/glusterfsd/src/glusterfsd-mem-types.h
+++ b/glusterfsd/src/glusterfsd-mem-types.h
@@ -30,6 +30,7 @@ enum gfd_mem_types_ {
         gfd_mt_xlator_cmdline_option_t,
         gfd_mt_char,
         gfd_mt_call_pool_t,
+        gfd_mt_vol_top_priv_t,
         gfd_mt_end
 
 };
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 330b83fa3..43d49b3f4 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -22,6 +22,7 @@
 #include <sys/wait.h>
 #include <stdlib.h>
 #include <signal.h>
+#include <pthread.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -38,11 +39,12 @@
 #include "protocol-common.h"
 #include "glusterfs3.h"
 #include "portmap-xdr.h"
-#include "glusterd1-xdr.h"
 #include "xdr-generic.h"
 
 #include "glusterfsd.h"
+#include "glusterfsd-mem-types.h"
 #include "rpcsvc.h"
+#include "cli1-xdr.h"
 
 static char is_mgmt_rpc_reconnect;
 
@@ -253,28 +255,83 @@ glusterfs_translator_info_response_send (rpcsvc_request_t *req, int ret,
 }
 
 int
+glusterfs_handle_translator_info_get_cont (gfd_vol_top_priv_t *priv)
+{
+        int                     ret = -1;
+        xlator_t                *any = NULL;
+        xlator_t                *xlator = NULL;
+        glusterfs_graph_t       *active = NULL;
+        glusterfs_ctx_t         *ctx = NULL;
+        char                    msg[2048] = {0,};
+        dict_t                  *output = NULL;
+        dict_t                  *dict = NULL;
+
+        GF_ASSERT (priv);
+
+        dict = dict_new ();
+        ret = dict_unserialize (priv->xlator_req.input.input_val,
+                                priv->xlator_req.input.input_len, &dict);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to unserialize dict");
+                goto cont;
+        }
+        ret = dict_set_double (dict, "time", priv->time);
+        if (ret)
+                goto cont;
+        ret = dict_set_double (dict, "throughput", priv->throughput);
+        if (ret)
+                goto cont;
+
+cont:
+        ctx = glusterfs_ctx_get ();
+        GF_ASSERT (ctx);
+        active = ctx->active;
+        any = active->first;
+
+        xlator = xlator_search_by_name (any, priv->xlator_req.name);
+        if (!xlator) {
+                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
+                          priv->xlator_req.name);
+                goto out;
+        }
+
+        output = dict_new ();
+        ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_INFO, dict, output);
+
+out:
+        ret = glusterfs_translator_info_response_send (priv->req, ret,
+                                                       msg, output);
+
+        if (priv->xlator_req.name)
+                free (priv->xlator_req.name);
+        if (priv->xlator_req.input.input_val)
+                free (priv->xlator_req.input.input_val);
+        if (dict)
+                dict_unref (dict);
+        if (output)
+                dict_unref (output);
+        GF_FREE (priv);
+
+        return ret;
+}
+
+int
 glusterfs_handle_translator_info_get (rpcsvc_request_t *req)
 {
         int32_t                  ret     = -1;
         gd1_mgmt_brick_op_req    xlator_req = {0,};
         dict_t                   *dict    = NULL;
-        xlator_t                 *xlator = NULL;
-        xlator_t                 *any = NULL;
-        dict_t                   *output = NULL;
-        char                     msg[2048] = {0};
-        glusterfs_ctx_t          *ctx = NULL;
-        glusterfs_graph_t        *active = NULL;
         xlator_t                 *this = NULL;
+        gf1_cli_top_op            top_op = 0;
+        int32_t                  blk_size = 0;
+        int32_t                  blk_count = 0;
+        gfd_vol_top_priv_t       *priv = NULL;
+        pthread_t                tid = -1;
 
         GF_ASSERT (req);
         this = THIS;
         GF_ASSERT (this);
 
-        ctx = glusterfs_ctx_get ();
-        GF_ASSERT (ctx);
-
-        active = ctx->active;
-        any = active->first;
         if (!xdr_to_generic (req->msg[0], &xlator_req,
                              (xdrproc_t)xdr_gd1_mgmt_brick_op_req)) {
                 //failed to decode msg;
@@ -293,29 +350,270 @@ glusterfs_handle_translator_info_get (rpcsvc_request_t *req)
                 goto out;
         }
 
-        xlator = xlator_search_by_name (any, xlator_req.name);
-        if (!xlator) {
-                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
-                          xlator_req.name);
-                ret = -1;
+        priv = GF_MALLOC (sizeof (gfd_vol_top_priv_t), gfd_mt_vol_top_priv_t);
+        if (!priv) {
+                gf_log ("glusterd", GF_LOG_ERROR, "failed to allocate memory");
                 goto out;
         }
-
-        output = dict_new ();
-        ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_INFO, dict, output);
+        priv->xlator_req = xlator_req;
+        priv->req = req;
+
+        ret = dict_get_int32 (dict, "top-op", (int32_t *)&top_op);
+        if ((!ret) && (GF_CLI_TOP_READ_PERF == top_op ||
+            GF_CLI_TOP_WRITE_PERF == top_op)) {
+                ret = dict_get_int32 (dict, "blk-size", &blk_size);
+                if (ret)
+                        goto cont;
+                ret = dict_get_int32 (dict, "blk-cnt", &blk_count);
+                if (ret)
+                        goto cont;
+                priv->blk_size = blk_size;
+                priv->blk_count = blk_count;
+                if (GF_CLI_TOP_READ_PERF == top_op) {
+                        ret = pthread_create (&tid, NULL,
+                                              glusterfs_volume_top_read_perf,
+                                              priv);
+                } else if ( GF_CLI_TOP_WRITE_PERF == top_op) {
+                        ret = pthread_create (&tid, NULL,
+                                              glusterfs_volume_top_write_perf,
+                                              priv);
+                }
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Thread create failed");
+                        goto cont;
+                }
+                gf_log ("glusterd", GF_LOG_DEBUG, "Created new thread with "
+                        "tid %u", (unsigned int)tid);
+                goto out;
+        }
+cont:
+        priv->throughput = 0;
+        priv->time = 0;
+        ret = glusterfs_handle_translator_info_get_cont (priv);
 out:
-        ret = glusterfs_translator_info_response_send (req, ret, msg, output);
         if (dict)
                 dict_unref (dict);
-        if (xlator_req.input.input_val)
-                free (xlator_req.input.input_val); // malloced by xdr
-        if (output)
-                dict_unref (output);
-        if (xlator_req.name)
-                free (xlator_req.name); //malloced by xdr
         return ret;
 }
 
+void *
+glusterfs_volume_top_write_perf (void *args)
+{
+        int32_t                 fd = -1;
+        int32_t                 input_fd = -1;
+        char                    export_path[PATH_MAX];
+        char                    *buf = NULL;
+        int32_t                 blk_size = 0;
+        int32_t                 blk_count = 0;
+        int32_t                 iter = 0;
+        int32_t                 ret = -1;
+        int64_t                 total_blks = 0;
+        struct timeval          begin, end = {0,};
+        double                  throughput = 0;
+        double                  time = 0;
+        gfd_vol_top_priv_t      *priv = NULL;
+
+        GF_ASSERT (args);
+        priv = (gfd_vol_top_priv_t *)args;
+
+        blk_size = priv->blk_size;
+        blk_count = priv->blk_count;
+
+        snprintf (export_path, sizeof (export_path), "%s/%s",
+                  priv->xlator_req.name, ".gf-tmp-stats-perf");
+
+        fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
+        if (-1 == fd) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not open tmp file");
+                goto out;
+        }
+
+        buf = GF_MALLOC (blk_size * sizeof(*buf), gf_common_mt_char);
+        if (!buf) {
+                ret = -1;
+                goto out;
+        }
+
+        input_fd = open ("/dev/urandom", O_RDONLY);
+        if (-1 == input_fd) {
+                ret = -1;
+                gf_log ("glusterd",GF_LOG_ERROR, "Unable to open input file");
+                goto out;
+        }
+
+        gettimeofday (&begin, NULL);
+        for (iter = 0; iter < blk_count; iter++) {
+                ret = read (input_fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = write (fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+                total_blks += ret;
+        }
+        ret = 0;
+        if (total_blks != (blk_size * blk_count)) {
+                gf_log ("glusterd", GF_LOG_WARNING, "Error in write");
+                ret = -1;
+                goto out;
+        }
+
+        gettimeofday (&end, NULL);
+        time = (end.tv_sec - begin.tv_sec) * 1e6
+                     + (end.tv_usec - begin.tv_usec);
+        throughput = total_blks / time;
+        gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs "
+                "bytes written %"PRId64, throughput, time, total_blks);
+
+out:
+        priv->throughput = throughput;
+        priv->time = time;
+
+        if (fd >= 0)
+                close (fd);
+        if (input_fd >= 0)
+                close (input_fd);
+        if (buf)
+                GF_FREE (buf);
+        unlink (export_path);
+
+        (void)glusterfs_handle_translator_info_get_cont (priv);
+
+        return NULL;
+}
+
+void *
+glusterfs_volume_top_read_perf (void *args)
+{
+        int32_t                 fd = -1;
+        int32_t                 input_fd = -1;
+        int32_t                 output_fd = -1;
+        char                    export_path[PATH_MAX];
+        char                    *buf = NULL;
+        int32_t                 blk_size = 0;
+        int32_t                 blk_count = 0;
+        int32_t                 iter = 0;
+        int32_t                 ret = -1;
+        int64_t                 total_blks = 0;
+        struct timeval          begin, end = {0,};
+        double                  throughput = 0;
+        double                  time = 0;
+        gfd_vol_top_priv_t      *priv = NULL;
+
+        GF_ASSERT (args);
+        priv = (gfd_vol_top_priv_t *)args;
+
+        blk_size = priv->blk_size;
+        blk_count = priv->blk_count;
+
+        snprintf (export_path, sizeof (export_path), "%s/%s",
+                  priv->xlator_req.name, ".gf-tmp-stats-perf");
+        fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
+        if (-1 == fd) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not open tmp file");
+                goto out;
+        }
+
+        buf = GF_MALLOC (blk_size * sizeof(*buf), gf_common_mt_char);
+        if (!buf) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not allocate memory");
+                goto out;
+        }
+
+        input_fd = open ("/dev/urandom", O_RDONLY);
+        if (-1 == input_fd) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not open input file");
+                goto out;
+        }
+
+        output_fd = open ("/dev/null", O_RDWR);
+        if (-1 == output_fd) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not open output file");
+                goto out;
+        }
+
+        for (iter = 0; iter < blk_count; iter++) {
+                ret = read (input_fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = write (fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = fsync (fd);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "could not flush cache");
+                goto out;
+        }
+        ret = lseek (fd, 0L, 0);
+        if (ret != 0) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "could not seek back to start");
+                ret = -1;
+                goto out;
+        }
+        gettimeofday (&begin, NULL);
+        for (iter = 0; iter < blk_count; iter++) {
+                ret = read (fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = write (output_fd, buf, blk_size);
+                if (ret != blk_size) {
+                        ret = -1;
+                        goto out;
+                }
+                total_blks += ret;
+        }
+        ret = 0;
+        if ((blk_size * blk_count) != total_blks) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_WARNING, "Error in write");
+                goto out;
+        }
+
+        gettimeofday (&end, NULL);
+        time = (end.tv_sec - begin.tv_sec) * 1e6
+               + (end.tv_usec - begin.tv_usec);
+        throughput = total_blks / time;
+        gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs "
+                "bytes read %"PRId64, throughput, time, total_blks);
+
+out:
+        priv->throughput = throughput;
+        priv->time = time;
+
+        if (fd >= 0)
+                close (fd);
+        if (input_fd >= 0)
+                close (input_fd);
+        if (output_fd >= 0)
+                close (output_fd);
+        if (buf)
+                GF_FREE (buf);
+        unlink (export_path);
+
+        (void)glusterfs_handle_translator_info_get_cont (priv);
+
+        return NULL;
+}
+
 int
 glusterfs_handle_rpc_msg (rpcsvc_request_t *req)
 {
diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h
index 5106559c1..0e68f7f0e 100644
--- a/glusterfsd/src/glusterfsd.h
+++ b/glusterfsd/src/glusterfsd.h
@@ -24,7 +24,8 @@
 #define _CONFIG_H
 #include "config.h"
 #endif
-
+#include "rpcsvc.h"
+#include "glusterd1-xdr.h"
 
 #define DEFAULT_GLUSTERD_VOLFILE              CONFDIR "/glusterd.vol"
 #define DEFAULT_CLIENT_VOLFILE                CONFDIR "/glusterfs.vol"
@@ -81,9 +82,22 @@ enum argp_option_keys {
         ARGP_USER_MAP_ROOT_KEY            = 156,
 };
 
+struct _gfd_vol_top_priv_t {
+        rpcsvc_request_t        *req;
+        gd1_mgmt_brick_op_req   xlator_req;
+        int32_t                 blk_count;
+        int32_t                 blk_size;
+        double                  throughput;
+        double                  time;
+        int32_t                 ret;
+};
+typedef struct _gfd_vol_top_priv_t gfd_vol_top_priv_t;
+
 int glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx);
 int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx);
 int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx);
 void cleanup_and_exit (int signum);
 
+void *glusterfs_volume_top_read_perf (void *args);
+void *glusterfs_volume_top_write_perf (void *args);
 #endif /* __GLUSTERFSD_H__ */