glusterfsd: Remove 'top {read,write}-perf' thread

'volume top {read,write}-perf' were performed in a different thread on the brick
process, to prevent them from blocking the main polling thread. This was
essential earlier when the rpc actor functions were run in the epoll thread. Now
rpc actors can be launched on their own threads with the introduction of
synctask support for rpcsvc. The brick-op actors are now launched using this
feature, which makes the need for '{read,write}-perf' to be performed in a new
thread.

Change-Id: Idad7ab623f0bfb0c3a5cfae796c06e0fa3cff1b9
BUG: 764890
Signed-off-by: Kaushal M <kaushal@redhat.com>
Reviewed-on: http://review.gluster.org/4510
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Reviewed-by: Anand Avati <avati@redhat.com>

3 files changed, 84 insertions, 145 deletions

diff --git a/glusterfsd/src/glusterfsd-mem-types.h b/glusterfsd/src/glusterfsd-mem-types.h
index 73a91c4a073..83f97b5d32a 100644
--- a/glusterfsd/src/glusterfsd-mem-types.h
+++ b/glusterfsd/src/glusterfsd-mem-types.h
@@ -20,7 +20,6 @@ enum gfd_mem_types_ {
         gfd_mt_xlator_cmdline_option_t,
         gfd_mt_char,
         gfd_mt_call_pool_t,
-        gfd_mt_vol_top_priv_t,
         gfd_mt_end
 
 };
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 8082ad3d1d4..86412efaffa 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -12,7 +12,6 @@
 #include <sys/wait.h>
 #include <stdlib.h>
 #include <signal.h>
-#include <pthread.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -32,7 +31,6 @@
 #include "xdr-generic.h"
 
 #include "glusterfsd.h"
-#include "glusterfsd-mem-types.h"
 #include "rpcsvc.h"
 #include "cli1-xdr.h"
 #include "statedump.h"
@@ -232,65 +230,6 @@ glusterfs_translator_info_response_send (rpcsvc_request_t *req, int ret,
 }
 
 int
-glusterfs_handle_translator_info_get_cont (gfd_vol_top_priv_t *priv)
-{
-        int                     ret = -1;
-        xlator_t                *any = NULL;
-        xlator_t                *xlator = NULL;
-        glusterfs_graph_t       *active = NULL;
-        glusterfs_ctx_t         *ctx = NULL;
-        char                    msg[2048] = {0,};
-        dict_t                  *output = NULL;
-        dict_t                  *dict = NULL;
-
-        GF_ASSERT (priv);
-
-        dict = dict_new ();
-        ret = dict_unserialize (priv->xlator_req.input.input_val,
-                                priv->xlator_req.input.input_len, &dict);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to unserialize dict");
-                goto cont;
-        }
-        ret = dict_set_double (dict, "time", priv->time);
-        if (ret)
-                goto cont;
-        ret = dict_set_double (dict, "throughput", priv->throughput);
-        if (ret)
-                goto cont;
-
-cont:
-        ctx = glusterfsd_ctx;
-        GF_ASSERT (ctx);
-        active = ctx->active;
-        any = active->first;
-
-        xlator = xlator_search_by_name (any, priv->xlator_req.name);
-        if (!xlator) {
-                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
-                          priv->xlator_req.name);
-                goto out;
-        }
-
-        output = dict_new ();
-        ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_INFO, dict, output);
-
-out:
-        ret = glusterfs_translator_info_response_send (priv->req, ret,
-                                                       msg, output);
-
-        free (priv->xlator_req.name);
-        free (priv->xlator_req.input.input_val);
-        if (dict)
-                dict_unref (dict);
-        if (output)
-                dict_unref (output);
-        GF_FREE (priv);
-
-        return ret;
-}
-
-int
 glusterfs_xlator_op_response_send (rpcsvc_request_t *req, int op_ret,
                                    char *msg, dict_t *output)
 {
@@ -326,15 +265,21 @@ glusterfs_xlator_op_response_send (rpcsvc_request_t *req, int op_ret,
 int
 glusterfs_handle_translator_info_get (rpcsvc_request_t *req)
 {
-        int32_t                  ret     = -1;
-        gd1_mgmt_brick_op_req    xlator_req = {0,};
-        dict_t                   *dict    = NULL;
-        xlator_t                 *this = NULL;
-        gf1_cli_top_op           top_op = 0;
-        uint32_t                 blk_size = 0;
-        uint32_t                 blk_count = 0;
-        gfd_vol_top_priv_t       *priv = NULL;
-        pthread_t                tid = -1;
+        int32_t                 ret     = -1;
+        gd1_mgmt_brick_op_req   xlator_req = {0,};
+        dict_t                  *dict    = NULL;
+        xlator_t                *this = NULL;
+        gf1_cli_top_op          top_op = 0;
+        uint32_t                blk_size = 0;
+        uint32_t                blk_count = 0;
+        double                  time = 0;
+        double                  throughput = 0;
+        xlator_t                *any = NULL;
+        xlator_t                *xlator = NULL;
+        glusterfs_graph_t       *active = NULL;
+        glusterfs_ctx_t         *ctx = NULL;
+        char                    msg[2048] = {0,};
+        dict_t                  *output = NULL;
 
         GF_ASSERT (req);
         this = THIS;
@@ -347,8 +292,8 @@ glusterfs_handle_translator_info_get (rpcsvc_request_t *req)
                 req->rpc_err = GARBAGE_ARGS;
                 goto out;
         }
-        dict = dict_new ();
 
+        dict = dict_new ();
         ret = dict_unserialize (xlator_req.input.input_val,
                                 xlator_req.input.input_len,
                                 &dict);
@@ -359,14 +304,6 @@ glusterfs_handle_translator_info_get (rpcsvc_request_t *req)
                 goto out;
         }
 
-        priv = GF_MALLOC (sizeof (gfd_vol_top_priv_t), gfd_mt_vol_top_priv_t);
-        if (!priv) {
-                gf_log ("glusterd", GF_LOG_ERROR, "failed to allocate memory");
-                goto out;
-        }
-        priv->xlator_req = xlator_req;
-        priv->req = req;
-
         ret = dict_get_int32 (dict, "top-op", (int32_t *)&top_op);
         if ((!ret) && (GF_CLI_TOP_READ_PERF == top_op ||
             GF_CLI_TOP_WRITE_PERF == top_op)) {
@@ -376,61 +313,73 @@ glusterfs_handle_translator_info_get (rpcsvc_request_t *req)
                 ret = dict_get_uint32 (dict, "blk-cnt", &blk_count);
                 if (ret)
                         goto cont;
-                priv->blk_size = blk_size;
-                priv->blk_count = blk_count;
+
                 if (GF_CLI_TOP_READ_PERF == top_op) {
-                        ret = pthread_create (&tid, NULL,
-                                              glusterfs_volume_top_read_perf,
-                                              priv);
+                        ret = glusterfs_volume_top_read_perf
+                                (blk_size, blk_count, xlator_req.name,
+                                 &throughput, &time);
                 } else if ( GF_CLI_TOP_WRITE_PERF == top_op) {
-                        ret = pthread_create (&tid, NULL,
-                                              glusterfs_volume_top_write_perf,
-                                              priv);
+                        ret = glusterfs_volume_top_write_perf
+                                (blk_size, blk_count, xlator_req.name,
+                                 &throughput, &time);
                 }
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "Thread create failed");
+                ret = dict_set_double (dict, "time", time);
+                if (ret)
+                        goto cont;
+                ret = dict_set_double (dict, "throughput", throughput);
+                if (ret)
                         goto cont;
-                }
-                gf_log ("glusterd", GF_LOG_DEBUG, "Created new thread with "
-                        "tid %u", (unsigned int)tid);
-                goto out;
         }
 cont:
-        priv->throughput = 0;
-        priv->time = 0;
-        ret = glusterfs_handle_translator_info_get_cont (priv);
+        ctx = glusterfsd_ctx;
+        GF_ASSERT (ctx);
+        active = ctx->active;
+        any = active->first;
+
+        xlator = xlator_search_by_name (any, xlator_req.name);
+        if (!xlator) {
+                snprintf (msg, sizeof (msg), "xlator %s is not loaded",
+                          xlator_req.name);
+                goto out;
+        }
+
+        output = dict_new ();
+        ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_INFO, dict, output);
+
 out:
+        ret = glusterfs_translator_info_response_send (req, ret, msg, output);
+
+        free (xlator_req.name);
+        free (xlator_req.input.input_val);
+        if (output)
+                dict_unref (output);
         if (dict)
                 dict_unref (dict);
         return ret;
 }
 
-void *
-glusterfs_volume_top_write_perf (void *args)
+int
+glusterfs_volume_top_write_perf (uint32_t blk_size, uint32_t blk_count,
+                                 char *brick_path, double *throughput,
+                                 double *time)
 {
         int32_t                 fd = -1;
         int32_t                 input_fd = -1;
         char                    export_path[PATH_MAX];
         char                    *buf = NULL;
-        uint32_t                blk_size = 0;
-        uint32_t                blk_count = 0;
         int32_t                 iter = 0;
         int32_t                 ret = -1;
         uint64_t                total_blks = 0;
         struct timeval          begin, end = {0,};
-        double                  throughput = 0;
-        double                  time = 0;
-        gfd_vol_top_priv_t      *priv = NULL;
-
-        GF_ASSERT (args);
-        priv = (gfd_vol_top_priv_t *)args;
 
-        blk_size = priv->blk_size;
-        blk_count = priv->blk_count;
+        GF_ASSERT (brick_path);
+        GF_ASSERT (throughput);
+        GF_ASSERT (time);
+        if (!(blk_size > 0) || ! (blk_count > 0))
+                goto out;
 
         snprintf (export_path, sizeof (export_path), "%s/%s",
-                  priv->xlator_req.name, ".gf-tmp-stats-perf");
+                  brick_path, ".gf-tmp-stats-perf");
 
         fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
         if (-1 == fd) {
@@ -474,16 +423,13 @@ glusterfs_volume_top_write_perf (void *args)
         }
 
         gettimeofday (&end, NULL);
-        time = (end.tv_sec - begin.tv_sec) * 1e6
+        *time = (end.tv_sec - begin.tv_sec) * 1e6
                      + (end.tv_usec - begin.tv_usec);
-        throughput = total_blks / time;
+        *throughput = total_blks / *time;
         gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs "
-                "bytes written %"PRId64, throughput, time, total_blks);
+                "bytes written %"PRId64, *throughput, *time, total_blks);
 
 out:
-        priv->throughput = throughput;
-        priv->time = time;
-
         if (fd >= 0)
                 close (fd);
         if (input_fd >= 0)
@@ -491,37 +437,32 @@ out:
         GF_FREE (buf);
         unlink (export_path);
 
-        (void)glusterfs_handle_translator_info_get_cont (priv);
-
-        return NULL;
+        return ret;
 }
 
-void *
-glusterfs_volume_top_read_perf (void *args)
+int
+glusterfs_volume_top_read_perf (uint32_t blk_size, uint32_t blk_count,
+                                char *brick_path, double *throughput,
+                                double *time)
 {
         int32_t                 fd = -1;
         int32_t                 input_fd = -1;
         int32_t                 output_fd = -1;
         char                    export_path[PATH_MAX];
         char                    *buf = NULL;
-        uint32_t                blk_size = 0;
-        uint32_t                blk_count = 0;
         int32_t                 iter = 0;
         int32_t                 ret = -1;
         uint64_t                total_blks = 0;
         struct timeval          begin, end = {0,};
-        double                  throughput = 0;
-        double                  time = 0;
-        gfd_vol_top_priv_t      *priv = NULL;
-
-        GF_ASSERT (args);
-        priv = (gfd_vol_top_priv_t *)args;
 
-        blk_size = priv->blk_size;
-        blk_count = priv->blk_count;
+        GF_ASSERT (brick_path);
+        GF_ASSERT (throughput);
+        GF_ASSERT (time);
+        if (!(blk_size > 0) || ! (blk_count > 0))
+                goto out;
 
         snprintf (export_path, sizeof (export_path), "%s/%s",
-                  priv->xlator_req.name, ".gf-tmp-stats-perf");
+                  brick_path, ".gf-tmp-stats-perf");
         fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU);
         if (-1 == fd) {
                 ret = -1;
@@ -597,16 +538,13 @@ glusterfs_volume_top_read_perf (void *args)
         }
 
         gettimeofday (&end, NULL);
-        time = (end.tv_sec - begin.tv_sec) * 1e6
-               + (end.tv_usec - begin.tv_usec);
-        throughput = total_blks / time;
+        *time = (end.tv_sec - begin.tv_sec) * 1e6
+                + (end.tv_usec - begin.tv_usec);
+        *throughput = total_blks / *time;
         gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs "
-                "bytes read %"PRId64, throughput, time, total_blks);
+                "bytes read %"PRId64, *throughput, *time, total_blks);
 
 out:
-        priv->throughput = throughput;
-        priv->time = time;
-
         if (fd >= 0)
                 close (fd);
         if (input_fd >= 0)
@@ -616,9 +554,7 @@ out:
         GF_FREE (buf);
         unlink (export_path);
 
-        (void)glusterfs_handle_translator_info_get_cont (priv);
-
-        return NULL;
+        return ret;
 }
 
 int
diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h
index ff3998a4a29..5b5805d14f4 100644
--- a/glusterfsd/src/glusterfsd.h
+++ b/glusterfsd/src/glusterfsd.h
@@ -102,8 +102,12 @@ int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx);
 int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx);
 void cleanup_and_exit (int signum);
 
-void *glusterfs_volume_top_read_perf (void *args);
-void *glusterfs_volume_top_write_perf (void *args);
+int glusterfs_volume_top_write_perf (uint32_t blk_size, uint32_t blk_count,
+                                     char *brick_path, double *throughput,
+                                     double *time);
+int glusterfs_volume_top_read_perf (uint32_t blk_size, uint32_t blk_count,
+                                    char *brick_path, double *throughput,
+                                    double *time);
 
 extern glusterfs_ctx_t *glusterfsd_ctx;
 #endif /* __GLUSTERFSD_H__ */