glusterd: Add client details to get-state output

This commit optionally adds client details corresponding to the
locally running bricks to the get-state output. Since getting
the client details involves sending RPC requests to the respective
local bricks, this is a relatively more costly operation. These
client details would be added to the get-state output only if the
get-state command is invoked with the 'detail' option.

This commit therefore also changes the get-state CLI usage. The
modified usage is as follows:

 # gluster get-state [<daemon>] [[odir </path/to/output/dir/>] \
[file <filename>]] [detail]

Change-Id: I42cd4ef160f9e96d55a08a10d32c8ba44e4cd3d8
BUG: 1431183
Signed-off-by: Samikshan Bairagya <samikshan@gmail.com>
Reviewed-on: https://review.gluster.org/17003
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

8 files changed, 262 insertions, 24 deletions

diff --git a/cli/src/cli-cmd-global.c b/cli/src/cli-cmd-global.c
index 2f9d24501d4..0f08985318e 100644
--- a/cli/src/cli-cmd-global.c
+++ b/cli/src/cli-cmd-global.c
@@ -41,7 +41,8 @@ struct cli_cmd global_cmds[] = {
            cli_cmd_global_help_cbk,
            "list global commands",
         },
-        { "get-state [<daemon>] [odir </path/to/output/dir/>] [file <filename>]",
+        { "get-state [<daemon>] [[odir </path/to/output/dir/>] "
+          "[file <filename>]] [detail]",
           cli_cmd_get_state_cbk,
           "Get local state representation of mentioned daemon",
         },
diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index be53cd2ed40..c43b52b63a2 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -829,6 +829,7 @@ cli_cmd_get_state_parse (struct cli_state *state,
         char      *filename             = NULL;
         char      *daemon_name          = NULL;
         int        count                = 0;
+        uint32_t   cmd                  = 0;
 
         GF_VALIDATE_OR_GOTO ("cli", options, out);
         GF_VALIDATE_OR_GOTO ("cli", words, out);
@@ -837,7 +838,7 @@ cli_cmd_get_state_parse (struct cli_state *state,
         if (!dict)
                 goto out;
 
-        if (wordcount < 1 || wordcount > 6) {
+        if (wordcount < 1 || wordcount > 7) {
                 *op_errstr = gf_strdup ("Problem parsing arguments."
                                         " Check usage.");
                 goto out;
@@ -868,16 +869,28 @@ cli_cmd_get_state_parse (struct cli_state *state,
                                 }
                         } else {
                                 if (count > 1) {
-                                        *op_errstr = gf_strdup ("Problem "
-                                                        "parsing arguments. "
+                                        if (count == wordcount-1 &&
+                                            !strcmp (words[count], "detail")) {
+                                                cmd = GF_CLI_GET_STATE_DETAIL;
+                                                continue;
+                                        } else {
+                                                *op_errstr = gf_strdup ("Problem"
+                                                        " parsing arguments. "
                                                         "Check usage.");
-                                        ret = -1;
-                                        goto out;
+                                                ret = -1;
+                                                goto out;
+                                        }
 
                                 }
                                 if (strcmp (words[count], "glusterd") == 0) {
                                         continue;
                                 } else {
+                                        if (count == wordcount-1 &&
+                                            !strcmp (words[count], "detail")) {
+                                                cmd = GF_CLI_GET_STATE_DETAIL;
+                                                continue;
+                                        }
+
                                         *op_errstr = gf_strdup ("glusterd is "
                                                  "the only supported daemon.");
                                         ret = -1;
@@ -919,6 +932,19 @@ cli_cmd_get_state_parse (struct cli_state *state,
                                 goto out;
                         }
                 }
+
+                if (cmd) {
+                        ret = dict_set_uint32 (dict, "getstate-cmd", cmd);
+                        if (ret) {
+                                *op_errstr = gf_strdup ("Command failed. Please"
+                                                        " check log file for"
+                                                        " more details.");
+                                gf_log (THIS->name, GF_LOG_ERROR, "Setting "
+                                        "get-state command type to dictionary "
+                                        "failed");
+                                goto out;
+                        }
+                }
         }
 
  out:
diff --git a/doc/gluster.8 b/doc/gluster.8
index c9a9d500a29..c7b6f221f91 100644
--- a/doc/gluster.8
+++ b/doc/gluster.8
@@ -269,7 +269,7 @@ Selects <HOSTNAME:BRICKNAME> as the source for all the files that are in split-b
 Selects the split-brained <FILE> present in <HOSTNAME:BRICKNAME> as source and completes heal.
 .SS "Other Commands"
 .TP
-\fB\ get-state [<daemon>] [odir </path/to/output/dir/>] [file <filename>] \fR
+\fB\ get-state [<daemon>] [[odir </path/to/output/dir/>] [file <filename>]] [detail] \fR
 Get local state representation of mentioned daemon and store data in provided path information
 .TP
 \fB\ help \fR
diff --git a/rpc/xdr/src/cli1-xdr.x b/rpc/xdr/src/cli1-xdr.x
index 0feb9360284..b7d07c2853c 100644
--- a/rpc/xdr/src/cli1-xdr.x
+++ b/rpc/xdr/src/cli1-xdr.x
@@ -153,6 +153,10 @@ enum gf1_cli_info_op {
         GF_CLI_INFO_CLEAR = 4
 };
 
+enum gf_cli_get_state_op {
+        GF_CLI_GET_STATE_DETAIL = 1
+};
+
 enum gf1_cli_top_op {
         GF_CLI_TOP_NONE = 0,
         GF_CLI_TOP_OPEN,
diff --git a/tests/bugs/cli/bug-1353156-get-state-cli-validations.t b/tests/bugs/cli/bug-1353156-get-state-cli-validations.t
index 6ab7a084da0..f6e72a591ce 100644
--- a/tests/bugs/cli/bug-1353156-get-state-cli-validations.t
+++ b/tests/bugs/cli/bug-1353156-get-state-cli-validations.t
@@ -77,7 +77,15 @@ TEST positive_test $CLI get-state odir $ODIR file gdstate
 
 TEST positive_test $CLI get-state glusterd odir $ODIR file gdstate
 
-TEST positive_test $CLI get-state glusterd odir $ODIR file gdstate
+TEST positive_test $CLI get-state detail
+
+TEST positive_test $CLI get-state glusterd detail
+
+TEST positive_test $CLI get-state odir $ODIR detail
+
+TEST positive_test $CLI get-state glusterd odir $ODIR detail
+
+TEST positive_test $CLI get-state glusterd odir $ODIR file gdstate detail
 
 TEST ! $CLI get-state glusterfsd odir $ODIR;
 ERRSTR=$($CLI get-state glusterfsd odir $ODIR 2>&1 >/dev/null);
@@ -111,19 +119,12 @@ TEST ! $CLI get-state glusterd foo bar;
 ERRSTR=$($CLI get-state glusterd foo bar 2>&1 >/dev/null);
 EXPECT 'Problem' get_parsing_arguments_part $ERRSTR;
 
-cleanup;
+TEST ! $CLI get-state glusterd detail file gdstate;
+ERRSTR=$($CLI get-state glusterd foo bar 2>&1 >/dev/null);
+EXPECT 'Problem' get_parsing_arguments_part $ERRSTR;
+
+TEST ! $CLI get-state glusterd foo bar detail;
+ERRSTR=$($CLI get-state glusterd foo bar 2>&1 >/dev/null);
+EXPECT 'Problem' get_parsing_arguments_part $ERRSTR;
 
-# I've cleaned this up as much as I can - making sure the gdstates directory
-# gets cleaned up, checking whether the CLI command actually succeeded before
-# parsing its output, etc. - but it still fails in Jenkins.  Specifically, the
-# first get-state request that hits the server (i.e. doesn't bail out with a
-# parse error first) succeeds, but any others time out.  They don't even get as
-# far as the glusterd log message that says we received a get-state request.
-# There doesn't seem to be a core file, so glusterd doesn't seem to have
-# crashed, but it's not responding either.  Even worse, the problem seems to be
-# environment-dependent; Jenkins is the only place I've seen it, and that's
-# just about the worst environment ever for debugging anything.
-#
-# I'm marking this test bad so progress can be made elsewhere.  If anybody else
-# thinks this functionality is important, and wants to make it debuggable, good
-# luck to you.
+cleanup;
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 8368354e65f..429999ef584 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -4947,6 +4947,8 @@ glusterd_handle_get_vol_opt (rpcsvc_request_t *req)
         return glusterd_big_locked_handler (req, __glusterd_handle_get_vol_opt);
 }
 
+extern struct rpc_clnt_program gd_brick_prog;
+
 static int
 glusterd_print_global_options (dict_t *opts, char *key, data_t *val, void *data)
 {
@@ -5031,6 +5033,178 @@ out:
 }
 
 static int
+glusterd_print_client_details (FILE *fp, dict_t *dict,
+                               glusterd_volinfo_t *volinfo, int volcount,
+                               glusterd_brickinfo_t *brickinfo, int brickcount)
+{
+        int              ret = -1;
+        xlator_t        *this = NULL;
+        int              brick_index = -1;
+        int              client_count = 0;
+        char             key[1024] = {0,};
+        char            *clientname = NULL;
+        uint64_t         bytesread = 0;
+        uint64_t         byteswrite = 0;
+        uint32_t         opversion = 0;
+
+        glusterd_pending_node_t     *pending_node = NULL;
+        rpc_clnt_t                  *rpc = NULL;
+        struct syncargs              args = {0,};
+        gd1_mgmt_brick_op_req       *brick_req = NULL;
+
+        this = THIS;
+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+
+        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+
+        if (gf_uuid_compare (brickinfo->uuid, MY_UUID) ||
+                             !glusterd_is_brick_started (brickinfo)) {
+                ret = 0;
+                goto out;
+        }
+
+        brick_index++;
+        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                  gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+                ret = -1;
+                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                        "Unable to allocate memory");
+                goto out;
+        }
+
+        pending_node->node = brickinfo;
+        pending_node->type = GD_NODE_BRICK;
+        pending_node->index = brick_index;
+
+        rpc = glusterd_pending_node_get_rpc (pending_node);
+        if (!rpc) {
+                ret = -1;
+                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_FAILURE,
+                        "Failed to retrieve rpc object");
+                goto out;
+        }
+
+        brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                               gf_gld_mt_mop_brick_req_t);
+        if (!brick_req) {
+                ret = -1;
+                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                        "Unable to allocate memory");
+                goto out;
+        }
+
+        brick_req->op = GLUSTERD_BRICK_STATUS;
+        brick_req->name = "";
+
+        ret = dict_set_int32 (dict, "cmd", GF_CLI_STATUS_CLIENTS);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dict, "volname", volinfo->volname);
+        if (ret)
+                goto out;
+
+        ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
+                                           &brick_req->input.input_len);
+        if (ret)
+                goto out;
+
+        GD_SYNCOP (rpc, (&args), NULL, gd_syncop_brick_op_cbk, brick_req,
+                   &gd_brick_prog, brick_req->op, xdr_gd1_mgmt_brick_op_req);
+
+        if (args.op_ret)
+                goto out;
+
+        ret = dict_get_int32 (args.dict, "clientcount", &client_count);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                        "Couldn't get client count");
+                goto out;
+        }
+
+        fprintf (fp, "Volume%d.Brick%d.client_count: %d\n",
+                 volcount, brickcount, client_count);
+
+        if (client_count == 0) {
+                ret = 0;
+                goto out;
+        }
+
+        int i;
+        for (i = 1; i <= client_count; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "client%d.hostname", i-1);
+                ret = dict_get_str (args.dict, key, &clientname);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_DICT_GET_FAILED,
+                                "Failed to get client hostname");
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "Client%d.hostname", i);
+                fprintf (fp, "Volume%d.Brick%d.%s: %s\n",
+                         volcount, brickcount, key, clientname);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "client%d.bytesread", i-1);
+                ret = dict_get_uint64 (args.dict, key, &bytesread);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_DICT_GET_FAILED,
+                                "Failed to get bytesread from client");
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "Client%d.bytesread", i);
+                fprintf (fp, "Volume%d.Brick%d.%s: %"PRIu64"\n",
+                         volcount, brickcount, key, bytesread);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "client%d.byteswrite", i-1);
+                ret = dict_get_uint64 (args.dict, key, &byteswrite);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_DICT_GET_FAILED,
+                                "Failed to get byteswrite from client");
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "Client%d.byteswrite", i);
+                fprintf (fp, "Volume%d.Brick%d.%s: %"PRIu64"\n",
+                         volcount, brickcount, key, byteswrite);
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "client%d.opversion", i-1);
+                ret = dict_get_uint32 (args.dict, key, &opversion);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_DICT_GET_FAILED,
+                                "Failed to get client opversion");
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "Client%d.opversion", i);
+                fprintf (fp, "Volume%d.Brick%d.%s: %"PRIu32"\n",
+                         volcount, brickcount, key, opversion);
+        }
+
+out:
+        if (pending_node)
+                GF_FREE (pending_node);
+
+        if (brick_req)
+                GF_FREE (brick_req);
+
+        return ret;
+}
+
+static int
 glusterd_get_state (rpcsvc_request_t *req, dict_t *dict)
 {
         int32_t                      ret = -1;
@@ -5052,6 +5226,7 @@ glusterd_get_state (rpcsvc_request_t *req, dict_t *dict)
         int                          odirlen = 0;
         time_t                       now = 0;
         char                         timestamp[16] = {0,};
+        uint32_t                     get_state_cmd = 0;
 
         char    *vol_type_str = NULL;
         char    *hot_tier_type_str = NULL;
@@ -5174,6 +5349,12 @@ glusterd_get_state (rpcsvc_request_t *req, dict_t *dict)
         }
         rcu_read_unlock ();
 
+        ret = dict_get_uint32 (dict, "getstate-cmd", &get_state_cmd);
+        if (ret) {
+                gf_msg_debug (this->name, 0, "get-state command type not set");
+                ret = 0;
+        }
+
         count = 0;
         fprintf (fp, "\n[Volumes]\n");
 
@@ -5264,6 +5445,19 @@ glusterd_get_state (rpcsvc_request_t *req, dict_t *dict)
                                       count <= volinfo->tier_info.hot_brick_count ?
                                       "Hot" : "Cold");
                         }
+
+                        if (get_state_cmd != GF_CLI_GET_STATE_DETAIL)
+                                continue;
+
+                        ret = glusterd_print_client_details (fp, dict,
+                                                             volinfo, count_bkp,
+                                                             brickinfo, count);
+                        if (ret) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0,
+                                        GD_MSG_CLIENTS_GET_STATE_FAILED,
+                                        "Failed to get client details");
+                                goto out;
+                        }
                 }
 
                 count = count_bkp;
diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h
index 15b2071ca5c..efeda0d4edf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-messages.h
+++ b/xlators/mgmt/glusterd/src/glusterd-messages.h
@@ -41,7 +41,7 @@
 
 #define GLUSTERD_COMP_BASE      GLFS_MSGID_GLUSTERD
 
-#define GLFS_NUM_MESSAGES       598
+#define GLFS_NUM_MESSAGES       599
 
 #define GLFS_MSGID_END          (GLUSTERD_COMP_BASE + GLFS_NUM_MESSAGES + 1)
 /* Messaged with message IDs */
@@ -4836,6 +4836,14 @@
 
 #define GD_MSG_TIER_WATERMARK_RESET_FAIL           (GLUSTERD_COMP_BASE + 598)
 
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+#define GD_MSG_CLIENTS_GET_STATE_FAILED            (GLUSTERD_COMP_BASE + 599)
+
 /*------------*/
 
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.h b/xlators/mgmt/glusterd/src/glusterd-syncop.h
index f3425c2f538..6c5fbf965e6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.h
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.h
@@ -49,6 +49,10 @@
         }                                                                      \
 } while (0)
 
+int32_t
+gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe);
+
 int gd_syncop_submit_request (struct rpc_clnt *rpc, void *req, void *local,
                               void *cookie, rpc_clnt_prog_t *prog, int procnum,
                               fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);