glusterd: add a cli command to trigger a statedump on a client

With this, we will be able to trigger statedumps on remote Gluster
clients, mainly targetted for applications using libgfapi.

Design:
SIGUSR signal is the most comman way of taking a statedump in Gluster.
But it cannot be used for libgfapi based processes, as the process
loading the library might have already consumed SIGUSR signal. Hence
going by the command way.

One has to issue a Gluster command to initiate a statedump on the
libgfapi based client. The command takes hostname and PID as an
argument. All the glusterds in the cluster, check if they are connected
to the specified hostname, and send an RPC request to all the connected
clients from that hostname (via the mgmt connection).

> URL: http://review.gluster.org/16357
> BUG: 1169302
> Signed-off-by: Poornima G <pgurusid@redhat.com>
> [ndevos: minor fixes and split patch in smaller pieces]
> Reviewed-on-master: https://review.gluster.org/9228
> Reviewed-by: Niels de Vos <ndevos@redhat.com>
> Tested-by: Niels de Vos <ndevos@redhat.com>
> Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
> Reviewed-by: Samikshan Bairagya <samikshan@gmail.com>

BUG: 1418981
Change-Id: Icbe4d2f026b32a2c7d5535e1bfb2cdaaff042e91
Signed-off-by: Shyam <srangana@redhat.com>
Reviewed-on: https://review.gluster.org/16601
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Niels de Vos <ndevos@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

15 files changed, 239 insertions, 13 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 9b1f1db88e8..d234ad09c4e 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -3541,19 +3541,44 @@ cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
         dict_t  *dict = NULL;
         int     option_cnt = 0;
         char    *option = NULL;
-        char    option_str[100] = {0,};
-
-        for (i = 3; i < wordcount; i++, option_cnt++) {
-                if (!cli_cmd_validate_dumpoption (words[i], &option)) {
+        char    option_str[_POSIX_HOST_NAME_MAX + 100] = {0,};
+        char    *tmp = NULL;
+        char    *ip_addr = NULL;
+        char    *pid = NULL;
+
+        if ((wordcount >= 5) && ((strcmp (words[3], "client")) == 0)) {
+                tmp = gf_strdup(words[4]);
+                if (!tmp) {
+                        ret = -1;
+                        goto out;
+                }
+                ip_addr = strtok(tmp, ":");
+                pid = strtok(NULL, ":");
+                if (valid_internet_address (ip_addr, _gf_true)
+                   && pid && gf_valid_pid (pid, strlen(pid))) {
+                        strncat (option_str, words[3], strlen (words[3]));
+                        strncat (option_str, " ", 1);
+                        strncat (option_str, ip_addr, strlen (ip_addr));
+                        strncat (option_str, " ", 1);
+                        strncat (option_str, pid, strlen (pid));
+                        option_cnt = 3;
+                } else {
+                        ret = -1;
+                        goto out;
+                }
+        } else {
+                for (i = 3; i < wordcount; i++, option_cnt++) {
+                        if (!cli_cmd_validate_dumpoption (words[i], &option)) {
+                                ret = -1;
+                                goto out;
+                        }
+                        strncat (option_str, option, strlen (option));
+                        strncat (option_str, " ", 1);
+                }
+                if ((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
                         ret = -1;
                         goto out;
                 }
-                strncat (option_str, option, strlen (option));
-                strncat (option_str, " ", 1);
-        }
-        if((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
-                ret = -1;
-                goto out;
         }
 
         dict = dict_new ();
@@ -3570,6 +3595,7 @@ cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
 
         *options = dict;
 out:
+        GF_FREE (tmp);
         if (ret && dict)
                 dict_unref (dict);
         if (ret)
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index dbf1e3e2326..5f07a057d47 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -3285,8 +3285,8 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_volume_heal_cbk,
           "self-heal commands on volume specified by <VOLNAME>"},
 
-        {"volume statedump <VOLNAME> [nfs|quotad] [all|mem|iobuf|callpool|priv|fd|"
-         "inode|history]...",
+        {"volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|"
+         "priv|fd|inode|history]... | [client <hostname:process-id>]]",
          cli_cmd_volume_statedump_cbk,
          "perform statedump on bricks"},
 
diff --git a/doc/debugging/statedump.md b/doc/debugging/statedump.md
index 18437f1144c..9939576e270 100644
--- a/doc/debugging/statedump.md
+++ b/doc/debugging/statedump.md
@@ -19,6 +19,19 @@ For quotad:     `gluster volume statedump <volname> quotad`
 
 For brick-processes files will be created in `statedump-directory` with name of the file as `hyphenated-brick-path.<pid>.dump.timestamp`. For all other processes it will be `glusterdump.<pid>.dump.timestamp`.
 
+For applications using libgfapi, `SIGUSR1` cannot be used, eg: smbd/libvirtd
+processes could have used the `SIGUSR1` signal already for other purposes.
+To generate statedump for the processes, using libgfapi, below command can be
+executed from one of the nodes in the gluster cluster to which the libgfapi
+application is connected to.
+
+    gluster volume statedump <volname> client <hostname>:<process id>
+
+The statedumps can be found in the `statedump-directory`, the name of the
+statedumps being `glusterdump.<pid>.dump.timestamp`. For a process there can be
+multiple such files created depending on the number of times the volume is
+accessed by the process (related to the number of `glfs_init()` calls).
+
 ##How to read statedump
 We shall see snippets of each type of statedump.
 
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index fa03d23b17b..8237c23811a 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1558,6 +1558,7 @@ rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
         [GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
         [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
                                  mgmt_cbk_event},
+        [GF_CBK_STATEDUMP] = {"STATEDUMP", GF_CBK_STATEDUMP, mgmt_cbk_event},
 };
 
 
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 39b145ae21a..c0884de8a5e 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -3677,6 +3677,27 @@ out:
         return running;
 }
 
+/* Check if the pid is > 0 */
+gf_boolean_t
+gf_valid_pid (const char *pid, int length)
+{
+        gf_boolean_t    ret = _gf_true;
+        pid_t           value = 0;
+        char           *end_ptr = NULL;
+
+        if (length <= 0) {
+                ret = _gf_false;
+                goto out;
+        }
+
+        value = strtol (pid, &end_ptr, 10);
+        if (value <= 0) {
+                ret = _gf_false;
+        }
+out:
+        return ret;
+}
+
 static int
 dht_is_linkfile_key (dict_t *this, char *key, data_t *value, void *data)
 {
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index dbd2489ec8c..67baa852c45 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -824,6 +824,8 @@ int gf_thread_create_detached (pthread_t *thread,
 
 gf_boolean_t
 gf_is_service_running (char *pidfile, int *pid);
+gf_boolean_t
+gf_valid_pid (const char *pid, int length);
 int
 gf_skip_header_section (int fd, int header_len);
 
diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h
index cd21ad8564f..8865baf759b 100644
--- a/rpc/rpc-lib/src/protocol-common.h
+++ b/rpc/rpc-lib/src/protocol-common.h
@@ -146,6 +146,7 @@ enum gf_cbk_procnum {
         GF_CBK_CHILD_UP,
         GF_CBK_CHILD_DOWN,
         GF_CBK_RECALL_LEASE,
+        GF_CBK_STATEDUMP,
         GF_CBK_MAXVALUE,
 };
 
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index 717c40af13a..e3b630e2919 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -70,7 +70,7 @@ struct peer_info {
         uint32_t max_op_version;
         uint32_t min_op_version;
         //Volume mounted by client
-        char volname[1024];
+        char volname[NAME_MAX];
 };
 typedef struct peer_info peer_info_t;
 
diff --git a/rpc/xdr/src/rpc-common-xdr.x b/rpc/xdr/src/rpc-common-xdr.x
index 464a7478c73..7ccfbb11a51 100644
--- a/rpc/xdr/src/rpc-common-xdr.x
+++ b/rpc/xdr/src/rpc-common-xdr.x
@@ -39,6 +39,9 @@ struct gf_dump_req {
 	u_quad_t gfs_id;
 };
 
+struct gf_statedump {
+	unsigned int pid;
+};
 
 struct gf_prog_detail {
 	string progname<>;
diff --git a/tests/bugs/cli/bug-1169302.t b/tests/bugs/cli/bug-1169302.t
new file mode 100755
index 00000000000..92252aa7887
--- /dev/null
+++ b/tests/bugs/cli/bug-1169302.t
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../cluster.rc
+
+function check_peers {
+    $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+cleanup;
+
+#setup cluster and test volume
+TEST launch_cluster 3; # start 3-node virtual cluster
+TEST $CLI_1 peer probe $H2; # peer probe server 2 from server 1 cli
+TEST $CLI_1 peer probe $H3; # peer probe server 3 from server 1 cli
+
+EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers;
+
+TEST $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0 $H3:$B3/$V0
+TEST $CLI_1 volume start $V0
+
+# there is no gfapi application to take statedumps yet, it will get added in
+# the next patch, this only tests the CLI for correctness
+
+cleanup_statedump
+
+TEST ! $CLI_1 volume statedump $V0 client $H2:0
+TEST ! $CLI_2 volume statedump $V0 client $H2:-1
+TEST $CLI_3 volume statedump $V0 client $H2:765
+TEST ! $CLI_1 volume statedump $V0 client $H2:
+TEST ! $CLI_2 volume statedump $V0 client
+
+cleanup_statedump
+cleanup;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 703ca49e326..f64b25cccd1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -7584,6 +7584,49 @@ out:
 }
 
 int
+glusterd_client_statedump (char *volname, char *options, int option_cnt,
+                           char **op_errstr)
+{
+        int                      ret                = 0;
+        char                    *dup_options        = NULL;
+        char                    *option             = NULL;
+        char                    *tmpptr             = NULL;
+        char                     msg[256]           = {0,};
+        char                    *target_ip          = NULL;
+        char                    *pid                = NULL;
+
+        dup_options = gf_strdup (options);
+        option = strtok_r (dup_options, " ", &tmpptr);
+        if (strcmp (option, "client")) {
+                snprintf (msg, sizeof (msg), "for gluster client statedump, options "
+                          "should be after the key 'client'");
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+        target_ip = strtok_r (NULL, " ", &tmpptr);
+        if (target_ip == NULL) {
+                snprintf (msg, sizeof (msg), "ip address not specified");
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        pid = strtok_r (NULL, " ", &tmpptr);
+        if (pid == NULL) {
+                snprintf (msg, sizeof (msg), "pid not specified");
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_client_statedump_submit_req (volname, target_ip, pid);
+out:
+        GF_FREE (dup_options);
+        return ret;
+}
+
+int
 glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr)
 {
         int                     ret = -1;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 76c963443df..db13c4c8ad4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -396,6 +396,10 @@ int
 glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr);
 
 int
+glusterd_client_statedump (char *volname, char *options, int option_cnt,
+                           char **op_errstr);
+
+int
 glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr);
 
 gf_boolean_t
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index d2f724be7c7..ad5fe909578 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2867,6 +2867,13 @@ glusterd_op_statedump_volume (dict_t *dict, char **op_errstr)
                                                  op_errstr);
                 if (ret)
                         goto out;
+
+        } else if (strstr (options, "client")) {
+                ret = glusterd_client_statedump (volname, options, option_cnt,
+                                                op_errstr);
+                if (ret)
+                        goto out;
+
         } else {
                 cds_list_for_each_entry (brickinfo, &volinfo->bricks,
                                          brick_list) {
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 18dcf10f2a3..d6f8baff4f2 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -47,6 +47,7 @@
 #include "glusterd-geo-rep.h"
 #include "run.h"
 #include "rpc-clnt-ping.h"
+#include "rpc-common-xdr.h"
 
 #include "syncop.h"
 
@@ -233,6 +234,72 @@ out:
 }
 
 int
+glusterd_client_statedump_submit_req (char *volname, char *target_ip,
+                                      char *pid)
+{
+        gf_statedump             statedump_req      = {0, };
+        glusterd_conf_t         *conf               = NULL;
+        int                      ret                = 0;
+        char                    *end_ptr            = NULL;
+        rpc_transport_t         *trans              = NULL;
+        char                    *ip_addr            = NULL;
+        xlator_t                *this               = NULL;
+        char                     tmp[UNIX_PATH_MAX] = {0, };
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        if (target_ip == NULL || pid == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        statedump_req.pid = strtol (pid, &end_ptr, 10);
+
+        gf_msg_debug (this->name, 0, "Performing statedump on volume %s "
+                      "client with pid:%d host:%s", volname, statedump_req.pid,
+                      target_ip);
+
+        pthread_mutex_lock (&conf->xprt_lock);
+        {
+                list_for_each_entry (trans, &conf->xprt_list, list) {
+                        /* check if this connection matches "all" or the
+                         * volname */
+                        if (strncmp (volname, "all", NAME_MAX) &&
+                            strncmp (trans->peerinfo.volname, volname,
+                                     NAME_MAX)) {
+                                /* no match, try next trans */
+                                continue;
+                        }
+
+                        strcpy (tmp, trans->peerinfo.identifier);
+                        ip_addr = strtok (tmp, ":");
+                        if (gf_is_same_address (ip_addr, target_ip)) {
+                                /* Every gluster client would have
+                                 * connected to glusterd(volfile server). This
+                                 * connection is used to send the statedump
+                                 * request rpc to the application.
+                                 */
+                                gf_msg_trace (this->name, 0, "Submitting "
+                                        "statedump rpc request for %s",
+                                        trans->peerinfo.identifier);
+                                rpcsvc_request_submit (conf->rpc, trans,
+                                                       &glusterd_cbk_prog,
+                                                       GF_CBK_STATEDUMP,
+                                                       &statedump_req, this->ctx,
+                                                       (xdrproc_t)xdr_gf_statedump);
+                        }
+                }
+        }
+        pthread_mutex_unlock (&conf->xprt_lock);
+out:
+        return ret;
+
+}
+
+int
 glusterd_fetchspec_notify (xlator_t *this)
 {
         int              ret   = -1;
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 4f2c8f287df..f3c7e1d6891 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -1000,6 +1000,10 @@ glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
                                 char *hostname, dict_t *dict);
 
 int
+glusterd_client_statedump_submit_req (char *volname, char *target_ip,
+                                      char *pid);
+
+int
 glusterd_fetchspec_notify (xlator_t *this);
 
 int