glusterd: add a cli command to trigger a statedump on a client

With this, we will be able to trigger statedumps on remote Gluster
clients, mainly targetted for applications using libgfapi.

Design:
SIGUSR signal is the most comman way of taking a statedump in Gluster.
But it cannot be used for libgfapi based processes, as the process
loading the library might have already consumed SIGUSR signal. Hence
going by the command way.

One has to issue a Gluster command to initiate a statedump on the
libgfapi based client. The command takes hostname and PID as an
argument. All the glusterds in the cluster, check if they are connected
to the specified hostname, and send an RPC request to all the connected
clients from that hostname (via the mgmt connection).

URL: http://review.gluster.org/16357
Change-Id: Icbe4d2f026b32a2c7d5535e1bfb2cdaaff042e91
BUG: 1169302
Signed-off-by: Poornima G <pgurusid@redhat.com>
[ndevos: minor fixes and split patch in smaller pieces]
Reviewed-on: https://review.gluster.org/9228
Reviewed-by: Niels de Vos <ndevos@redhat.com>
Tested-by: Niels de Vos <ndevos@redhat.com>
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Samikshan Bairagya <samikshan@gmail.com>

15 files changed, 239 insertions, 13 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 3cc6ca97d37..d93af0e79d3 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -3541,19 +3541,44 @@ cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
         dict_t  *dict = NULL;
         int     option_cnt = 0;
         char    *option = NULL;
-        char    option_str[100] = {0,};
-
-        for (i = 3; i < wordcount; i++, option_cnt++) {
-                if (!cli_cmd_validate_dumpoption (words[i], &option)) {
+        char    option_str[_POSIX_HOST_NAME_MAX + 100] = {0,};
+        char    *tmp = NULL;
+        char    *ip_addr = NULL;
+        char    *pid = NULL;
+
+        if ((wordcount >= 5) && ((strcmp (words[3], "client")) == 0)) {
+                tmp = gf_strdup(words[4]);
+                if (!tmp) {
+                        ret = -1;
+                        goto out;
+                }
+                ip_addr = strtok(tmp, ":");
+                pid = strtok(NULL, ":");
+                if (valid_internet_address (ip_addr, _gf_true)
+                   && pid && gf_valid_pid (pid, strlen(pid))) {
+                        strncat (option_str, words[3], strlen (words[3]));
+                        strncat (option_str, " ", 1);
+                        strncat (option_str, ip_addr, strlen (ip_addr));
+                        strncat (option_str, " ", 1);
+                        strncat (option_str, pid, strlen (pid));
+                        option_cnt = 3;
+                } else {
+                        ret = -1;
+                        goto out;
+                }
+        } else {
+                for (i = 3; i < wordcount; i++, option_cnt++) {
+                        if (!cli_cmd_validate_dumpoption (words[i], &option)) {
+                                ret = -1;
+                                goto out;
+                        }
+                        strncat (option_str, option, strlen (option));
+                        strncat (option_str, " ", 1);
+                }
+                if ((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
                         ret = -1;
                         goto out;
                 }
-                strncat (option_str, option, strlen (option));
-                strncat (option_str, " ", 1);
-        }
-        if((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
-                ret = -1;
-                goto out;
         }
 
         dict = dict_new ();
@@ -3570,6 +3595,7 @@ cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
 
         *options = dict;
 out:
+        GF_FREE (tmp);
         if (ret && dict)
                 dict_unref (dict);
         if (ret)
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index dbf1e3e2326..5f07a057d47 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -3285,8 +3285,8 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_volume_heal_cbk,
           "self-heal commands on volume specified by <VOLNAME>"},
 
-        {"volume statedump <VOLNAME> [nfs|quotad] [all|mem|iobuf|callpool|priv|fd|"
-         "inode|history]...",
+        {"volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|"
+         "priv|fd|inode|history]... | [client <hostname:process-id>]]",
          cli_cmd_volume_statedump_cbk,
          "perform statedump on bricks"},
 
diff --git a/doc/debugging/statedump.md b/doc/debugging/statedump.md
index 18437f1144c..9939576e270 100644
--- a/doc/debugging/statedump.md
+++ b/doc/debugging/statedump.md
@@ -19,6 +19,19 @@ For quotad:     `gluster volume statedump <volname> quotad`
 
 For brick-processes files will be created in `statedump-directory` with name of the file as `hyphenated-brick-path.<pid>.dump.timestamp`. For all other processes it will be `glusterdump.<pid>.dump.timestamp`.
 
+For applications using libgfapi, `SIGUSR1` cannot be used, eg: smbd/libvirtd
+processes could have used the `SIGUSR1` signal already for other purposes.
+To generate statedump for the processes, using libgfapi, below command can be
+executed from one of the nodes in the gluster cluster to which the libgfapi
+application is connected to.
+
+    gluster volume statedump <volname> client <hostname>:<process id>
+
+The statedumps can be found in the `statedump-directory`, the name of the
+statedumps being `glusterdump.<pid>.dump.timestamp`. For a process there can be
+multiple such files created depending on the number of times the volume is
+accessed by the process (related to the number of `glfs_init()` calls).
+
 ##How to read statedump
 We shall see snippets of each type of statedump.
 
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 92c3343ad21..07c375d275f 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1457,6 +1457,7 @@ rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
         [GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
         [GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
                                  mgmt_cbk_event},
+        [GF_CBK_STATEDUMP] = {"STATEDUMP", GF_CBK_STATEDUMP, mgmt_cbk_event},
 };
 
 
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 18c2a39d60e..0486409a849 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -3669,6 +3669,27 @@ out:
         return running;
 }
 
+/* Check if the pid is > 0 */
+gf_boolean_t
+gf_valid_pid (const char *pid, int length)
+{
+        gf_boolean_t    ret = _gf_true;
+        pid_t           value = 0;
+        char           *end_ptr = NULL;
+
+        if (length <= 0) {
+                ret = _gf_false;
+                goto out;
+        }
+
+        value = strtol (pid, &end_ptr, 10);
+        if (value <= 0) {
+                ret = _gf_false;
+        }
+out:
+        return ret;
+}
+
 static int
 dht_is_linkfile_key (dict_t *this, char *key, data_t *value, void *data)
 {
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index b77b7ad97de..14cd6964f76 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -796,6 +796,8 @@ int gf_thread_create_detached (pthread_t *thread,
 
 gf_boolean_t
 gf_is_service_running (char *pidfile, int *pid);
+gf_boolean_t
+gf_valid_pid (const char *pid, int length);
 int
 gf_skip_header_section (int fd, int header_len);
 
diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h
index 89a7bb0bcde..69a39b2c7a4 100644
--- a/rpc/rpc-lib/src/protocol-common.h
+++ b/rpc/rpc-lib/src/protocol-common.h
@@ -146,6 +146,7 @@ enum gf_cbk_procnum {
         GF_CBK_CHILD_UP,
         GF_CBK_CHILD_DOWN,
         GF_CBK_RECALL_LEASE,
+        GF_CBK_STATEDUMP,
         GF_CBK_MAXVALUE,
 };
 
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index 717c40af13a..e3b630e2919 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -70,7 +70,7 @@ struct peer_info {
         uint32_t max_op_version;
         uint32_t min_op_version;
         //Volume mounted by client
-        char volname[1024];
+        char volname[NAME_MAX];
 };
 typedef struct peer_info peer_info_t;
 
diff --git a/rpc/xdr/src/rpc-common-xdr.x b/rpc/xdr/src/rpc-common-xdr.x
index 464a7478c73..7ccfbb11a51 100644
--- a/rpc/xdr/src/rpc-common-xdr.x
+++ b/rpc/xdr/src/rpc-common-xdr.x
@@ -39,6 +39,9 @@ struct gf_dump_req {
 	u_quad_t gfs_id;
 };
 
+struct gf_statedump {
+	unsigned int pid;
+};
 
 struct gf_prog_detail {
 	string progname<>;
diff --git a/tests/bugs/cli/bug-1169302.t b/tests/bugs/cli/bug-1169302.t
new file mode 100755
index 00000000000..92252aa7887
--- /dev/null
+++ b/tests/bugs/cli/bug-1169302.t
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../cluster.rc
+
+function check_peers {
+    $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+cleanup;
+
+#setup cluster and test volume
+TEST launch_cluster 3; # start 3-node virtual cluster
+TEST $CLI_1 peer probe $H2; # peer probe server 2 from server 1 cli
+TEST $CLI_1 peer probe $H3; # peer probe server 3 from server 1 cli
+
+EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers;
+
+TEST $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0 $H3:$B3/$V0
+TEST $CLI_1 volume start $V0
+
+# there is no gfapi application to take statedumps yet, it will get added in
+# the next patch, this only tests the CLI for correctness
+
+cleanup_statedump
+
+TEST ! $CLI_1 volume statedump $V0 client $H2:0
+TEST ! $CLI_2 volume statedump $V0 client $H2:-1
+TEST $CLI_3 volume statedump $V0 client $H2:765
+TEST ! $CLI_1 volume statedump $V0 client $H2:
+TEST ! $CLI_2 volume statedump $V0 client
+
+cleanup_statedump
+cleanup;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 21482752c53..a77cc674e63 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -7044,6 +7044,49 @@ out:
 }
 
 int
+glusterd_client_statedump (char *volname, char *options, int option_cnt,
+                           char **op_errstr)
+{
+        int                      ret                = 0;
+        char                    *dup_options        = NULL;
+        char                    *option             = NULL;
+        char                    *tmpptr             = NULL;
+        char                     msg[256]           = {0,};
+        char                    *target_ip          = NULL;
+        char                    *pid                = NULL;
+
+        dup_options = gf_strdup (options);
+        option = strtok_r (dup_options, " ", &tmpptr);
+        if (strcmp (option, "client")) {
+                snprintf (msg, sizeof (msg), "for gluster client statedump, options "
+                          "should be after the key 'client'");
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+        target_ip = strtok_r (NULL, " ", &tmpptr);
+        if (target_ip == NULL) {
+                snprintf (msg, sizeof (msg), "ip address not specified");
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        pid = strtok_r (NULL, " ", &tmpptr);
+        if (pid == NULL) {
+                snprintf (msg, sizeof (msg), "pid not specified");
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_client_statedump_submit_req (volname, target_ip, pid);
+out:
+        GF_FREE (dup_options);
+        return ret;
+}
+
+int
 glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr)
 {
         int                     ret = -1;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 5f490534ef5..e801c1a03a3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -390,6 +390,10 @@ int
 glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr);
 
 int
+glusterd_client_statedump (char *volname, char *options, int option_cnt,
+                           char **op_errstr);
+
+int
 glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr);
 
 gf_boolean_t
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 0c3ac5816e7..ecc4f9609c1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2867,6 +2867,13 @@ glusterd_op_statedump_volume (dict_t *dict, char **op_errstr)
                                                  op_errstr);
                 if (ret)
                         goto out;
+
+        } else if (strstr (options, "client")) {
+                ret = glusterd_client_statedump (volname, options, option_cnt,
+                                                op_errstr);
+                if (ret)
+                        goto out;
+
         } else {
                 cds_list_for_each_entry (brickinfo, &volinfo->bricks,
                                          brick_list) {
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 18dcf10f2a3..d6f8baff4f2 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -47,6 +47,7 @@
 #include "glusterd-geo-rep.h"
 #include "run.h"
 #include "rpc-clnt-ping.h"
+#include "rpc-common-xdr.h"
 
 #include "syncop.h"
 
@@ -233,6 +234,72 @@ out:
 }
 
 int
+glusterd_client_statedump_submit_req (char *volname, char *target_ip,
+                                      char *pid)
+{
+        gf_statedump             statedump_req      = {0, };
+        glusterd_conf_t         *conf               = NULL;
+        int                      ret                = 0;
+        char                    *end_ptr            = NULL;
+        rpc_transport_t         *trans              = NULL;
+        char                    *ip_addr            = NULL;
+        xlator_t                *this               = NULL;
+        char                     tmp[UNIX_PATH_MAX] = {0, };
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        if (target_ip == NULL || pid == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        statedump_req.pid = strtol (pid, &end_ptr, 10);
+
+        gf_msg_debug (this->name, 0, "Performing statedump on volume %s "
+                      "client with pid:%d host:%s", volname, statedump_req.pid,
+                      target_ip);
+
+        pthread_mutex_lock (&conf->xprt_lock);
+        {
+                list_for_each_entry (trans, &conf->xprt_list, list) {
+                        /* check if this connection matches "all" or the
+                         * volname */
+                        if (strncmp (volname, "all", NAME_MAX) &&
+                            strncmp (trans->peerinfo.volname, volname,
+                                     NAME_MAX)) {
+                                /* no match, try next trans */
+                                continue;
+                        }
+
+                        strcpy (tmp, trans->peerinfo.identifier);
+                        ip_addr = strtok (tmp, ":");
+                        if (gf_is_same_address (ip_addr, target_ip)) {
+                                /* Every gluster client would have
+                                 * connected to glusterd(volfile server). This
+                                 * connection is used to send the statedump
+                                 * request rpc to the application.
+                                 */
+                                gf_msg_trace (this->name, 0, "Submitting "
+                                        "statedump rpc request for %s",
+                                        trans->peerinfo.identifier);
+                                rpcsvc_request_submit (conf->rpc, trans,
+                                                       &glusterd_cbk_prog,
+                                                       GF_CBK_STATEDUMP,
+                                                       &statedump_req, this->ctx,
+                                                       (xdrproc_t)xdr_gf_statedump);
+                        }
+                }
+        }
+        pthread_mutex_unlock (&conf->xprt_lock);
+out:
+        return ret;
+
+}
+
+int
 glusterd_fetchspec_notify (xlator_t *this)
 {
         int              ret   = -1;
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 32f29526fb4..d00e4e20811 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -1000,6 +1000,10 @@ glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
                                 char *hostname, dict_t *dict);
 
 int
+glusterd_client_statedump_submit_req (char *volname, char *target_ip,
+                                      char *pid);
+
+int
 glusterd_fetchspec_notify (xlator_t *this);
 
 int