cluster/afr: split-brain resolution CLI

Extend the AFR heal command to include automated split-brain resolution.

This patch [3/3] is the final patch for afr automated split-brain resolution
implementation.

"gluster volume heal <VOLNAME> [full | statistics [heal-count [replica
<HOSTNAME:BRICKNAME>]] |info [healed | heal-failed | split-brain]| split-brain
{bigger-file <FILE> |source-brick <HOSTNAME:BRICKNAME> [<FILE>]}]"

The new additions being:
1.gluster volume heal <VOLNAME> split-brain bigger-file <FILE>
Locates the replica containing the FILE, selects bigger-file as source and
completes heal.

2.gluster volume heal <VOLNAME> split-brain source-brick <HOSTNAME:BRICKNAME>
<FILE>
Selects <FILE> present in <HOSTNAME:BRICKNAME> as source and completes heal.

3.gluster volume heal <VOLNAME> split-brain <HOSTNAME:BRICKNAME>
Selects all split-brained files in <HOSTNAME:BRICKNAME> as source and completes
heal.

Note: <FILE> can be either the full file name as seen from the root of the
volume (or) the gfid-string representation of the file, which sometimes gets
displayed in the heal info command's output.

Entry/gfid split-brain resolution is not supported.

Example can be found in the test case.

Change-Id: I4649733922d406f14f28ee9033a5cb627b9538b3
BUG: 1136769
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-on: http://review.gluster.org/9377
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Tested-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>

3 files changed, 147 insertions, 29 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 28888ba656d..53b14d27708 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -2929,6 +2929,43 @@ out:
         return ret;
 }
 
+static int
+set_hostname_path_in_dict (const char *token, dict_t *dict, int heal_op)
+{
+        char *hostname = NULL;
+        char *path     = NULL;
+        int   ret      = 0;
+
+        ret = extract_hostname_path_from_token (token, &hostname, &path);
+        if (ret)
+                goto out;
+
+        switch (heal_op) {
+        case GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK:
+                ret = dict_set_dynstr (dict, "heal-source-hostname",
+                                       hostname);
+                if (ret)
+                        goto out;
+                ret = dict_set_dynstr (dict, "heal-source-brickpath",
+                                       path);
+                break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                ret = dict_set_dynstr (dict, "per-replica-cmd-hostname",
+                                       hostname);
+                if (ret)
+                        goto out;
+                ret = dict_set_dynstr (dict, "per-replica-cmd-path",
+                                       path);
+                break;
+        default:
+                ret = -1;
+                break;
+        }
+
+out:
+        return ret;
+
+}
 
 int
 cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
@@ -2936,8 +2973,6 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
 {
         int     ret = 0;
         dict_t  *dict = NULL;
-        char    *hostname = NULL;
-        char    *path = NULL;
 
         dict = dict_new ();
         if (!dict)
@@ -3008,6 +3043,35 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                 ret = -1;
                 goto out;
         }
+        if (wordcount == 6) {
+                if (strcmp (words[3], "split-brain")) {
+                        ret = -1;
+                        goto out;
+                }
+                if (!strcmp (words[4], "bigger-file")) {
+                        ret = dict_set_int32 (dict, "heal-op",
+                                        GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE);
+                        if (ret)
+                                goto out;
+                        ret = dict_set_str (dict, "file", (char *)words[5]);
+                        if (ret)
+                                goto out;
+                        goto done;
+                }
+                if (!strcmp (words[4], "source-brick")) {
+                        ret = dict_set_int32 (dict, "heal-op",
+                                              GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK);
+                        if (ret)
+                                goto out;
+                        ret = set_hostname_path_in_dict (words[5], dict,
+                                              GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK);
+                        if (ret)
+                                goto out;
+                        goto done;
+                }
+                ret = -1;
+                goto out;
+        }
         if (wordcount == 7) {
                 if (!strcmp (words[3], "statistics")
                     && !strcmp (words[4], "heal-count")
@@ -3017,21 +3081,26 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                                    GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
                         if (ret)
                                 goto out;
-                        ret = extract_hostname_path_from_token (words[6],
-                                                              &hostname, &path);
+                        ret = set_hostname_path_in_dict (words[6], dict,
+                                   GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
                         if (ret)
                                 goto out;
-                        ret = dict_set_dynstr (dict, "per-replica-cmd-hostname",
-                                               hostname);
+                        goto done;
+
+                }
+                if (!strcmp (words[3], "split-brain") &&
+                    !strcmp (words[4], "source-brick")) {
+                        ret = dict_set_int32 (dict, "heal-op",
+                                              GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK);
+                        ret = set_hostname_path_in_dict (words[5], dict,
+                                              GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK);
                         if (ret)
                                 goto out;
-                        ret = dict_set_dynstr (dict, "per-replica-cmd-path",
-                                               path);
+                        ret = dict_set_str (dict, "file",
+                                            (char *) words[6]);
                         if (ret)
                                 goto out;
-                        else
-                                goto done;
-
+                        goto done;
                 }
         }
         ret = -1;
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 238c8673d75..501b5776dec 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -1879,6 +1879,60 @@ cli_print_brick_status (cli_volume_status_t *status)
         return 0;
 }
 
+#define NEEDS_GLFS_HEAL(op) ((op == GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) || \
+                             (op == GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK) ||      \
+                             (op == GF_AFR_OP_INDEX_SUMMARY))
+
+int
+cli_launch_glfs_heal (int heal_op, dict_t *options)
+{
+        char      buff[PATH_MAX] = {0};
+        runner_t  runner         = {0};
+        char      *filename      = NULL;
+        char      *hostname      = NULL;
+        char      *path          = NULL;
+        char      *volname       = NULL;
+        char      *out           = NULL;
+        int        ret           = 0;
+
+        runinit (&runner);
+        ret = dict_get_str (options, "volname", &volname);
+        runner_add_args (&runner, SBIN_DIR"/glfsheal", volname, NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+
+        switch (heal_op) {
+        case GF_AFR_OP_INDEX_SUMMARY:
+                break;
+        case GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+                ret = dict_get_str (options, "file", &filename);
+                runner_add_args (&runner, "bigger-file", filename, NULL);
+                break;
+        case  GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK:
+                ret = dict_get_str (options, "heal-source-hostname",
+                                    &hostname);
+                ret = dict_get_str (options, "heal-source-brickpath",
+                                    &path);
+                runner_add_args (&runner, "source-brick", NULL);
+                runner_argprintf (&runner, "%s:%s", hostname, path);
+                if (dict_get_str (options, "file", &filename) == 0)
+                        runner_argprintf (&runner, filename);
+                break;
+        default:
+                ret = -1;
+        }
+        ret = runner_start (&runner);
+        if (ret == -1)
+                goto out;
+        while ((out = fgets (buff, sizeof(buff),
+                             runner_chio (&runner, STDOUT_FILENO)))) {
+                printf ("%s", out);
+        }
+        ret = runner_end (&runner);
+        ret = WEXITSTATUS (ret);
+
+out:
+        return ret;
+}
 int
 cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word,
                           const char **words, int wordcount)
@@ -1892,9 +1946,6 @@ cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word,
         xlator_t                *this = NULL;
         cli_local_t             *local = NULL;
         int                     heal_op = 0;
-        runner_t                runner = {0};
-        char                    buff[PATH_MAX] = {0};
-        char                    *out = NULL;
 
         this = THIS;
         frame = create_frame (this, this->ctx->pool);
@@ -1916,21 +1967,10 @@ cli_cmd_volume_heal_cbk (struct cli_state *state, struct cli_cmd_word *word,
         ret = dict_get_int32 (options, "heal-op", &heal_op);
         if (ret < 0)
                 goto out;
-
-        if (heal_op == GF_AFR_OP_INDEX_SUMMARY) {
-                runinit (&runner);
-                runner_add_args (&runner, SBIN_DIR"/glfsheal", words[2], NULL);
-                runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-                ret = runner_start (&runner);
+        if (NEEDS_GLFS_HEAL (heal_op)) {
+                ret = cli_launch_glfs_heal (heal_op, options);
                 if (ret == -1)
                         goto out;
-                while ((out = fgets(buff, sizeof(buff),
-                                   runner_chio (&runner, STDOUT_FILENO)))) {
-                        printf ("%s", out);
-                }
-
-                ret = runner_end (&runner);
-                ret = WEXITSTATUS (ret);
         }
         else {
                 proc = &cli_rpc_prog->proctable[GLUSTER_CLI_HEAL_VOLUME];
@@ -1946,7 +1986,7 @@ out:
         if (ret) {
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
-                        cli_out ("Volume heal failed");
+                        cli_out ("Volume heal failed.");
         }
 
         CLI_STACK_DESTROY (frame);
@@ -2316,7 +2356,10 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_volume_status_cbk,
           "display status of all or specified volume(s)/brick"},
 
-        { "volume heal <VOLNAME> [{full | statistics {heal-count {replica <hostname:brickname>}} |info {healed | heal-failed | split-brain}}]",
+        { "volume heal <VOLNAME> [full | statistics [heal-count "\
+          "[replica <HOSTNAME:BRICKNAME>]] |info [healed | heal-failed | "\
+          "split-brain]| split-brain {bigger-file <FILE> |source-brick "\
+          "<HOSTNAME:BRICKNAME> [<FILE>]}]",
           cli_cmd_volume_heal_cbk,
           "self-heal commands on volume specified by <VOLNAME>"},
 
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 1d8cf23ff42..72ffaf4129a 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -7358,6 +7358,12 @@ gf_cli_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 case    GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
                         heal_op_str = "count of entries to be healed per replica";
                         break;
+                /* The below 2 cases are never hit; they're coded only to make
+                 * compiler warnings go away.*/
+                case    GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+                case    GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK:
+                        break;
+
                 case    GF_AFR_OP_INVALID:
                         heal_op_str = "invalid heal op";
                         break;