glusterd : Introduce reset brick

The command basically allows replace brick with src and
dst bricks as same.

Usage:
gluster v reset-brick <volname> <hostname:brick-path> start
This command kills the brick to be reset. Once this command is run,
admin can do other manual operations that they need to do,
like configuring some options for the brick. Once this is done,
resetting the brick can be continued with the following options.

gluster v reset-brick <vname> <hostname:brick> <hostname:brick> commit {force}

Does the job of resetting the brick. 'force' option should be used
when the brick already contains volinfo id.

Problem: On doing a disk-replacement of a brick in a replicate volume
the following 2 scenarios may occur :

a) there is a chance that reads are served from this replaced-disk brick,
which leads to empty reads. b) potential data loss if next writes succeed
only on replaced brick, and heal is done to other bricks from this one.

Solution: After disk-replacement, make sure that reset-brick command is
run for that brick so that pending markers are set for the brick and it
is not chosen as source for reads and heal. But, as of now replace-brick
for the same brick-path is not allowed. In order to fix the above
mentioned problem, same brick-path replace-brick is needed.
With this patch reset-brick commit {force} will be allowed even when
source and destination <hostname:brickpath> are identical as long as
1) destination brick is not alive
2) source and destination brick have the same brick uuid and path.
Also, the destination brick after replace-brick will use the same port
as the source brick.

Change-Id: I440b9e892ffb781ea4b8563688c3f85c7a7c89de
BUG: 1266876
Signed-off-by: Anuradha Talur <atalur@redhat.com>
Reviewed-on: http://review.gluster.org/12250
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Ashish Pandey <aspandey@redhat.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

5 files changed, 415 insertions, 42 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index be0a8e756d5..86b02877159 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -2134,6 +2134,137 @@ out:
         return ret;
 }
 
+int32_t
+cli_cmd_brick_op_validate_bricks (const char **words, dict_t *dict,
+				  int src, int dst)
+{
+        int             ret = -1;
+        char            *delimiter  = NULL;
+
+        if (validate_brick_name ((char *)words[src])) {
+                cli_err ("wrong brick type: %s, use "
+                         "<HOSTNAME>:<export-dir-abs-path>", words[3]);
+                ret = -1;
+                goto out;
+        } else {
+                delimiter = strrchr ((char *)words[src], '/');
+                ret = gf_canonicalize_path (delimiter);
+                if (ret)
+                        goto out;
+        }
+
+        ret = dict_set_str (dict, "src-brick", (char *)words[src]);
+        if (ret)
+                goto out;
+
+        if (dst == -1) {
+                ret = 0;
+                goto out;
+        }
+
+        if (validate_brick_name ((char *)words[dst])) {
+                cli_err ("wrong brick type: %s, use "
+                         "<HOSTNAME>:<export-dir-abs-path>", words[dst]);
+                ret = -1;
+                goto out;
+        } else {
+                delimiter = strrchr ((char *)words[dst], '/');
+                ret = gf_canonicalize_path (delimiter);
+                if (ret)
+                        goto out;
+        }
+
+        ret = dict_set_str (dict, "dst-brick", (char *)words[dst]);
+        if (ret)
+                goto out;
+        ret = 0;
+out:
+	return ret;
+}
+
+int32_t
+cli_cmd_volume_reset_brick_parse (const char **words, int wordcount,
+                                  dict_t **options)
+{
+        int                   ret        = -1;
+        char                 *volname    = NULL;
+        dict_t               *dict       = NULL;
+
+        if (wordcount < 5 || wordcount > 7)
+                goto out;
+
+        dict = dict_new ();
+
+        if (!dict)
+                goto out;
+
+        volname = (char *)words[2];
+
+        ret = dict_set_str (dict, "volname", volname);
+        if (ret)
+                goto out;
+
+        if (wordcount == 5) {
+                if (strcmp (words[4], "start")) {
+                        cli_err ("Invalid option '%s' for reset-brick. Please "
+                                 "enter valid reset-brick command", words[4]);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = cli_cmd_brick_op_validate_bricks (words, dict, 3, -1);
+                if (ret)
+                        goto out;
+
+                ret = dict_set_str (dict, "operation", "GF_RESET_OP_START");
+                if (ret)
+                        goto out;
+        } else if (wordcount == 6) {
+                if (strcmp (words[5], "commit")) {
+                        cli_err ("Invalid option '%s' for reset-brick. Please "
+                                 "enter valid reset-brick command", words[5]);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = cli_cmd_brick_op_validate_bricks (words, dict, 3, 4);
+                if (ret)
+                        goto out;
+
+                ret = dict_set_str (dict, "operation", "GF_RESET_OP_COMMIT");
+                if (ret)
+                        goto out;
+        } else if (wordcount == 7) {
+                if (strcmp (words[5], "commit") || strcmp (words[6], "force")) {
+                        cli_err ("Invalid option '%s %s' for reset-brick. Please "
+                                 "enter valid reset-brick command",
+                                  words[5], words[6]);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = cli_cmd_brick_op_validate_bricks (words, dict, 3, 4);
+                if (ret)
+                        goto out;
+
+                ret = dict_set_str (dict, "operation",
+                                    "GF_RESET_OP_COMMIT_FORCE");
+                if (ret)
+                        goto out;
+        }
+
+        *options = dict;
+
+out:
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR,
+                        "Unable to parse reset-brick CLI");
+                if (dict)
+                        dict_unref (dict);
+        }
+
+        return ret;
+}
 
 int32_t
 cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
@@ -2141,7 +2272,6 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
 {
         int                   ret        = -1;
         char                 *volname    = NULL;
-        char                 *delimiter  = NULL;
         dict_t               *dict       = NULL;
 
         GF_ASSERT (words);
@@ -2167,35 +2297,7 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
         if (ret)
                 goto out;
 
-        if (validate_brick_name ((char *)words[3])) {
-                cli_err ("wrong brick type: %s, use "
-                         "<HOSTNAME>:<export-dir-abs-path>", words[3]);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr ((char *)words[3], ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret)
-                        goto out;
-        }
-
-        ret = dict_set_str (dict, "src-brick", (char *)words[3]);
-        if (ret)
-                goto out;
-
-        if (validate_brick_name ((char *)words[4])) {
-                cli_err ("wrong brick type: %s, use "
-                         "<HOSTNAME>:<export-dir-abs-path>", words[4]);
-                ret = -1;
-                goto out;
-        } else {
-                delimiter = strrchr ((char *)words[4], ':');
-                ret = gf_canonicalize_path (delimiter + 1);
-                if (ret)
-                        goto out;
-        }
-
-        ret = dict_set_str (dict, "dst-brick", (char *)words[4]);
+        ret = cli_cmd_brick_op_validate_bricks (words, dict, 3, 4);
         if (ret)
                 goto out;
 
@@ -2216,7 +2318,7 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
 
 out:
         if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "Unable to parse replace-brick CLI");
+                gf_log ("cli", GF_LOG_ERROR, "Unable to parse reset-brick CLI");
                 if (dict)
                         dict_unref (dict);
         }
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 020697ff56b..2171d357aa5 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -1829,6 +1829,59 @@ out:
 }
 
 int
+cli_cmd_volume_reset_brick_cbk (struct cli_state *state,
+                                struct cli_cmd_word *word,
+                                const char **words,
+                                int wordcount)
+{
+        int                      ret          = -1;
+        rpc_clnt_procedure_t    *proc         = NULL;
+        call_frame_t            *frame        = NULL;
+        dict_t                  *options = NULL;
+        int                      sent = 0;
+        int                      parse_error = 0;
+        cli_local_t             *local = NULL;
+
+#ifdef GF_SOLARIS_HOST_OS
+        cli_out ("Command not supported on Solaris");
+        goto out;
+#endif
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_RESET_BRICK];
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (!frame)
+                goto out;
+
+        ret = cli_cmd_volume_reset_brick_parse (words, wordcount, &options);
+
+        if (ret) {
+                cli_usage_out (word->pattern);
+                parse_error = 1;
+                goto out;
+        }
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
+        if (proc->fn) {
+                ret = proc->fn (frame, THIS, options);
+        }
+
+out:
+        if (ret) {
+                gf_event (EVENT_BRICK_RESET, "Volume reset-brick failed.");
+                cli_cmd_sent_status_get (&sent);
+                if ((sent == 0) && (parse_error == 0))
+                        cli_out ("Volume reset-brick failed");
+        } else {
+                gf_event (EVENT_BRICK_RESET, "Volume reset-brick succeeded.");
+        }
+
+        CLI_STACK_DESTROY (frame);
+
+        return ret;
+}
+
+int
 cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
                                   struct cli_cmd_word *word,
                                   const char **words,
@@ -1868,9 +1921,12 @@ cli_cmd_volume_replace_brick_cbk (struct cli_state *state,
 
 out:
         if (ret) {
+                gf_event (EVENT_BRICK_REPLACE, "Volume replace-brick failed.");
                 cli_cmd_sent_status_get (&sent);
                 if ((sent == 0) && (parse_error == 0))
                         cli_out ("Volume replace-brick failed");
+        } else {
+                gf_event (EVENT_BRICK_RESET, "Volume replace-brick succeeded.");
         }
 
         CLI_STACK_DESTROY (frame);
@@ -3016,6 +3072,11 @@ struct cli_cmd volume_cmds[] = {
          "Bitrot translator specific operation. For more information about "
          "bitrot command type  'man gluster'"
         },
+        { "volume reset-brick <VOLNAME> <SOURCE-BRICK> {{start} |"
+          " {<NEW-BRICK> commit}}",
+          cli_cmd_volume_reset_brick_cbk,
+          "reset-brick operations"},
+
         { NULL, NULL, NULL }
 };
 
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 6259546b6e9..6b5277ced95 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -2961,6 +2961,149 @@ out:
 }
 
 int
+gf_cli_reset_brick_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        gf_cli_rsp                       rsp              = {0,};
+        int                              ret              = -1;
+        cli_local_t                     *local            = NULL;
+        call_frame_t                    *frame            = NULL;
+        char                            *src_brick        = NULL;
+        char                            *dst_brick        = NULL;
+        char                            *status_reply     = NULL;
+        char                            *rb_operation_str = NULL;
+        dict_t                          *rsp_dict         = NULL;
+        char                             msg[1024]        = {0,};
+        char                            *task_id_str      = NULL;
+        char                            *reset_op         = NULL;
+
+        GF_ASSERT (myframe);
+
+        if (-1 == req->rpc_status) {
+                goto out;
+        }
+
+        frame = myframe;
+
+        GF_ASSERT (frame->local);
+
+        local = frame->local;
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_cli_rsp);
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "Failed to decode xdr response");
+                goto out;
+        }
+
+        ret = dict_get_str (local->dict, "operation", &reset_op);
+        if (ret) {
+                gf_log (frame->this->name, GF_LOG_ERROR,
+                        "dict_get on operation failed");
+                goto out;
+        }
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                rsp.dict.dict_len,
+                                &rsp_dict);
+                if (ret < 0) {
+                        gf_log (frame->this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize rsp buffer to dictionary");
+                        goto out;
+                }
+        }
+
+        if (strcmp (reset_op, "GF_RESET_OP_START") &&
+            strcmp (reset_op, "GF_RESET_OP_COMMIT") &&
+            strcmp (reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
+                rb_operation_str = gf_strdup ("Unknown operation");
+                ret = -1;
+                goto out;
+        }
+
+        if (rsp.op_ret && (strcmp (rsp.op_errstr, ""))) {
+                rb_operation_str = gf_strdup (rsp.op_errstr);
+        } else {
+                if (!strcmp (reset_op, "GF_RESET_OP_START")) {
+                        if (rsp.op_ret)
+                                rb_operation_str = gf_strdup ("reset-brick "
+                                                              "start "
+                                                              "operation "
+                                                              "failed");
+                        else
+                                rb_operation_str = gf_strdup ("reset-brick "
+                                                              "start "
+                                                              "operation "
+                                                              "successful");
+                } else if (!strcmp (reset_op, "GF_RESET_OP_COMMIT")) {
+
+                        if (rsp.op_ret)
+                                rb_operation_str = gf_strdup ("reset-brick "
+                                                              "commit "
+                                                              "operation "
+                                                              "failed");
+                        else
+                                rb_operation_str = gf_strdup ("reset-brick "
+                                                              "commit "
+                                                              "operation "
+                                                              "successful");
+                } else if (!strcmp (reset_op, "GF_RESET_OP_COMMIT_FORCE")) {
+
+                        if (rsp.op_ret)
+                                rb_operation_str = gf_strdup ("reset-brick "
+                                                              "commit "
+                                                              "force operation "
+                                                              "failed");
+                        else
+                                rb_operation_str = gf_strdup ("reset-brick "
+                                                              "commit "
+                                                              "force operation "
+                                                              "successful");
+                }
+        }
+
+        gf_log ("cli", GF_LOG_INFO, "Received resp to reset brick");
+        snprintf (msg, sizeof (msg), "%s",
+                  rb_operation_str ? rb_operation_str : "Unknown operation");
+
+        if (global_state->mode & GLUSTER_MODE_XML) {
+                ret = cli_xml_output_vol_replace_brick (rsp_dict,
+                                                        rsp.op_ret,
+                                                        rsp.op_errno, msg);
+                if (ret)
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Error outputting to xml");
+                goto out;
+        }
+
+        if (rsp.op_ret)
+                cli_err ("volume reset-brick: failed: %s", msg);
+        else
+                cli_out ("volume reset-brick: success: %s", msg);
+        ret = rsp.op_ret;
+
+out:
+        if (frame)
+                frame->local = NULL;
+
+        if (local)
+                cli_local_wipe (local);
+
+        if (rb_operation_str)
+                GF_FREE (rb_operation_str);
+
+        cli_cmd_broadcast_response (ret);
+        free (rsp.dict.dict_val);
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        return ret;
+}
+int
 gf_cli_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
@@ -2971,7 +3114,7 @@ gf_cli_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
         char                            *rb_operation_str = NULL;
         dict_t                          *rsp_dict         = NULL;
         char                             msg[1024]        = {0,};
-        char                            *replace_op       = 0;
+        char                            *replace_op       = NULL;
 
         GF_ASSERT (myframe);
 
@@ -3013,7 +3156,7 @@ gf_cli_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
                 }
         }
 
-        if (!strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        if (!strcmp (replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
 
                 if (rsp.op_ret || ret)
                         rb_operation_str = gf_strdup ("replace-brick commit "
@@ -3035,7 +3178,7 @@ gf_cli_replace_brick_cbk (struct rpc_req *req, struct iovec *iov,
                   rb_operation_str ? rb_operation_str : "Unknown operation");
 
         if (global_state->mode & GLUSTER_MODE_XML) {
-                ret = cli_xml_output_vol_replace_brick (replace_op, rsp_dict,
+                ret = cli_xml_output_vol_replace_brick (rsp_dict,
                                                         rsp.op_ret,
                                                         rsp.op_errno, msg);
                 if (ret)
@@ -3054,10 +3197,8 @@ out:
         if (frame)
                 frame->local = NULL;
 
-        if (local) {
-                dict_unref (local->dict);
+        if (local)
                 cli_local_wipe (local);
-        }
 
         if (rb_operation_str)
                 GF_FREE (rb_operation_str);
@@ -4889,8 +5030,73 @@ out:
 }
 
 int32_t
+gf_cli_reset_brick (call_frame_t *frame, xlator_t *this, void *data)
+{
+        gf_cli_req                  req        =  { {0,} };
+        int                         ret        = 0;
+        dict_t                     *dict       = NULL;
+        char                       *dst_brick  = NULL;
+        char                       *src_brick  = NULL;
+        char                       *volname    = NULL;
+        char                       *op         = NULL;
+
+        if (!frame || !this ||  !data) {
+                ret = -1;
+                goto out;
+        }
+
+        dict = data;
+
+        ret = dict_get_str (dict, "operation", &op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dict_get on operation failed");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dict_get on volname failed");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "src-brick", &src_brick);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dict_get on src-brick failed");
+                goto out;
+        }
+
+        if (!strcmp (op, "GF_RESET_OP_COMMIT") ||
+            !strcmp (op, "GF_RESET_OP_COMMIT_FORCE")) {
+                ret = dict_get_str (dict, "dst-brick", &dst_brick);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "dict_get on dst-brick failed");
+                        goto out;
+                }
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Received command reset-brick %s on %s.", op, src_brick);
+
+        ret = cli_to_glusterd (&req, frame, gf_cli_reset_brick_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_RESET_BRICK, this, cli_rpc_prog,
+                               NULL);
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+
+        GF_FREE (req.dict.dict_val);
+
+        return ret;
+}
+
+int32_t
 gf_cli_replace_brick (call_frame_t *frame, xlator_t *this,
-                         void *data)
+                      void *data)
 {
         gf_cli_req                  req        =  {{0,}};
         int                         ret        = 0;
@@ -11345,7 +11551,8 @@ struct rpc_clnt_procedure gluster_cli_actors[GLUSTER_CLI_MAXVALUE] = {
         [GLUSTER_CLI_ATTACH_TIER]      = {"ATTACH_TIER", gf_cli_attach_tier},
         [GLUSTER_CLI_DETACH_TIER]      = {"DETACH_TIER", gf_cli_detach_tier},
         [GLUSTER_CLI_TIER]             = {"TIER", gf_cli_tier},
-        [GLUSTER_CLI_GET_STATE]        = {"GET_STATE", gf_cli_get_state}
+        [GLUSTER_CLI_GET_STATE]        = {"GET_STATE", gf_cli_get_state},
+        [GLUSTER_CLI_RESET_BRICK]      = {"RESET_BRICK", gf_cli_reset_brick}
 };
 
 struct rpc_clnt_program cli_prog = {
diff --git a/cli/src/cli-xml-output.c b/cli/src/cli-xml-output.c
index f2b1941b77b..6639262066c 100644
--- a/cli/src/cli-xml-output.c
+++ b/cli/src/cli-xml-output.c
@@ -3861,7 +3861,7 @@ out:
 }
 
 int
-cli_xml_output_vol_replace_brick (char *op, dict_t *dict,
+cli_xml_output_vol_replace_brick (dict_t *dict,
                                   int op_ret, int op_errno, char *op_errstr)
 {
 #if (HAVE_LIB_XML)
diff --git a/cli/src/cli.h b/cli/src/cli.h
index f9c642ee4d0..ba0d845381a 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -284,6 +284,10 @@ cli_cmd_volume_replace_brick_parse (const char **words, int wordcount,
                                    dict_t **options);
 
 int32_t
+cli_cmd_volume_reset_brick_parse (const char **words, int wordcount,
+                                  dict_t **options);
+
+int32_t
 cli_cmd_log_rotate_parse (const char **words, int wordcount, dict_t **options);
 int32_t
 cli_cmd_log_locate_parse (const char **words, int wordcount, dict_t **options);
@@ -424,7 +428,7 @@ cli_xml_output_vol_remove_brick_detach_tier (gf_boolean_t status_op,
                                              const char *op);
 
 int
-cli_xml_output_vol_replace_brick (char *op, dict_t *dict, int op_ret,
+cli_xml_output_vol_replace_brick (dict_t *dict, int op_ret,
                                   int op_errno, char *op_errstr);
 
 int
@@ -489,5 +493,4 @@ print_quota_list_empty (char *path, int type);
 
 int
 gf_gsync_status_t_comparator (const void *p, const void *q);
-
 #endif /* __CLI_H__ */