Tier: "tier start force" command implementation

	back port of : http://review.gluster.org/#/c/12983/

The start command doesnt restart the tier deamon if the deamon
is running at one node. hence to bring up the tierd on the nodes
where the deamon is down, the force command is implemented.
It skips the check for tierd running.

>Change-Id: I0037d3e5ecfe56637d0da201a97903c435d26436
>BUG: 1292112
>Signed-off-by: hari gowtham <hgowtham@redhat.com>

Change-Id: Idaca442c1a41ded8bf555a6e34eed0ebb9ea4034
BUG: 1293698
Signed-off-by: hari <hgowtham@redhat.com>
Reviewed-on: http://review.gluster.org/13069
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>
Tested-by: Dan Lambright <dlambrig@redhat.com>

10 files changed, 185 insertions, 59 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 1d4d6cf89c7..68c407d5b4f 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -1701,9 +1701,9 @@ cli_cmd_volume_tier_parse (const char **words, int wordcount,
 {
         dict_t  *dict    = NULL;
         char    *volname = NULL;
-        char    *word    = NULL;
         int      ret     = -1;
         int32_t  command = GF_OP_CMD_NONE;
+        int32_t  is_force    = 0;
 
         GF_ASSERT (words);
         GF_ASSERT (options);
@@ -1713,7 +1713,7 @@ cli_cmd_volume_tier_parse (const char **words, int wordcount,
         if (!dict)
                 goto out;
 
-        if (wordcount != 4) {
+        if (!(wordcount == 4 || wordcount == 5)) {
                 gf_log ("cli", GF_LOG_ERROR, "Invalid Syntax");
                 ret = -1;
                 goto out;
@@ -1735,11 +1735,28 @@ cli_cmd_volume_tier_parse (const char **words, int wordcount,
                 goto out;
 
         volname = (char *)words[2];
-
-        word = (char *)words[3];
-        if (!strcmp(word, "status"))
-                command = GF_DEFRAG_CMD_STATUS_TIER;
-        else {
+        if (wordcount == 4) {
+                if (!strcmp(words[3], "status"))
+                        command = GF_DEFRAG_CMD_STATUS_TIER;
+                else if (!strcmp(words[3], "start"))
+                        command = GF_DEFRAG_CMD_START_TIER;
+                else {
+                        ret = -1;
+                        goto out;
+                }
+        } else if (wordcount == 5) {
+                if ((!strcmp (words[3], "start")) &&
+                    (!strcmp (words[4], "force"))) {
+                        command = GF_DEFRAG_CMD_START_TIER;
+                        is_force = 1;
+                        ret = dict_set_int32 (dict, "force", is_force);
+                        if (ret)
+                                goto out;
+                } else {
+                        ret = -1;
+                        goto out;
+                }
+        } else {
                 ret = -1;
                 goto out;
         }
@@ -3661,7 +3678,7 @@ cli_cmd_volume_defrag_parse (const char **words, int wordcount,
                     strcmp (words[3], "status"))
                             goto out;
         } else if ((strcmp (words[3], "tier") == 0) &&
-               (strcmp (words[4], "start") == 0)) {
+                   (strcmp (words[4], "start") == 0)) {
                 volname = (char *) words[2];
                 cmd = GF_DEFRAG_CMD_START_TIER;
                 goto done;
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 92edc356705..3707e130183 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -2662,6 +2662,7 @@ struct cli_cmd volume_cmds[] = {
           "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
 
         { "volume tier <VOLNAME> status\n"
+        "volume tier <VOLNAME> start [force]"
         "volume tier <VOLNAME> attach [<replica COUNT>] <NEW-BRICK>...\n"
         "volume tier <VOLNAME> detach <start|stop|status|commit|[force]>\n",
         cli_cmd_volume_tier_cbk,
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 6dcf0f37e3f..b7e0aedbcbb 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -1780,7 +1780,6 @@ gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                  * case since unlock failures can be highlighted
                                  * event though rebalance command was successful
                                  */
-
                                  if (cmd == GF_DEFRAG_CMD_START_TIER) {
                                          snprintf (msg, sizeof (msg),
                                                          "Attach tier is successful "
diff --git a/tests/basic/tier/tierd_check.t b/tests/basic/tier/tierd_check.t
new file mode 100644
index 00000000000..ee97883688d
--- /dev/null
+++ b/tests/basic/tier/tierd_check.t
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../tier.rc
+
+
+# Creates a tiered volume with pure distribute hot and cold tiers
+# Both hot and cold tiers will have an equal number of bricks.
+
+function create_dist_tier_vol () {
+        mkdir $B0/cold
+        mkdir $B0/hot
+        TEST $CLI volume create $V0 $H0:$B0/cold/${V0}{1..3}
+        TEST $CLI volume set $V0 performance.quick-read off
+        TEST $CLI volume set $V0 performance.io-cache off
+        TEST $CLI volume start $V0
+        TEST $CLI volume attach-tier $V0 $H0:$B0/hot/${V0}{1..2}
+        TEST $CLI volume set $V0 cluster.tier-mode test
+}
+
+function tier_deamon_check () {
+pgrep -f "rebalance/$V0"
+echo "$?"
+}
+
+function tier_deamon_kill () {
+pkill -f "rebalance/$V0"
+echo "$?"
+}
+
+cleanup;
+
+#Basic checks
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume status
+
+
+#Create and start a tiered volume
+create_dist_tier_vol
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_deamon_check
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 0 tier_deamon_kill
+
+TEST $CLI volume tier $V0 start
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_check
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_kill
+
+TEST $CLI volume tier $V0 start force
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_check
+
+TEST $CLI volume tier $V0 start force
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT "0" tier_deamon_check
+
+cleanup
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index bc7390b8b40..6e3320f9931 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -4532,7 +4532,8 @@ dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
         }
 
         if ((conf->defrag && conf->defrag->cmd == GF_DEFRAG_CMD_START_TIER) ||
-            (conf->defrag && conf->defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) ||
+            (conf->defrag && conf->defrag->cmd ==
+             GF_DEFRAG_CMD_START_DETACH_TIER) ||
             (!(conf->local_subvols_cnt) || !conf->defrag)) {
                 local->call_cnt = conf->subvolume_cnt;
 
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 6375286f636..239ac26fe24 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -2929,8 +2929,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                                  * CTR DB Lookup Heal triggered on existing
                                  * data.
                                  * */
-                                if (defrag->cmd ==
-                                        GF_DEFRAG_CMD_START_TIER) {
+                                if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) {
                                         gf_fix_layout_tier_attach_lookup
                                                 (this, loc, entry);
                                 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index ea443aaa05e..5f49e4b8752 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -270,7 +270,8 @@ glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
                                                    priv);
                 snprintf (logfile, PATH_MAX, "%s/%s-%s-%s.log",
                           DEFAULT_LOG_FILE_DIRECTORY, volinfo->volname,
-                          (cmd == GF_DEFRAG_CMD_START_TIER ? "tier":"rebalance"),
+                          (cmd == GF_DEFRAG_CMD_START_TIER ?
+                           "tier":"rebalance"),
                           uuid_utoa(MY_UUID));
 
         } else {
@@ -605,6 +606,7 @@ glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
         char                    *task_id_str = NULL;
         dict_t                  *op_ctx      = NULL;
         xlator_t                *this        = 0;
+        int32_t                 is_force     = 0;
 
         this = THIS;
         GF_ASSERT (this);
@@ -629,16 +631,20 @@ glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
         }
         switch (cmd) {
         case GF_DEFRAG_CMD_START_TIER:
+                ret = dict_get_int32 (dict, "force", &is_force);
+                if (ret)
+                        is_force = 0;
+
                 if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
                         gf_asprintf (op_errstr, "volume %s is not a tier "
                                      "volume.", volinfo->volname);
                         ret = -1;
                         goto out;
                 }
-                if (glusterd_is_tier_daemon_running (volinfo)) {
+                if ((!is_force) && glusterd_is_tier_daemon_running (volinfo)) {
                         ret = gf_asprintf (op_errstr, "A Tier daemon is "
-                                           "already running on volume %s",
-                                           volname);
+                                        "already running on volume %s",
+                                        volname);
                         ret = -1;
                         goto out;
                 }
@@ -797,6 +803,7 @@ glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         dict_t                  *ctx = NULL;
         xlator_t                *this = NULL;
         uint32_t                commit_hash;
+        int32_t                 is_force    = 0;
 
         this = THIS;
         GF_ASSERT (this);
@@ -860,39 +867,79 @@ glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
         case GF_DEFRAG_CMD_START_FORCE:
         case GF_DEFRAG_CMD_START_TIER:
-                /* Reset defrag status to 'NOT STARTED' whenever a
-                 * remove-brick/rebalance command is issued to remove
-                 * stale information from previous run.
-                 */
-                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
 
-                ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
-                if (ret) {
-                        gf_msg_debug (this->name, 0, "Missing rebalance "
-                                "id");
-                        ret = 0;
+
+                ret = dict_get_int32 (dict, "force", &is_force);
+                if (ret)
+                        is_force = 0;
+                if (!is_force) {
+                        /* Reset defrag status to 'NOT STARTED' whenever a
+                         * remove-brick/rebalance command is issued to remove
+                         * stale information from previous run.
+                         */
+                        volinfo->rebal.defrag_status =
+                               GF_DEFRAG_STATUS_NOT_STARTED;
+
+                        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_msg_debug (this->name, 0, "Missing rebalance"
+                                              " id");
+                                ret = 0;
+                        } else {
+                                gf_uuid_parse (task_id_str,
+                                               volinfo->rebal.rebalance_id);
+                                volinfo->rebal.op = GD_OP_REBALANCE;
+                        }
+                        if (!gd_should_i_start_rebalance (volinfo)) {
+                                /* Store the rebalance-id and rebalance command
+                                 * even if the peer isn't starting a rebalance
+                                 * process. On peers where a rebalance process
+                                 * is started, glusterd_handle_defrag_start
+                                 * performs the storing.
+                                 * Storing this is needed for having
+                                 * 'volume status' work correctly.
+                                 */
+                                glusterd_store_perform_node_state_store
+                                        (volinfo);
+                                break;
+                        }
+                        if (dict_get_uint32 (dict, "commit-hash", &commit_hash)
+                                             == 0) {
+                                volinfo->rebal.commit_hash = commit_hash;
+                        }
+                        ret = glusterd_handle_defrag_start (volinfo, msg,
+                                        sizeof (msg),
+                                        cmd, NULL, GD_OP_REBALANCE);
+                        break;
                 } else {
-                        gf_uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
-                        volinfo->rebal.op = GD_OP_REBALANCE;
-                }
-                if (!gd_should_i_start_rebalance (volinfo)) {
-                        /* Store the rebalance-id and rebalance command even if
-                         * the peer isn't starting a rebalance process. On peers
-                         * where a rebalance process is started,
-                         * glusterd_handle_defrag_start performs the storing.
-                         *
-                         * Storing this is needed for having 'volume status'
-                         * work correctly.
+                        /* Reset defrag status to 'STARTED' so that the
+                         * pid is checked and restarted accordingly.
+                         * If the pid is not running it executes the
+                         * "NOT_STARTED" case and restarts the process
                          */
-                        glusterd_store_perform_node_state_store (volinfo);
+                        volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
+                        volinfo->rebal.defrag_cmd = cmd;
+                        volinfo->rebal.op = GD_OP_REBALANCE;
+
+                        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_msg_debug (this->name, 0, "Missing rebalance"
+                                              " id");
+                                ret = 0;
+                        } else {
+                                gf_uuid_parse (task_id_str,
+                                               volinfo->rebal.rebalance_id);
+                                volinfo->rebal.op = GD_OP_REBALANCE;
+                        }
+                        if (dict_get_uint32 (dict, "commit-hash", &commit_hash)
+                                             == 0) {
+                                volinfo->rebal.commit_hash = commit_hash;
+                        }
+                        ret = glusterd_restart_rebalance_for_volume (volinfo);
                         break;
                 }
-                if (dict_get_uint32 (dict, "commit-hash", &commit_hash) == 0) {
-                        volinfo->rebal.commit_hash = commit_hash;
-                }
-                ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
-                                                    cmd, NULL, GD_OP_REBALANCE);
-                break;
         case GF_DEFRAG_CMD_STOP:
         case GF_DEFRAG_CMD_STOP_DETACH_TIER:
                 /* Clear task-id only on explicitly stopping rebalance.
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index c9457e89b45..6fc0dcdc603 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -5647,7 +5647,7 @@ glusterd_is_tier_daemon_running (glusterd_volinfo_t *volinfo)
                 return _gf_false;
 
         if (volinfo->rebal.defrag &&
-             volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER) {
+            volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER) {
                 return _gf_true;
         }
 
@@ -7030,12 +7030,12 @@ glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
         case GF_DEFRAG_STATUS_STARTED:
                 GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
                 if (gf_is_service_running (pidfile, &pid)) {
-                        glusterd_rebalance_rpc_create (volinfo, _gf_true);
+                        ret = glusterd_rebalance_rpc_create (volinfo, _gf_true);
                         break;
                 }
         case GF_DEFRAG_STATUS_NOT_STARTED:
-                glusterd_handle_defrag_start (volinfo, op_errstr, len, cmd,
-                                              cbk, volinfo->rebal.op);
+                ret = glusterd_handle_defrag_start (volinfo, op_errstr, len,
+                                cmd, cbk, volinfo->rebal.op);
                 break;
         default:
                 gf_msg (this->name, GF_LOG_ERROR, 0,
@@ -7102,14 +7102,14 @@ out:
 }
 
 
-void
+int
 glusterd_restart_rebalance_for_volume (glusterd_volinfo_t *volinfo)
 {
-
+        int             ret = -1;
         char          op_errstr[PATH_MAX];
 
         if (!volinfo->rebal.defrag_cmd)
-                return;
+                return -1;
         if (!gd_should_i_start_rebalance (volinfo)) {
 
                 /* Store the rebalance-id and rebalance command even if
@@ -7123,10 +7123,11 @@ glusterd_restart_rebalance_for_volume (glusterd_volinfo_t *volinfo)
                 if (volinfo->type == GF_CLUSTER_TYPE_TIER)
                         glusterd_store_perform_node_state_store (volinfo);
 
-                return;
+                return 0;
         }
-        glusterd_volume_defrag_restart (volinfo, op_errstr, PATH_MAX,
+        ret = glusterd_volume_defrag_restart (volinfo, op_errstr, PATH_MAX,
                                 volinfo->rebal.defrag_cmd, NULL);
+        return ret;
 }
 int
 glusterd_restart_rebalance (glusterd_conf_t *conf)
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 99225c674eb..e912f64ae23 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -374,7 +374,7 @@ glusterd_create_sub_tier_volinfo (glusterd_volinfo_t *volinfo,
                                    glusterd_volinfo_t **dup_volinfo,
                                    gf_boolean_t is_hot_tier,
                                    const char *new_name);
-void
+int
 glusterd_restart_rebalance_for_volume (glusterd_volinfo_t *volinfo);
 
 void
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 1e573f8c5ee..ceb472284b2 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -632,11 +632,11 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
                 STACK_DESTROY (frame->root);                            \
         } while (0)
 
-#define GLUSTERD_GET_DEFRAG_PROCESS(path, volinfo) do {                     \
-                if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER)  \
-                        snprintf (path, NAME_MAX, "tier"); \
-                else                                                        \
-                        snprintf (path, NAME_MAX, "rebalance"); \
+#define GLUSTERD_GET_DEFRAG_PROCESS(path, volinfo) do {                 \
+                if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_START_TIER) \
+                        snprintf (path, NAME_MAX, "tier");              \
+                else                                                    \
+                        snprintf (path, NAME_MAX, "rebalance");         \
         } while (0)
 
 #define GLUSTERD_GET_DEFRAG_DIR(path, volinfo, priv) do {               \