glusterd: support for tier volumes 'detach start' and 'detach commit'

        Back port of http://review.gluster.org/10108

These commands work in a manner analagous to rebalancing when removing a
brick. The existing migration daemon detects "detach start" and switches
to moving data off the hot tier. While in this state all lookups are
directed to the cold tier.

gluster v detach-tier <vol> start
gluster v detach-tier <vol> commit

The status and stop cli commands shall be submitted separately.

>Change-Id: I24fda5cc3ba74f5fb8aa9a3234ad51f18b80a8a0
>BUG: 1205540
>Signed-off-by: Dan Lambright <dlambrig@redhat.com>
>Signed-off-by: root <root@localhost.localdomain>
>Signed-off-by: Dan Lambright <dlambrig@redhat.com>
>Reviewed-on: http://review.gluster.org/10108
>Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

Change-Id: I212d748d077fb5870ee84b316c653acbafbea3f7
BUG: 1220047
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
Reviewed-on: http://review.gluster.org/10708
Reviewed-by: Dan Lambright <dlambrig@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

18 files changed, 259 insertions, 75 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 38b3bbde13d..e5fb4089dab 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -1647,6 +1647,66 @@ out:
         return ret;
 }
 
+int32_t
+cli_cmd_volume_detach_tier_parse (const char **words, int wordcount,
+                                  dict_t **options)
+{
+        int      ret = -1;
+        char    *word = NULL;
+        dict_t  *dict = NULL;
+        int32_t  command = GF_OP_CMD_NONE;
+        int      force = 0;
+
+        if (!((wordcount == 4) || (wordcount == 5)))
+                goto out;
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_str (dict, "volname", (char *)words[2]);
+        if (ret)
+                goto out;
+
+        if (wordcount == 5) {
+                word = (char *)words[4];
+                if (!strcmp(word, "force"))
+                        force = 1;
+        }
+
+        word = (char *)words[3];
+
+        ret = -1;
+
+        if (!strcmp(word, "start")) {
+                command = GF_OP_CMD_DETACH_START;
+        } else if (!strcmp(word, "commit")) {
+                if (force)
+                        command = GF_OP_CMD_DETACH_COMMIT_FORCE;
+                else
+                        command = GF_OP_CMD_DETACH_COMMIT;
+        } else if (!strcmp(word, "stop"))
+                command = GF_DEFRAG_CMD_STOP_DETACH_TIER;
+        else if (!strcmp(word, "status"))
+                command = GF_DEFRAG_CMD_STATUS;
+        else
+                goto out;
+
+        ret = dict_set_int32 (dict, "command", command);
+        if (ret)
+                goto out;
+
+        *options = dict;
+        ret = 0;
+out:
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "Unable to parse detach-tier CLI");
+                if (dict)
+                        dict_unref (dict);
+        }
+
+        return ret;
+}
 
 int32_t
 cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index f9d4041f3ca..91b2a1988d9 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -924,26 +924,16 @@ cli_cmd_volume_detach_tier_cbk (struct cli_state *state,
         const char *question = "Removing tier can result in data loss. "
                                "Do you want to Continue?";
 
-        if (wordcount != 3)
-                goto out;
-
         frame = create_frame (THIS, THIS->ctx->pool);
         if (!frame)
                 goto out;
 
-        options = dict_new ();
-        if (!options)
-                goto out;
+        ret = cli_cmd_volume_detach_tier_parse(words, wordcount, &options);
 
-        ret = dict_set_int32 (options, "force", 1);
         if (ret)
                 goto out;
 
-        ret = dict_set_int32 (options, "command", GF_OP_CMD_DETACH);
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (options, "volname", (char *)words[2]);
+        ret = dict_set_int32 (options, "force", 1);
         if (ret)
                 goto out;
 
@@ -2523,7 +2513,8 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_volume_attach_tier_cbk,
           "attach tier to volume <VOLNAME>"},
 
-        { "volume detach-tier <VOLNAME>",
+        { "volume detach-tier <VOLNAME> "
+          " <start|stop|status|commit|[force]>",
           cli_cmd_volume_detach_tier_cbk,
           "detach tier from volume <VOLNAME>"},
 
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 5361e83a1f6..932fe5f8cb7 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -56,6 +56,10 @@ extern rpc_clnt_prog_t *cli_rpc_prog;
 extern int              cli_op_ret;
 extern int              connected;
 
+int32_t
+gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
+                     void *data);
+
 char *cli_vol_type_str[] = {"Distribute",
                             "Stripe",
                             "Replicate",
@@ -1355,11 +1359,14 @@ gf_cli_print_rebalance_status (dict_t *dict, enum gf_task_types task_type)
                 /* Check if status is NOT_STARTED, and continue early */
                 memset (key, 0, 256);
                 snprintf (key, 256, "status-%d", i);
+
                 ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
                 if (ret) {
+                        gf_log ("cli", GF_LOG_TRACE, "count %d %d", count, i);
                         gf_log ("cli", GF_LOG_TRACE, "failed to get status");
                         goto out;
                 }
+
                 if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
                         continue;
 
@@ -2245,6 +2252,7 @@ gf_cli_remove_brick_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
         switch (cmd) {
+        case GF_OP_CMD_DETACH_START:
         case GF_OP_CMD_START:
                 cmd_str = "start";
 
@@ -3835,30 +3843,7 @@ int32_t
 gf_cli_detach_tier (call_frame_t *frame, xlator_t *this,
                     void *data)
 {
-        gf_cli_req              req =  {{0,} };
-        int                     ret = 0;
-        dict_t                  *dict = NULL;
-        char                    *volname = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-
-        ret = cli_to_glusterd (&req, frame, gf_cli_remove_brick_cbk,
-                              (xdrproc_t) xdr_gf_cli_req, dict,
-                               GLUSTER_CLI_DETACH_TIER, this,
-                               cli_rpc_prog, NULL);
-
-
-out:
-        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
-
-        GF_FREE (req.dict.dict_val);
-
-        return ret;
+        return gf_cli_remove_brick(frame, this, data);
 }
 
 
diff --git a/cli/src/cli.h b/cli/src/cli.h
index 71bf3add3d2..3da58dd22d8 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -259,6 +259,10 @@ cli_cmd_volume_add_brick_parse (const char **words, int wordcount,
                                 dict_t **options, int *type);
 
 int32_t
+cli_cmd_volume_detach_tier_parse (const char **words, int wordcount,
+                                  dict_t **options);
+
+int32_t
 cli_cmd_volume_remove_brick_parse (const char **words, int wordcount,
                                    dict_t **options, int *question);
 
diff --git a/rpc/xdr/src/cli1-xdr.x b/rpc/xdr/src/cli1-xdr.x
index 57d0bb2195f..cdf8e725788 100644
--- a/rpc/xdr/src/cli1-xdr.x
+++ b/rpc/xdr/src/cli1-xdr.x
@@ -5,7 +5,9 @@
         GF_DEFRAG_CMD_START_LAYOUT_FIX,
         GF_DEFRAG_CMD_START_FORCE, /* used by remove-brick data migration */
         GF_DEFRAG_CMD_START_TIER,
-        GF_DEFRAG_CMD_STATUS_TIER
+        GF_DEFRAG_CMD_STATUS_TIER,
+        GF_DEFRAG_CMD_START_DETACH_TIER,
+        GF_DEFRAG_CMD_STOP_DETACH_TIER
 };
 
  enum gf_defrag_status_t {
@@ -48,7 +50,9 @@ enum gf_bitrot_type {
         GF_OP_CMD_STOP,
         GF_OP_CMD_STATUS,
         GF_OP_CMD_COMMIT_FORCE,
-        GF_OP_CMD_DETACH
+        GF_OP_CMD_DETACH_START,
+        GF_OP_CMD_DETACH_COMMIT,
+        GF_OP_CMD_DETACH_COMMIT_FORCE
 };
 
 enum gf_quota_type {
diff --git a/tests/basic/tier/tier.t b/tests/basic/tier/tier.t
index 6097ad6e7d6..fe15395cf4b 100755
--- a/tests/basic/tier/tier.t
+++ b/tests/basic/tier/tier.t
@@ -36,7 +36,7 @@ function file_on_fast_tier {
 
 function confirm_tier_removed {
     $CLI system getspec $V0 | grep $1
-    if [ $? == 0 ] ; then
+    if [ $? == 0 ]; then
         echo "1"
     else
         echo "0"
@@ -52,6 +52,11 @@ function confirm_vol_stopped {
     fi
 }
 
+LAST_BRICK=1
+CACHE_BRICK=2
+DEMOTE_TIMEOUT=12
+PROMOTE_TIMEOUT=5
+MIGRATION_TIMEOUT=10
 cleanup
 
 
@@ -108,21 +113,18 @@ sleep 5
 EXPECT_WITHIN $PROMOTE_TIMEOUT "0" file_on_fast_tier d1/data2.txt
 EXPECT_WITHIN $PROMOTE_TIMEOUT "0" file_on_fast_tier d1/data3.txt
 
-# Test rebalance commands
-TEST $CLI volume rebalance $V0 tier status
-TEST $CLI volume rebalance $V0 stop
-
 # stop gluster, when it comes back info file should have tiered volume
 killall glusterd
 TEST glusterd
 
-# TODO: Remove force. Gracefully migrate data off hot tier.
-# Rebalance+promotion/demotion is under construction.
+# Test rebalance commands
+TEST $CLI volume rebalance $V0 tier status
+
+TEST $CLI volume detach-tier $V0 start
 
-TEST $CLI volume detach-tier $V0
+TEST $CLI volume detach-tier $V0 commit
 
-# temporarily comment out
-#TEST ! [ -e $M0/d1/data.txt ]
+EXPECT "0" file_on_slow_tier d1/data.txt
 
 EXPECT "0" confirm_tier_removed ${V0}${CACHE_BRICK_FIRST}
 
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 9b23cb11c06..6c0afdbec90 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -6417,6 +6417,8 @@ dht_notify (xlator_t *this, int event, void *data, ...)
                                 goto unlock;
                         if (cmd == GF_DEFRAG_CMD_STATUS)
                                 gf_defrag_status_get (defrag, output);
+                        else if (cmd == GF_DEFRAG_CMD_START_DETACH_TIER)
+                                gf_defrag_start_detach_tier(defrag);
                         else if (cmd == GF_DEFRAG_CMD_STOP)
                                 gf_defrag_stop (defrag,
                                                 GF_DEFRAG_STATUS_STOPPED, output);
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 3a1906c0108..9a6ed1a889a 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -266,6 +266,9 @@ enum gf_defrag_type {
         GF_DEFRAG_CMD_START_FORCE = 1 + 4,
         GF_DEFRAG_CMD_START_TIER = 1 + 5,
         GF_DEFRAG_CMD_STATUS_TIER = 1 + 6,
+	GF_DEFRAG_CMD_START_DETACH_TIER = 1 + 7,
+	GF_DEFRAG_CMD_STOP_DETACH_TIER = 1 + 8,
+
 };
 typedef enum gf_defrag_type gf_defrag_type;
 
@@ -893,6 +896,9 @@ int
 gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict);
 
 int
+gf_defrag_start_detach_tier (gf_defrag_info_t *defrag);
+
+int
 gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
                 dict_t *output);
 
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 33f6099c6d1..60f7314efe0 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -2682,6 +2682,22 @@ gf_defrag_start_crawl (void *data)
                         goto out;
                 }
                 methods->migration_other(this, defrag);
+                if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) {
+                        migrate_data = dict_new ();
+                        if (!migrate_data) {
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = dict_set_str (migrate_data,
+                                            GF_XATTR_FILE_MIGRATE_KEY,
+                                            "force");
+                        if (ret)
+                                goto out;
+
+                        ret = gf_defrag_fix_layout (this, defrag, &loc,
+                                                    fix_layout,
+                                                    migrate_data);
+                }
         }
         gf_log ("DHT", GF_LOG_INFO, "crawling file-system completed");
 out:
@@ -2901,6 +2917,14 @@ out:
 }
 
 int
+gf_defrag_start_detach_tier (gf_defrag_info_t *defrag)
+{
+        defrag->cmd = GF_DEFRAG_CMD_START_DETACH_TIER;
+
+        return 0;
+}
+
+int
 gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
                 dict_t *output)
 {
diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c
index 48604374aa7..92d13f4f04b 100644
--- a/xlators/cluster/dht/src/tier.c
+++ b/xlators/cluster/dht/src/tier.c
@@ -812,6 +812,15 @@ tier_start (xlator_t *this, gf_defrag_info_t *defrag)
                         goto out;
                 }
 
+                if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) {
+                        ret = 1;
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                DHT_MSG_LOG_TIER_ERROR,
+                                "defrag->defrag_cmd == "
+                                "GF_DEFRAG_CMD_START_DETACH_TIER");
+                        goto out;
+                }
+
                 tick = (tick + 1) % TIMER_SECS;
                 if ((next_demote != tick) && (next_promote != tick))
                         continue;
@@ -911,15 +920,19 @@ tier_migration_get_dst (xlator_t *this, dht_local_t *local)
 {
         dht_conf_t              *conf   = NULL;
         int32_t                  ret = -1;
+        gf_defrag_info_t        *defrag = NULL;
 
         GF_VALIDATE_OR_GOTO("tier", this, out);
         GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
         conf = this->private;
-        if (!conf)
-                goto out;
 
-        if (conf->subvolumes[0] == local->cached_subvol)
+        defrag = conf->defrag;
+
+        if (defrag && defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) {
+                local->rebalance.target_node = conf->subvolumes[0];
+
+        } else if (conf->subvolumes[0] == local->cached_subvol)
                 local->rebalance.target_node =
                         conf->subvolumes[1];
         else
@@ -936,16 +949,25 @@ out:
 xlator_t *
 tier_search (xlator_t *this, dht_layout_t *layout, const char *name)
 {
-        xlator_t  *subvol = NULL;
-        void      *value;
-        int        search_first_subvol = 0;
+        xlator_t                *subvol = NULL;
+        void                    *value;
+        int                      search_first_subvol = 0;
+        dht_conf_t              *conf   = NULL;
+        gf_defrag_info_t        *defrag = NULL;
 
         GF_VALIDATE_OR_GOTO("tier", this, out);
         GF_VALIDATE_OR_GOTO(this->name, layout, out);
         GF_VALIDATE_OR_GOTO(this->name, name, out);
+        GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+        conf = this->private;
+
+        defrag = conf->defrag;
+        if (defrag && defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER)
+                search_first_subvol = 1;
 
-        if (!dict_get_ptr (this->options, "rule", &value) &&
-            !strcmp(layout->list[0].xlator->name, value)) {
+        else if (!dict_get_ptr (this->options, "rule", &value) &&
+                 !strcmp(layout->list[0].xlator->name, value)) {
                 search_first_subvol = 1;
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index 1178f3684a3..cffabfe34e2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -1591,6 +1591,7 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
                 ret = 0;
                 goto out;
 
+        case GF_OP_CMD_DETACH_START:
         case GF_OP_CMD_START:
         {
                 if ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
@@ -1723,7 +1724,8 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
 
                 break;
 
-        case GF_OP_CMD_DETACH:
+        case GF_OP_CMD_DETACH_COMMIT:
+        case GF_OP_CMD_DETACH_COMMIT_FORCE:
         case GF_OP_CMD_COMMIT_FORCE:
                 break;
         }
@@ -1948,7 +1950,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
         xlator_t                *this          = NULL;
         dict_t                  *bricks_dict   = NULL;
         char                    *brick_tmpstr  = NULL;
-
+        int                      start_remove  = 0;
         this = THIS;
         GF_ASSERT (this);
 
@@ -1972,10 +1974,15 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
         }
         cmd = flag;
 
+        if ((GF_OP_CMD_START == cmd) ||
+            (GF_OP_CMD_DETACH_START == cmd))
+                start_remove = 1;
+
         /* Set task-id, if available, in ctx dict for operations other than
          * start
          */
-        if (is_origin_glusterd (dict) && (cmd != GF_OP_CMD_START)) {
+
+        if (is_origin_glusterd (dict) && (!start_remove)) {
                 if (!gf_uuid_is_null (volinfo->rebal.rebalance_id)) {
                         ret = glusterd_copy_uuid_to_dict
                                 (volinfo->rebal.rebalance_id, dict,
@@ -1990,7 +1997,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
 
         /* Clear task-id, rebal.op and stored bricks on commmitting/stopping
          * remove-brick */
-        if ((cmd != GF_OP_CMD_START) || (cmd != GF_OP_CMD_STATUS)) {
+        if ((!start_remove) && (cmd != GF_OP_CMD_STATUS)) {
                 gf_uuid_clear (volinfo->rebal.rebalance_id);
                 volinfo->rebal.op = GD_OP_NONE;
                 dict_unref (volinfo->rebal.dict);
@@ -2034,6 +2041,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        case GF_OP_CMD_DETACH_START:
         case GF_OP_CMD_START:
                 /* Reset defrag status to 'NOT STARTED' whenever a
                  * remove-brick/rebalance command is issued to remove
@@ -2056,7 +2064,8 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 force = 1;
                 break;
 
-        case GF_OP_CMD_DETACH:
+        case GF_OP_CMD_DETACH_COMMIT:
+        case GF_OP_CMD_DETACH_COMMIT_FORCE:
                 glusterd_op_perform_detach_tier (volinfo);
                 /* fall through */
 
@@ -2092,7 +2101,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
          * remove-brick. Right now this is required for displaying the task
          * parameters with task status in volume status.
          */
-        if (GF_OP_CMD_START == cmd) {
+        if (start_remove) {
                 bricks_dict = dict_new ();
                 if (!bricks_dict) {
                         ret = -1;
@@ -2105,6 +2114,10 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 }
         }
+
+        if (volinfo->type == GF_CLUSTER_TYPE_TIER)
+                count = glusterd_set_detach_bricks(dict, volinfo);
+
         while ( i <= count) {
                 snprintf (key, 256, "brick%d", i);
                 ret = dict_get_str (dict, key, &brick);
@@ -2114,7 +2127,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 }
 
-                if (GF_OP_CMD_START == cmd) {
+                if (start_remove) {
                         brick_tmpstr = gf_strdup (brick);
                         if (!brick_tmpstr) {
                                 ret = -1;
@@ -2137,7 +2150,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 i++;
         }
-        if (GF_OP_CMD_START == cmd)
+        if (start_remove)
                 volinfo->rebal.dict = dict_ref (bricks_dict);
 
         volinfo->subvol_count = (volinfo->brick_count /
@@ -2157,7 +2170,8 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                  * volumes undergoing a detach operation, they should not
                  * be modified here.
                  */
-                if ((replica_count == 1) && (cmd != GF_OP_CMD_DETACH)) {
+                if ((replica_count == 1) && (cmd != GF_OP_CMD_DETACH_COMMIT) &&
+                    (cmd != GF_OP_CMD_DETACH_COMMIT_FORCE)) {
                         if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
                                 volinfo->type = GF_CLUSTER_TYPE_NONE;
                                 /* backward compatibility */
@@ -2182,8 +2196,8 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        if (GF_OP_CMD_START == cmd &&
-                        volinfo->status == GLUSTERD_STATUS_STARTED) {
+        if (start_remove &&
+            volinfo->status == GLUSTERD_STATUS_STARTED) {
                 ret = glusterd_svcs_reconfigure (volinfo);
                 if (ret) {
                         gf_log (this->name, GF_LOG_WARNING,
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index e55e6793f4a..fbd74585229 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -77,6 +77,11 @@ static struct cds_list_head gd_op_sm_queue;
 synclock_t gd_op_sm_lock;
 glusterd_op_info_t    opinfo = {{0},};
 
+int
+glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
+                                         struct cds_list_head *selected);
+
+
 int32_t
 glusterd_txn_opinfo_dict_init ()
 {
@@ -5166,10 +5171,10 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr,
         int32_t                                 i = 1;
         char                                    key[256] = {0,};
         glusterd_pending_node_t                 *pending_node = NULL;
+        int32_t                                 command = 0;
         int32_t                                 force = 0;
 
 
-
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
@@ -5190,6 +5195,15 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr,
                 goto out;
         }
 
+        ret = dict_get_int32 (dict, "command", &command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get command");
+                goto out;
+        }
+
+        if (command == GF_OP_CMD_DETACH_START)
+                return glusterd_bricks_select_rebalance_volume(dict, op_errstr, selected);
+
         ret = dict_get_int32 (dict, "force", &force);
         if (ret) {
                 gf_log (THIS->name, GF_LOG_INFO, "force flag is not set");
@@ -5868,7 +5882,7 @@ out:
 
 }
 
-static int
+int
 glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
                                          struct cds_list_head *selected)
 {
@@ -5905,7 +5919,7 @@ glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
         } else {
                 pending_node->node = volinfo;
                 pending_node->type = GD_NODE_REBALANCE;
-                cds_list_add_tail (&pending_node->list, &opinfo.pending_bricks);
+                cds_list_add_tail (&pending_node->list, selected);
                 pending_node = NULL;
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 0978c7c3149..84d9210a105 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -206,6 +206,10 @@ glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
         GF_ASSERT (volinfo);
         GF_ASSERT (op_errstr);
 
+        if ((cmd == GF_OP_CMD_DETACH_START) &&
+            (volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_STARTED))
+                return 0;
+
         ret = glusterd_defrag_start_validate (volinfo, op_errstr, len, op);
         if (ret)
                 goto out;
@@ -486,6 +490,7 @@ __glusterd_handle_defrag_volume (rpcsvc_request_t *req)
 
         if ((cmd == GF_DEFRAG_CMD_STATUS) ||
             (cmd == GF_DEFRAG_CMD_STATUS_TIER) ||
+            (cmd == GF_DEFRAG_CMD_STOP_DETACH_TIER) ||
               (cmd == GF_DEFRAG_CMD_STOP)) {
                 ret = glusterd_op_begin (req, GD_OP_DEFRAG_BRICK_VOLUME,
                                          dict, msg, sizeof (msg));
@@ -835,6 +840,7 @@ glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                 ret = 0;
                 break;
 
+        case GF_DEFRAG_CMD_START_DETACH_TIER:
         case GF_DEFRAG_CMD_STATUS:
         case GF_DEFRAG_CMD_STATUS_TIER:
                 break;
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 4c28dd8b8ea..f6e31b24943 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -2569,10 +2569,10 @@ glusterd_store_update_volinfo (glusterd_volinfo_t *volinfo)
                         volinfo->tier_info.cold_disperse_count = atoi (value);
                 } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_COUNT,
                                      strlen (key))) {
-                        volinfo->tier_info.cold_brick_count = atoi (value);
+                        volinfo->tier_info.hot_brick_count = atoi (value);
                 } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_REPLICA_COUNT,
                                      strlen (key))) {
-                        volinfo->tier_info.cold_replica_count = atoi (value);
+                        volinfo->tier_info.hot_replica_count = atoi (value);
                 } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_TYPE,
                                      strlen (key))) {
                         volinfo->tier_info.hot_type = atoi (value);
@@ -2664,7 +2664,10 @@ glusterd_store_update_volinfo (glusterd_volinfo_t *volinfo)
                         break;
 
                         case GF_CLUSTER_TYPE_TIER:
-                        break;
+                                volinfo->tier_info.cold_dist_leaf_count =
+                                        glusterd_calc_dist_leaf_count (
+                                                volinfo->tier_info.cold_replica_count, 1);
+                                break;
 
                         default:
                                 GF_ASSERT (0);
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index 859aa636885..43539643c0b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -1592,6 +1592,7 @@ gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
         rpc_clnt_t              *rpc = NULL;
         dict_t                  *rsp_dict = NULL;
         glusterd_conf_t         *conf = NULL;
+        int32_t                 cmd = GF_OP_CMD_NONE;
 
         this = THIS;
         conf = this->private;
@@ -1635,8 +1636,24 @@ gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
                                 "due to rpc failure.");
                         goto out;
                 }
+
+                /* Redirect operation to be detach tier via rebalance flow. */
+                ret = dict_get_int32 (req_dict, "command", &cmd);
+                if (!ret) {
+                        if (cmd == GF_OP_CMD_DETACH_START) {
+                                op = GD_OP_REBALANCE;
+                                ret = dict_set_int32 (req_dict, "rebalance-command",
+                                                      GF_DEFRAG_CMD_START_DETACH_TIER);
+                                if (ret)
+                                        goto out;
+                        }
+                }
                 ret = gd_syncop_mgmt_brick_op (rpc, pending_node, op, req_dict,
                                                op_ctx, op_errstr);
+                if (cmd == GF_OP_CMD_DETACH_START) {
+                        op = GD_OP_REMOVE_BRICK;
+                        dict_del (req_dict, "rebalance-command");
+                }
                 if (ret)
                         goto out;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index f532a9c4714..8209e779c52 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -4426,6 +4426,12 @@ glusterd_restart_gsyncds (glusterd_conf_t *conf)
         return ret;
 }
 
+int
+glusterd_calc_dist_leaf_count (int rcount, int scount)
+{
+        return (rcount ? rcount : 1) * (scount ? scount : 1);
+}
+
 inline int
 glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo)
 {
@@ -4435,7 +4441,7 @@ glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo)
     if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE)
         return volinfo->disperse_count;
 
-    return (rcount ? rcount : 1) * (scount ? scount : 1);
+    return glusterd_calc_dist_leaf_count (rcount, scount);
 }
 
 int
@@ -5149,6 +5155,10 @@ out:
 int
 glusterd_is_defrag_on (glusterd_volinfo_t *volinfo)
 {
+        /* Defrag is never enabled for tiered volumes. */
+        if (volinfo->type == GF_CLUSTER_TYPE_TIER)
+                return 0;
+
         return (volinfo->rebal.defrag != NULL);
 }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 689ba9ae751..fcfddd5bffa 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -368,6 +368,9 @@ glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
                            dict_t *vol_opts);
 
 int
+glusterd_calc_dist_leaf_count (int rcount, int scount);
+
+int
 glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo);
 
 glusterd_brickinfo_t*
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 8b37c5ff325..58328b553a4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -3063,6 +3063,11 @@ volgen_graph_build_dht_cluster (volgen_graph_t *graph,
         if (clusters < 0)
                 goto out;
 
+        if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
+                ret = 0;
+                goto out;
+        }
+
         dht = first_of (graph);
         ret = _graph_get_decommissioned_children (dht, volinfo,
                                                   &decommissioned_children);
@@ -3336,6 +3341,7 @@ volume_volgen_graph_build_clusters_tier (volgen_graph_t *graph,
         int                st_type = 0;
         char               st_volname[GD_VOLUME_NAME_MAX];
         int                dist_count = 0;
+        char              *decommissioned_children = NULL;
 
         st_brick_count     = volinfo->brick_count;
         st_replica_count   = volinfo->replica_count;
@@ -3413,6 +3419,17 @@ volume_volgen_graph_build_clusters_tier (volgen_graph_t *graph,
 
         st_type = GF_CLUSTER_TYPE_TIER;
 
+        ret = _graph_get_decommissioned_children (xl, volinfo,
+                                                  &decommissioned_children);
+        if (ret)
+                goto out;
+        if (decommissioned_children) {
+                ret = xlator_set_option (xl, "decommissioned-bricks",
+                                         decommissioned_children);
+                if (ret)
+                        goto out;
+        }
+
  out:
         volinfo->brick_count     = st_brick_count;
         volinfo->replica_count   = st_replica_count;