glusterd: support for tier volumes 'detach start' and 'detach commit'

These commands work in a manner analagous to rebalancing when removing a
brick. The existing migration daemon detects "detach start" and switches
to moving data off the hot tier. While in this state all lookups are
directed to the cold tier.

gluster v detach-tier <vol> start
gluster v detach-tier <vol> commit

The status and stop cli commands shall be submitted separately.

Change-Id: I24fda5cc3ba74f5fb8aa9a3234ad51f18b80a8a0
BUG: 1205540
Signed-off-by: Dan Lambright <dlambrig@redhat.com>
Signed-off-by: root <root@localhost.localdomain>
Signed-off-by: Dan Lambright <dlambrig@redhat.com>
Reviewed-on: http://review.gluster.org/10108
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Tested-by: NetBSD Build System

8 files changed, 102 insertions, 18 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index 165622f87ef..c69d2ada238 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -1604,6 +1604,7 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
                 ret = 0;
                 goto out;
 
+        case GF_OP_CMD_DETACH_START:
         case GF_OP_CMD_START:
         {
                 if ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
@@ -1736,7 +1737,8 @@ glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
 
                 break;
 
-        case GF_OP_CMD_DETACH:
+        case GF_OP_CMD_DETACH_COMMIT:
+        case GF_OP_CMD_DETACH_COMMIT_FORCE:
         case GF_OP_CMD_COMMIT_FORCE:
                 break;
         }
@@ -1961,7 +1963,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
         xlator_t                *this          = NULL;
         dict_t                  *bricks_dict   = NULL;
         char                    *brick_tmpstr  = NULL;
-
+        int                      start_remove  = 0;
         this = THIS;
         GF_ASSERT (this);
 
@@ -1985,10 +1987,15 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
         }
         cmd = flag;
 
+        if ((GF_OP_CMD_START == cmd) ||
+            (GF_OP_CMD_DETACH_START == cmd))
+                start_remove = 1;
+
         /* Set task-id, if available, in ctx dict for operations other than
          * start
          */
-        if (is_origin_glusterd (dict) && (cmd != GF_OP_CMD_START)) {
+
+        if (is_origin_glusterd (dict) && (!start_remove)) {
                 if (!gf_uuid_is_null (volinfo->rebal.rebalance_id)) {
                         ret = glusterd_copy_uuid_to_dict
                                 (volinfo->rebal.rebalance_id, dict,
@@ -2003,7 +2010,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
 
         /* Clear task-id, rebal.op and stored bricks on commmitting/stopping
          * remove-brick */
-        if ((cmd != GF_OP_CMD_START) || (cmd != GF_OP_CMD_STATUS)) {
+        if ((!start_remove) && (cmd != GF_OP_CMD_STATUS)) {
                 gf_uuid_clear (volinfo->rebal.rebalance_id);
                 volinfo->rebal.op = GD_OP_NONE;
                 dict_unref (volinfo->rebal.dict);
@@ -2047,6 +2054,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        case GF_OP_CMD_DETACH_START:
         case GF_OP_CMD_START:
                 /* Reset defrag status to 'NOT STARTED' whenever a
                  * remove-brick/rebalance command is issued to remove
@@ -2069,7 +2077,8 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 force = 1;
                 break;
 
-        case GF_OP_CMD_DETACH:
+        case GF_OP_CMD_DETACH_COMMIT:
+        case GF_OP_CMD_DETACH_COMMIT_FORCE:
                 glusterd_op_perform_detach_tier (volinfo);
                 /* fall through */
 
@@ -2105,7 +2114,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
          * remove-brick. Right now this is required for displaying the task
          * parameters with task status in volume status.
          */
-        if (GF_OP_CMD_START == cmd) {
+        if (start_remove) {
                 bricks_dict = dict_new ();
                 if (!bricks_dict) {
                         ret = -1;
@@ -2118,6 +2127,10 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 }
         }
+
+        if (volinfo->type == GF_CLUSTER_TYPE_TIER)
+                count = glusterd_set_detach_bricks(dict, volinfo);
+
         while ( i <= count) {
                 snprintf (key, 256, "brick%d", i);
                 ret = dict_get_str (dict, key, &brick);
@@ -2127,7 +2140,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 }
 
-                if (GF_OP_CMD_START == cmd) {
+                if (start_remove) {
                         brick_tmpstr = gf_strdup (brick);
                         if (!brick_tmpstr) {
                                 ret = -1;
@@ -2150,7 +2163,7 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                         goto out;
                 i++;
         }
-        if (GF_OP_CMD_START == cmd)
+        if (start_remove)
                 volinfo->rebal.dict = dict_ref (bricks_dict);
 
         volinfo->subvol_count = (volinfo->brick_count /
@@ -2170,7 +2183,8 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                  * volumes undergoing a detach operation, they should not
                  * be modified here.
                  */
-                if ((replica_count == 1) && (cmd != GF_OP_CMD_DETACH)) {
+                if ((replica_count == 1) && (cmd != GF_OP_CMD_DETACH_COMMIT) &&
+                    (cmd != GF_OP_CMD_DETACH_COMMIT_FORCE)) {
                         if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
                                 volinfo->type = GF_CLUSTER_TYPE_NONE;
                                 /* backward compatibility */
@@ -2195,8 +2209,8 @@ glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        if (GF_OP_CMD_START == cmd &&
-                        volinfo->status == GLUSTERD_STATUS_STARTED) {
+        if (start_remove &&
+            volinfo->status == GLUSTERD_STATUS_STARTED) {
                 ret = glusterd_svcs_reconfigure (volinfo);
                 if (ret) {
                         gf_log (this->name, GF_LOG_WARNING,
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 5bfdb0bb43e..c91fdbcea4f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -77,6 +77,11 @@ static struct cds_list_head gd_op_sm_queue;
 synclock_t gd_op_sm_lock;
 glusterd_op_info_t    opinfo = {{0},};
 
+int
+glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
+                                         struct cds_list_head *selected);
+
+
 int32_t
 glusterd_txn_opinfo_dict_init ()
 {
@@ -5161,10 +5166,10 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr,
         int32_t                                 i = 1;
         char                                    key[256] = {0,};
         glusterd_pending_node_t                 *pending_node = NULL;
+        int32_t                                 command = 0;
         int32_t                                 force = 0;
 
 
-
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
@@ -5185,6 +5190,15 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr,
                 goto out;
         }
 
+        ret = dict_get_int32 (dict, "command", &command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get command");
+                goto out;
+        }
+
+        if (command == GF_OP_CMD_DETACH_START)
+                return glusterd_bricks_select_rebalance_volume(dict, op_errstr, selected);
+
         ret = dict_get_int32 (dict, "force", &force);
         if (ret) {
                 gf_log (THIS->name, GF_LOG_INFO, "force flag is not set");
@@ -5863,7 +5877,7 @@ out:
 
 }
 
-static int
+int
 glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
                                          struct cds_list_head *selected)
 {
@@ -5900,7 +5914,7 @@ glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
         } else {
                 pending_node->node = volinfo;
                 pending_node->type = GD_NODE_REBALANCE;
-                cds_list_add_tail (&pending_node->list, &opinfo.pending_bricks);
+                cds_list_add_tail (&pending_node->list, selected);
                 pending_node = NULL;
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index f5bb319cb7d..840438df29a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -215,6 +215,10 @@ glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
         GF_ASSERT (volinfo);
         GF_ASSERT (op_errstr);
 
+        if ((cmd == GF_OP_CMD_DETACH_START) &&
+            (volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_STARTED))
+                return 0;
+
         ret = glusterd_defrag_start_validate (volinfo, op_errstr, len, op);
         if (ret)
                 goto out;
@@ -495,6 +499,7 @@ __glusterd_handle_defrag_volume (rpcsvc_request_t *req)
 
         if ((cmd == GF_DEFRAG_CMD_STATUS) ||
             (cmd == GF_DEFRAG_CMD_STATUS_TIER) ||
+            (cmd == GF_DEFRAG_CMD_STOP_DETACH_TIER) ||
               (cmd == GF_DEFRAG_CMD_STOP)) {
                 ret = glusterd_op_begin (req, GD_OP_DEFRAG_BRICK_VOLUME,
                                          dict, msg, sizeof (msg));
@@ -844,6 +849,7 @@ glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
                 ret = 0;
                 break;
 
+        case GF_DEFRAG_CMD_START_DETACH_TIER:
         case GF_DEFRAG_CMD_STATUS:
         case GF_DEFRAG_CMD_STATUS_TIER:
                 break;
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 0e581b198a9..fdf3365056b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -2795,10 +2795,10 @@ glusterd_store_update_volinfo (glusterd_volinfo_t *volinfo)
                         volinfo->tier_info.cold_disperse_count = atoi (value);
                 } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_COUNT,
                                      strlen (key))) {
-                        volinfo->tier_info.cold_brick_count = atoi (value);
+                        volinfo->tier_info.hot_brick_count = atoi (value);
                 } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_REPLICA_COUNT,
                                      strlen (key))) {
-                        volinfo->tier_info.cold_replica_count = atoi (value);
+                        volinfo->tier_info.hot_replica_count = atoi (value);
                 } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_TYPE,
                                      strlen (key))) {
                         volinfo->tier_info.hot_type = atoi (value);
@@ -2890,7 +2890,10 @@ glusterd_store_update_volinfo (glusterd_volinfo_t *volinfo)
                         break;
 
                         case GF_CLUSTER_TYPE_TIER:
-                        break;
+                                volinfo->tier_info.cold_dist_leaf_count =
+                                        glusterd_calc_dist_leaf_count (
+                                                volinfo->tier_info.cold_replica_count, 1);
+                                break;
 
                         default:
                                 GF_ASSERT (0);
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index c5066b015a3..663b0a73875 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -1542,6 +1542,7 @@ gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
         rpc_clnt_t              *rpc = NULL;
         dict_t                  *rsp_dict = NULL;
         glusterd_conf_t         *conf = NULL;
+        int32_t                 cmd = GF_OP_CMD_NONE;
 
         this = THIS;
         conf = this->private;
@@ -1585,8 +1586,24 @@ gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
                                 "due to rpc failure.");
                         goto out;
                 }
+
+                /* Redirect operation to be detach tier via rebalance flow. */
+                ret = dict_get_int32 (req_dict, "command", &cmd);
+                if (!ret) {
+                        if (cmd == GF_OP_CMD_DETACH_START) {
+                                op = GD_OP_REBALANCE;
+                                ret = dict_set_int32 (req_dict, "rebalance-command",
+                                                      GF_DEFRAG_CMD_START_DETACH_TIER);
+                                if (ret)
+                                        goto out;
+                        }
+                }
                 ret = gd_syncop_mgmt_brick_op (rpc, pending_node, op, req_dict,
                                                op_ctx, op_errstr);
+                if (cmd == GF_OP_CMD_DETACH_START) {
+                        op = GD_OP_REMOVE_BRICK;
+                        dict_del (req_dict, "rebalance-command");
+                }
                 if (ret)
                         goto out;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index afc3faaefb5..50c5c58bcfa 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -4535,6 +4535,12 @@ glusterd_restart_gsyncds (glusterd_conf_t *conf)
         return ret;
 }
 
+int
+glusterd_calc_dist_leaf_count (int rcount, int scount)
+{
+        return (rcount ? rcount : 1) * (scount ? scount : 1);
+}
+
 inline int
 glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo)
 {
@@ -4544,7 +4550,7 @@ glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo)
     if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE)
         return volinfo->disperse_count;
 
-    return (rcount ? rcount : 1) * (scount ? scount : 1);
+    return glusterd_calc_dist_leaf_count (rcount, scount);
 }
 
 int
@@ -5258,6 +5264,10 @@ out:
 int
 glusterd_is_defrag_on (glusterd_volinfo_t *volinfo)
 {
+        /* Defrag is never enabled for tiered volumes. */
+        if (volinfo->type == GF_CLUSTER_TYPE_TIER)
+                return 0;
+
         return (volinfo->rebal.defrag != NULL);
 }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index d2dbddec3f1..59a07bacc1c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -378,6 +378,9 @@ glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
                            dict_t *vol_opts);
 
 int
+glusterd_calc_dist_leaf_count (int rcount, int scount);
+
+int
 glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo);
 
 glusterd_brickinfo_t*
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index ce63e4fc0a4..38c425e8440 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -3025,6 +3025,11 @@ volgen_graph_build_dht_cluster (volgen_graph_t *graph,
         if (clusters < 0)
                 goto out;
 
+        if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
+                ret = 0;
+                goto out;
+        }
+
         dht = first_of (graph);
         ret = _graph_get_decommissioned_children (dht, volinfo,
                                                   &decommissioned_children);
@@ -3271,6 +3276,7 @@ volume_volgen_graph_build_clusters_tier (volgen_graph_t *graph,
         int                st_type = 0;
         char               st_volname[GD_VOLUME_NAME_MAX];
         int                dist_count = 0;
+        char              *decommissioned_children = NULL;
 
         st_brick_count     = volinfo->brick_count;
         st_replica_count   = volinfo->replica_count;
@@ -3338,6 +3344,17 @@ volume_volgen_graph_build_clusters_tier (volgen_graph_t *graph,
 
         st_type = GF_CLUSTER_TYPE_TIER;
 
+        ret = _graph_get_decommissioned_children (xl, volinfo,
+                                                  &decommissioned_children);
+        if (ret)
+                goto out;
+        if (decommissioned_children) {
+                ret = xlator_set_option (xl, "decommissioned-bricks",
+                                         decommissioned_children);
+                if (ret)
+                        goto out;
+        }
+
  out:
         volinfo->brick_count     = st_brick_count;
         volinfo->replica_count   = st_replica_count;