glusterd: CLI commands to create and manage tiered volumes.

A tiered volume is a normal volume with some number of new bricks
representing "hot" storage. The "hot" bricks can be attached or
detached dynamically to a normal volume. When this happens, a new graph
is constructed. The root of the new graph is an instance of the tier
translator. One subvolume of the tier translator leads to the old volume,
and another leads to the new hot bricks.

attach-tier <VOLNAME> [<replica> <COUNT>] <NEW-BRICK> ... [force]

volume detach-tier <VOLNAME> [replica <COUNT>] <BRICK>
... <start|stop|status|commit|force>

gluster volume rebalance <volume> tier start

gluster volume rebalance <volume> tier stop

gluster volume rebalance <volume> tier status

The "tier start" CLI command starts a server side daemon. The daemon
initiates file level migration based on caching policies. The daemon's
status can be monitored and stopped.

Note development on the "tier status" command is incomplete. It will be
added in a subsequent patch.

When the "hot" storage is detached, the tier translator is removed
from the graph and the tiered volume reverts to its original state as
described in the volume's info file.

For more background and design see the feature page [1].

[1]
http://www.gluster.org/community/documentation/index.php/Features/data-classification

Change-Id: Ic8042ce37327b850b9e199236e5be3dae95d2472
BUG: 1194753
Signed-off-by: Dan Lambright <dlambrig@redhat.com>
Reviewed-on: http://review.gluster.org/9753
Reviewed-by: Vijay Bellur <vbellur@redhat.com>
Tested-by: Vijay Bellur <vbellur@redhat.com>

3 files changed, 374 insertions, 8 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 5520c9e46b1..54a57008457 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -355,6 +355,10 @@ cli_validate_disperse_volume (char *word, gf1_cluster_type type,
                 cli_err ("striped-replicated-dispersed volume "
                          "is not supported");
                 goto out;
+        case GF_CLUSTER_TYPE_TIER:
+                cli_err ("tier-dispersed volume is not "
+                         "supported");
+                goto out;
         case GF_CLUSTER_TYPE_STRIPE:
                 cli_err ("striped-dispersed volume is not "
                          "supported");
@@ -490,6 +494,11 @@ cli_cmd_volume_create_parse (struct cli_state *state, const char **words,
                         case GF_CLUSTER_TYPE_STRIPE:
                                 type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
                                 break;
+                        case GF_CLUSTER_TYPE_TIER:
+                                cli_err ("replicated-tiered volume is not "
+                                         "supported");
+                                goto out;
+                                break;
                         case GF_CLUSTER_TYPE_DISPERSE:
                                 cli_err ("replicated-dispersed volume is not "
                                          "supported");
@@ -529,6 +538,10 @@ cli_cmd_volume_create_parse (struct cli_state *state, const char **words,
                                 cli_err ("striped-dispersed volume is not "
                                          "supported");
                                 goto out;
+                        case GF_CLUSTER_TYPE_TIER:
+                                cli_err ("striped-tier volume is not "
+                                         "supported");
+                                goto out;
                         }
                         if (wordcount < (index + 2)) {
                                 ret = -1;
@@ -3384,6 +3397,16 @@ cli_cmd_volume_defrag_parse (const char **words, int wordcount,
                 if (strcmp (words[3], "start") && strcmp (words[3], "stop") &&
                     strcmp (words[3], "status"))
                             goto out;
+        } else if ((strcmp (words[3], "tier") == 0) &&
+               (strcmp (words[4], "start") == 0)) {
+                volname = (char *) words[2];
+                cmd = GF_DEFRAG_CMD_START_TIER;
+                goto done;
+        } else if ((strcmp (words[3], "tier") == 0) &&
+               (strcmp (words[4], "status") == 0)) {
+                volname = (char *) words[2];
+                cmd = GF_DEFRAG_CMD_STATUS_TIER;
+                goto done;
         } else {
                 if (strcmp (words[3], "fix-layout") &&
                     strcmp (words[3], "start"))
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 6c950da4e97..3098d74491c 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -840,6 +840,142 @@ out:
         return ret;
 }
 
+int
+cli_cmd_volume_attach_tier_cbk (struct cli_state *state,
+                                struct cli_cmd_word *word, const char **words,
+                                int wordcount)
+{
+        int                     ret = -1;
+        rpc_clnt_procedure_t    *proc = NULL;
+        call_frame_t            *frame = NULL;
+        dict_t                  *options = NULL;
+        int                     sent = 0;
+        int                     parse_error = 0;
+        gf_answer_t             answer = GF_ANSWER_NO;
+        cli_local_t             *local = NULL;
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (!frame)
+                goto out;
+
+        ret = cli_cmd_volume_add_brick_parse (words, wordcount, &options);
+        if (ret) {
+                cli_usage_out (word->pattern);
+                parse_error = 1;
+                goto out;
+        }
+
+        if (state->mode & GLUSTER_MODE_WIGNORE) {
+                ret = dict_set_int32 (options, "force", _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to set force "
+                                "option");
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (options, "attach-tier", 1);
+        if (ret)
+                goto out;
+
+        ret = dict_set_int32 (options, "type", GF_CLUSTER_TYPE_TIER);
+        if (ret)
+                goto out;
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_ATTACH_TIER];
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
+        if (proc->fn) {
+                ret = proc->fn (frame, THIS, options);
+        }
+
+out:
+        if (ret) {
+                cli_cmd_sent_status_get (&sent);
+                if ((sent == 0) && (parse_error == 0))
+                        cli_out ("attach-tier failed");
+        }
+
+        CLI_STACK_DESTROY (frame);
+
+        return ret;
+}
+
+int
+cli_cmd_volume_detach_tier_cbk (struct cli_state *state,
+                              struct cli_cmd_word *word, const char **words,
+                              int wordcount)
+{
+        int                     ret = -1;
+        rpc_clnt_procedure_t    *proc = NULL;
+        call_frame_t            *frame = NULL;
+        dict_t                  *options = NULL;
+        int                     sent = 0;
+        int                     parse_error = 0;
+        gf_answer_t             answer = GF_ANSWER_NO;
+        cli_local_t             *local = NULL;
+        int                     need_question = 0;
+
+        const char *question = "Removing tier can result in data loss. "
+                               "Do you want to Continue?";
+
+        if (wordcount != 3)
+                goto out;
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (!frame)
+                goto out;
+
+        options = dict_new ();
+        if (!options)
+                goto out;
+
+        ret = dict_set_int32 (options, "force", 1);
+        if (ret)
+                goto out;
+
+        ret = dict_set_int32 (options, "command", GF_OP_CMD_DETACH);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (options, "volname", (char *)words[2]);
+        if (ret)
+                goto out;
+
+        ret = dict_set_int32 (options, "count", 1);
+        if (ret)
+                goto out;
+
+        if (!(state->mode & GLUSTER_MODE_SCRIPT) && need_question) {
+                /* we need to ask question only in case of 'commit or force' */
+                answer = cli_cmd_get_confirmation (state, question);
+                if (GF_ANSWER_NO == answer) {
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DETACH_TIER];
+
+        CLI_LOCAL_INIT (local, words, frame, options);
+
+        if (proc->fn) {
+                ret = proc->fn (frame, THIS, options);
+        }
+
+out:
+        if (ret) {
+                cli_cmd_sent_status_get (&sent);
+                if ((sent == 0) && (parse_error == 0))
+                        cli_out ("Volume detach-tier failed");
+        }
+
+        CLI_STACK_DESTROY (frame);
+
+        return ret;
+}
+
 static int
 gf_cli_create_auxiliary_mount (char *volname)
 {
@@ -2435,6 +2571,14 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_volume_rename_cbk,
           "rename volume <VOLNAME> to <NEW-VOLNAME>"},*/
 
+        { "volume attach-tier <VOLNAME> [<replica COUNT>] <NEW-BRICK>...",
+          cli_cmd_volume_attach_tier_cbk,
+          "attach tier to volume <VOLNAME>"},
+
+        { "volume detach-tier <VOLNAME>",
+          cli_cmd_volume_detach_tier_cbk,
+          "detach tier from volume <VOLNAME>"},
+
         { "volume add-brick <VOLNAME> [<stripe|replica> <COUNT>] <NEW-BRICK> ... [force]",
           cli_cmd_volume_add_brick_cbk,
           "add brick to volume <VOLNAME>"},
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 6e66e377ed5..c9b01694436 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -61,6 +61,7 @@ char *cli_vol_type_str[] = {"Distribute",
                             "Replicate",
                             "Striped-Replicate",
                             "Disperse",
+                            "Tier",
                             "Distributed-Stripe",
                             "Distributed-Replicate",
                             "Distributed-Striped-Replicate",
@@ -739,8 +740,9 @@ xml_output:
                 vol_type = type;
 
                 // Distributed (stripe/replicate/stripe-replica) setups
-                if ((type > 0) && ( dist_count < brick_count))
-                        vol_type = type + 4;
+                if ((type != GF_CLUSTER_TYPE_TIER) && (type > 0) &&
+                    (dist_count < brick_count))
+                       vol_type = type + 5;
 
                 cli_out ("Volume Name: %s", volname);
                 cli_out ("Type: %s", cli_vol_type_str[vol_type]);
@@ -1441,6 +1443,134 @@ out:
 }
 
 int
+gf_cli_print_tier_status (dict_t *dict, enum gf_task_types task_type)
+{
+        int                ret          = -1;
+        int                count        = 0;
+        int                i            = 1;
+        char               key[256]     = {0,};
+        gf_defrag_status_t status_rcd   = GF_DEFRAG_STATUS_NOT_STARTED;
+        uint64_t           files        = 0;
+        uint64_t           size         = 0;
+        uint64_t           lookup       = 0;
+        char               *node_name   = NULL;
+        uint64_t           failures     = 0;
+        uint64_t           skipped      = 0;
+        double             elapsed      = 0;
+        char               *status_str  = NULL;
+        char               *size_str    = NULL;
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret) {
+                gf_log ("cli", GF_LOG_ERROR, "count not set");
+                goto out;
+        }
+
+
+        cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s", "Node",
+                 "Rebalanced-files", "size", "scanned", "failures", "skipped",
+                 "status", "run time in secs");
+        cli_out ("%40s %16s %13s %13s %13s %13s %20s %18s", "---------",
+                 "-----------", "-----------", "-----------", "-----------",
+                 "-----------", "------------", "--------------");
+        for (i = 1; i <= count; i++) {
+                /* Reset the variables to prevent carryover of values */
+                node_name = NULL;
+                files = 0;
+                size = 0;
+                lookup = 0;
+                skipped = 0;
+                status_str = NULL;
+                elapsed = 0;
+
+                /* Check if status is NOT_STARTED, and continue early */
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", i);
+                ret = dict_get_int32 (dict, key, (int32_t *)&status_rcd);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_TRACE, "failed to get status");
+                        goto out;
+                }
+                if (GF_DEFRAG_STATUS_NOT_STARTED == status_rcd)
+                        continue;
+
+
+                snprintf (key, 256, "node-name-%d", i);
+                ret = dict_get_str (dict, key, &node_name);
+                if (ret)
+                        gf_log ("cli", GF_LOG_TRACE, "failed to get node-name");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "files-%d", i);
+                ret = dict_get_uint64 (dict, key, &files);
+                if (ret)
+                        gf_log ("cli", GF_LOG_TRACE,
+                                "failed to get file count");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "size-%d", i);
+                ret = dict_get_uint64 (dict, key, &size);
+                if (ret)
+                        gf_log ("cli", GF_LOG_TRACE,
+                                "failed to get size of xfer");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "lookups-%d", i);
+                ret = dict_get_uint64 (dict, key, &lookup);
+                if (ret)
+                        gf_log ("cli", GF_LOG_TRACE,
+                                "failed to get lookedup file count");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "failures-%d", i);
+                ret = dict_get_uint64 (dict, key, &failures);
+                if (ret)
+                        gf_log ("cli", GF_LOG_TRACE,
+                                "failed to get failures count");
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "skipped-%d", i);
+                ret = dict_get_uint64 (dict, key, &skipped);
+                if (ret)
+                        gf_log ("cli", GF_LOG_TRACE,
+                                "failed to get skipped count");
+
+                /* For remove-brick include skipped count into failure count*/
+                if (task_type != GF_TASK_TYPE_REBALANCE) {
+                        failures += skipped;
+                        skipped = 0;
+                }
+
+                memset (key, 0, 256);
+                snprintf (key, 256, "run-time-%d", i);
+                ret = dict_get_double (dict, key, &elapsed);
+                if (ret)
+                        gf_log ("cli", GF_LOG_TRACE, "failed to get run-time");
+
+                /* Check for array bound */
+                if (status_rcd >= GF_DEFRAG_STATUS_MAX)
+                        status_rcd = GF_DEFRAG_STATUS_MAX;
+
+                status_str = cli_vol_task_status_str[status_rcd];
+                size_str = gf_uint64_2human_readable(size);
+                if (size_str) {
+                        cli_out ("%40s %16"PRIu64 " %13s" " %13"PRIu64 " %13"
+                                 PRIu64" %13"PRIu64 " %20s %18.2f", node_name,
+                                 files, size_str, lookup, failures, skipped,
+                                 status_str, elapsed);
+                } else {
+                        cli_out ("%40s %16"PRIu64 " %13"PRIu64 " %13"PRIu64
+                                 " %13"PRIu64" %13"PRIu64 " %20s %18.2f",
+                                 node_name, files, size, lookup, failures,
+                                 skipped, status_str, elapsed);
+                }
+                GF_FREE(size_str);
+        }
+out:
+        return ret;
+}
+
+int
 gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                              int count, void *myframe)
 {
@@ -1504,7 +1634,9 @@ gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 }
         }
 
-        if (!((cmd == GF_DEFRAG_CMD_STOP) || (cmd == GF_DEFRAG_CMD_STATUS)) &&
+        if (!((cmd == GF_DEFRAG_CMD_STOP) ||
+              (cmd == GF_DEFRAG_CMD_STATUS) ||
+              (cmd == GF_DEFRAG_CMD_STATUS_TIER)) &&
              !(global_state->mode & GLUSTER_MODE_XML)) {
                 /* All other possibilites are about starting a rebalance */
                 ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
@@ -1577,7 +1709,12 @@ gf_cli_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 goto out;
         }
 
-        ret = gf_cli_print_rebalance_status (dict, GF_TASK_TYPE_REBALANCE);
+        if (cmd == GF_DEFRAG_CMD_STATUS_TIER)
+                ret = gf_cli_print_tier_status (dict, GF_TASK_TYPE_REBALANCE);
+        else
+                ret = gf_cli_print_rebalance_status (dict,
+                                                     GF_TASK_TYPE_REBALANCE);
+
         if (ret)
                 gf_log ("cli", GF_LOG_ERROR,
                         "Failed to print rebalance status");
@@ -3616,7 +3753,7 @@ int32_t
 gf_cli_reset_volume (call_frame_t *frame, xlator_t *this,
                         void *data)
 {
-        gf_cli_req              req =  {{0,}};
+        gf_cli_req              req =  {{0,} };
         int                     ret = 0;
         dict_t                  *dict = NULL;
 
@@ -3665,7 +3802,7 @@ int32_t
 gf_cli_set_volume (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
-        gf_cli_req              req =  {{0,}};
+        gf_cli_req              req =  {{0,} };
         int                     ret = 0;
         dict_t                  *dict = NULL;
 
@@ -3691,7 +3828,7 @@ int32_t
 gf_cli_add_brick (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
-        gf_cli_req              req =  {{0,}};
+        gf_cli_req              req =  {{0,} };
         int                     ret = 0;
         dict_t                  *dict = NULL;
         char                    *volname = NULL;
@@ -3726,6 +3863,66 @@ out:
 }
 
 int32_t
+gf_cli_attach_tier (call_frame_t *frame, xlator_t *this,
+                    void *data)
+{
+        gf_cli_req              req =  {{0,} };
+        int                     ret = 0;
+        dict_t                  *dict = NULL;
+
+        if (!frame || !this ||  !data) {
+                ret = -1;
+                goto out;
+        }
+
+        dict = data;
+
+        if (ret)
+                goto out;
+
+        ret = cli_to_glusterd (&req, frame, gf_cli_add_brick_cbk,
+                               (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_ATTACH_TIER, this,
+                               cli_rpc_prog, NULL);
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+
+        GF_FREE (req.dict.dict_val);
+        return ret;
+}
+
+int32_t
+gf_cli_detach_tier (call_frame_t *frame, xlator_t *this,
+                    void *data)
+{
+        gf_cli_req              req =  {{0,} };
+        int                     ret = 0;
+        dict_t                  *dict = NULL;
+        char                    *volname = NULL;
+
+        if (!frame || !this ||  !data) {
+                ret = -1;
+                goto out;
+        }
+
+        dict = data;
+
+        ret = cli_to_glusterd (&req, frame, gf_cli_remove_brick_cbk,
+                              (xdrproc_t) xdr_gf_cli_req, dict,
+                               GLUSTER_CLI_DETACH_TIER, this,
+                               cli_rpc_prog, NULL);
+
+
+out:
+        gf_log ("cli", GF_LOG_DEBUG, "Returning %d", ret);
+
+        GF_FREE (req.dict.dict_val);
+
+        return ret;
+}
+
+
+int32_t
 gf_cli_remove_brick (call_frame_t *frame, xlator_t *this,
                          void *data)
 {
@@ -9965,7 +10162,9 @@ struct rpc_clnt_procedure gluster_cli_actors[GLUSTER_CLI_MAXVALUE] = {
         [GLUSTER_CLI_BARRIER_VOLUME]   = {"BARRIER VOLUME", gf_cli_barrier_volume},
         [GLUSTER_CLI_GANESHA]          = {"GANESHA", gf_cli_ganesha},
         [GLUSTER_CLI_GET_VOL_OPT]      = {"GET_VOL_OPT", gf_cli_get_vol_opt},
-        [GLUSTER_CLI_BITROT]           = {"BITROT", gf_cli_bitrot}
+        [GLUSTER_CLI_BITROT]           = {"BITROT", gf_cli_bitrot},
+        [GLUSTER_CLI_ATTACH_TIER]      = {"ATTACH_TIER", gf_cli_attach_tier},
+        [GLUSTER_CLI_DETACH_TIER]      = {"DETACH_TIER", gf_cli_detach_tier}
 };
 
 struct rpc_clnt_program cli_prog = {