cluster/ec: Add self-heal-daemon command handlers

This patch introduces the changes required in ec xlator to handle
index/full heal.

Index healer threads:
Ec xlator start an index healer thread per local brick. This thread keeps
waking up every minute to check if there are any files to be healed based on
the indices kept in index directory. Whenever child_up event comes, then also
this index healer thread wakes up and crawls the indices and triggers heal.
When self-heal-daemon is disabled on this particular volume then the healer
thread keeps waiting until it is enabled again to perform heals.

Full healer threads:
Ec xlator starts a full healer thread for the local subvolume provided by
glusterd to perform full crawl on the directory hierarchy to perform heals.
Once the crawl completes the thread exits if no more full heals are issued.

Changed xl-op prefix GF_AFR_OP to GF_SHD_OP to make it more generic.

Change-Id: Idf9b2735d779a6253717be064173dfde6f8f824b
BUG: 1177601
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/9787
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

3 files changed, 124 insertions, 124 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 9c887fa78a5..bbec6aa5f8d 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -2984,7 +2984,7 @@ set_hostname_path_in_dict (const char *token, dict_t *dict, int heal_op)
                 goto out;
 
         switch (heal_op) {
-        case GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK:
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
                 ret = dict_set_dynstr (dict, "heal-source-hostname",
                                        hostname);
                 if (ret)
@@ -2992,7 +2992,7 @@ set_hostname_path_in_dict (const char *token, dict_t *dict, int heal_op)
                 ret = dict_set_dynstr (dict, "heal-source-brickpath",
                                        path);
                 break;
-        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
                 ret = dict_set_dynstr (dict, "per-replica-cmd-hostname",
                                        hostname);
                 if (ret)
@@ -3014,29 +3014,29 @@ heal_command_type_get (const char *command)
 {
         int     i = 0;
         /* subcommands are set as NULL */
-        char    *heal_cmds[GF_AFR_OP_HEAL_DISABLE + 1] = {
-                [GF_AFR_OP_INVALID]                            = NULL,
-                [GF_AFR_OP_HEAL_INDEX]                         = NULL,
-                [GF_AFR_OP_HEAL_FULL]                          = "full",
-                [GF_AFR_OP_INDEX_SUMMARY]                      = "info",
-                [GF_AFR_OP_HEALED_FILES]                       = NULL,
-                [GF_AFR_OP_HEAL_FAILED_FILES]                  = NULL,
-                [GF_AFR_OP_SPLIT_BRAIN_FILES]                  = NULL,
-                [GF_AFR_OP_STATISTICS]                         = "statistics",
-                [GF_AFR_OP_STATISTICS_HEAL_COUNT]              = NULL,
-                [GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA]  = NULL,
-                [GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE]       = "split-brain",
-                [GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK]             = "split-brain",
-                [GF_AFR_OP_HEAL_ENABLE]                        = "enable",
-                [GF_AFR_OP_HEAL_DISABLE]                       = "disable",
+        char    *heal_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
+                [GF_SHD_OP_INVALID]                            = NULL,
+                [GF_SHD_OP_HEAL_INDEX]                         = NULL,
+                [GF_SHD_OP_HEAL_FULL]                          = "full",
+                [GF_SHD_OP_INDEX_SUMMARY]                      = "info",
+                [GF_SHD_OP_HEALED_FILES]                       = NULL,
+                [GF_SHD_OP_HEAL_FAILED_FILES]                  = NULL,
+                [GF_SHD_OP_SPLIT_BRAIN_FILES]                  = NULL,
+                [GF_SHD_OP_STATISTICS]                         = "statistics",
+                [GF_SHD_OP_STATISTICS_HEAL_COUNT]              = NULL,
+                [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA]  = NULL,
+                [GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE]       = "split-brain",
+                [GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK]             = "split-brain",
+                [GF_SHD_OP_HEAL_ENABLE]                        = "enable",
+                [GF_SHD_OP_HEAL_DISABLE]                       = "disable",
         };
 
-        for (i = 0; i <= GF_AFR_OP_HEAL_DISABLE; i++) {
+        for (i = 0; i <= GF_SHD_OP_HEAL_DISABLE; i++) {
                 if (heal_cmds[i] && (strcmp (heal_cmds[i], command) == 0))
                         return i;
         }
 
-        return GF_AFR_OP_INVALID;
+        return GF_SHD_OP_INVALID;
 }
 
 int
@@ -3047,7 +3047,7 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
         dict_t  *dict = NULL;
         char    *hostname = NULL;
         char    *path = NULL;
-        gf_xl_afr_op_t op = GF_AFR_OP_INVALID;
+        gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
 
         dict = dict_new ();
         if (!dict)
@@ -3060,13 +3060,13 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
         }
 
         if (wordcount == 3) {
-                ret = dict_set_int32 (dict, "heal-op", GF_AFR_OP_HEAL_INDEX);
+                ret = dict_set_int32 (dict, "heal-op", GF_SHD_OP_HEAL_INDEX);
                 goto done;
         }
 
         if (wordcount == 4) {
                 op = heal_command_type_get (words[3]);
-                if (op == GF_AFR_OP_INVALID) {
+                if (op == GF_SHD_OP_INVALID) {
                         ret = -1;
                         goto out;
                 }
@@ -3085,17 +3085,17 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                 if (!strcmp (words[3], "info")) {
                         if (!strcmp (words[4], "healed")) {
                                 ret = dict_set_int32 (dict, "heal-op",
-                                                      GF_AFR_OP_HEALED_FILES);
+                                                      GF_SHD_OP_HEALED_FILES);
                                 goto done;
                         }
                         if (!strcmp (words[4], "heal-failed")) {
                                 ret = dict_set_int32 (dict, "heal-op",
-                                                   GF_AFR_OP_HEAL_FAILED_FILES);
+                                                   GF_SHD_OP_HEAL_FAILED_FILES);
                                 goto done;
                         }
                         if (!strcmp (words[4], "split-brain")) {
                                 ret = dict_set_int32 (dict, "heal-op",
-                                                   GF_AFR_OP_SPLIT_BRAIN_FILES);
+                                                   GF_SHD_OP_SPLIT_BRAIN_FILES);
                                 goto done;
                         }
                 }
@@ -3103,7 +3103,7 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                 if (!strcmp (words[3], "statistics")) {
                         if (!strcmp (words[4], "heal-count")) {
                                 ret = dict_set_int32 (dict, "heal-op",
-                                               GF_AFR_OP_STATISTICS_HEAL_COUNT);
+                                               GF_SHD_OP_STATISTICS_HEAL_COUNT);
                                 goto done;
                         }
                 }
@@ -3117,7 +3117,7 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                 }
                 if (!strcmp (words[4], "bigger-file")) {
                         ret = dict_set_int32 (dict, "heal-op",
-                                        GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE);
+                                        GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE);
                         if (ret)
                                 goto out;
                         ret = dict_set_str (dict, "file", (char *)words[5]);
@@ -3127,11 +3127,11 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                 }
                 if (!strcmp (words[4], "source-brick")) {
                         ret = dict_set_int32 (dict, "heal-op",
-                                              GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK);
+                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
                         if (ret)
                                 goto out;
                         ret = set_hostname_path_in_dict (words[5], dict,
-                                              GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK);
+                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
                         if (ret)
                                 goto out;
                         goto done;
@@ -3145,11 +3145,11 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                     && !strcmp (words[5], "replica")) {
 
                         ret = dict_set_int32 (dict, "heal-op",
-                                   GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+                                   GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
                         if (ret)
                                 goto out;
                         ret = set_hostname_path_in_dict (words[6], dict,
-                                   GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
+                                   GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA);
                         if (ret)
                                 goto out;
                         goto done;
@@ -3158,9 +3158,9 @@ cli_cmd_volume_heal_options_parse (const char **words, int wordcount,
                 if (!strcmp (words[3], "split-brain") &&
                     !strcmp (words[4], "source-brick")) {
                         ret = dict_set_int32 (dict, "heal-op",
-                                              GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK);
+                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
                         ret = set_hostname_path_in_dict (words[5], dict,
-                                              GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK);
+                                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
                         if (ret)
                                 goto out;
                         ret = dict_set_str (dict, "file",
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 1b235bade6f..af9cc6a5aa6 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -1924,10 +1924,10 @@ cli_print_brick_status (cli_volume_status_t *status)
         return 0;
 }
 
-#define NEEDS_GLFS_HEAL(op) ((op == GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) || \
-                             (op == GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK) ||      \
-                             (op == GF_AFR_OP_INDEX_SUMMARY) ||               \
-                             (op == GF_AFR_OP_SPLIT_BRAIN_FILES))
+#define NEEDS_GLFS_HEAL(op) ((op == GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE) || \
+                             (op == GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK) ||      \
+                             (op == GF_SHD_OP_INDEX_SUMMARY) ||               \
+                             (op == GF_SHD_OP_SPLIT_BRAIN_FILES))
 
 int
 cli_launch_glfs_heal (int heal_op, dict_t *options)
@@ -1947,13 +1947,13 @@ cli_launch_glfs_heal (int heal_op, dict_t *options)
         runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
 
         switch (heal_op) {
-        case GF_AFR_OP_INDEX_SUMMARY:
+        case GF_SHD_OP_INDEX_SUMMARY:
                 break;
-        case GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
                 ret = dict_get_str (options, "file", &filename);
                 runner_add_args (&runner, "bigger-file", filename, NULL);
                 break;
-        case GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK:
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
                 ret = dict_get_str (options, "heal-source-hostname",
                                     &hostname);
                 ret = dict_get_str (options, "heal-source-brickpath",
@@ -1963,7 +1963,7 @@ cli_launch_glfs_heal (int heal_op, dict_t *options)
                 if (dict_get_str (options, "file", &filename) == 0)
                         runner_argprintf (&runner, filename);
                 break;
-        case GF_AFR_OP_SPLIT_BRAIN_FILES:
+        case GF_SHD_OP_SPLIT_BRAIN_FILES:
                 runner_add_args (&runner, "split-brain-info", NULL);
                 break;
         default:
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 4e1c6873cee..b2964b68ff6 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -7379,22 +7379,22 @@ gf_is_cli_heal_get_command (gf_xl_afr_op_t heal_op)
 {
         /* If the command is get command value is 1 otherwise 0, for
            invalid commands -1 */
-        int    get_cmds[GF_AFR_OP_HEAL_DISABLE + 1] = {
-                       [GF_AFR_OP_INVALID] = -1,
-                       [GF_AFR_OP_HEAL_INDEX] = 0,
-                       [GF_AFR_OP_HEAL_FULL] = 0,
-                       [GF_AFR_OP_INDEX_SUMMARY] = 1,
-                       [GF_AFR_OP_HEALED_FILES] = 1,
-                       [GF_AFR_OP_HEAL_FAILED_FILES] = 1,
-                       [GF_AFR_OP_SPLIT_BRAIN_FILES] = 1,
-                       [GF_AFR_OP_STATISTICS] = 1,
-                       [GF_AFR_OP_STATISTICS_HEAL_COUNT] = 1,
-                       [GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = 1,
-                       [GF_AFR_OP_HEAL_ENABLE] = 0,
-                       [GF_AFR_OP_HEAL_DISABLE] = 0,
+        int    get_cmds[GF_SHD_OP_HEAL_DISABLE + 1] = {
+                       [GF_SHD_OP_INVALID] = -1,
+                       [GF_SHD_OP_HEAL_INDEX] = 0,
+                       [GF_SHD_OP_HEAL_FULL] = 0,
+                       [GF_SHD_OP_INDEX_SUMMARY] = 1,
+                       [GF_SHD_OP_HEALED_FILES] = 1,
+                       [GF_SHD_OP_HEAL_FAILED_FILES] = 1,
+                       [GF_SHD_OP_SPLIT_BRAIN_FILES] = 1,
+                       [GF_SHD_OP_STATISTICS] = 1,
+                       [GF_SHD_OP_STATISTICS_HEAL_COUNT] = 1,
+                       [GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA] = 1,
+                       [GF_SHD_OP_HEAL_ENABLE] = 0,
+                       [GF_SHD_OP_HEAL_DISABLE] = 0,
                 };
 
-        if (heal_op > GF_AFR_OP_INVALID && heal_op <= GF_AFR_OP_HEAL_DISABLE)
+        if (heal_op > GF_SHD_OP_INVALID && heal_op <= GF_SHD_OP_HEAL_DISABLE)
                 return get_cmds[heal_op] == 1;
         return _gf_false;
 }
@@ -7412,7 +7412,7 @@ gf_cli_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
         dict_t                  *dict = NULL;
         int                     brick_count = 0;
         int                     i = 0;
-        gf_xl_afr_op_t          heal_op = GF_AFR_OP_INVALID;
+        gf_xl_afr_op_t          heal_op = GF_SHD_OP_INVALID;
         char                    *operation = NULL;
         char                    *substr = NULL;
         char                    *heal_op_str = NULL;
@@ -7461,56 +7461,56 @@ gf_cli_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
         operation = "Gathering ";
         substr = "";
         switch (heal_op) {
-                case    GF_AFR_OP_HEAL_INDEX:
-                        operation   = "Launching heal operation ";
-                        heal_op_str = "to perform index self heal";
-                        substr      = "\nUse heal info commands to check"
-                                      " status";
-                        break;
-                case    GF_AFR_OP_HEAL_FULL:
-                        operation   = "Launching heal operation ";
-                        heal_op_str = "to perform full self heal";
-                        substr      = "\nUse heal info commands to check"
-                                      " status";
-                        break;
-                case    GF_AFR_OP_INDEX_SUMMARY:
-                        heal_op_str = "list of entries to be healed";
-                        break;
-                case    GF_AFR_OP_HEALED_FILES:
-                        heal_op_str = "list of healed entries";
-                        break;
-                case    GF_AFR_OP_HEAL_FAILED_FILES:
-                        heal_op_str = "list of heal failed entries";
-                        break;
-                case    GF_AFR_OP_SPLIT_BRAIN_FILES:
-                        heal_op_str = "list of split brain entries";
-                        break;
-                case    GF_AFR_OP_STATISTICS:
-                        heal_op_str =  "crawl statistics";
-                        break;
-                case    GF_AFR_OP_STATISTICS_HEAL_COUNT:
-                        heal_op_str = "count of entries to be healed";
-                        break;
-                case    GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-                        heal_op_str = "count of entries to be healed per replica";
-                        break;
-                /* The below 2 cases are never hit; they're coded only to make
-                 * compiler warnings go away.*/
-                case    GF_AFR_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
-                case    GF_AFR_OP_SBRAIN_HEAL_FROM_BRICK:
-                        break;
+        case    GF_SHD_OP_HEAL_INDEX:
+                operation   = "Launching heal operation ";
+                heal_op_str = "to perform index self heal";
+                substr      = "\nUse heal info commands to check"
+                              " status";
+                break;
+        case    GF_SHD_OP_HEAL_FULL:
+                operation   = "Launching heal operation ";
+                heal_op_str = "to perform full self heal";
+                substr      = "\nUse heal info commands to check"
+                              " status";
+                break;
+        case    GF_SHD_OP_INDEX_SUMMARY:
+                heal_op_str = "list of entries to be healed";
+                break;
+        case    GF_SHD_OP_HEALED_FILES:
+                heal_op_str = "list of healed entries";
+                break;
+        case    GF_SHD_OP_HEAL_FAILED_FILES:
+                heal_op_str = "list of heal failed entries";
+                break;
+        case    GF_SHD_OP_SPLIT_BRAIN_FILES:
+                heal_op_str = "list of split brain entries";
+                break;
+        case    GF_SHD_OP_STATISTICS:
+                heal_op_str =  "crawl statistics";
+                break;
+        case    GF_SHD_OP_STATISTICS_HEAL_COUNT:
+                heal_op_str = "count of entries to be healed";
+                break;
+        case    GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                heal_op_str = "count of entries to be healed per replica";
+                break;
+        /* The below 2 cases are never hit; they're coded only to make
+         * compiler warnings go away.*/
+        case    GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+        case    GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+                break;
 
-                case    GF_AFR_OP_INVALID:
-                        heal_op_str = "invalid heal op";
-                        break;
-                case    GF_AFR_OP_HEAL_ENABLE:
-                        operation   = "";
-                        heal_op_str = "Enable heal";
-                        break;
-                case    GF_AFR_OP_HEAL_DISABLE:
-                        operation   = "";
-                        heal_op_str = "Disable heal";
-                        break;
+        case    GF_SHD_OP_INVALID:
+                heal_op_str = "invalid heal op";
+                break;
+        case    GF_SHD_OP_HEAL_ENABLE:
+                operation   = "";
+                heal_op_str = "Enable heal";
+                break;
+        case    GF_SHD_OP_HEAL_DISABLE:
+                operation   = "";
+                heal_op_str = "Disable heal";
+                break;
         }
 
         if (rsp.op_ret) {
@@ -7559,25 +7559,25 @@ gf_cli_heal_volume_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
         switch (heal_op) {
-                case GF_AFR_OP_STATISTICS:
-                        for (i = 0; i < brick_count; i++)
-                                cmd_heal_volume_statistics_out (dict, i);
-                        break;
-                case GF_AFR_OP_STATISTICS_HEAL_COUNT:
-                case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-                        for (i = 0; i < brick_count; i++)
-                                cmd_heal_volume_statistics_heal_count_out (dict,
-                                                                           i);
-                        break;
-                case GF_AFR_OP_INDEX_SUMMARY:
-                case GF_AFR_OP_HEALED_FILES:
-                case GF_AFR_OP_HEAL_FAILED_FILES:
-                case GF_AFR_OP_SPLIT_BRAIN_FILES:
-                        for (i = 0; i < brick_count; i++)
-                                cmd_heal_volume_brick_out (dict, i);
-                        break;
-                default:
-                        break;
+        case GF_SHD_OP_STATISTICS:
+                for (i = 0; i < brick_count; i++)
+                        cmd_heal_volume_statistics_out (dict, i);
+                break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT:
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                for (i = 0; i < brick_count; i++)
+                        cmd_heal_volume_statistics_heal_count_out (dict,
+                                                                   i);
+                break;
+        case GF_SHD_OP_INDEX_SUMMARY:
+        case GF_SHD_OP_HEALED_FILES:
+        case GF_SHD_OP_HEAL_FAILED_FILES:
+        case GF_SHD_OP_SPLIT_BRAIN_FILES:
+                for (i = 0; i < brick_count; i++)
+                        cmd_heal_volume_brick_out (dict, i);
+                break;
+        default:
+                break;
         }
 
         ret = rsp.op_ret;