cluster/afr: Make data selfheal trigger to be configurable.

By default, lookup triggers data self-heal but that is not the preferred way
of operating replicated volumes. We would like the data self heals to be
triggered in open instead.
Number of back-ground self-heals allowed is 16 and lookups block until
self-heal is completed. We want to prevent blocking in fops. We can not make
lookups independent of self-heal frames because when there are gfid conflicts
the decision of which file is correct is determined in self-heal phase.
So in afr, lookup self-heal is going to guarantee name space consistency
and open/fd fops will take responsibility for data consistency, these
are non blocking. The user needs to set the option cluster.data-self-heal
"open" for this behavior.

Change-Id: If9463cdb9ebac114708558ec13bbca0270acd659
BUG: 3503
Reviewed-on: http://review.gluster.com/334
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@gluster.com>

11 files changed, 217 insertions, 112 deletions

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index caf724593..2d5e98196 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -980,9 +980,7 @@ afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
         read_child = afr_inode_get_read_ctx (this, local->cont.lookup.inode,
                                              NULL);
         if (read_child < 0) {
-                ret = -EIO;
-                local->op_ret = -1;
-                local->op_errno = EIO;
+                ret = -1;
                 goto out;
         }
         gf_log (this->name, GF_LOG_DEBUG, "Building lookup response from %d",
@@ -1024,28 +1022,28 @@ afr_lookup_update_lk_counts (afr_local_t *local, xlator_t *this,
 }
 
 static void
-afr_lookup_set_self_heal_data_by_xattr (afr_local_t *local, xlator_t *this,
-                                        dict_t *xattr)
+afr_lookup_set_self_heal_params_by_xattr (afr_local_t *local, xlator_t *this,
+                                          dict_t *xattr)
 {
         GF_ASSERT (local);
         GF_ASSERT (this);
         GF_ASSERT (xattr);
 
         if (afr_sh_has_metadata_pending (xattr, this)) {
-                local->self_heal.need_metadata_self_heal = _gf_true;
+                local->self_heal.do_metadata_self_heal = _gf_true;
                 gf_log(this->name, GF_LOG_DEBUG,
                        "metadata self-heal is pending for %s.",
                        local->loc.path);
         }
 
         if (afr_sh_has_entry_pending (xattr, this)) {
-                local->self_heal.need_entry_self_heal = _gf_true;
+                local->self_heal.do_entry_self_heal = _gf_true;
                 gf_log(this->name, GF_LOG_DEBUG,
                        "entry self-heal is pending for %s.", local->loc.path);
         }
 
         if (afr_sh_has_data_pending (xattr, this)) {
-                local->self_heal.need_data_self_heal = _gf_true;
+                local->self_heal.do_data_self_heal = _gf_true;
                 gf_log(this->name, GF_LOG_DEBUG,
                        "data self-heal is pending for %s.", local->loc.path);
         }
@@ -1059,12 +1057,12 @@ afr_detect_self_heal_by_iatt (afr_local_t *local, xlator_t *this,
                 /* mismatching permissions */
                 gf_log (this->name, GF_LOG_INFO,
                         "permissions differ for %s ", local->loc.path);
-                local->self_heal.need_metadata_self_heal = _gf_true;
+                local->self_heal.do_metadata_self_heal = _gf_true;
         }
 
         if (OWNERSHIP_DIFFERS (buf, lookup_buf)) {
                 /* mismatching permissions */
-                local->self_heal.need_metadata_self_heal = _gf_true;
+                local->self_heal.do_metadata_self_heal = _gf_true;
                 gf_log (this->name, GF_LOG_INFO,
                         "ownership differs for %s ", local->loc.path);
         }
@@ -1073,7 +1071,7 @@ afr_detect_self_heal_by_iatt (afr_local_t *local, xlator_t *this,
             && IA_ISREG (buf->ia_type)) {
                 gf_log (this->name, GF_LOG_INFO,
                         "size differs for %s ", local->loc.path);
-                local->self_heal.need_data_self_heal = _gf_true;
+                local->self_heal.do_data_self_heal = _gf_true;
         }
 
         if (uuid_compare (buf->ia_gfid, lookup_buf->ia_gfid)) {
@@ -1084,17 +1082,18 @@ afr_detect_self_heal_by_iatt (afr_local_t *local, xlator_t *this,
 }
 
 static void
-afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this)
+afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this,
+                                       gf_boolean_t split_brain)
 {
         GF_ASSERT (local);
         GF_ASSERT (this);
 
         if ((local->success_count > 0) && (local->enoent_count > 0)) {
-                local->self_heal.need_metadata_self_heal = _gf_true;
-                local->self_heal.need_data_self_heal     = _gf_true;
-                local->self_heal.need_entry_self_heal    = _gf_true;
-                local->self_heal.need_gfid_self_heal    = _gf_true;
-                local->self_heal.need_missing_entry_self_heal    = _gf_true;
+                local->self_heal.do_metadata_self_heal = _gf_true;
+                local->self_heal.do_data_self_heal     = _gf_true;
+                local->self_heal.do_entry_self_heal    = _gf_true;
+                local->self_heal.do_gfid_self_heal    = _gf_true;
+                local->self_heal.do_missing_entry_self_heal    = _gf_true;
                 gf_log(this->name, GF_LOG_INFO,
                        "entries are missing in lookup of %s.",
                        local->loc.path);
@@ -1102,12 +1101,11 @@ afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this)
                 goto out;
         }
 
-        if ((local->success_count > 0) &&
-            afr_is_split_brain (this, local->cont.lookup.inode) &&
+        if ((local->success_count > 0) && split_brain &&
             IA_ISREG (local->cont.lookup.inode->ia_type)) {
-                local->self_heal.need_data_self_heal = _gf_true;
-                local->self_heal.need_gfid_self_heal    = _gf_true;
-                local->self_heal.need_missing_entry_self_heal    = _gf_true;
+                local->self_heal.do_data_self_heal = _gf_true;
+                local->self_heal.do_gfid_self_heal    = _gf_true;
+                local->self_heal.do_missing_entry_self_heal    = _gf_true;
                 gf_log (this->name, GF_LOG_WARNING,
                         "split brain detected during lookup of %s.",
                         local->loc.path);
@@ -1123,11 +1121,12 @@ afr_can_self_heal_proceed (afr_self_heal_t *sh, afr_private_t *priv)
         GF_ASSERT (sh);
         GF_ASSERT (priv);
 
-        return (sh->need_gfid_self_heal
-                || sh->need_missing_entry_self_heal
-                || (priv->data_self_heal && sh->need_data_self_heal)
-                || (priv->metadata_self_heal && sh->need_metadata_self_heal)
-                || (priv->entry_self_heal && sh->need_entry_self_heal));
+        return (sh->do_gfid_self_heal
+                || sh->do_missing_entry_self_heal
+                || (afr_data_self_heal_enabled (priv->data_self_heal) &&
+                    sh->do_data_self_heal)
+                || (priv->metadata_self_heal && sh->do_metadata_self_heal)
+                || (priv->entry_self_heal && sh->do_entry_self_heal));
 }
 
 afr_transaction_type
@@ -1193,7 +1192,7 @@ afr_is_transaction_running (afr_local_t *local)
 
 void
 afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                      gf_boolean_t is_background, ia_type_t ia_type,
+                      gf_boolean_t background, ia_type_t ia_type, char *reason,
                       void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
                                                    xlator_t *this),
                       int (*unwind) (call_frame_t *frame, xlator_t *this,
@@ -1201,6 +1200,7 @@ afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
 {
         afr_local_t             *local = NULL;
         char                    sh_type_str[256] = {0,};
+        char                    *bg = "";
 
         GF_ASSERT (frame);
         GF_ASSERT (this);
@@ -1208,7 +1208,7 @@ afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
         GF_ASSERT (ia_type != IA_INVAL);
 
         local = frame->local;
-        local->self_heal.background = is_background;
+        local->self_heal.background = background;
         local->self_heal.type       = ia_type;
         local->self_heal.unwind     = unwind;
         local->self_heal.gfid_sh_success_cbk     = gfid_sh_success_cbk;
@@ -1217,9 +1217,11 @@ afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
                                     sh_type_str,
                                     sizeof (sh_type_str));
 
+        if (background)
+                bg = "background";
         gf_log (this->name, GF_LOG_INFO,
-                "background %s self-heal triggered. path: %s",
-                sh_type_str, local->loc.path);
+                "%s %s self-heal triggered. path: %s, reason: %s", bg,
+                sh_type_str, local->loc.path, reason);
 
         afr_self_heal (frame, this, inode);
 }
@@ -1351,8 +1353,27 @@ afr_lookup_conflicting_entries (afr_local_t *local, xlator_t *this)
         return conflict;
 }
 
+gf_boolean_t
+afr_open_only_data_self_heal (char *data_self_heal)
+{
+        return !strcmp (data_self_heal, "open");
+}
+
+gf_boolean_t
+afr_data_self_heal_enabled (char *data_self_heal)
+{
+        gf_boolean_t    enabled = _gf_false;
+
+        if (gf_string2boolean (data_self_heal, &enabled) == -1) {
+                enabled = !strcmp (data_self_heal, "open");
+                GF_ASSERT (enabled);
+        }
+
+        return enabled;
+}
+
 static void
-afr_lookup_set_self_heal_data (afr_local_t *local, xlator_t *this)
+afr_lookup_set_self_heal_params (afr_local_t *local, xlator_t *this)
 {
         int                     i = 0;
         struct iatt             *bufs = NULL;
@@ -1360,15 +1381,20 @@ afr_lookup_set_self_heal_data (afr_local_t *local, xlator_t *this)
         afr_private_t           *priv = NULL;
         int32_t                 child1 = -1;
         int32_t                 child2 = -1;
+        afr_self_heal_t         *sh = NULL;
+        gf_boolean_t            split_brain = _gf_false;
 
         priv  = this->private;
-        afr_detect_self_heal_by_lookup_status (local, this);
+        sh = &local->self_heal;
+
+        split_brain = afr_is_split_brain (this, local->cont.lookup.inode);
+        afr_detect_self_heal_by_lookup_status (local, this, split_brain);
 
         if (afr_lookup_gfid_missing_count (local, this))
-                local->self_heal.need_gfid_self_heal    = _gf_true;
+                local->self_heal.do_gfid_self_heal    = _gf_true;
 
         if (_gf_true == afr_lookup_conflicting_entries (local, this))
-                local->self_heal.need_missing_entry_self_heal    = _gf_true;
+                local->self_heal.do_missing_entry_self_heal    = _gf_true;
         else
                 afr_update_gfid_from_iatts (local->self_heal.sh_gfid_req,
                                             local->cont.lookup.bufs,
@@ -1386,9 +1412,12 @@ afr_lookup_set_self_heal_data (afr_local_t *local, xlator_t *this)
         xattr = local->cont.lookup.xattrs;
         for (i = 0; i < local->success_count; i++) {
                 child1 = local->cont.lookup.success_children[i];
-                afr_lookup_set_self_heal_data_by_xattr (local, this,
-                                                        xattr[child1]);
+                afr_lookup_set_self_heal_params_by_xattr (local, this,
+                                                          xattr[child1]);
         }
+        if (afr_open_only_data_self_heal (priv->data_self_heal)
+            && !split_brain)
+                sh->do_data_self_heal = _gf_false;
 }
 
 int
@@ -1461,12 +1490,13 @@ afr_post_gfid_sh_success (call_frame_t *sh_frame, xlator_t *this)
 }
 
 static void
-afr_lookup_perform_self_heal_if_needed (call_frame_t *frame, xlator_t *this,
-                                        gf_boolean_t *sh_launched)
+afr_lookup_perform_self_heal (call_frame_t *frame, xlator_t *this,
+                              gf_boolean_t *sh_launched)
 {
         unsigned int         up_count = 0;
         afr_private_t       *priv    = NULL;
         afr_local_t         *local   = NULL;
+        char                *reason  = NULL;
 
         GF_ASSERT (sh_launched);
         *sh_launched = _gf_false;
@@ -1480,14 +1510,15 @@ afr_lookup_perform_self_heal_if_needed (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
 
-        afr_lookup_set_self_heal_data (local, this);
+        afr_lookup_set_self_heal_params (local, this);
         if (afr_can_self_heal_proceed (&local->self_heal, priv)) {
                 if  (afr_is_transaction_running (local))
                         goto out;
 
+                reason = "lookup detected pending operations";
                 afr_launch_self_heal (frame, this, local->cont.lookup.inode,
                                       _gf_true, local->cont.lookup.buf.ia_type,
-                                      afr_post_gfid_sh_success,
+                                      reason, afr_post_gfid_sh_success,
                                       afr_self_heal_lookup_unwind);
                 *sh_launched = _gf_true;
         }
@@ -1554,29 +1585,19 @@ afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
                                      local->cont.lookup.success_children,
                                      priv->child_count, local->loc.path,
                                      this->name)) {
-                if (fail_conflict == _gf_false) {
+                if (fail_conflict == _gf_false)
                         ret = 0;
-                } else {
-                        local->op_ret = -1;
-                        local->op_errno = EIO;
-                }
                 goto out;
         }
 
         if (!afr_is_transaction_running (local)) {
                 ret = afr_lookup_select_read_child (local, this, &read_child);
-                if (ret) {
-                        local->op_ret = -1;
-                        local->op_errno = EIO;
+                if (ret)
                         goto out;
-                }
 
                 ret = afr_lookup_set_read_ctx (local, this, read_child);
-                if (ret) {
-                        local->op_ret = -1;
-                        local->op_errno = EIO;
+                if (ret)
                         goto out;
-                }
         }
 
         ret = afr_lookup_build_response_params (local, this);
@@ -1590,6 +1611,10 @@ afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
 
         ret = 0;
 out:
+        if (ret) {
+                local->op_ret = -1;
+                local->op_errno = EIO;
+        }
         return ret;
 }
 
@@ -1629,7 +1654,7 @@ afr_lookup_done (call_frame_t *frame, xlator_t *this)
                 goto unwind;
         uuid_copy (local->self_heal.sh_gfid_req, local->cont.lookup.gfid_req);
 
-        afr_lookup_perform_self_heal_if_needed (frame, this, &sh_launched);
+        afr_lookup_perform_self_heal (frame, this, &sh_launched);
         if (sh_launched) {
                 unwind = 0;
                 goto unwind;
@@ -3311,7 +3336,7 @@ afr_priv_dump (xlator_t *this)
                 gf_proc_dump_write(key, "%s", priv->pending_key[i]);
         }
         gf_proc_dump_build_key(key, key_prefix, "data_self_heal");
-        gf_proc_dump_write(key, "%d", priv->data_self_heal);
+        gf_proc_dump_write(key, "%s", priv->data_self_heal);
         gf_proc_dump_build_key(key, key_prefix, "metadata_self_heal");
         gf_proc_dump_write(key, "%d", priv->metadata_self_heal);
         gf_proc_dump_build_key(key, key_prefix, "entry_self_heal");
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index d323e6d5f..57f0a03fa 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -106,15 +106,17 @@ afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
         afr_self_heal_t * sh          = NULL;
         gf_dirent_t *     entry       = NULL;
         gf_dirent_t *     tmp         = NULL;
+        char              *reason     = NULL;
         int               child_index = 0;
         uint32_t          entry_cksum = 0;
         int               call_count  = 0;
         off_t             last_offset = 0;
-        char              sh_type_str[256] = {0,};
+        inode_t           *inode      = NULL;
 
         priv  = this->private;
         local = frame->local;
         sh    = &local->self_heal;
+        inode = local->fd->inode;
 
         child_index = (long) cookie;
 
@@ -162,23 +164,15 @@ out:
                                         priv->child_count,
                                         local->child_up)) {
 
-                        sh->need_entry_self_heal  = _gf_true;
+                        sh->do_entry_self_heal  = _gf_true;
                         sh->forced_merge          = _gf_true;
 
-                        afr_self_heal_type_str_get(&local->self_heal,
-                                                   sh_type_str,
-                                                   sizeof(sh_type_str));
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%s self-heal triggered. path: %s, "
-                                "reason: checksums of directory differ,"
-                                " forced merge option set",
-                                sh_type_str, local->loc.path);
-
-                        afr_launch_self_heal (frame, this, local->fd->inode,
-                                              _gf_false, local->fd->inode->ia_type,
-                                              NULL, afr_examine_dir_sh_unwind);
+                        reason = "checksums of directory differ";
+                        afr_launch_self_heal (frame, this, inode, _gf_false,
+                                              inode->ia_type, reason, NULL,
+                                              afr_examine_dir_sh_unwind);
                 } else {
-                        afr_set_opendir_done (this, local->fd->inode);
+                        afr_set_opendir_done (this, inode);
 
                         AFR_STACK_UNWIND (opendir, frame, local->op_ret,
                                           local->op_errno, local->fd);
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index b40af1b06..c6d7b5f22 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -325,22 +325,20 @@ afr_trigger_open_fd_self_heal (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t             *local = NULL;
         afr_self_heal_t         *sh = NULL;
-        char                    sh_type_str[256] = {0};
+        inode_t                 *inode = NULL;
+        char                    *reason = NULL;
 
         local = frame->local;
         sh    = &local->self_heal;
+        inode = local->fd->inode;
 
-        sh->need_missing_entry_self_heal = _gf_true;
-        sh->need_gfid_self_heal = _gf_true;
-        sh->need_data_self_heal = _gf_true;
-        afr_self_heal_type_str_get(&local->self_heal, sh_type_str,
-                                   sizeof(sh_type_str));
-        gf_log (this->name, GF_LOG_INFO, "%s self-heal triggered. "
-                "path: %s, reason: Replicate up down flush, data lock "
-                "is held", sh_type_str, local->loc.path);
-
-        afr_launch_self_heal (frame, this, local->fd->inode, _gf_true,
-                              local->fd->inode->ia_type, NULL, NULL);
+        sh->do_missing_entry_self_heal = _gf_true;
+        sh->do_gfid_self_heal = _gf_true;
+        sh->do_data_self_heal = _gf_true;
+
+        reason = "subvolume came online";
+        afr_launch_self_heal (frame, this, inode, _gf_true, inode->ia_type,
+                              reason, NULL, NULL);
 }
 
 int
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index ae5851860..4b328e184 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -55,6 +55,72 @@
 #include "afr-self-heal.h"
 #include "afr-self-heal-common.h"
 
+int
+afr_stale_child_up (afr_local_t *local, xlator_t *this)
+{
+        int             i = 0;
+        afr_private_t   *priv = NULL;
+        int             up = -1;
+
+        priv = this->private;
+
+        if (!local->fresh_children)
+                local->fresh_children = afr_children_create (priv->child_count);
+        if (!local->fresh_children)
+                goto out;
+
+        afr_inode_get_read_ctx (this, local->fd->inode, local->fresh_children);
+        if (priv->child_count == afr_get_children_count (local->fresh_children,
+                                                         priv->child_count))
+                goto out;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->child_up[i])
+                        continue;
+                if (afr_is_child_present (local->fresh_children,
+                                          priv->child_count, i))
+                        continue;
+                up = i;
+                break;
+        }
+out:
+        return up;
+}
+
+void
+afr_perform_data_self_heal (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        inode_t         *inode = NULL;
+        int             st_child = -1;
+        char            reason[64] = {0};
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+        inode = local->fd->inode;
+
+        if (!IA_ISREG (inode->ia_type))
+                goto out;
+
+        st_child = afr_stale_child_up (local, this);
+        if (st_child < 0)
+                goto out;
+
+        sh->do_data_self_heal          = _gf_true;
+        sh->do_metadata_self_heal      = _gf_true;
+        sh->do_gfid_self_heal          = _gf_true;
+        sh->do_missing_entry_self_heal = _gf_true;
+
+        snprintf (reason, sizeof (reason), "stale subvolume %d detected",
+                  st_child);
+        afr_launch_self_heal (frame, this, inode, _gf_true, inode->ia_type,
+                              reason, NULL, NULL);
+out:
+        return;
+}
 
 int
 afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
@@ -62,7 +128,11 @@ afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         struct iatt *postbuf)
 {
         afr_local_t * local = frame->local;
+        afr_private_t *priv = NULL;
 
+        priv = this->private;
+        if (afr_open_only_data_self_heal (priv->data_self_heal))
+                afr_perform_data_self_heal (frame, this);
         AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
                           local->fd);
         return 0;
@@ -80,7 +150,9 @@ afr_open_cbk (call_frame_t *frame, void *cookie,
         int            ret         = 0;
         int            call_count  = -1;
         int            child_index = (long) cookie;
+        afr_private_t *priv        = NULL;
 
+        priv = this->private;
         local = frame->local;
 
         LOCK (&frame->lock);
@@ -133,6 +205,8 @@ unlock:
                                     this, this->fops->ftruncate,
                                     fd, 0);
                 } else {
+                        if (afr_open_only_data_self_heal (priv->data_self_heal))
+                                afr_perform_data_self_heal (frame, this);
                         AFR_STACK_UNWIND (open, frame, local->op_ret,
                                           local->op_errno, local->fd);
                 }
@@ -153,6 +227,7 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         int32_t         op_ret     = -1;
         int32_t         op_errno   = 0;
         int32_t         wind_flags = flags & (~O_TRUNC);
+        //We can't let truncation to happen outside transaction.
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (this, out);
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 169d549d9..489a5bf44 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1924,11 +1924,11 @@ afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
 
         shc->unwind = sh->unwind;
         shc->gfid_sh_success_cbk = sh->gfid_sh_success_cbk;
-        shc->need_missing_entry_self_heal = sh->need_missing_entry_self_heal;
-        shc->need_gfid_self_heal = sh->need_gfid_self_heal;
-        shc->need_data_self_heal = sh->need_data_self_heal;
-        shc->need_metadata_self_heal = sh->need_metadata_self_heal;
-        shc->need_entry_self_heal = sh->need_entry_self_heal;
+        shc->do_missing_entry_self_heal = sh->do_missing_entry_self_heal;
+        shc->do_gfid_self_heal = sh->do_gfid_self_heal;
+        shc->do_data_self_heal = sh->do_data_self_heal;
+        shc->do_metadata_self_heal = sh->do_metadata_self_heal;
+        shc->do_entry_self_heal = sh->do_entry_self_heal;
         shc->forced_merge = sh->forced_merge;
         shc->data_lock_held = sh->data_lock_held;
         shc->background = sh->background;
@@ -2065,9 +2065,9 @@ afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
         gf_log (this->name, GF_LOG_TRACE,
                 "performing self heal on %s (metadata=%d data=%d entry=%d)",
                 local->loc.path,
-                local->self_heal.need_metadata_self_heal,
-                local->self_heal.need_data_self_heal,
-                local->self_heal.need_entry_self_heal);
+                local->self_heal.do_metadata_self_heal,
+                local->self_heal.do_data_self_heal,
+                local->self_heal.do_entry_self_heal);
 
         sh_frame        = copy_frame (frame);
         afr_set_lk_owner (sh_frame, this);
@@ -2120,9 +2120,9 @@ afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
 
 
         FRAME_SU_DO (sh_frame, afr_local_t);
-        if (sh->need_missing_entry_self_heal) {
+        if (sh->do_missing_entry_self_heal) {
                 afr_self_heal_conflicting_entries (sh_frame, this);
-        } else if (sh->need_gfid_self_heal) {
+        } else if (sh->do_gfid_self_heal) {
                 GF_ASSERT (!uuid_is_null (sh->sh_gfid_req));
                 afr_self_heal_gfids (sh_frame, this);
         } else {
@@ -2143,24 +2143,24 @@ afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
         GF_ASSERT (str && (size > strlen (" missing-entry gfid "
                                           "meta-data data entry")));
 
-        if (self_heal_p->need_metadata_self_heal) {
+        if (self_heal_p->do_metadata_self_heal) {
                 snprintf (str, size, " meta-data");
         }
 
-        if (self_heal_p->need_data_self_heal) {
+        if (self_heal_p->do_data_self_heal) {
                 snprintf (str + strlen(str), size - strlen(str), " data");
         }
 
-        if (self_heal_p->need_entry_self_heal) {
+        if (self_heal_p->do_entry_self_heal) {
                 snprintf (str + strlen(str), size - strlen(str), " entry");
         }
 
-        if (self_heal_p->need_missing_entry_self_heal) {
+        if (self_heal_p->do_missing_entry_self_heal) {
                 snprintf (str + strlen(str), size - strlen(str),
                          " missing-entry");
         }
 
-        if (self_heal_p->need_gfid_self_heal) {
+        if (self_heal_p->do_gfid_self_heal) {
                 snprintf (str + strlen(str), size - strlen(str), " gfid");
         }
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index cbc6d6e39..216017cbb 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -1336,7 +1336,8 @@ afr_self_heal_data (call_frame_t *frame, xlator_t *this)
         local = frame->local;
         sh = &local->self_heal;
 
-        if (sh->need_data_self_heal && priv->data_self_heal) {
+        if (sh->do_data_self_heal &&
+            afr_data_self_heal_enabled (priv->data_self_heal)) {
                 afr_sh_data_open (frame, this);
         } else {
                 gf_log (this->name, GF_LOG_TRACE,
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 8f1c96986..356b15e63 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -2328,7 +2328,7 @@ afr_self_heal_entry (call_frame_t *frame, xlator_t *this)
         priv = this->private;
         local = frame->local;
 
-        if (local->self_heal.need_entry_self_heal && priv->entry_self_heal) {
+        if (local->self_heal.do_entry_self_heal && priv->entry_self_heal) {
                 afr_sh_entrylk (frame, this, &local->loc, NULL,
                                 afr_sh_post_nonblocking_entry_cbk);
         } else {
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index ba4118535..9999fdcdb 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -637,7 +637,7 @@ afr_self_heal_metadata (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        if (local->self_heal.need_metadata_self_heal && priv->metadata_self_heal) {
+        if (local->self_heal.do_metadata_self_heal && priv->metadata_self_heal) {
                 afr_sh_metadata_lock (frame, this);
         } else {
                 afr_sh_metadata_done (frame, this);
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 46091523e..044213e07 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -115,7 +115,7 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("metadata-self-heal",
                           priv->metadata_self_heal, options, bool, out);
 
-        GF_OPTION_RECONF ("data-self-heal", priv->data_self_heal, options, bool,
+        GF_OPTION_RECONF ("data-self-heal", priv->data_self_heal, options, str,
                           out);
 
         GF_OPTION_RECONF ("entry-self-heal", priv->entry_self_heal, options,
@@ -227,7 +227,7 @@ init (xlator_t *this)
         GF_OPTION_INIT ("background-self-heal-count",
                         priv->background_self_heal_count, uint32, out);
 
-        GF_OPTION_INIT ("data-self-heal", priv->data_self_heal, bool, out);
+        GF_OPTION_INIT ("data-self-heal", priv->data_self_heal, str, out);
 
         GF_OPTION_INIT ("data-self-heal-algorithm",
                         priv->data_self_heal_algorithm, str, out);
@@ -414,7 +414,11 @@ struct volume_options options[] = {
           .default_value = "16",
         },
         { .key  = {"data-self-heal"},
-          .type = GF_OPTION_TYPE_BOOL,
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "",
+          .value = {"1", "on", "yes", "true", "enable",
+                    "0", "off", "no", "false", "disable",
+                    "open"},
           .default_value = "on",
         },
         { .key  = {"data-self-heal-algorithm"},
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index bd0a08842..b9a11c486 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -92,7 +92,7 @@ typedef struct _afr_private {
 
         char **pending_key;
 
-        gf_boolean_t data_self_heal;              /* on/off */
+        char         *data_self_heal;              /* on/off/open */
         char *       data_self_heal_algorithm;    /* name of algorithm */
         unsigned int data_self_heal_window_size;  /* max number of pipelined
                                                      read/writes */
@@ -140,11 +140,11 @@ typedef struct {
         /* External interface: These are variables (some optional) that
            are set by whoever has triggered self-heal */
 
-        gf_boolean_t need_data_self_heal;
-        gf_boolean_t need_metadata_self_heal;
-        gf_boolean_t need_entry_self_heal;
-        gf_boolean_t need_gfid_self_heal;
-        gf_boolean_t need_missing_entry_self_heal;
+        gf_boolean_t do_data_self_heal;
+        gf_boolean_t do_metadata_self_heal;
+        gf_boolean_t do_entry_self_heal;
+        gf_boolean_t do_gfid_self_heal;
+        gf_boolean_t do_missing_entry_self_heal;
 
         gf_boolean_t forced_merge;        /* Is this a self-heal triggered to
                                              forcibly merge the directories? */
@@ -979,7 +979,7 @@ afr_inode_rm_stale_children (xlator_t *this, inode_t *inode, int32_t read_child,
                              int32_t *stale_children);
 void
 afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                      gf_boolean_t is_background, ia_type_t ia_type,
+                      gf_boolean_t background, ia_type_t ia_type, char *reason,
                       void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
                                                    xlator_t *this),
                       int (*unwind) (call_frame_t *frame, xlator_t *this,
@@ -991,6 +991,14 @@ int
 afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop);
 int
 afr_set_elem_count_get (unsigned char *elems, int child_count);
+
 afr_fd_ctx_t *
 afr_fd_ctx_get (fd_t *fd, xlator_t *this);
+
+gf_boolean_t
+afr_open_only_data_self_heal (char *data_self_heal);
+
+gf_boolean_t
+afr_data_self_heal_enabled (char *data_self_heal);
+
 #endif /* __AFR_H__ */
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
index 9cc4cf3c3..ede9f3b49 100644
--- a/xlators/cluster/afr/src/pump.c
+++ b/xlators/cluster/afr/src/pump.c
@@ -2497,7 +2497,7 @@ init (xlator_t *this)
         priv->favorite_child = source_child;
         priv->background_self_heal_count = 0;
 
-	priv->data_self_heal     = 1;
+	priv->data_self_heal     = "on";
 	priv->metadata_self_heal = 1;
 	priv->entry_self_heal    = 1;