tests: Fix split-brain-favorite-child-policy.t failures

Problem: In CentOS-7, the file was receving an extra removexattr(security.ima) FOP which changed its ctime, breaking the assumption that a particular brick had the latest ctime based on the writevs done in the .t Fix: 1. Compare the ctime of both files in the backend and pick the one with the latest ctime for the fav-child policy. Also unmount the volume before comparing, to avoid any further FOPS on the file that can possibly modify the timestamps. 2. Added floating point handling in stat function. Thanks to Pranith for the helping debugging the regex. Change-Id: I06041a0f39a29d2593b867af8685d65c7cd99150 BUG: 1408757 Signed-off-by: Ravishankar N <ravishankar@redhat.com> Reviewed-on: http://review.gluster.org/16288 Smoke: Gluster Build System <jenkins@build.gluster.org> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org> CentOS-regression: Gluster Build System <jenkins@build.gluster.org> Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
author: Ravishankar N <ravishankar@redhat.com> 2016-12-29 12:10:00 +0000
committer: Pranith Kumar Karampuri <pkarampu@redhat.com> 2017-01-02 22:13:51 -0800
commit: 76fff8cb2a164b596ca67e65c99623f5b68361fd (patch)
tree: 667273fe2b0f96cd817697478ab4fb118e0a8078 /tests/basic/afr
parent: 11b6a2c9fc5232b58774cab29873406c0fbfef19 (diff)
1 files changed, 14 insertions, 4 deletions
diff --git a/tests/basic/afr/split-brain-favorite-child-policy.t b/tests/basic/afr/split-brain-favorite-child-policy.t
index 7a14852685c..3df8e718bf0 100644
--- a/tests/basic/afr/split-brain-favorite-child-policy.t
+++ b/tests/basic/afr/split-brain-favorite-child-policy.t
@@ -42,8 +42,15 @@ TEST $CLI volume heal $V0
 cat $M0/file > /dev/null
 EXPECT "1" echo $?
 
-#We know that the first brick has latest ctime.
-LATEST_CTIME_MD5=$(md5sum $B0/${V0}0/file | cut -d\  -f1)
+# Umount to prevent further FOPS on the file, then find the brick with latest ctime.
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
+ctime1=`stat -c "%.Z"  $B0/${V0}0/file`
+ctime2=`stat -c "%.Z"  $B0/${V0}1/file`
+if (( $(echo "$ctime1 > $ctime2" | bc -l) )); then
+        LATEST_CTIME_MD5=$(md5sum $B0/${V0}0/file | cut -d\  -f1)
+else
+        LATEST_CTIME_MD5=$(md5sum $B0/${V0}1/file | cut -d\  -f1)
+fi
 TEST $CLI volume set $V0 cluster.favorite-child-policy ctime
 TEST $CLI volume start $V0 force
 EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
@@ -51,10 +58,13 @@ EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0
 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1
 TEST $CLI volume heal $V0
 EXPECT_WITHIN $HEAL_TIMEOUT "^0$" get_pending_heal_count $V0
+B0_MD5=$(md5sum $B0/${V0}0/file | cut -d\  -f1)
+B1_MD5=$(md5sum $B0/${V0}1/file | cut -d\  -f1)
+TEST [ "$LATEST_CTIME_MD5" == "$B0_MD5" ]
+TEST [ "$LATEST_CTIME_MD5" == "$B1_MD5" ]
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
 cat $M0/file > /dev/null
 EXPECT "0" echo $?
-HEALED_MD5=$(md5sum $B0/${V0}1/file | cut -d\  -f1)
-TEST [ "$LATEST_CTIME_MD5" == "$HEALED_MD5" ]
 
 ############ Healing using favorite-child-policy = mtime #################
 TEST $CLI volume set $V0 cluster.favorite-child-policy none
diff --git a/xlators/Makefile.am b/xlators/Makefile.am
index b1643d26ca0..ef20cbb64fa 100644
--- a/xlators/Makefile.am
+++ b/xlators/Makefile.am
@@ -1,3 +1,13 @@
-SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system
+if BUILD_GNFS
+  GNFS_DIR = nfs
+endif
+
+DIST_SUBDIRS = cluster storage protocol performance debug features \
+          mount nfs mgmt system playground meta
+
+SUBDIRS = cluster storage protocol performance debug features \
+          mount ${GNFS_DIR} mgmt system playground meta
+
+EXTRA_DIST = xlator.sym
 
 CLEANFILES =
diff --git a/xlators/bindings/Makefile.am b/xlators/bindings/Makefile.am
deleted file mode 100644
index f7766580257..00000000000
--- a/xlators/bindings/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS = $(BINDINGS_SUBDIRS)
diff --git a/xlators/bindings/python/src/Makefile.am b/xlators/bindings/python/src/Makefile.am
deleted file mode 100644
index c0b9141c667..00000000000
--- a/xlators/bindings/python/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-
-xlator_PROGRAMS = python.so
-
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/bindings
-
-python_PYTHON = gluster.py glustertypes.py glusterstack.py
-
-pythondir = $(xlatordir)/python
-
-python_so_SOURCES = python.c
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \
-	$(PYTHON_CPPLAGS) -DGLUSTER_PYTHON_PATH=\"$(pythondir)\"
-
-AM_LDFLAGS = $(PYTHON_LDFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/bindings/python/src/gluster.py b/xlators/bindings/python/src/gluster.py
deleted file mode 100644
index ee0eb131011..00000000000
--- a/xlators/bindings/python/src/gluster.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-from glustertypes import *
-from glusterstack import *
-import sys
-import inspect
-
-libglusterfs = CDLL("libglusterfs.so")
-_gf_log = libglusterfs._gf_log
-_gf_log.restype = c_int32
-_gf_log.argtypes = [c_char_p, c_char_p, c_char_p, c_int32, c_int, c_char_p]
-
-gf_log_loglevel = c_int.in_dll(libglusterfs, "gf_log_loglevel")
-
-GF_LOG_NONE = 0
-GF_LOG_CRITICAL = 1
-GF_LOG_ERROR = 2
-GF_LOG_WARNING = 3
-GF_LOG_DEBUG = 4
-
-def gf_log(module, level, fmt, *params):
-    if level <= gf_log_loglevel:
-        frame = sys._getframe(1)
-        _gf_log(module, frame.f_code.co_filename, frame.f_code.co_name,
-                frame.f_lineno, level, fmt, *params)
-
-class ComplexTranslator(object):
-    def __init__(self, xlator):
-        self.xlator = xlator_t.from_address(xlator)
-
-    def __getattr__(self, item):
-        return getattr(self.xlator, item)
diff --git a/xlators/bindings/python/src/glusterstack.py b/xlators/bindings/python/src/glusterstack.py
deleted file mode 100644
index ba24c81652e..00000000000
--- a/xlators/bindings/python/src/glusterstack.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-from glustertypes import *
-
-libc = CDLL("libc.so.6")
-calloc = libc.calloc
-calloc.argtypes = [c_int, c_int]
-calloc.restype = c_void_p
-
-# TODO: Can these be done in C somehow?
-def stack_wind(frame, rfn, obj, fn, *params):
-    """Frame is a frame object"""
-    _new = cast(calloc(1, sizeof(call_frame_t)), POINTER(call_frame_t))
-    _new[0].root = frame.root
-    _new[0].next = frame.root[0].frames.next
-    _new[0].prev = pointer(frame.root[0].frames)
-    if frame.root[0].frames.next:
-        frame.root[0].frames.next[0].prev = _new
-    frame.root[0].frames.next = _new
-    _new[0].this = obj
-    # TODO: Type checking like tmp_cbk?
-    _new[0].ret = rfn
-    _new[0].parent = pointer(frame)
-    _new[0].cookie = cast(_new, c_void_p)
-    # TODO: Initialize lock
-    #_new.lock.init()
-    frame.ref_count += 1
-    fn(_new, obj, *params)
-
-def stack_unwind(frame, *params):
-    """Frame is a frame object"""
-    fn = frame[0].ret
-    parent = frame[0].parent[0]
-    parent.ref_count -= 1
-
-    op_ret = params[0]
-    op_err = params[1]
-    params = params[2:]
-    fn(parent, call_frame_t.from_address(frame[0].cookie), parent.this,
-            op_ret, op_err, *params)
diff --git a/xlators/bindings/python/src/glustertypes.py b/xlators/bindings/python/src/glustertypes.py
deleted file mode 100644
index e9069d07c72..00000000000
--- a/xlators/bindings/python/src/glustertypes.py
+++ /dev/null
@@ -1,167 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-import collections
-
-#
-# Forward declaration of some gluster types
-#
-class call_frame_t(Structure):
-    pass
-
-class call_ctx_t(Structure):
-    pass
-
-class call_pool_t(Structure):
-    pass
-
-class xlator_t(Structure):
-    def _getFirstChild(self):
-        return self.children[0].xlator
-    firstChild = property(_getFirstChild)
-
-class xlator_list_t(Structure):
-    pass
-
-class xlator_fops(Structure):
-    pass
-
-class xlator_mops(Structure):
-    pass
-
-class glusterfs_ctx_t(Structure):
-    pass
-
-class list_head(Structure):
-    pass
-
-class dict_t(Structure):
-    pass
-
-class inode_table_t(Structure):
-    pass
-
-class fd_t(Structure):
-    pass
-
-class iovec(Structure):
-    _fields_ = [
-            ("iov_base", c_void_p),
-            ("iov_len", c_size_t),
-            ]
-
-    def __init__(self, s):
-        self.iov_base = cast(c_char_p(s), c_void_p)
-        self.iov_len = len(s)
-
-    def getBytes(self):
-        return string_at(self.iov_base, self.iov_len)
-
-# This is a pthread_spinlock_t
-# TODO: what happens to volatile-ness?
-gf_lock_t = c_int
-
-uid_t = c_uint32
-gid_t = c_uint32
-pid_t = c_int32
-
-off_t = c_int64
-
-#
-# Function pointer types
-#
-ret_fn_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(call_frame_t),
-                              POINTER(xlator_t), c_int32, c_int32)
-
-fini_fn_t = CFUNCTYPE(None, POINTER(xlator_t))
-init_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t))
-event_notify_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t), c_int32, c_void_p)
-
-list_head._fields_ = [
-        ("next", POINTER(list_head)),
-        ("prev", POINTER(list_head)),
-        ]
-
-call_frame_t._fields_ = [
-        ("root", POINTER(call_ctx_t)),
-        ("parent", POINTER(call_frame_t)),
-        ("next", POINTER(call_frame_t)),
-        ("prev", POINTER(call_frame_t)),
-        ("local", c_void_p),
-        ("this", POINTER(xlator_t)),
-        ("ret", ret_fn_t),
-        ("ref_count", c_int32),
-        ("lock", gf_lock_t),
-        ("cookie", c_void_p),
-        ("op", c_int32),
-        ("type", c_int8),
-        ]
-
-call_ctx_t._fields_ = [
-        ("all_frames", list_head),
-        ("trans", c_void_p),
-        ("pool", call_pool_t),
-        ("unique", c_uint64),
-        ("state", c_void_p),
-        ("uid", uid_t),
-        ("gid", gid_t),
-        ("pid", pid_t),
-        ("frames", call_frame_t),
-        ("req_refs", POINTER(dict_t)),
-        ("rsp_refs", POINTER(dict_t)),
-        ]
-
-xlator_t._fields_ = [
-        ("name", c_char_p),
-        ("type", c_char_p),
-        ("next", POINTER(xlator_t)),
-        ("prev", POINTER(xlator_t)),
-        ("parent", POINTER(xlator_t)),
-        ("children", POINTER(xlator_list_t)),
-        ("fops", POINTER(xlator_fops)),
-        ("mops", POINTER(xlator_mops)),
-        ("fini", fini_fn_t),
-        ("init", init_fn_t),
-        ("notify", event_notify_fn_t),
-        ("options", POINTER(dict_t)),
-        ("ctx", POINTER(glusterfs_ctx_t)),
-        ("itable", POINTER(inode_table_t)),
-        ("ready", c_char),
-        ("private", c_void_p),
-        ]
-
-xlator_list_t._fields_ = [
-        ("xlator", POINTER(xlator_t)),
-        ("next", POINTER(xlator_list_t)),
-        ]
-
-fop_functions = collections.defaultdict(lambda: c_void_p)
-fop_function_names = ['lookup', 'forget', 'stat', 'fstat', 'chmod', 'fchmod',
-        'chown', 'fchown', 'truncate', 'ftruncate', 'utimens', 'access',
-        'readlink', 'mknod', 'mkdir', 'unlink', 'rmdir', 'symlink',
-        'rename', 'link', 'create', 'open', 'readv', 'writev', 'flush',
-        'close', 'fsync', 'opendir', 'readdir', 'closedir', 'fsyncdir',
-        'statfs', 'setxattr', 'getxattr', 'removexattr', 'lk', 'writedir',
-        # TODO: Call backs?
-        ]
-
-fop_writev_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(xlator_t),
-                                  POINTER(fd_t), POINTER(iovec), c_int32,
-                                  off_t)
-
-fop_functions['writev'] = fop_writev_t
-xlator_fops._fields_ = [(f, fop_functions[f]) for f in fop_function_names]
diff --git a/xlators/bindings/python/src/python.c b/xlators/bindings/python/src/python.c
deleted file mode 100644
index 3310a211513..00000000000
--- a/xlators/bindings/python/src/python.c
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
-   Copyright (c) 2007-2010 Chris AtLee <chris@atlee.ca>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#include <Python.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-
-typedef struct
-{
-    char        *scriptname;
-    PyObject    *pXlator;
-    PyObject    *pScriptModule;
-    PyObject    *pGlusterModule;
-    PyThreadState *pInterp;
-
-    PyObject    *pFrameType, *pVectorType, *pFdType;
-} python_private_t;
-
-int32_t
-python_writev (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              struct iovec *vector,
-              int32_t count, 
-              off_t offset)
-{
-  python_private_t *priv = (python_private_t *)this->private;
-  gf_log("python", GF_LOG_DEBUG, "In writev");
-  if (PyObject_HasAttrString(priv->pXlator, "writev"))
-  {
-
-      PyObject *retval = PyObject_CallMethod(priv->pXlator, "writev",
-              "O O O i l",
-              PyObject_CallMethod(priv->pFrameType, "from_address", "O&", PyLong_FromVoidPtr, frame),
-              PyObject_CallMethod(priv->pFdType, "from_address", "O&", PyLong_FromVoidPtr, fd),
-              PyObject_CallMethod(priv->pVectorType, "from_address", "O&", PyLong_FromVoidPtr, vector),
-              count,
-              offset);
-      if (PyErr_Occurred())
-      {
-          PyErr_Print();
-      }
-      Py_XDECREF(retval);
-  }
-  else
-  {
-      return default_writev(frame, this, fd, vector, count, offset);
-  }
-  return 0;
-}
-
-struct xlator_fops fops = {
-    .writev       = python_writev
-};
-
-static PyObject *
-AnonModule_FromFile (const char* fname)
-{
-    // Get the builtins
-    PyThreadState* pThread = PyThreadState_Get();
-    PyObject *pBuiltins = pThread->interp->builtins;
-
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        return NULL;
-    }
-
-    // Create a new dictionary for running code in
-    PyObject *pModuleDict = PyDict_New();
-    PyDict_SetItemString(pModuleDict, "__builtins__", pBuiltins);
-    Py_INCREF(pBuiltins);
-
-    // Run the file in the new context
-    FILE* fp = fopen(fname, "r");
-    PyRun_File(fp, fname, Py_file_input, pModuleDict, pModuleDict);
-    fclose(fp);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        return NULL;
-    }
-
-    // Create an object to hold the new context
-    PyRun_String("class ModuleWrapper(object):\n\tpass\n", Py_single_input, pModuleDict, pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        return NULL;
-    }
-    PyObject *pModule = PyRun_String("ModuleWrapper()", Py_eval_input, pModuleDict, pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        Py_XDECREF(pModule);
-        return NULL;
-    }
-
-    // Set the new context's dictionary to the one we used to run the code
-    // inside
-    PyObject_SetAttrString(pModule, "__dict__", pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        Py_DECREF(pModule);
-        return NULL;
-    }
-
-    return pModule;
-}
-
-int32_t
-init (xlator_t *this)
-{
-  // This is ok to call more than once per process
-  Py_InitializeEx(0);
-
-  if (!this->children) {
-    gf_log ("python", GF_LOG_ERROR, 
-            "FATAL: python should have exactly one child");
-    return -1;
-  }
-
-  python_private_t *priv = CALLOC (sizeof (python_private_t), 1);
-  ERR_ABORT (priv);
-
-  data_t *scriptname = dict_get (this->options, "scriptname");
-  if (scriptname) {
-      priv->scriptname = data_to_str(scriptname);
-  } else {
-      gf_log("python", GF_LOG_ERROR,
-              "FATAL: python requires the scriptname parameter");
-      return -1;
-  }
-
-  priv->pInterp = Py_NewInterpreter();
-    
-  // Adjust python's path
-  PyObject *syspath = PySys_GetObject("path");
-  PyObject *path = PyString_FromString(GLUSTER_PYTHON_PATH);
-  PyList_Append(syspath, path);
-  Py_DECREF(path);
-
-  gf_log("python", GF_LOG_DEBUG,
-          "Loading gluster module");
-
-  priv->pGlusterModule = PyImport_ImportModule("gluster");
-  if (PyErr_Occurred())
-  {
-      PyErr_Print();
-      return -1;
-  }
-
-  priv->pFrameType = PyObject_GetAttrString(priv->pGlusterModule, "call_frame_t");
-  priv->pFdType = PyObject_GetAttrString(priv->pGlusterModule, "fd_t");
-  priv->pVectorType = PyObject_GetAttrString(priv->pGlusterModule, "iovec");
-
-  gf_log("python", GF_LOG_DEBUG, "Loading script...%s", priv->scriptname);
-  
-  priv->pScriptModule = AnonModule_FromFile(priv->scriptname);
-  if (!priv->pScriptModule || PyErr_Occurred())
-  {
-      gf_log("python", GF_LOG_ERROR, "Error loading %s", priv->scriptname);
-      PyErr_Print();
-      return -1;
-  }
-
-  if (!PyObject_HasAttrString(priv->pScriptModule, "xlator"))
-  {
-      gf_log("python", GF_LOG_ERROR, "%s does not have a xlator attribute", priv->scriptname);
-      return -1;
-  }
-  gf_log("python", GF_LOG_DEBUG, "Instantiating translator");
-  priv->pXlator = PyObject_CallMethod(priv->pScriptModule, "xlator", "O&",
-          PyLong_FromVoidPtr, this);
-  if (PyErr_Occurred() || !priv->pXlator)
-  {
-      PyErr_Print();
-      return -1;
-  }
-
-  this->private = priv;
-
-  gf_log ("python", GF_LOG_DEBUG, "python xlator loaded");
-  return 0;
-}
-
-void 
-fini (xlator_t *this)
-{
-  python_private_t *priv = (python_private_t*)(this->private);
-  Py_DECREF(priv->pXlator);
-  Py_DECREF(priv->pScriptModule);
-  Py_DECREF(priv->pGlusterModule);
-  Py_DECREF(priv->pFrameType);
-  Py_DECREF(priv->pFdType);
-  Py_DECREF(priv->pVectorType);
-  Py_EndInterpreter(priv->pInterp);
-  return;
-}
diff --git a/xlators/bindings/python/src/testxlator.py b/xlators/bindings/python/src/testxlator.py
deleted file mode 100644
index 507455c856a..00000000000
--- a/xlators/bindings/python/src/testxlator.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-
-"""
-This is a test translator written in python.
-
-Important things to note:
-    This file must be import-able from glusterfsd.  This probably means
-    setting PYTHONPATH to where this file is located.
-
-    This file must have a top-level xlator class object that will be
-    used to instantiate individual translators.
-"""
-from gluster import *
-
-class MyXlator(ComplexTranslator):
-    name = "MyXlator"
-    def writev_cbk(self, frame, cookie, op_ret, op_errno, buf):
-        stack_unwind(frame, op_ret, op_errno, buf)
-        return 0
-
-    def writev(self, frame, fd, vector, count, offset):
-        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
-        # TODO: Use cookie to pass this to writev_cbk
-        old_count = vector.iov_len
-
-        data = vector.getBytes().encode("zlib")
-
-        vector = iovec(data)
-        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
-
-        @ret_fn_t
-        def rfn(frame, prev, this, op_ret, op_errno, *params):
-            if len(params) == 0:
-                params = [0]
-            return self.writev_cbk(frame, prev, old_count, op_errno, *params)
-
-        stack_wind(frame, rfn, self.firstChild,
-                self.firstChild[0].fops[0].writev, fd, vector, count, offset)
-        return 0
-
-xlator = MyXlator
diff --git a/xlators/cluster/Makefile.am b/xlators/cluster/Makefile.am
index 0990822a7d3..8e067d5ab58 100644
--- a/xlators/cluster/Makefile.am
+++ b/xlators/cluster/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = stripe afr dht
+SUBDIRS = afr dht ec
 
 CLEANFILES = 
diff --git a/xlators/cluster/afr/src/Makefile.am b/xlators/cluster/afr/src/Makefile.am
index 95db5dd9645..610819b28fc 100644
--- a/xlators/cluster/afr/src/Makefile.am
+++ b/xlators/cluster/afr/src/Makefile.am
@@ -1,35 +1,35 @@
-xlator_LTLIBRARIES = afr.la pump.la
+xlator_LTLIBRARIES = afr.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
 afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c \
-	afr-inode-write.c afr-open.c afr-transaction.c afr-self-heal-data.c \
-	afr-self-heal-common.c afr-self-heal-metadata.c afr-self-heal-entry.c \
-	afr-self-heal-algorithm.c afr-lk-common.c afr-self-heald.c \
+	afr-inode-write.c afr-open.c afr-transaction.c afr-lk-common.c \
+	afr-read-txn.c \
 	$(top_builddir)/xlators/lib/src/libxlator.c
 
-afr_la_LDFLAGS = -module -avoidversion
-afr_la_SOURCES = $(afr_common_source) afr.c
-afr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+AFR_SELFHEAL_SOURCES = afr-self-heal-common.c afr-self-heal-data.c \
+	afr-self-heal-entry.c afr-self-heal-metadata.c afr-self-heald.c \
+	afr-self-heal-name.c
 
-pump_la_LDFLAGS = -module -avoidversion
-pump_la_SOURCES =  $(afr_common_source) pump.c
-pump_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+afr_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+afr_la_SOURCES = $(afr_common_source) $(AFR_SELFHEAL_SOURCES) afr.c
+afr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h \
-	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-self-heal-common.h \
-	afr-self-heal-algorithm.h pump.h afr-mem-types.h afr-common.c \
-	afr-self-heald.h $(top_builddir)/xlators/lib/src/libxlator.h \
-	$(top_builddir)/glusterfsd/src/glusterfsd.h
+	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-mem-types.h \
+	afr-common.c afr-self-heald.h \
+	$(top_builddir)/xlators/lib/src/libxlator.h afr-messages.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src -shared -nostartfiles $(GF_CFLAGS)
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
 
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/replicate.so
-	rm -f $(DESTDIR)$(xlatordir)/pump.so
 
 install-data-hook:
 	ln -sf afr.so $(DESTDIR)$(xlatordir)/replicate.so
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index b6e16074364..032ab5c8001 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -15,29 +15,20 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "statedump.h"
-#include "inode.h"
-
-#include "fd.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/hashfn.h>
+#include <glusterfs/list.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/events.h>
+#include <glusterfs/upcall-utils.h>
 
 #include "afr-inode-read.h"
 #include "afr-inode-write.h"
@@ -45,4202 +36,7843 @@
 #include "afr-dir-write.h"
 #include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
 #include "afr-self-heald.h"
-#include "pump.h"
+#include "afr-messages.h"
 
-#define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000200000000ULL
-#define AFR_ICTX_SPLIT_BRAIN_MASK      0x0000000100000000ULL
-#define AFR_ICTX_READ_CHILD_MASK       0x00000000FFFFFFFFULL
+int32_t
+afr_quorum_errno(afr_private_t *priv)
+{
+    return ENOTCONN;
+}
 
-int
-afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
-                                gf_boolean_t fail_conflict);
-void
-afr_children_copy (int32_t *dst, int32_t *src, unsigned int child_count)
+gf_boolean_t
+afr_is_private_directory(afr_private_t *priv, uuid_t pargfid, const char *name,
+                         pid_t pid)
 {
-        int     i = 0;
+    if (!__is_root_gfid(pargfid)) {
+        return _gf_false;
+    }
 
-        for (i = 0; i < child_count; i++)
-                dst[i] = src[i];
+    if (strcmp(name, GF_REPLICATE_TRASH_DIR) == 0) {
+        /*For backward compatibility /.landfill is private*/
+        return _gf_true;
+    }
+
+    if (pid == GF_CLIENT_PID_GSYNCD) {
+        /*geo-rep needs to create/sync private directory on slave because
+         * it appears in changelog*/
+        return _gf_false;
+    }
+
+    if (pid == GF_CLIENT_PID_GLFS_HEAL || pid == GF_CLIENT_PID_SELF_HEALD) {
+        if (strcmp(name, priv->anon_inode_name) == 0) {
+            /* anonymous-inode dir is private*/
+            return _gf_true;
+        }
+    } else {
+        if (strncmp(name, AFR_ANON_DIR_PREFIX, strlen(AFR_ANON_DIR_PREFIX)) ==
+            0) {
+            /* anonymous-inode dir prefix is private for geo-rep to work*/
+            return _gf_true;
+        }
+    }
+
+    return _gf_false;
 }
 
 void
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req, const char *path)
+afr_fill_success_replies(afr_local_t *local, afr_private_t *priv,
+                         unsigned char *replies)
 {
-        int             i           = 0;
-        afr_private_t   *priv       = NULL;
-        int             ret         = 0;
-
-        priv   = this->private;
+    int i = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_uint64 (xattr_req, priv->pending_key[i],
-                                       3 * sizeof(int32_t));
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: Unable to set dict value for %s",
-                                path, priv->pending_key[i]);
-                /* 3 = data+metadata+entry */
-        }
-        ret = dict_set_int32 (xattr_req, GF_GFIDLESS_LOOKUP, 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s: failed to set gfidless "
-                        "lookup", path);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+            replies[i] = 1;
+        } else {
+            replies[i] = 0;
         }
+    }
 }
 
 int
-afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
-                              dict_t *xattr_req, loc_t *loc, void **gfid_req)
+afr_fav_child_reset_sink_xattrs(void *opaque);
+
+int
+afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *frame, void *opaque);
+
+static void
+afr_discover_done(call_frame_t *frame, xlator_t *this);
+
+int
+afr_dom_lock_acquire_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-        int     ret = -ENOMEM;
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = (long)cookie;
 
-        GF_ASSERT (gfid_req);
+    local->cont.lk.dom_lock_op_ret[i] = op_ret;
+    local->cont.lk.dom_lock_op_errno[i] = op_errno;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to acquire %s on %s",
+               uuid_utoa(local->fd->inode->gfid), AFR_LK_HEAL_DOM,
+               priv->children[i]->name);
+    } else {
+        local->cont.lk.dom_locked_nodes[i] = 1;
+    }
 
-        *gfid_req = NULL;
-        local->xattr_req = dict_new ();
-        if (!local->xattr_req)
-                goto out;
-        if (xattr_req)
-                dict_copy (xattr_req, local->xattr_req);
+    syncbarrier_wake(&local->barrier);
 
-        afr_xattr_req_prepare (this, local->xattr_req, loc->path);
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_INODELK_COUNT);
-        }
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_ENTRYLK_COUNT);
-        }
+    return 0;
+}
 
-        ret = dict_set_uint32 (local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_PARENT_ENTRYLK);
+int
+afr_dom_lock_acquire(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0;
+
+    priv = frame->this->private;
+    local = frame->local;
+    local->cont.lk.dom_locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.locked_nodes),
+        gf_afr_mt_char);
+    if (!local->cont.lk.dom_locked_nodes) {
+        return -ENOMEM;
+    }
+    local->cont.lk.dom_lock_op_ret = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.dom_lock_op_ret),
+        gf_afr_mt_int32_t);
+    if (!local->cont.lk.dom_lock_op_ret) {
+        return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
+    }
+    local->cont.lk.dom_lock_op_errno = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.dom_lock_op_errno),
+        gf_afr_mt_int32_t);
+    if (!local->cont.lk.dom_lock_op_errno) {
+        return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
+    }
+    flock.l_type = F_WRLCK;
+
+    AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
+              local->fd, F_SETLK, &flock, NULL);
+
+    if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, frame->this, NULL))
+        goto blocking_lock;
+
+    /*If any of the bricks returned EAGAIN, we still need blocking locks.*/
+    if (AFR_COUNT(local->cont.lk.dom_locked_nodes, priv->child_count) !=
+        priv->child_count) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->cont.lk.dom_lock_op_ret[i] == -1 &&
+                local->cont.lk.dom_lock_op_errno[i] == EAGAIN)
+                goto blocking_lock;
         }
+    }
 
-        ret = dict_get_ptr (local->xattr_req, "gfid-req", gfid_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%s: failed to get the gfid from dict", loc->path);
-                *gfid_req = NULL;
-        } else {
-                if (loc->parent != NULL)
-                        dict_del (local->xattr_req, "gfid-req");
-        }
-        ret = 0;
-out:
-        return ret;
-}
+    return 0;
 
-void
-afr_lookup_save_gfid (uuid_t dst, void* new, const loc_t *loc)
-{
-        inode_t  *inode = NULL;
+blocking_lock:
+    afr_dom_lock_release(frame);
+    AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
+              local->fd, F_SETLKW, &flock, NULL);
+    if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, frame->this, NULL)) {
+        afr_dom_lock_release(frame);
+        return -afr_quorum_errno(priv);
+    }
 
-        inode = loc->inode;
-        if (inode && !uuid_is_null (inode->gfid))
-                uuid_copy (dst, inode->gfid);
-        else if (!uuid_is_null (loc->gfid))
-                uuid_copy (dst, loc->gfid);
-        else if (new && !uuid_is_null (new))
-                uuid_copy (dst, new);
+    return 0;
 }
 
 int
-afr_errno_count (int32_t *children, int *child_errno,
-                 unsigned int child_count, int32_t op_errno)
-{
-        int i = 0;
-        int errno_count = 0;
-        int child = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (children) {
-                        child = children[i];
-                        if (child == -1)
-                                break;
-                } else {
-                        child = i;
-                }
-                if (child_errno[child] == op_errno)
-                        errno_count++;
-        }
-        return errno_count;
+afr_dom_lock_release_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = (long)cookie;
+
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to release %s on %s", local->loc.path,
+               AFR_LK_HEAL_DOM, priv->children[i]->name);
+    }
+    local->cont.lk.dom_locked_nodes[i] = 0;
+
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
 }
 
-int32_t
-afr_set_dict_gfid (dict_t *dict, uuid_t gfid)
-{
-        int ret       = 0;
-        uuid_t *pgfid = NULL;
+void
+afr_dom_lock_release(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    local = frame->local;
+    priv = frame->this->private;
+    locked_on = local->cont.lk.dom_locked_nodes;
+    if (AFR_COUNT(locked_on, priv->child_count) == 0)
+        return;
+    flock.l_type = F_UNLCK;
 
-        GF_ASSERT (gfid);
+    AFR_ONLIST(locked_on, frame, afr_dom_lock_release_cbk, finodelk,
+               AFR_LK_HEAL_DOM, local->fd, F_SETLK, &flock, NULL);
 
-        pgfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
-        if (!pgfid) {
-                ret = -1;
-                goto out;
-        }
+    return;
+}
 
-        uuid_copy (*pgfid, gfid);
+static void
+afr_lk_heal_info_cleanup(afr_lk_heal_info_t *info)
+{
+    if (!info)
+        return;
+    if (info->xdata_req)
+        dict_unref(info->xdata_req);
+    if (info->fd)
+        fd_unref(info->fd);
+    GF_FREE(info->locked_nodes);
+    GF_FREE(info->child_up_event_gen);
+    GF_FREE(info->child_down_event_gen);
+    GF_FREE(info);
+}
 
-        ret = dict_set_dynptr (dict, "gfid-req", pgfid, sizeof (uuid_t));
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "gfid set failed");
+static int
+afr_add_lock_to_saved_locks(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+    afr_lk_heal_info_t *info = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = -ENOMEM;
+
+    info = GF_CALLOC(sizeof(*info), 1, gf_afr_mt_lk_heal_info_t);
+    if (!info) {
+        goto cleanup;
+    }
+    INIT_LIST_HEAD(&info->pos);
+    info->fd = fd_ref(local->fd);
+    info->cmd = local->cont.lk.cmd;
+    info->pid = frame->root->pid;
+    info->flock = local->cont.lk.user_flock;
+    info->xdata_req = dict_copy_with_ref(local->xdata_req, NULL);
+    if (!info->xdata_req) {
+        goto cleanup;
+    }
+    info->lk_owner = frame->root->lk_owner;
+    info->locked_nodes = GF_MALLOC(
+        sizeof(*info->locked_nodes) * priv->child_count, gf_afr_mt_char);
+    if (!info->locked_nodes) {
+        goto cleanup;
+    }
+    memcpy(info->locked_nodes, local->cont.lk.locked_nodes,
+           sizeof(*info->locked_nodes) * priv->child_count);
+    info->child_up_event_gen = GF_CALLOC(sizeof(*info->child_up_event_gen),
+                                         priv->child_count, gf_afr_mt_int32_t);
+    if (!info->child_up_event_gen) {
+        goto cleanup;
+    }
+    info->child_down_event_gen = GF_CALLOC(sizeof(*info->child_down_event_gen),
+                                           priv->child_count,
+                                           gf_afr_mt_int32_t);
+    if (!info->child_down_event_gen) {
+        goto cleanup;
+    }
+
+    LOCK(&local->fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(local->fd, this);
+        if (fd_ctx)
+            fd_ctx->lk_heal_info = info;
+    }
+    UNLOCK(&local->fd->lock);
+    if (!fd_ctx) {
+        goto cleanup;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_add_tail(&info->pos, &priv->saved_locks);
+    }
+    UNLOCK(&priv->lock);
+
+    return 0;
+cleanup:
+    gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
+           "%s: Failed to add lock to healq",
+           uuid_utoa(local->fd->inode->gfid));
+    if (info) {
+        afr_lk_heal_info_cleanup(info);
+        if (fd_ctx) {
+            LOCK(&local->fd->lock);
+            {
+                fd_ctx->lk_heal_info = NULL;
+            }
+            UNLOCK(&local->fd->lock);
+        }
+    }
+    return ret;
+}
 
+static int
+afr_remove_lock_from_saved_locks(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    struct gf_flock flock = local->cont.lk.user_flock;
+    afr_lk_heal_info_t *info = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = -EINVAL;
+
+    fd_ctx = afr_fd_ctx_get(local->fd, this);
+    if (!fd_ctx || !fd_ctx->lk_heal_info) {
+        goto out;
+    }
+
+    info = fd_ctx->lk_heal_info;
+    if ((info->flock.l_start != flock.l_start) ||
+        (info->flock.l_whence != flock.l_whence) ||
+        (info->flock.l_len != flock.l_len)) {
+        /*TODO: Compare lkowners too.*/
+        goto out;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_del(&fd_ctx->lk_heal_info->pos);
+    }
+    UNLOCK(&priv->lock);
+
+    afr_lk_heal_info_cleanup(info);
+    fd_ctx->lk_heal_info = NULL;
+    ret = 0;
 out:
-        if (ret && pgfid)
-                GF_FREE (pgfid);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to remove lock from healq",
+               uuid_utoa(local->fd->inode->gfid));
+    return ret;
+}
 
-        return ret;
+int
+afr_lock_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                  dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "Failed to heal lock on child %d for %s", i,
+               uuid_utoa(local->fd->inode->gfid));
+    }
+    syncbarrier_wake(&local->barrier);
+    return 0;
 }
 
-afr_inode_ctx_t*
-afr_inode_ctx_get_from_addr (uint64_t addr, int32_t child_count)
+int
+afr_getlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        int             ret  = -1;
-        afr_inode_ctx_t *ctx = NULL;
-        size_t          size = 0;
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
 
-        GF_ASSERT (child_count > 0);
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "Failed getlk for %s", uuid_utoa(local->fd->inode->gfid));
+    } else {
+        local->cont.lk.getlk_rsp[i] = *lock;
+    }
 
-        if (!addr) {
-                ctx = GF_CALLOC (1, sizeof (*ctx),
-                                 gf_afr_mt_inode_ctx_t);
-                if (!ctx)
-                        goto out;
-                size = sizeof (*ctx->fresh_children);
-                ctx->fresh_children = GF_CALLOC (child_count, size,
-                                                 gf_afr_mt_int32_t);
-                if (!ctx->fresh_children)
-                        goto out;
-        } else {
-                ctx = (afr_inode_ctx_t*) (long) addr;
-        }
-        ret = 0;
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+static gf_boolean_t
+afr_does_lk_owner_match(call_frame_t *frame, afr_private_t *priv,
+                        afr_lk_heal_info_t *info)
+{
+    int i = 0;
+    afr_local_t *local = frame->local;
+    struct gf_flock flock = {
+        0,
+    };
+    gf_boolean_t ret = _gf_true;
+    char *wind_on = alloca0(priv->child_count);
+    unsigned char *success_replies = alloca0(priv->child_count);
+    local->cont.lk.getlk_rsp = GF_CALLOC(sizeof(*local->cont.lk.getlk_rsp),
+                                         priv->child_count, gf_afr_mt_gf_lock);
+
+    flock = info->flock;
+    for (i = 0; i < priv->child_count; i++) {
+        if (info->locked_nodes[i])
+            wind_on[i] = 1;
+    }
+
+    AFR_ONLIST(wind_on, frame, afr_getlk_cbk, lk, info->fd, F_GETLK, &flock,
+               info->xdata_req);
+
+    afr_fill_success_replies(local, priv, success_replies);
+    if (AFR_COUNT(success_replies, priv->child_count) == 0) {
+        ret = _gf_false;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid || local->replies[i].op_ret != 0)
+            continue;
+        if (local->cont.lk.getlk_rsp[i].l_type == F_UNLCK)
+            continue;
+        /*TODO: Do we really need to compare lkowner if F_UNLCK is true?*/
+        if (!is_same_lkowner(&local->cont.lk.getlk_rsp[i].l_owner,
+                             &info->lk_owner)) {
+            ret = _gf_false;
+            break;
+        }
+    }
 out:
-        if (ret && ctx) {
-                GF_FREE (ctx->fresh_children);
-                GF_FREE (ctx);
-                ctx = NULL;
-        }
-        return ctx;
+    afr_local_replies_wipe(local, priv);
+    GF_FREE(local->cont.lk.getlk_rsp);
+    local->cont.lk.getlk_rsp = NULL;
+    return ret;
 }
 
-void
-afr_inode_get_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
+static void
+afr_mark_fd_bad(fd_t *fd, xlator_t *this)
 {
-        GF_ASSERT (inode);
-        GF_ASSERT (params);
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        int             ret = 0;
-        afr_inode_ctx_t *ctx = NULL;
-        afr_private_t   *priv = NULL;
-        int             i = 0;
-        uint64_t        ctx_addr   = 0;
-        int32_t         read_child = -1;
-        int32_t         *fresh_children = NULL;
-
-        priv = this->private;
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        goto unlock;
-                ctx = afr_inode_ctx_get_from_addr (ctx_addr, priv->child_count);
-                if (!ctx)
-                        goto unlock;
-                switch (params->op) {
-                case AFR_INODE_GET_READ_CTX:
-                        fresh_children = params->u.read_ctx.children;
-                        read_child = (int32_t)(ctx->masks &
-                                               AFR_ICTX_READ_CHILD_MASK);
-                        params->u.read_ctx.read_child = read_child;
-                        if (!fresh_children)
-                                goto unlock;
-                        for (i = 0; i < priv->child_count; i++)
-                                fresh_children[i] = ctx->fresh_children[i];
-                        break;
-                case AFR_INODE_GET_OPENDIR_DONE:
-                        params->u.value = _gf_false;
-                        if (ctx->masks & AFR_ICTX_OPENDIR_DONE_MASK)
-                                params->u.value = _gf_true;
-                        break;
-                case AFR_INODE_GET_SPLIT_BRAIN:
-                        params->u.value = _gf_false;
-                        if (ctx->masks & AFR_ICTX_SPLIT_BRAIN_MASK)
-                                params->u.value = _gf_true;
-                        ;
-                        break;
-                default:
-                        GF_ASSERT (0);
-                        break;
-                }
+    if (!fd)
+        return;
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(fd, this);
+        if (fd_ctx) {
+            fd_ctx->is_fd_bad = _gf_true;
+            fd_ctx->lk_heal_info = NULL;
         }
-unlock:
-        UNLOCK (&inode->lock);
+    }
+    UNLOCK(&fd->lock);
 }
 
-gf_boolean_t
-afr_is_split_brain (xlator_t *this, inode_t *inode)
+static void
+afr_add_lock_to_lkhealq(afr_private_t *priv, afr_lk_heal_info_t *info)
 {
-        afr_inode_params_t params = {0};
+    LOCK(&priv->lock);
+    {
+        list_del(&info->pos);
+        list_add_tail(&info->pos, &priv->lk_healq);
+    }
+    UNLOCK(&priv->lock);
+}
 
-        params.op = AFR_INODE_GET_SPLIT_BRAIN;
-        afr_inode_get_ctx (this, inode, &params);
-        return params.u.value;
+static void
+afr_lock_heal_do(call_frame_t *frame, afr_private_t *priv,
+                 afr_lk_heal_info_t *info)
+{
+    int i = 0;
+    int op_errno = 0;
+    int32_t *current_event_gen = NULL;
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    char *wind_on = alloca0(priv->child_count);
+    gf_boolean_t retry = _gf_true;
+
+    frame->root->pid = info->pid;
+    lk_owner_copy(&frame->root->lk_owner, &info->lk_owner);
+
+    op_errno = -afr_dom_lock_acquire(frame);
+    if ((op_errno != 0)) {
+        goto release;
+    }
+
+    if (!afr_does_lk_owner_match(frame, priv, info)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_LK_HEAL_DOM,
+               "Ignoring lock heal for %s since lk-onwers mismatch. "
+               "Lock possibly pre-empted by another client.",
+               uuid_utoa(info->fd->inode->gfid));
+        goto release;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (info->locked_nodes[i])
+            continue;
+        wind_on[i] = 1;
+    }
+
+    current_event_gen = alloca(priv->child_count);
+    memcpy(current_event_gen, info->child_up_event_gen,
+           priv->child_count * sizeof *current_event_gen);
+    AFR_ONLIST(wind_on, frame, afr_lock_heal_cbk, lk, info->fd, info->cmd,
+               &info->flock, info->xdata_req);
+
+    LOCK(&priv->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!wind_on[i])
+                continue;
+            if ((!local->replies[i].valid) || (local->replies[i].op_ret != 0)) {
+                continue;
+            }
+
+            if ((current_event_gen[i] == info->child_up_event_gen[i]) &&
+                (current_event_gen[i] > info->child_down_event_gen[i])) {
+                info->locked_nodes[i] = 1;
+                retry = _gf_false;
+                list_del_init(&info->pos);
+                list_add_tail(&info->pos, &priv->saved_locks);
+            } else {
+                /*We received subsequent child up/down events while heal was in
+                 * progress; don't mark child as healed. Attempt again on the
+                 * new child up*/
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_LK_HEAL_DOM,
+                       "Event gen mismatch: skipped healing lock on child %d "
+                       "for %s.",
+                       i, uuid_utoa(info->fd->inode->gfid));
+            }
+        }
+    }
+    UNLOCK(&priv->lock);
+
+release:
+    afr_dom_lock_release(frame);
+    if (retry)
+        afr_add_lock_to_lkhealq(priv, info);
+    return;
 }
 
-gf_boolean_t
-afr_is_opendir_done (xlator_t *this, inode_t *inode)
+static int
+afr_lock_heal_done(int ret, call_frame_t *frame, void *opaque)
 {
-        afr_inode_params_t params = {0};
-
-        params.op = AFR_INODE_GET_OPENDIR_DONE;
-        afr_inode_get_ctx (this, inode, &params);
-        return params.u.value;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
+static int
+afr_lock_heal(void *opaque)
+{
+    call_frame_t *frame = (call_frame_t *)opaque;
+    call_frame_t *iter_frame = NULL;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
+    struct list_head healq = {
+        0,
+    };
+    int ret = 0;
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame) {
+        return ENOMEM;
+    }
+
+    INIT_LIST_HEAD(&healq);
+    LOCK(&priv->lock);
+    {
+        list_splice_init(&priv->lk_healq, &healq);
+    }
+    UNLOCK(&priv->lock);
+
+    list_for_each_entry_safe(info, tmp, &healq, pos)
+    {
+        GF_ASSERT((AFR_COUNT(info->locked_nodes, priv->child_count) <
+                   priv->child_count));
+        ((afr_local_t *)(iter_frame->local))->fd = fd_ref(info->fd);
+        afr_lock_heal_do(iter_frame, priv, info);
+        AFR_STACK_RESET(iter_frame);
+        if (iter_frame->local == NULL) {
+            ret = ENOTCONN;
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOTCONN,
+                   AFR_MSG_LK_HEAL_DOM,
+                   "Aborting processing of lk_healq."
+                   "Healing will be reattempted on next child up for locks "
+                   "that are still in quorum.");
+            LOCK(&priv->lock);
+            {
+                list_add_tail(&healq, &priv->lk_healq);
+            }
+            UNLOCK(&priv->lock);
+            break;
+        }
+    }
+
+    AFR_STACK_DESTROY(iter_frame);
+    return ret;
+}
 
-int32_t
-afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children)
+static int
+__afr_lock_heal_synctask(xlator_t *this, afr_private_t *priv, int child)
 {
-        afr_inode_params_t      params = {0};
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
 
-        params.op = AFR_INODE_GET_READ_CTX;
-        params.u.read_ctx.children = fresh_children;
-        afr_inode_get_ctx (this, inode, &params);
-        return params.u.read_ctx.read_child;
+    if (priv->shd.iamshd)
+        return 0;
+
+    list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
+    {
+        info->child_up_event_gen[child] = priv->event_generation;
+        list_del_init(&info->pos);
+        list_add_tail(&info->pos, &priv->lk_healq);
+    }
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        return -1;
+
+    ret = synctask_new(this->ctx->env, afr_lock_heal, afr_lock_heal_done, frame,
+                       frame);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_LK_HEAL_DOM,
+               "Failed to launch lock heal synctask");
+
+    return ret;
 }
 
-void
-afr_inode_ctx_set_read_child (afr_inode_ctx_t *ctx, int32_t read_child)
+static int
+__afr_mark_pending_lk_heal(xlator_t *this, afr_private_t *priv, int child)
 {
-        uint64_t        remaining_mask = 0;
-        uint64_t        mask         = 0;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
 
-        remaining_mask = (~AFR_ICTX_READ_CHILD_MASK & ctx->masks);
-        mask = (AFR_ICTX_READ_CHILD_MASK & read_child);
-        ctx->masks = remaining_mask | mask;
+    if (priv->shd.iamshd)
+        return 0;
+    list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
+    {
+        info->child_down_event_gen[child] = priv->event_generation;
+        if (info->locked_nodes[child] == 1)
+            info->locked_nodes[child] = 0;
+        if (!afr_has_quorum(info->locked_nodes, this, NULL)) {
+            /* Since the lock was lost on quorum no. of nodes, we should
+             * not attempt to heal it anymore. Some other client could have
+             * acquired the lock, modified data and released it and this
+             * client wouldn't know about it if we heal it.*/
+            afr_mark_fd_bad(info->fd, this);
+            list_del(&info->pos);
+            afr_lk_heal_info_cleanup(info);
+            /* We're not winding an unlock on the node where the lock is still
+             * present because when fencing logic switches over to the new
+             * client (since we marked the fd bad), it should preempt any
+             * existing lock. */
+        }
+    }
+    return 0;
 }
 
-void
-afr_inode_ctx_set_read_ctx (afr_inode_ctx_t *ctx, int32_t read_child,
-                            int32_t *fresh_children, int32_t child_count)
+gf_boolean_t
+afr_is_consistent_io_possible(afr_local_t *local, afr_private_t *priv,
+                              int32_t *op_errno)
+{
+    if (priv->consistent_io && local->call_count != priv->child_count) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
+               "All subvolumes are not up");
+        if (op_errno)
+            *op_errno = ENOTCONN;
+        return _gf_false;
+    }
+    return _gf_true;
+}
+
+gf_boolean_t
+afr_is_lock_mode_mandatory(dict_t *xdata)
 {
-        int             i            = 0;
+    int ret = 0;
+    uint32_t lk_mode = GF_LK_ADVISORY;
 
-        afr_inode_ctx_set_read_child (ctx, read_child);
-        for (i = 0; i < child_count; i++) {
-                if (fresh_children)
-                        ctx->fresh_children[i] = fresh_children[i];
-                else
-                        ctx->fresh_children[i] = -1;
-        }
+    ret = dict_get_uint32(xdata, GF_LOCK_MODE, &lk_mode);
+    if (!ret && lk_mode == GF_LK_MANDATORY)
+        return _gf_true;
+
+    return _gf_false;
 }
 
-void
-afr_inode_ctx_rm_stale_children (afr_inode_ctx_t *ctx, int32_t *stale_children,
-                                 int32_t child_count)
+call_frame_t *
+afr_copy_frame(call_frame_t *base)
 {
-        int             i            = 0;
-        int32_t         read_child   = -1;
+    afr_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int op_errno = 0;
 
-        GF_ASSERT (stale_children);
-        for (i = 0; i < child_count; i++) {
-                if (stale_children[i] == -1)
-                        break;
-                afr_children_rm_child (ctx->fresh_children,
-                                       stale_children[i], child_count);
-        }
-        read_child = (int32_t)(ctx->masks & AFR_ICTX_READ_CHILD_MASK);
-        if (!afr_is_child_present (ctx->fresh_children, child_count,
-                                   read_child))
-                afr_inode_ctx_set_read_child (ctx, ctx->fresh_children[0]);
+    frame = copy_frame(base);
+    if (!frame)
+        return NULL;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local) {
+        AFR_STACK_DESTROY(frame);
+        return NULL;
+    }
+
+    return frame;
 }
 
-void
-afr_inode_ctx_set_opendir_done (afr_inode_ctx_t *ctx)
+/* Check if an entry or inode could be undergoing a transaction. */
+gf_boolean_t
+afr_is_possibly_under_txn(afr_transaction_type type, afr_local_t *local,
+                          xlator_t *this)
+{
+    int i = 0;
+    int tmp = 0;
+    afr_private_t *priv = NULL;
+    GF_UNUSED char *key = NULL;
+    int keylen = 0;
+
+    priv = this->private;
+
+    if (type == AFR_ENTRY_TRANSACTION) {
+        key = GLUSTERFS_PARENT_ENTRYLK;
+        keylen = SLEN(GLUSTERFS_PARENT_ENTRYLK);
+    } else if (type == AFR_DATA_TRANSACTION) {
+        /*FIXME: Use GLUSTERFS_INODELK_DOM_COUNT etc. once
+         * pl_inodelk_xattr_fill supports separate keys for different
+         * domains.*/
+        key = GLUSTERFS_INODELK_COUNT;
+        keylen = SLEN(GLUSTERFS_INODELK_COUNT);
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].xdata)
+            continue;
+        if (dict_get_int32n(local->replies[i].xdata, key, keylen, &tmp) == 0)
+            if (tmp)
+                return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static void
+afr_inode_ctx_destroy(afr_inode_ctx_t *ctx)
 {
-        uint64_t        remaining_mask = 0;
-        uint64_t        mask = 0;
+    int i = 0;
+
+    if (!ctx)
+        return;
+
+    for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+        GF_FREE(ctx->pre_op_done[i]);
+    }
 
-        remaining_mask = (~AFR_ICTX_OPENDIR_DONE_MASK & ctx->masks);
-        mask = (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_OPENDIR_DONE_MASK);
-        ctx->masks = remaining_mask | mask;
+    GF_FREE(ctx);
 }
 
-void
-afr_inode_ctx_set_splitbrain (afr_inode_ctx_t *ctx, gf_boolean_t set)
+int
+__afr_inode_ctx_get(xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx)
+{
+    uint64_t ctx_int = 0;
+    int ret = -1;
+    int i = -1;
+    int num_locks = -1;
+    afr_inode_ctx_t *ictx = NULL;
+    afr_lock_t *lock = NULL;
+    afr_private_t *priv = this->private;
+
+    ret = __inode_ctx_get(inode, this, &ctx_int);
+    if (ret == 0) {
+        *ctx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
+        return 0;
+    }
+
+    ictx = GF_CALLOC(1, sizeof(afr_inode_ctx_t), gf_afr_mt_inode_ctx_t);
+    if (!ictx)
+        goto out;
+
+    for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+        ictx->pre_op_done[i] = GF_CALLOC(sizeof *ictx->pre_op_done[i],
+                                         priv->child_count, gf_afr_mt_int32_t);
+        if (!ictx->pre_op_done[i]) {
+            ret = -ENOMEM;
+            goto out;
+        }
+    }
+
+    num_locks = sizeof(ictx->lock) / sizeof(afr_lock_t);
+    for (i = 0; i < num_locks; i++) {
+        lock = &ictx->lock[i];
+        INIT_LIST_HEAD(&lock->post_op);
+        INIT_LIST_HEAD(&lock->frozen);
+        INIT_LIST_HEAD(&lock->waiting);
+        INIT_LIST_HEAD(&lock->owners);
+    }
+
+    ctx_int = (uint64_t)(uintptr_t)ictx;
+    ret = __inode_ctx_set(inode, this, &ctx_int);
+    if (ret) {
+        goto out;
+    }
+
+    ictx->spb_choice = -1;
+    ictx->read_subvol = 0;
+    ictx->write_subvol = 0;
+    ictx->lock_count = 0;
+    ret = 0;
+    *ctx = ictx;
+out:
+    if (ret) {
+        afr_inode_ctx_destroy(ictx);
+    }
+    return ret;
+}
+
+/*
+ * INODE CTX 64-bit VALUE FORMAT FOR SMALL (<= 16) SUBVOL COUNTS:
+ *
+ * |<----------   64bit   ------------>|
+ *  63           32 31    16 15       0
+ * |   EVENT_GEN   |  DATA  | METADATA |
+ *
+ *
+ *  METADATA (bit-0 .. bit-15): bitmap representing subvolumes from which
+ *                              metadata can be attempted to be read.
+ *
+ *                              bit-0 => priv->subvolumes[0]
+ *                              bit-1 => priv->subvolumes[1]
+ *                              ... etc. till bit-15
+ *
+ *  DATA (bit-16 .. bit-31): bitmap representing subvolumes from which data
+ *                           can be attempted to be read.
+ *
+ *                           bit-16 => priv->subvolumes[0]
+ *                           bit-17 => priv->subvolumes[1]
+ *                           ... etc. till bit-31
+ *
+ *  EVENT_GEN (bit-32 .. bit-63): event generation (i.e priv->event_generation)
+ *                                when DATA and METADATA was last updated.
+ *
+ *                                If EVENT_GEN is < priv->event_generation,
+ *                                or is 0, it means afr_inode_refresh() needs
+ *                                to be called to recalculate the bitmaps.
+ */
+
+int
+__afr_set_in_flight_sb_status(xlator_t *this, afr_local_t *local,
+                              inode_t *inode)
+{
+    int i = 0;
+    int txn_type = 0;
+    int count = 0;
+    int index = -1;
+    uint16_t datamap_old = 0;
+    uint16_t metadatamap_old = 0;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint16_t tmp_map = 0;
+    uint16_t mask = 0;
+    uint32_t event = 0;
+    uint64_t val = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    txn_type = local->transaction.type;
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        val = local->inode_ctx->write_subvol;
+    else
+        val = local->inode_ctx->read_subvol;
+
+    metadatamap_old = metadatamap = (val & 0x000000000000ffff);
+    datamap_old = datamap = (val & 0x00000000ffff0000) >> 16;
+    event = (val & 0xffffffff00000000) >> 32;
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        tmp_map = datamap;
+    else if (txn_type == AFR_METADATA_TRANSACTION)
+        tmp_map = metadatamap;
+
+    count = gf_bits_count(tmp_map);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.failed_subvols[i])
+            continue;
+
+        mask = 1 << i;
+        if (txn_type == AFR_METADATA_TRANSACTION)
+            metadatamap &= ~mask;
+        else if (txn_type == AFR_DATA_TRANSACTION)
+            datamap &= ~mask;
+    }
+
+    switch (txn_type) {
+        case AFR_METADATA_TRANSACTION:
+            if ((metadatamap_old != 0) && (metadatamap == 0) && (count == 1)) {
+                index = gf_bits_index(tmp_map);
+                local->transaction.in_flight_sb_errno = local->replies[index]
+                                                            .op_errno;
+                local->transaction.in_flight_sb = _gf_true;
+                metadatamap |= (1 << index);
+            }
+            if (metadatamap_old != metadatamap) {
+                __afr_inode_need_refresh_set(inode, this);
+            }
+            break;
+
+        case AFR_DATA_TRANSACTION:
+            if ((datamap_old != 0) && (datamap == 0) && (count == 1)) {
+                index = gf_bits_index(tmp_map);
+                local->transaction.in_flight_sb_errno = local->replies[index]
+                                                            .op_errno;
+                local->transaction.in_flight_sb = _gf_true;
+                datamap |= (1 << index);
+            }
+            if (datamap_old != datamap)
+                __afr_inode_need_refresh_set(inode, this);
+            break;
+
+        default:
+            break;
+    }
+
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        local->inode_ctx->write_subvol = val;
+    local->inode_ctx->read_subvol = val;
+
+    return 0;
+}
+
+gf_boolean_t
+afr_is_symmetric_error(call_frame_t *frame, xlator_t *this)
 {
-        uint64_t        remaining_mask = 0;
-        uint64_t        mask = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int op_errno = 0;
+    int i_errno = 0;
+    gf_boolean_t matching_errors = _gf_true;
+    int i = 0;
 
-        if (set) {
-                remaining_mask = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx->masks);
-                mask = (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_SPLIT_BRAIN_MASK);
-                ctx->masks = remaining_mask | mask;
-        } else {
-                ctx->masks = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx->masks);
+    priv = this->private;
+    local = frame->local;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret != -1) {
+            /* Operation succeeded on at least one subvol,
+               so it is not a failed-everywhere situation.
+            */
+            matching_errors = _gf_false;
+            break;
         }
-}
+        i_errno = local->replies[i].op_errno;
 
-void
-afr_inode_set_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
-{
-        GF_ASSERT (inode);
-        GF_ASSERT (params);
-
-        int             ret = 0;
-        afr_inode_ctx_t *ctx            = NULL;
-        afr_private_t   *priv           = NULL;
-        uint64_t        ctx_addr        = 0;
-        gf_boolean_t    set             = _gf_false;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children = NULL;
-        int32_t         *stale_children = NULL;
-
-        priv = this->private;
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        ctx_addr = 0;
-                ctx = afr_inode_ctx_get_from_addr (ctx_addr, priv->child_count);
-                if (!ctx)
-                        goto unlock;
-                switch (params->op) {
-                case AFR_INODE_SET_READ_CTX:
-                        read_child = params->u.read_ctx.read_child;
-                        fresh_children = params->u.read_ctx.children;
-                        afr_inode_ctx_set_read_ctx (ctx, read_child,
-                                                    fresh_children,
-                                                    priv->child_count);
-                        break;
-                case AFR_INODE_RM_STALE_CHILDREN:
-                        stale_children = params->u.read_ctx.children;
-                        afr_inode_ctx_rm_stale_children (ctx,
-                                                         stale_children,
-                                                         priv->child_count);
-                        break;
-                case AFR_INODE_SET_OPENDIR_DONE:
-                        afr_inode_ctx_set_opendir_done (ctx);
-                        break;
-                case AFR_INODE_SET_SPLIT_BRAIN:
-                        set = params->u.value;
-                        afr_inode_ctx_set_splitbrain (ctx, set);
-                        break;
-                default:
-                        GF_ASSERT (0);
-                        break;
-                }
-                ret = __inode_ctx_put (inode, this, (uint64_t)ctx);
-                if (ret) {
-                        gf_log_callingfn (this->name, GF_LOG_ERROR, "failed to "
-                                          "set the inode ctx (%s)",
-                                          uuid_utoa (inode->gfid));
-                }
+        if (i_errno == ENOTCONN) {
+            /* ENOTCONN is not a symmetric error. We do not
+               know if the operation was performed on the
+               backend or not.
+            */
+            matching_errors = _gf_false;
+            break;
         }
-unlock:
-        UNLOCK (&inode->lock);
+
+        if (!op_errno) {
+            op_errno = i_errno;
+        } else if (op_errno != i_errno) {
+            /* Mismatching op_errno's */
+            matching_errors = _gf_false;
+            break;
+        }
+    }
+
+    return matching_errors;
 }
 
-void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set)
+int
+afr_set_in_flight_sb_status(xlator_t *this, call_frame_t *frame, inode_t *inode)
 {
-        afr_inode_params_t      params = {0};
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    /* If this transaction saw no failures, then exit. */
+    if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count) == 0)
+        return 0;
+
+    if (afr_is_symmetric_error(frame, this))
+        return 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_set_in_flight_sb_status(this, local, inode);
+    }
+    UNLOCK(&inode->lock);
 
-        params.op = AFR_INODE_SET_SPLIT_BRAIN;
-        params.u.value          = set;
-        afr_inode_set_ctx (this, inode, &params);
+    return ret;
 }
 
-void
-afr_set_opendir_done (xlator_t *this, inode_t *inode)
-{
-        afr_inode_params_t params = {0};
+int
+__afr_inode_read_subvol_get_small(inode_t *inode, xlator_t *this,
+                                  unsigned char *data, unsigned char *metadata,
+                                  int *event_p)
+{
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint32_t event = 0;
+    uint64_t val = 0;
+    int i = 0;
+    afr_inode_ctx_t *ctx = NULL;
+
+    priv = this->private;
+
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret < 0)
+        return ret;
 
-        params.op = AFR_INODE_SET_OPENDIR_DONE;
-        afr_inode_set_ctx (this, inode, &params);
+    val = ctx->read_subvol;
+
+    metadatamap = (val & 0x000000000000ffff);
+    datamap = (val & 0x00000000ffff0000) >> 16;
+    event = (val & 0xffffffff00000000) >> 32;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (metadata)
+            metadata[i] = (metadatamap >> i) & 1;
+        if (data)
+            data[i] = (datamap >> i) & 1;
+    }
+
+    if (event_p)
+        *event_p = event;
+    return ret;
 }
 
-void
-afr_inode_set_read_ctx (xlator_t *this, inode_t *inode, int32_t read_child,
-                        int32_t *fresh_children)
+int
+__afr_inode_read_subvol_set_small(inode_t *inode, xlator_t *this,
+                                  unsigned char *data, unsigned char *metadata,
+                                  int event)
 {
-        afr_inode_params_t params = {0};
-        afr_private_t      *priv  = NULL;
+    afr_private_t *priv = NULL;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint64_t val = 0;
+    int i = 0;
+    int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
+
+    priv = this->private;
 
-        priv = this->private;
-        GF_ASSERT (read_child >= 0);
-        GF_ASSERT (fresh_children);
-        GF_ASSERT (afr_is_child_present (fresh_children, priv->child_count,
-                                         read_child));
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret)
+        goto out;
 
-        params.op = AFR_INODE_SET_READ_CTX;
-        params.u.read_ctx.read_child     = read_child;
-        params.u.read_ctx.children = fresh_children;
-        afr_inode_set_ctx (this, inode, &params);
+    for (i = 0; i < priv->child_count; i++) {
+        if (data[i])
+            datamap |= (1 << i);
+        if (metadata[i])
+            metadatamap |= (1 << i);
+    }
+
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
+
+    ctx->read_subvol = val;
+
+    ret = 0;
+out:
+    return ret;
 }
 
-void
-afr_inode_rm_stale_children (xlator_t *this, inode_t *inode,
-                             int32_t *stale_children)
+int
+__afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
+                            unsigned char *metadata, int *event_p)
 {
-        afr_inode_params_t params = {0};
+    afr_private_t *priv = NULL;
+    int ret = -1;
 
-        GF_ASSERT (stale_children);
+    priv = this->private;
 
-        params.op = AFR_INODE_RM_STALE_CHILDREN;
-        params.u.read_ctx.children = stale_children;
-        afr_inode_set_ctx (this, inode, &params);
+    if (priv->child_count <= 16)
+        ret = __afr_inode_read_subvol_get_small(inode, this, data, metadata,
+                                                event_p);
+    else
+        /* TBD: allocate structure with array and read from it */
+        ret = -1;
+
+    return ret;
 }
 
-gf_boolean_t
-afr_is_source_child (int32_t *sources, int32_t child_count, int32_t child)
+int
+__afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this,
+                                   int *spb_choice)
 {
-        gf_boolean_t             source_xattrs = _gf_false;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        GF_ASSERT (child < child_count);
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret < 0)
+        return ret;
 
-        if ((child >= 0) && (child < child_count) &&
-             sources[child]) {
-                source_xattrs = _gf_true;
-        }
-        return source_xattrs;
+    *spb_choice = ctx->spb_choice;
+    return 0;
 }
 
-gf_boolean_t
-afr_is_child_present (int32_t *success_children, int32_t child_count,
-                      int32_t child)
+int
+__afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
+                            unsigned char *metadata, int event)
 {
-        gf_boolean_t             success_child = _gf_false;
-        int                      i = 0;
+    afr_private_t *priv = NULL;
+    int ret = -1;
 
-        GF_ASSERT (child < child_count);
+    priv = this->private;
 
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                if (child == success_children[i]) {
-                        success_child = _gf_true;
-                        break;
-                }
-        }
-        return success_child;
+    if (priv->child_count <= 16)
+        ret = __afr_inode_read_subvol_set_small(inode, this, data, metadata,
+                                                event);
+    else
+        ret = -1;
+
+    return ret;
 }
 
-gf_boolean_t
-afr_is_read_child (int32_t *success_children, int32_t *sources,
-                   int32_t child_count, int32_t child)
+int
+__afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this,
+                                   int spb_choice)
 {
-        gf_boolean_t             success_child = _gf_false;
-        gf_boolean_t             source        = _gf_false;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        if (child < 0) {
-                return _gf_false;
-        }
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret)
+        goto out;
 
-        GF_ASSERT (success_children);
-        GF_ASSERT (child_count > 0);
+    ctx->spb_choice = spb_choice;
 
-        success_child = afr_is_child_present (success_children, child_count,
-                                              child);
-        if (!success_child)
-                goto out;
-        if (NULL == sources) {
-                source = _gf_true;
-                goto out;
-        }
-        source = afr_is_source_child (sources, child_count, child);
+    ret = 0;
 out:
-        return (success_child && source);
+    return ret;
 }
 
-int32_t
-afr_hash_child (int32_t *success_children, int32_t child_count,
-                unsigned int hmode, uuid_t gfid)
+int
+afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
+                          unsigned char *metadata, int *event_p)
 {
-        uuid_t  gfid_copy = {0,};
+    int ret = -1;
 
-        if (!hmode) {
-                return -1;
-        }
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        if (gfid) {
-               uuid_copy(gfid_copy,gfid);
-        }
-        if (hmode > 1) {
-                /*
-                 * Why getpid?  Because it's one of the cheapest calls
-                 * available - faster than gethostname etc. - and returns a
-                 * constant-length value that's sure to be shorter than a UUID.
-                 * It's still very unlikely to be the same across clients, so
-                 * it still provides good mixing.  We're not trying for
-                 * perfection here.  All we need is a low probability that
-                 * multiple clients won't converge on the same subvolume.
-                 */
-                *((pid_t *)gfid_copy) = getpid();
-        }
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_read_subvol_get(inode, this, data, metadata, event_p);
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
+}
 
-        return SuperFastHash((char *)gfid_copy,
-                             sizeof(gfid_copy)) % child_count;
+int
+afr_inode_get_readable(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                       unsigned char *readable, int *event_p, int type)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+    unsigned char *data = alloca0(priv->child_count);
+    unsigned char *metadata = alloca0(priv->child_count);
+    int data_count = 0;
+    int metadata_count = 0;
+    int event_generation = 0;
+    int ret = 0;
+
+    ret = afr_inode_read_subvol_get(inode, this, data, metadata,
+                                    &event_generation);
+    if (ret == -1)
+        return -EIO;
+
+    data_count = AFR_COUNT(data, priv->child_count);
+    metadata_count = AFR_COUNT(metadata, priv->child_count);
+
+    if (inode->ia_type == IA_IFDIR) {
+        /* For directories, allow even if it is in data split-brain. */
+        if (type == AFR_METADATA_TRANSACTION || local->op == GF_FOP_STAT ||
+            local->op == GF_FOP_FSTAT) {
+            if (!metadata_count)
+                return -EIO;
+        }
+    } else {
+        /* For files, abort in case of data/metadata split-brain. */
+        if (!data_count || !metadata_count) {
+            return -EIO;
+        }
+    }
+
+    if (type == AFR_METADATA_TRANSACTION && readable)
+        memcpy(readable, metadata, priv->child_count * sizeof *metadata);
+    if (type == AFR_DATA_TRANSACTION && readable) {
+        if (!data_count)
+            memcpy(readable, local->child_up,
+                   priv->child_count * sizeof *readable);
+        else
+            memcpy(readable, data, priv->child_count * sizeof *data);
+    }
+    if (event_p)
+        *event_p = event_generation;
+    return 0;
 }
 
-/* If sources is NULL the xattrs are assumed to be of source for all
- * success_children.
+static int
+afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this,
+                                 int *spb_choice)
+{
+    int ret = -1;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_split_brain_choice_get(inode, this, spb_choice);
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
+}
+
+/*
+ * frame is used to get the favourite policy. Since
+ * afr_inode_split_brain_choice_get was called with afr_open, it is possible to
+ * have a frame with out local->replies. So in that case, frame is passed as
+ * null, hence this function will handle the frame NULL case.
  */
 int
-afr_select_read_child_from_policy (int32_t *success_children,
-                                   int32_t child_count, int32_t prev_read_child,
-                                   int32_t config_read_child, int32_t *sources,
-                                   unsigned int hmode, uuid_t gfid)
+afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this,
+                                call_frame_t *frame, int *spb_subvol)
 {
-        int32_t                  read_child   = -1;
-        int                      i            = 0;
+    int ret = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        GF_ASSERT (success_children);
+    GF_VALIDATE_OR_GOTO("afr", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, spb_subvol, out);
 
-        read_child = config_read_child;
-        if (afr_is_read_child (success_children, sources, child_count,
-                               read_child))
-                goto out;
-
-        read_child = prev_read_child;
-        if (afr_is_read_child (success_children, sources, child_count,
-                               read_child))
-                goto out;
+    priv = this->private;
 
-        read_child = afr_hash_child (success_children, child_count,
-                                     hmode, gfid);
-        if (afr_is_read_child (success_children, sources, child_count,
-                               read_child)) {
-                goto out;
+    ret = afr_inode_split_brain_choice_get(inode, this, spb_subvol);
+    if (*spb_subvol < 0 && priv->fav_child_policy && frame && frame->local) {
+        local = frame->local;
+        *spb_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
+                                               NULL);
+        if (*spb_subvol >= 0) {
+            ret = 0;
         }
+    }
 
-        for (i = 0; i < child_count; i++) {
-                read_child = success_children[i];
-                if (read_child < 0)
-                        break;
-                if (afr_is_read_child (success_children, sources, child_count,
-                                       read_child))
-                        goto out;
-        }
-        read_child = -1;
+out:
+    return ret;
+}
+int
+afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
+                          unsigned char *metadata, int event)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_read_subvol_set(inode, this, data, metadata, event);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return read_child;
+    return ret;
 }
 
-/* This function should be used when all the success_children are sources
- */
-void
-afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
-                              int32_t *fresh_children, int32_t prev_read_child,
-                              int32_t config_read_child, uuid_t gfid)
-{
-        int                      read_child = -1;
-        afr_private_t            *priv = NULL;
-
-        priv = this->private;
-        read_child = afr_select_read_child_from_policy (fresh_children,
-                                                        priv->child_count,
-                                                        prev_read_child,
-                                                        config_read_child,
-                                                        NULL,
-                                                        priv->hash_mode, gfid);
-        if (read_child >= 0)
-                afr_inode_set_read_ctx (this, inode, read_child,
-                                        fresh_children);
-}
-
-/* afr_next_call_child ()
- * This is a common function used by all the read-type fops
- * This function should not be called with the inode's read_children array.
- * The fop's handler should make a copy of the inode's read_children,
- * preferred read_child into the local vars, because while this function is
- * in execution there is a chance for inode's read_ctx to change.
- */
-int32_t
-afr_next_call_child (int32_t *fresh_children, unsigned char *child_up,
-                     size_t child_count, int32_t *last_index,
-                     int32_t read_child)
+int
+afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this, int spb_choice)
 {
-        int             next_index      = 0;
-        int32_t         next_call_child = -1;
+    int ret = -1;
 
-        GF_ASSERT (last_index);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        next_index = *last_index;
-retry:
-        next_index++;
-        if ((next_index >= child_count) ||
-           (fresh_children[next_index] == -1))
-                goto out;
-        if ((fresh_children[next_index] == read_child) ||
-           (!child_up[fresh_children[next_index]]))
-                goto retry;
-        *last_index = next_index;
-        next_call_child = fresh_children[next_index];
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_split_brain_choice_set(inode, this, spb_choice);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return next_call_child;
+    return ret;
 }
 
- /* This function should not be called with the inode's read_children array.
- * The fop's handler should make a copy of the inode's read_children,
- * preferred read_child into the local vars, because while this function is
- * in execution there is a chance for inode's read_ctx to change.
+/* The caller of this should perform afr_inode_refresh, if this function
+ * returns _gf_true
  */
-int32_t
-afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
-                    int32_t *fresh_children,
-                    int32_t *call_child, int32_t *last_index)
+gf_boolean_t
+afr_is_inode_refresh_reqd(inode_t *inode, xlator_t *this, int event_gen1,
+                          int event_gen2)
 {
-        int             ret   = 0;
-        afr_private_t   *priv = NULL;
-        int             i     = 0;
+    gf_boolean_t need_refresh = _gf_false;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        GF_ASSERT (child_up);
-        GF_ASSERT (call_child);
-        GF_ASSERT (last_index);
-        GF_ASSERT (fresh_children);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        if (read_child < 0) {
-                ret = -EIO;
-                goto out;
-        }
-        priv = this->private;
-        *call_child = -1;
-        *last_index = -1;
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret)
+            goto unlock;
 
-        if (child_up[read_child]) {
-                *call_child = read_child;
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (fresh_children[i] == -1)
-                                break;
-                        if (child_up[fresh_children[i]]) {
-                                *call_child = fresh_children[i];
-                                ret = 0;
-                                break;
-                        }
-                }
+        need_refresh = ctx->need_refresh;
+        /* Hoping that the caller will do inode_refresh followed by
+         * this, hence setting the need_refresh to false */
+        ctx->need_refresh = _gf_false;
+    }
+unlock:
+    UNLOCK(&inode->lock);
 
-                if (*call_child == -1) {
-                        ret = -ENOTCONN;
-                        goto out;
-                }
+    if (event_gen1 != event_gen2)
+        need_refresh = _gf_true;
+out:
+    return need_refresh;
+}
 
-                *last_index = i;
-        }
+int
+__afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
+{
+    int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
+
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret == 0) {
+        ctx->need_refresh = _gf_true;
+    }
+
+    return ret;
+}
+
+int
+afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_need_refresh_set(inode, this);
+    }
+    UNLOCK(&inode->lock);
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d, call_child: %d, "
-                "last_index: %d", ret, *call_child, *last_index);
-        return ret;
+    return ret;
 }
 
-void
-afr_reset_xattr (dict_t **xattr, unsigned int child_count)
+int
+afr_spb_choice_timeout_cancel(xlator_t *this, inode_t *inode)
 {
-        unsigned int i = 0;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        if (!xattr)
-                goto out;
-        for (i = 0; i < child_count; i++) {
-                if (xattr[i]) {
-                        dict_unref (xattr[i]);
-                        xattr[i] = NULL;
-                }
+    if (!inode)
+        return ret;
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret < 0 || !ctx) {
+            UNLOCK(&inode->lock);
+            gf_msg(this->name, GF_LOG_WARNING, 0,
+                   AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+                   "Failed to cancel split-brain choice timer.");
+            goto out;
+        }
+        ctx->spb_choice = -1;
+        if (ctx->timer) {
+            gf_timer_call_cancel(this->ctx, ctx->timer);
+            ctx->timer = NULL;
         }
+        ret = 0;
+    }
+    UNLOCK(&inode->lock);
 out:
-        return;
+    return ret;
 }
 
 void
-afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
+afr_set_split_brain_choice_cbk(void *data)
 {
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
+    inode_t *inode = data;
+    xlator_t *this = THIS;
 
-        sh = &local->self_heal;
-        priv = this->private;
+    afr_spb_choice_timeout_cancel(this, inode);
+    inode_invalidate(inode);
+    inode_unref(inode);
+    return;
+}
 
-        GF_FREE (sh->buf);
+int
+afr_set_split_brain_choice(int ret, call_frame_t *frame, void *opaque)
+{
+    int op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_inode_ctx_t *ctx = NULL;
+    inode_t *inode = NULL;
+    loc_t *loc = NULL;
+    xlator_t *this = NULL;
+    afr_spbc_timeout_t *data = opaque;
+    struct timespec delta = {
+        0,
+    };
+    gf_boolean_t timer_set = _gf_false;
+    gf_boolean_t timer_cancelled = _gf_false;
+    gf_boolean_t timer_reset = _gf_false;
+    int old_spb_choice = -1;
+
+    frame = data->frame;
+    loc = data->loc;
+    this = frame->this;
+    priv = this->private;
+
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    delta.tv_sec = priv->spb_choice_timeout;
+    delta.tv_nsec = 0;
+
+    if (!loc->inode) {
+        ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    if (!(data->d_spb || data->m_spb)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+               "Cannot set "
+               "replica.split-brain-choice on %s. File is"
+               " not in data/metadata split-brain.",
+               uuid_utoa(loc->gfid));
+        ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /*
+     * we're ref'ing the inode before LOCK like it is done elsewhere in the
+     * code. If we ref after LOCK, coverity complains of possible deadlocks.
+     */
+    inode = inode_ref(loc->inode);
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret) {
+            UNLOCK(&inode->lock);
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+                   "Failed to get inode_ctx for %s", loc->name);
+            goto post_unlock;
+        }
 
-        GF_FREE (sh->parentbufs);
+        old_spb_choice = ctx->spb_choice;
+        ctx->spb_choice = data->spb_child_index;
 
-        if (sh->inode)
-                inode_unref (sh->inode);
+        /* Possible changes in spb-choice :
+         *         valid to -1    : cancel timer and unref
+         *         valid to valid : cancel timer and inject new one
+         *         -1    to -1    : unref and do not do anything
+         *         -1 to valid    : inject timer
+         */
 
-        if (sh->xattr) {
-                afr_reset_xattr (sh->xattr, priv->child_count);
-                GF_FREE (sh->xattr);
+        /* ctx->timer is NULL iff previous value of
+         * ctx->spb_choice is -1
+         */
+        if (ctx->timer) {
+            if (ctx->spb_choice == -1) {
+                if (!gf_timer_call_cancel(this->ctx, ctx->timer)) {
+                    ctx->timer = NULL;
+                    timer_cancelled = _gf_true;
+                }
+                /* If timer cancel failed here it means that the
+                 *  previous cbk will be executed which will set
+                 *  spb_choice to -1. So we can consider the
+                 *  'valid to -1' case to be a success
+                 *  (i.e. ret = 0) and goto unlock.
+                 */
+                goto unlock;
+            }
+            goto reset_timer;
+        } else {
+            if (ctx->spb_choice == -1)
+                goto unlock;
+            goto set_timer;
         }
 
-        GF_FREE (sh->child_errno);
+    reset_timer:
+        ret = gf_timer_call_cancel(this->ctx, ctx->timer);
+        if (ret != 0) {
+            /* We need to bail out now instead of launching a new
+             * timer. Otherwise the cbk of the previous timer event
+             * will cancel the new ctx->timer.
+             */
+            ctx->spb_choice = old_spb_choice;
+            ret = -1;
+            op_errno = EAGAIN;
+            goto unlock;
+        }
+        ctx->timer = NULL;
+        timer_reset = _gf_true;
+
+    set_timer:
+        ctx->timer = gf_timer_call_after(this->ctx, delta,
+                                         afr_set_split_brain_choice_cbk, inode);
+        if (!ctx->timer) {
+            ctx->spb_choice = old_spb_choice;
+            ret = -1;
+            op_errno = ENOMEM;
+        }
+        if (!timer_reset && ctx->timer)
+            timer_set = _gf_true;
+        if (timer_reset && !ctx->timer)
+            timer_cancelled = _gf_true;
+    }
+unlock:
+    UNLOCK(&inode->lock);
+post_unlock:
+    if (!timer_set)
+        inode_unref(inode);
+    if (timer_cancelled)
+        inode_unref(inode);
+    /*
+     * We need to invalidate the inode to prevent the kernel from serving
+     * reads from an older cached value despite a change in spb_choice to
+     * a new value.
+     */
+    inode_invalidate(inode);
+out:
+    GF_FREE(data);
+    AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    return 0;
+}
 
-        afr_matrix_cleanup (sh->pending_matrix, priv->child_count);
-        afr_matrix_cleanup (sh->delta_matrix, priv->child_count);
+int
+afr_accused_fill(xlator_t *this, dict_t *xdata, unsigned char *accused,
+                 afr_transaction_type type)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int idx = afr_index_for_transaction_type(type);
+    void *pending_raw = NULL;
+    int pending[3];
+    int ret = 0;
 
-        GF_FREE (sh->sources);
+    priv = this->private;
 
-        GF_FREE (sh->success);
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_get_ptr(xdata, priv->pending_key[i], &pending_raw);
+        if (ret) /* no pending flags */
+            continue;
+        memcpy(pending, pending_raw, sizeof(pending));
 
-        GF_FREE (sh->locked_nodes);
+        if (ntoh32(pending[idx]))
+            accused[i] = 1;
+    }
 
-        if (sh->healing_fd) {
-                fd_unref (sh->healing_fd);
-                sh->healing_fd = NULL;
-        }
-
-        GF_FREE ((char *)sh->linkname);
+    return 0;
+}
 
-        GF_FREE (sh->success_children);
+int
+afr_accuse_smallfiles(xlator_t *this, struct afr_reply *replies,
+                      unsigned char *data_accused)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t maxsize = 0;
 
-        GF_FREE (sh->fresh_children);
+    priv = this->private;
 
-        GF_FREE (sh->fresh_parent_dirs);
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].xdata &&
+            dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))
+            continue;
+        if (data_accused[i])
+            continue;
+        if (replies[i].poststat.ia_size > maxsize)
+            maxsize = replies[i].poststat.ia_size;
+    }
 
-        loc_wipe (&sh->parent_loc);
-        loc_wipe (&sh->lookup_loc);
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_accused[i])
+            continue;
+        if (AFR_IS_ARBITER_BRICK(priv, i))
+            continue;
+        if (replies[i].poststat.ia_size < maxsize)
+            data_accused[i] = 1;
+    }
 
-        GF_FREE (sh->checksum);
+    return 0;
+}
 
-        GF_FREE (sh->write_needed);
-        if (sh->healing_fd)
-                fd_unref (sh->healing_fd);
+int
+afr_readables_fill(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   unsigned char *data_accused, unsigned char *metadata_accused,
+                   unsigned char *data_readable,
+                   unsigned char *metadata_readable, struct afr_reply *replies)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+    int ret = 0;
+    ia_type_t ia_type = IA_INVAL;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        data_readable[i] = 1;
+        metadata_readable[i] = 1;
+    }
+    if (AFR_IS_ARBITER_BRICK(priv, ARBITER_BRICK_INDEX)) {
+        data_readable[ARBITER_BRICK_INDEX] = 0;
+        metadata_readable[ARBITER_BRICK_INDEX] = 0;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies) { /* Lookup */
+            if (!replies[i].valid || replies[i].op_ret == -1 ||
+                (replies[i].xdata &&
+                 dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))) {
+                data_readable[i] = 0;
+                metadata_readable[i] = 0;
+                continue;
+            }
+
+            xdata = replies[i].xdata;
+            ia_type = replies[i].poststat.ia_type;
+        } else { /* pre-op xattrop */
+            xdata = local->transaction.changelog_xdata[i];
+            ia_type = inode->ia_type;
+        }
+
+        if (!xdata)
+            continue; /* mkdir_cbk sends NULL xdata_rsp. */
+        afr_accused_fill(this, xdata, data_accused,
+                         (ia_type == IA_IFDIR) ? AFR_ENTRY_TRANSACTION
+                                               : AFR_DATA_TRANSACTION);
+
+        afr_accused_fill(this, xdata, metadata_accused,
+                         AFR_METADATA_TRANSACTION);
+    }
+
+    if (replies && ia_type != IA_INVAL && ia_type != IA_IFDIR &&
+        /* We want to accuse small files only when we know for
+         * sure that there is no IO happening. Otherwise, the
+         * ia_sizes obtained in post-refresh replies may
+         * mismatch due to a race between inode-refresh and
+         * ongoing writes, causing spurious heal launches*/
+        !afr_is_possibly_under_txn(AFR_DATA_TRANSACTION, local, this)) {
+        afr_accuse_smallfiles(this, replies, data_accused);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_accused[i]) {
+            data_readable[i] = 0;
+            ret = 1;
+        }
+        if (metadata_accused[i]) {
+            metadata_readable[i] = 0;
+            ret = 1;
+        }
+    }
+    return ret;
 }
 
+int
+afr_replies_interpret(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      gf_boolean_t *start_heal)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int event_generation = 0;
+    int i = 0;
+    unsigned char *data_accused = NULL;
+    unsigned char *metadata_accused = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    priv = this->private;
+    replies = local->replies;
+    event_generation = local->event_generation;
+
+    data_accused = alloca0(priv->child_count);
+    data_readable = alloca0(priv->child_count);
+    metadata_accused = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+
+    ret = afr_readables_fill(frame, this, inode, data_accused, metadata_accused,
+                             data_readable, metadata_readable, replies);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (start_heal && priv->child_up[i] &&
+            (data_accused[i] || metadata_accused[i])) {
+            *start_heal = _gf_true;
+            break;
+        }
+    }
+    afr_inode_read_subvol_set(inode, this, data_readable, metadata_readable,
+                              event_generation);
+    return ret;
+}
 
-void
-afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
+int
+afr_refresh_selfheal_done(int ret, call_frame_t *heal, void *opaque)
 {
-        afr_private_t * priv = NULL;
-
-        priv = this->private;
+    if (heal)
+        AFR_STACK_DESTROY(heal);
+    return 0;
+}
 
-        afr_matrix_cleanup (local->pending, priv->child_count);
-        afr_matrix_cleanup (local->transaction.txn_changelog,
-                            priv->child_count);
+int
+afr_inode_refresh_err(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int err = 0;
 
-        GF_FREE (local->internal_lock.locked_nodes);
+    local = frame->local;
+    priv = this->private;
 
-        GF_FREE (local->internal_lock.inode_locked_nodes);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && !local->replies[i].op_ret) {
+            err = 0;
+            goto ret;
+        }
+    }
 
-        GF_FREE (local->internal_lock.entry_locked_nodes);
+    err = afr_final_errno(local, priv);
+ret:
+    return err;
+}
 
-        GF_FREE (local->internal_lock.lower_locked_nodes);
+gf_boolean_t
+afr_selfheal_enabled(const xlator_t *this)
+{
+    const afr_private_t *priv = this->private;
 
+    return priv->data_self_heal || priv->metadata_self_heal ||
+           priv->entry_self_heal;
+}
 
-        GF_FREE (local->transaction.pre_op);
-        GF_FREE (local->transaction.eager_lock);
+int
+afr_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
+{
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *heal_local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    int event_generation = 0;
+    int read_subvol = -1;
+    int ret = 0;
+
+    local = frame->local;
+    inode = local->inode;
+    priv = this->private;
+
+    if (err)
+        goto refresh_done;
+
+    if (local->op == GF_FOP_LOOKUP)
+        goto refresh_done;
+
+    ret = afr_inode_get_readable(frame, inode, this, local->readable,
+                                 &event_generation, local->transaction.type);
+
+    if (ret == -EIO) {
+        /* No readable subvolume even after refresh ==> splitbrain.*/
+        if (!priv->fav_child_policy) {
+            err = EIO;
+            goto refresh_done;
+        }
+        read_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
+                                               NULL);
+        if (read_subvol == -1) {
+            err = EIO;
+            goto refresh_done;
+        }
+
+        heal_frame = afr_frame_create(this, NULL);
+        if (!heal_frame) {
+            err = EIO;
+            goto refresh_done;
+        }
+        heal_local = heal_frame->local;
+        heal_local->xdata_req = dict_new();
+        if (!heal_local->xdata_req) {
+            err = EIO;
+            AFR_STACK_DESTROY(heal_frame);
+            goto refresh_done;
+        }
+        heal_local->heal_frame = frame;
+        ret = synctask_new(this->ctx->env, afr_fav_child_reset_sink_xattrs,
+                           afr_fav_child_reset_sink_xattrs_cbk, heal_frame,
+                           heal_frame);
+        return 0;
+    }
 
-        GF_FREE (local->transaction.basename);
-        GF_FREE (local->transaction.new_basename);
+refresh_done:
+    afr_local_replies_wipe(local, this->private);
+    local->refreshfn(frame, this, err);
 
-        loc_wipe (&local->transaction.parent_loc);
-        loc_wipe (&local->transaction.new_parent_loc);
+    return 0;
 }
 
+int
+afr_inode_refresh_done(call_frame_t *frame, xlator_t *this, int error)
+{
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t start_heal = _gf_false;
+    afr_local_t *heal_local = NULL;
+    unsigned char *success_replies = NULL;
+    int ret = 0;
+
+    if (error != 0) {
+        goto refresh_done;
+    }
+
+    local = frame->local;
+    priv = this->private;
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+
+    if (priv->thin_arbiter_count && local->is_read_txn &&
+        AFR_COUNT(success_replies, priv->child_count) != priv->child_count) {
+        /* We need to query the good bricks and/or thin-arbiter.*/
+        if (success_replies[0]) {
+            local->read_txn_query_child = AFR_CHILD_ZERO;
+        } else if (success_replies[1]) {
+            local->read_txn_query_child = AFR_CHILD_ONE;
+        }
+        error = EINVAL;
+        goto refresh_done;
+    }
+
+    if (!afr_has_quorum(success_replies, this, frame)) {
+        error = afr_final_errno(frame->local, this->private);
+        if (!error)
+            error = afr_quorum_errno(priv);
+        goto refresh_done;
+    }
+
+    ret = afr_replies_interpret(frame, this, local->refreshinode, &start_heal);
+
+    if (ret && afr_selfheal_enabled(this) && start_heal) {
+        heal_frame = afr_frame_create(this, NULL);
+        if (!heal_frame)
+            goto refresh_done;
+        heal_local = heal_frame->local;
+        heal_local->refreshinode = inode_ref(local->refreshinode);
+        heal_local->heal_frame = heal_frame;
+        if (!afr_throttled_selfheal(heal_frame, this)) {
+            AFR_STACK_DESTROY(heal_frame);
+            goto refresh_done;
+        }
+    }
+
+refresh_done:
+    afr_txn_refresh_done(frame, this, error);
+
+    return 0;
+}
 
 void
-afr_local_cleanup (afr_local_t *local, xlator_t *this)
-{
-        afr_private_t * priv = NULL;
+afr_inode_refresh_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, struct iatt *buf,
+                             dict_t *xdata, struct iatt *par)
+{
+    afr_local_t *local = NULL;
+    int call_child = (long)cookie;
+    int8_t need_heal = 1;
+    int call_count = 0;
+    int ret = 0;
+
+    local = frame->local;
+    local->replies[call_child].valid = 1;
+    local->replies[call_child].op_ret = op_ret;
+    local->replies[call_child].op_errno = op_errno;
+    if (op_ret != -1) {
+        local->replies[call_child].poststat = *buf;
+        if (par)
+            local->replies[call_child].postparent = *par;
+        if (xdata)
+            local->replies[call_child].xdata = dict_ref(xdata);
+    }
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        if (ret) {
+            gf_msg_debug(this->name, -ret, "Unable to get link count");
+        }
+    }
 
-        if (!local)
-                return;
+    local->replies[call_child].need_heal = need_heal;
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        ret = afr_inode_refresh_err(frame, this);
+        if (ret) {
+            gf_msg_debug(this->name, ret, "afr_inode_refresh_err failed");
+        }
+        afr_inode_refresh_done(frame, this, ret);
+    }
+}
 
-        afr_local_sh_cleanup (local, this);
+int
+afr_inode_refresh_subvol_with_lookup_cbk(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int op_ret,
+                                         int op_errno, inode_t *inode,
+                                         struct iatt *buf, dict_t *xdata,
+                                         struct iatt *par)
+{
+    afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                                 xdata, par);
+    return 0;
+}
 
-        afr_local_transaction_cleanup (local, this);
+int
+afr_inode_refresh_subvol_with_lookup(call_frame_t *frame, xlator_t *this, int i,
+                                     inode_t *inode, uuid_t gfid, dict_t *xdata)
+{
+    loc_t loc = {
+        0,
+    };
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->newloc);
+    loc.inode = inode;
+    if (gf_uuid_is_null(inode->gfid) && gfid) {
+        /* To handle setattr/setxattr on yet to be linked inode from
+         * dht */
+        gf_uuid_copy(loc.gfid, gfid);
+    } else {
+        gf_uuid_copy(loc.gfid, inode->gfid);
+    }
 
-        if (local->fd)
-                fd_unref (local->fd);
+    STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_lookup_cbk,
+                      (void *)(long)i, priv->children[i],
+                      priv->children[i]->fops->lookup, &loc, xdata);
+    return 0;
+}
 
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
+int
+afr_inode_refresh_subvol_with_fstat_cbk(call_frame_t *frame, void *cookie,
+                                        xlator_t *this, int32_t op_ret,
+                                        int32_t op_errno, struct iatt *buf,
+                                        dict_t *xdata)
+{
+    afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                                 xdata, NULL);
+    return 0;
+}
 
-        if (local->dict)
-                dict_unref (local->dict);
+int
+afr_inode_refresh_subvol_with_fstat(call_frame_t *frame, xlator_t *this, int i,
+                                    dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        GF_FREE (local->child_up);
+    priv = this->private;
+    local = frame->local;
 
-        GF_FREE (local->child_errno);
+    STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_fstat_cbk,
+                      (void *)(long)i, priv->children[i],
+                      priv->children[i]->fops->fstat, local->fd, xdata);
+    return 0;
+}
 
-        GF_FREE (local->fresh_children);
+int
+afr_inode_refresh_do(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int i = 0;
+    int ret = 0;
+    dict_t *xdata = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    unsigned char *wind_subvols = NULL;
 
-        GF_FREE (local->fd_open_on);
+    priv = this->private;
+    local = frame->local;
+    wind_subvols = alloca0(priv->child_count);
 
-        { /* lookup */
-                if (local->cont.lookup.xattrs) {
-                        afr_reset_xattr (local->cont.lookup.xattrs,
-                                         priv->child_count);
-                        GF_FREE (local->cont.lookup.xattrs);
-                        local->cont.lookup.xattrs = NULL;
-                }
+    afr_local_replies_wipe(local, priv);
 
-                if (local->cont.lookup.xattr) {
-                        dict_unref (local->cont.lookup.xattr);
-                }
+    if (local->fd) {
+        fd_ctx = afr_fd_ctx_get(local->fd, this);
+        if (!fd_ctx) {
+            afr_inode_refresh_done(frame, this, EINVAL);
+            return 0;
+        }
+    }
 
-                if (local->cont.lookup.inode) {
-                        inode_unref (local->cont.lookup.inode);
-                }
+    xdata = dict_new();
+    if (!xdata) {
+        afr_inode_refresh_done(frame, this, ENOMEM);
+        return 0;
+    }
 
-                GF_FREE (local->cont.lookup.postparents);
+    ret = afr_xattr_req_prepare(this, xdata);
+    if (ret != 0) {
+        dict_unref(xdata);
+        afr_inode_refresh_done(frame, this, -ret);
+        return 0;
+    }
 
-                GF_FREE (local->cont.lookup.bufs);
+    ret = dict_set_sizen_str_sizen(xdata, "link-count", GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
 
-                GF_FREE (local->cont.lookup.success_children);
+    ret = dict_set_str_sizen(xdata, GLUSTERFS_INODELK_DOM_COUNT, this->name);
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "Unable to set inodelk-dom-count in dict ");
+    }
 
-                GF_FREE (local->cont.lookup.sources);
+    if (local->fd) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->child_up[i] && fd_ctx->opened_on[i] == AFR_FD_OPENED)
+                wind_subvols[i] = 1;
         }
+    } else {
+        memcpy(wind_subvols, local->child_up,
+               sizeof(*local->child_up) * priv->child_count);
+    }
 
-        { /* getxattr */
-                GF_FREE (local->cont.getxattr.name);
-        }
+    local->call_count = AFR_COUNT(wind_subvols, priv->child_count);
 
-        { /* lk */
-                GF_FREE (local->cont.lk.locked_nodes);
-        }
+    call_count = local->call_count;
+    if (!call_count) {
+        dict_unref(xdata);
+        if (local->fd && AFR_COUNT(local->child_up, priv->child_count))
+            afr_inode_refresh_done(frame, this, EBADFD);
+        else
+            afr_inode_refresh_done(frame, this, ENOTCONN);
+        return 0;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!wind_subvols[i])
+            continue;
 
-        { /* create */
-                if (local->cont.create.fd)
-                        fd_unref (local->cont.create.fd);
-                if (local->cont.create.params)
-                        dict_unref (local->cont.create.params);
-        }
+        if (local->fd)
+            afr_inode_refresh_subvol_with_fstat(frame, this, i, xdata);
+        else
+            afr_inode_refresh_subvol_with_lookup(
+                frame, this, i, local->refreshinode, local->refreshgfid, xdata);
 
-        { /* mknod */
-                if (local->cont.mknod.params)
-                        dict_unref (local->cont.mknod.params);
-        }
+        if (!--call_count)
+            break;
+    }
 
-        { /* mkdir */
-                if (local->cont.mkdir.params)
-                        dict_unref (local->cont.mkdir.params);
-        }
+    dict_unref(xdata);
 
-        { /* symlink */
-                if (local->cont.symlink.params)
-                        dict_unref (local->cont.symlink.params);
-        }
+    return 0;
+}
 
-        { /* writev */
-                GF_FREE (local->cont.writev.vector);
-        }
+int
+afr_inode_refresh(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  uuid_t gfid, afr_inode_refresh_cbk_t refreshfn)
+{
+    afr_local_t *local = NULL;
 
-        { /* setxattr */
-                if (local->cont.setxattr.dict)
-                        dict_unref (local->cont.setxattr.dict);
-        }
+    local = frame->local;
 
-        { /* fsetxattr */
-                if (local->cont.fsetxattr.dict)
-                        dict_unref (local->cont.fsetxattr.dict);
-        }
+    local->refreshfn = refreshfn;
 
-        { /* removexattr */
-                GF_FREE (local->cont.removexattr.name);
-        }
-        { /* xattrop */
-                if (local->cont.xattrop.xattr)
-                        dict_unref (local->cont.xattrop.xattr);
-        }
-        { /* fxattrop */
-                if (local->cont.fxattrop.xattr)
-                        dict_unref (local->cont.fxattrop.xattr);
-        }
-        { /* symlink */
-                GF_FREE (local->cont.symlink.linkpath);
-        }
+    if (local->refreshinode) {
+        inode_unref(local->refreshinode);
+        local->refreshinode = NULL;
+    }
 
-        { /* opendir */
-                GF_FREE (local->cont.opendir.checksum);
-        }
+    local->refreshinode = inode_ref(inode);
 
-        { /* readdirp */
-                if (local->cont.readdir.dict)
-                        dict_unref (local->cont.readdir.dict);
-        }
+    if (gfid)
+        gf_uuid_copy(local->refreshgfid, gfid);
+    else
+        gf_uuid_clear(local->refreshgfid);
 
-        if (local->xdata_req)
-                dict_unref (local->xdata_req);
+    afr_inode_refresh_do(frame, this);
 
-        if (local->xdata_rsp)
-                dict_unref (local->xdata_rsp);
+    return 0;
 }
 
-
 int
-afr_frame_return (call_frame_t *frame)
+afr_xattr_req_prepare(xlator_t *this, dict_t *xattr_req)
 {
-        afr_local_t *local = NULL;
-        int          call_count = 0;
-
-        local = frame->local;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
 
-        LOCK (&frame->lock);
-        {
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
+    priv = this->private;
 
-        return call_count;
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_uint64(xattr_req, priv->pending_key[i],
+                              AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Unable to set dict value for %s", priv->pending_key[i]);
+        /* 3 = data+metadata+entry */
+    }
+    ret = dict_set_uint64(xattr_req, AFR_DIRTY,
+                          AFR_NUM_CHANGE_LOGS * sizeof(int));
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "failed to set dirty "
+                     "query flag");
+    }
+
+    ret = dict_set_int32_sizen(xattr_req, "list-xattr", 1);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set list-xattr in dict ");
+    }
+
+    return ret;
 }
 
 int
-afr_set_elem_count_get (unsigned char *elems, int child_count)
-{
-        int i   = 0;
-        int ret = 0;
-
-        for (i = 0; i < child_count; i++)
-                if (elems[i])
-                        ret++;
-        return ret;
+afr_lookup_xattr_req_prepare(afr_local_t *local, xlator_t *this,
+                             dict_t *xattr_req, loc_t *loc)
+{
+    int ret = -ENOMEM;
+
+    if (!local->xattr_req)
+        local->xattr_req = dict_new();
+
+    if (!local->xattr_req)
+        goto out;
+
+    if (xattr_req && (xattr_req != local->xattr_req))
+        dict_copy(xattr_req, local->xattr_req);
+
+    ret = afr_xattr_req_prepare(this, local->xattr_req);
+
+    ret = dict_set_uint64(local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_INODELK_COUNT);
+    }
+    ret = dict_set_uint64(local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_ENTRYLK_COUNT);
+    }
+
+    ret = dict_set_uint32(local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_PARENT_ENTRYLK);
+    }
+
+    ret = dict_set_sizen_str_sizen(local->xattr_req, "link-count",
+                                   GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
+
+    ret = 0;
+out:
+    return ret;
 }
 
-/**
- * up_children_count - return the number of children that are up
- */
-
-unsigned int
-afr_up_children_count (unsigned char *child_up, unsigned int child_count)
+int
+afr_least_pending_reads_child(afr_private_t *priv, unsigned char *readable)
 {
-        return afr_set_elem_count_get (child_up, child_count);
-}
+    int i = 0;
+    int child = -1;
+    int64_t read_iter = -1;
+    int64_t pending_read = -1;
 
-unsigned int
-afr_locked_children_count (unsigned char *children, unsigned int child_count)
-{
-        return afr_set_elem_count_get (children, child_count);
-}
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i])
+            continue;
+        read_iter = GF_ATOMIC_GET(priv->pending_reads[i]);
+        if (child == -1 || read_iter < pending_read) {
+            pending_read = read_iter;
+            child = i;
+        }
+    }
 
-unsigned int
-afr_pre_op_done_children_count (unsigned char *pre_op,
-                                unsigned int child_count)
-{
-        return afr_set_elem_count_get (pre_op, child_count);
+    return child;
 }
 
-gf_boolean_t
-afr_is_fresh_lookup (loc_t *loc, xlator_t *this)
+static int32_t
+afr_least_latency_child(afr_private_t *priv, unsigned char *readable)
 {
-        uint64_t          ctx = 0;
-        int32_t           ret = 0;
+    int32_t i = 0;
+    int child = -1;
 
-        GF_ASSERT (loc);
-        GF_ASSERT (this);
-        GF_ASSERT (loc->inode);
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
+            priv->child_latency[i] < 0)
+            continue;
 
-        ret = inode_ctx_get (loc->inode, this, &ctx);
-        if (0 == ret)
-                return _gf_false;
-        return _gf_true;
+        if (child == -1 ||
+            priv->child_latency[i] < priv->child_latency[child]) {
+            child = i;
+        }
+    }
+    return child;
 }
 
-void
-afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
+static int32_t
+afr_least_latency_times_pending_reads_child(afr_private_t *priv,
+                                            unsigned char *readable)
 {
-        GF_ASSERT (loc);
-        GF_ASSERT (buf);
+    int32_t i = 0;
+    int child = -1;
+    int64_t pending_read = 0;
+    int64_t latency = -1;
+    int64_t least_latency = -1;
 
-        uuid_copy (loc->gfid, buf->ia_gfid);
-        if (postparent)
-                uuid_copy (loc->pargfid, postparent->ia_gfid);
-}
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
+            priv->child_latency[i] < 0)
+            continue;
 
-int
-afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
-{
-        struct iatt     *buf = NULL;
-        struct iatt     *postparent = NULL;
-        dict_t          **xattr = NULL;
-        int32_t         *success_children = NULL;
-        int32_t         *sources = NULL;
-        afr_private_t   *priv = NULL;
-        int32_t         read_child = -1;
-        int             ret = 0;
-        int             i = 0;
-
-        GF_ASSERT (local);
-
-        buf = &local->cont.lookup.buf;
-        postparent = &local->cont.lookup.postparent;
-        xattr = &local->cont.lookup.xattr;
-        priv = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->cont.lookup.inode,
-                                             local->fresh_children);
-        if (read_child < 0) {
-                ret = -1;
-                goto out;
-        }
-        success_children = local->cont.lookup.success_children;
-        sources = local->cont.lookup.sources;
-        memset (sources, 0, sizeof (*sources) * priv->child_count);
-        afr_children_intersection_get (local->fresh_children, success_children,
-                                       sources, priv->child_count);
-        if (!sources[read_child]) {
-                read_child = -1;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (sources[i]) {
-                                read_child = i;
-                                break;
-                        }
-                }
-        }
-        if (read_child < 0) {
-                ret = -1;
-                goto out;
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "Building lookup response from %d",
-                read_child);
-        if (!*xattr)
-                *xattr = dict_ref (local->cont.lookup.xattrs[read_child]);
-        *buf = local->cont.lookup.bufs[read_child];
-        *postparent = local->cont.lookup.postparents[read_child];
+        pending_read = GF_ATOMIC_GET(priv->pending_reads[i]);
+        latency = (pending_read + 1) * priv->child_latency[i];
 
-        if (IA_INVAL == local->cont.lookup.inode->ia_type) {
-                /* fix for RT #602 */
-                local->cont.lookup.inode->ia_type = buf->ia_type;
+        if (child == -1 || latency < least_latency) {
+            least_latency = latency;
+            child = i;
         }
-out:
-        return ret;
+    }
+    return child;
 }
 
-static void
-afr_lookup_update_lk_counts (afr_local_t *local, xlator_t *this,
-                            int child_index, dict_t *xattr)
-{
-        uint32_t inodelk_count = 0;
-        uint32_t entrylk_count = 0;
-        int      ret           = -1;
-        uint32_t parent_entrylk = 0;
-
-        GF_ASSERT (local);
-        GF_ASSERT (this);
-        GF_ASSERT (xattr);
-        GF_ASSERT (child_index >= 0);
-
-        ret = dict_get_uint32 (xattr, GLUSTERFS_INODELK_COUNT,
-                               &inodelk_count);
-        if (ret == 0)
-                local->inodelk_count += inodelk_count;
-
-        ret = dict_get_uint32 (xattr, GLUSTERFS_ENTRYLK_COUNT,
-                               &entrylk_count);
-        if (ret == 0)
-                local->entrylk_count += entrylk_count;
-        ret = dict_get_uint32 (xattr, GLUSTERFS_PARENT_ENTRYLK,
-                               &parent_entrylk);
-        if (!ret)
-                local->cont.lookup.parent_entrylk += parent_entrylk;
+int
+afr_hash_child(afr_read_subvol_args_t *args, afr_private_t *priv,
+               unsigned char *readable)
+{
+    uuid_t gfid_copy = {
+        0,
+    };
+    pid_t pid;
+    int child = -1;
+
+    switch (priv->hash_mode) {
+        case AFR_READ_POLICY_FIRST_UP:
+            break;
+        case AFR_READ_POLICY_GFID_HASH:
+            gf_uuid_copy(gfid_copy, args->gfid);
+            child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
+                    priv->child_count;
+            break;
+        case AFR_READ_POLICY_GFID_PID_HASH:
+            if (args->ia_type != IA_IFDIR) {
+                /*
+                 * Why getpid?  Because it's one of the cheapest calls
+                 * available - faster than gethostname etc. - and
+                 * returns a constant-length value that's sure to be
+                 * shorter than a UUID. It's still very unlikely to be
+                 * the same across clients, so it still provides good
+                 * mixing.  We're not trying for perfection here. All we
+                 * need is a low probability that multiple clients
+                 * won't converge on the same subvolume.
+                 */
+                gf_uuid_copy(gfid_copy, args->gfid);
+                pid = getpid();
+                *(pid_t *)gfid_copy ^= pid;
+            }
+            child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
+                    priv->child_count;
+            break;
+        case AFR_READ_POLICY_LESS_LOAD:
+            child = afr_least_pending_reads_child(priv, readable);
+            break;
+        case AFR_READ_POLICY_LEAST_LATENCY:
+            child = afr_least_latency_child(priv, readable);
+            break;
+        case AFR_READ_POLICY_LOAD_LATENCY_HYBRID:
+            child = afr_least_latency_times_pending_reads_child(priv, readable);
+            break;
+    }
+
+    return child;
 }
 
-static void
-afr_lookup_set_self_heal_params_by_xattr (afr_local_t *local, xlator_t *this,
-                                          dict_t *xattr)
+int
+afr_read_subvol_select_by_policy(inode_t *inode, xlator_t *this,
+                                 unsigned char *readable,
+                                 afr_read_subvol_args_t *args)
 {
-        GF_ASSERT (local);
-        GF_ASSERT (this);
-        GF_ASSERT (xattr);
+    int i = 0;
+    int read_subvol = -1;
+    afr_private_t *priv = NULL;
+    afr_read_subvol_args_t local_args = {
+        0,
+    };
 
-        if (afr_sh_has_metadata_pending (xattr, this)) {
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "metadata self-heal is pending for %s.",
-                       local->loc.path);
-        }
+    priv = this->private;
 
-        if (afr_sh_has_entry_pending (xattr, this)) {
-                local->self_heal.do_entry_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "entry self-heal is pending for %s.", local->loc.path);
-        }
+    /* first preference - explicitly specified or local subvolume */
+    if (priv->read_child >= 0 && readable[priv->read_child])
+        return priv->read_child;
 
-        if (afr_sh_has_data_pending (xattr, this)) {
-                local->self_heal.do_data_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "data self-heal is pending for %s.", local->loc.path);
-        }
-}
+    if (inode_is_linked(inode)) {
+        gf_uuid_copy(local_args.gfid, inode->gfid);
+        local_args.ia_type = inode->ia_type;
+    } else if (args) {
+        local_args = *args;
+    }
 
-static void
-afr_detect_self_heal_by_iatt (afr_local_t *local, xlator_t *this,
-                            struct iatt *buf, struct iatt *lookup_buf)
-{
-        if (PERMISSION_DIFFERS (buf, lookup_buf)) {
-                /* mismatching permissions */
-                gf_log (this->name, GF_LOG_INFO,
-                        "permissions differ for %s ", local->loc.path);
-                local->self_heal.do_metadata_self_heal = _gf_true;
-        }
+    /* second preference - use hashed mode */
+    read_subvol = afr_hash_child(&local_args, priv, readable);
+    if (read_subvol >= 0 && readable[read_subvol])
+        return read_subvol;
 
-        if (OWNERSHIP_DIFFERS (buf, lookup_buf)) {
-                /* mismatching permissions */
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                gf_log (this->name, GF_LOG_INFO,
-                        "ownership differs for %s ", local->loc.path);
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (readable[i])
+            return i;
+    }
 
-        if (SIZE_DIFFERS (buf, lookup_buf)
-            && IA_ISREG (buf->ia_type)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "size differs for %s ", local->loc.path);
-                local->self_heal.do_data_self_heal = _gf_true;
-        }
+    /* no readable subvolumes, either split brain or all subvols down */
 
-        if (uuid_compare (buf->ia_gfid, lookup_buf->ia_gfid)) {
-                /* mismatching gfid */
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: gfid different on subvolume", local->loc.path);
-        }
+    return -1;
 }
 
-static void
-afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this,
-                                       gf_boolean_t split_brain)
-{
-        GF_ASSERT (local);
-        GF_ASSERT (this);
-
-        if ((local->success_count > 0) && (local->enoent_count > 0)) {
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                local->self_heal.do_data_self_heal     = _gf_true;
-                local->self_heal.do_entry_self_heal    = _gf_true;
-                local->self_heal.do_gfid_self_heal    = _gf_true;
-                local->self_heal.do_missing_entry_self_heal    = _gf_true;
-                gf_log(this->name, GF_LOG_INFO,
-                       "entries are missing in lookup of %s.",
-                       local->loc.path);
-                //If all self-heals are needed no need to check for other rules
-                goto out;
-        }
-
-        if ((local->success_count > 0) && split_brain &&
-            IA_ISREG (local->cont.lookup.inode->ia_type)) {
-                local->self_heal.do_data_self_heal = _gf_true;
-                local->self_heal.do_gfid_self_heal    = _gf_true;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "split brain detected during lookup of %s.",
-                        local->loc.path);
-        }
+int
+afr_inode_read_subvol_type_get(inode_t *inode, xlator_t *this,
+                               unsigned char *readable, int *event_p, int type)
+{
+    int ret = -1;
 
-out:
-        return;
+    if (type == AFR_METADATA_TRANSACTION)
+        ret = afr_inode_read_subvol_get(inode, this, 0, readable, event_p);
+    else
+        ret = afr_inode_read_subvol_get(inode, this, readable, 0, event_p);
+    return ret;
 }
 
-gf_boolean_t
-afr_can_self_heal_proceed (afr_self_heal_t *sh, afr_private_t *priv)
+void
+afr_readables_intersect_get(inode_t *inode, xlator_t *this, int *event,
+                            unsigned char *intersection)
 {
-        GF_ASSERT (sh);
-        GF_ASSERT (priv);
+    afr_private_t *priv = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    unsigned char *intersect = NULL;
+
+    priv = this->private;
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+    intersect = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(inode, this, data_readable, metadata_readable,
+                              event);
 
-        return (sh->do_gfid_self_heal
-                || sh->do_missing_entry_self_heal
-                || (afr_data_self_heal_enabled (priv->data_self_heal) &&
-                    sh->do_data_self_heal)
-                || (priv->metadata_self_heal && sh->do_metadata_self_heal)
-                || (priv->entry_self_heal && sh->do_entry_self_heal));
+    AFR_INTERSECT(intersect, data_readable, metadata_readable,
+                  priv->child_count);
+    if (intersection)
+        memcpy(intersection, intersect,
+               sizeof(*intersection) * priv->child_count);
 }
 
-afr_transaction_type
-afr_transaction_type_get (ia_type_t ia_type)
+int
+afr_read_subvol_get(inode_t *inode, xlator_t *this, int *subvol_p,
+                    unsigned char *readables, int *event_p,
+                    afr_transaction_type type, afr_read_subvol_args_t *args)
 {
-        afr_transaction_type    type = AFR_METADATA_TRANSACTION;
+    afr_private_t *priv = NULL;
+    unsigned char *readable = NULL;
+    unsigned char *intersection = NULL;
+    int subvol = -1;
+    int event = 0;
 
-        GF_ASSERT (ia_type != IA_INVAL);
+    priv = this->private;
 
-        if (IA_ISDIR (ia_type)) {
-                type = AFR_ENTRY_TRANSACTION;
-        } else if (IA_ISREG (ia_type)) {
-                type = AFR_DATA_TRANSACTION;
-        }
-        return type;
-}
+    readable = alloca0(priv->child_count);
+    intersection = alloca0(priv->child_count);
 
-int
-afr_lookup_select_read_child (afr_local_t *local, xlator_t *this,
-                              int32_t *read_child)
-{
-        ia_type_t               ia_type        = IA_INVAL;
-        int32_t                 source         = -1;
-        int                     ret            = -1;
-        dict_t                  **xattrs       = NULL;
-        int32_t                 *success_children = NULL;
-        afr_transaction_type    type           = AFR_METADATA_TRANSACTION;
-        uuid_t                  *gfid          = NULL;
-
-        GF_ASSERT (local);
-        GF_ASSERT (this);
-        GF_ASSERT (local->success_count > 0);
-
-        success_children = local->cont.lookup.success_children;
-        /*We can take the success_children[0] only because we already
-         *handle the conflicting children other wise, we could select the
-         *read_child based on wrong file type
-         */
-        ia_type = local->cont.lookup.bufs[success_children[0]].ia_type;
-        type = afr_transaction_type_get (ia_type);
-        xattrs = local->cont.lookup.xattrs;
-        gfid = &local->cont.lookup.buf.ia_gfid;
-        source = afr_lookup_select_read_child_by_txn_type (this, local, xattrs,
-                                                           type, *gfid);
-        if (source < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "failed to select source "
-                        "for %s", local->loc.path);
-                goto out;
-        }
+    afr_inode_read_subvol_type_get(inode, this, readable, &event, type);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Source selected as %d for %s",
-                source, local->loc.path);
-        *read_child = source;
-        ret = 0;
-out:
-        return ret;
-}
+    afr_readables_intersect_get(inode, this, &event, intersection);
 
-static inline gf_boolean_t
-afr_is_transaction_running (afr_local_t *local)
-{
-        GF_ASSERT (local->fop == GF_FOP_LOOKUP);
-        return ((local->inodelk_count > 0) || (local->entrylk_count > 0));
+    if (AFR_COUNT(intersection, priv->child_count) > 0)
+        subvol = afr_read_subvol_select_by_policy(inode, this, intersection,
+                                                  args);
+    else
+        subvol = afr_read_subvol_select_by_policy(inode, this, readable, args);
+    if (subvol_p)
+        *subvol_p = subvol;
+    if (event_p)
+        *event_p = event;
+    if (readables)
+        memcpy(readables, readable, sizeof(*readables) * priv->child_count);
+    return subvol;
 }
 
 void
-afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                      gf_boolean_t background, ia_type_t ia_type, char *reason,
-                      void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
-                                                   xlator_t *this),
-                      int (*unwind) (call_frame_t *frame, xlator_t *this,
-                                     int32_t op_ret, int32_t op_errno,
-                                     int32_t sh_failed))
-{
-        afr_local_t             *local = NULL;
-        char                    sh_type_str[256] = {0,};
-        char                    *bg = "";
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-        GF_ASSERT (ia_type != IA_INVAL);
+afr_local_transaction_cleanup(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-        local = frame->local;
-        local->self_heal.background = background;
-        local->self_heal.type       = ia_type;
-        local->self_heal.unwind     = unwind;
-        local->self_heal.gfid_sh_success_cbk     = gfid_sh_success_cbk;
-
-        afr_self_heal_type_str_get (&local->self_heal,
-                                    sh_type_str,
-                                    sizeof (sh_type_str));
-
-        if (background)
-                bg = "background";
-        gf_log (this->name, GF_LOG_INFO,
-                "%s %s self-heal triggered. path: %s, reason: %s", bg,
-                sh_type_str, local->loc.path, reason);
-
-        afr_self_heal (frame, this, inode);
-}
-
-unsigned int
-afr_gfid_missing_count (const char *xlator_name, int32_t *success_children,
-                        struct iatt *bufs, unsigned int child_count,
-                        const char *path)
-{
-        unsigned int    gfid_miss_count   = 0;
-        int             i              = 0;
-        struct iatt     *child1        = NULL;
-
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                child1 = &bufs[success_children[i]];
-                if (uuid_is_null (child1->ia_gfid)) {
-                        gf_log (xlator_name, GF_LOG_DEBUG, "%s: gfid is null"
-                                " on subvolume %d", path, success_children[i]);
-                        gfid_miss_count++;
-                }
-        }
+    priv = this->private;
 
-        return gfid_miss_count;
-}
+    afr_matrix_cleanup(local->pending, priv->child_count);
 
-static int
-afr_lookup_gfid_missing_count (afr_local_t *local, xlator_t *this)
-{
-        int32_t         *success_children = NULL;
-        afr_private_t   *priv          = NULL;
-        struct iatt     *bufs          = NULL;
-        int             miss_count     = 0;
+    GF_FREE(local->internal_lock.lower_locked_nodes);
 
-        priv = this->private;
-        bufs = local->cont.lookup.bufs;
-        success_children = local->cont.lookup.success_children;
+    afr_lockees_cleanup(&local->internal_lock);
 
-        miss_count =  afr_gfid_missing_count (this->name, success_children,
-                                              bufs, priv->child_count,
-                                              local->loc.path);
-        return miss_count;
-}
+    GF_FREE(local->transaction.pre_op);
 
-gf_boolean_t
-afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
-                        unsigned int child_count, const char *path,
-                        const char *xlator_name)
-{
-        gf_boolean_t    conflicting    = _gf_false;
-        int             i              = 0;
-        struct iatt     *child1        = NULL;
-        struct iatt     *child2        = NULL;
-        uuid_t          *gfid          = NULL;
-
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                child1 = &bufs[success_children[i]];
-                if ((!gfid) && (!uuid_is_null (child1->ia_gfid)))
-                        gfid = &child1->ia_gfid;
-
-                if (i == 0)
-                        continue;
-
-                child2 = &bufs[success_children[i-1]];
-                if (FILETYPE_DIFFERS (child1, child2)) {
-                        gf_log (xlator_name, GF_LOG_WARNING, "%s: filetype "
-                                "differs on subvolumes (%d, %d)", path,
-                                success_children[i-1], success_children[i]);
-                        conflicting = _gf_true;
-                        goto out;
-                }
-                if (!gfid || uuid_is_null (child1->ia_gfid))
-                        continue;
-                if (uuid_compare (*gfid, child1->ia_gfid)) {
-                       gf_log (xlator_name, GF_LOG_WARNING, "%s: gfid differs"
-                               " on subvolume %d", path, success_children[i]);
-                       conflicting = _gf_true;
-                       goto out;
-                }
+    GF_FREE(local->transaction.pre_op_sources);
+    if (local->transaction.changelog_xdata) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->transaction.changelog_xdata[i])
+                continue;
+            dict_unref(local->transaction.changelog_xdata[i]);
         }
-out:
-        return conflicting;
+        GF_FREE(local->transaction.changelog_xdata);
+    }
+
+    GF_FREE(local->transaction.failed_subvols);
+
+    GF_FREE(local->transaction.basename);
+    GF_FREE(local->transaction.new_basename);
+
+    loc_wipe(&local->transaction.parent_loc);
+    loc_wipe(&local->transaction.new_parent_loc);
 }
 
-/* afr_update_gfid_from_iatts: This function should be called only if the
- * iatts are not conflicting.
- */
 void
-afr_update_gfid_from_iatts (uuid_t uuid, struct iatt *bufs,
-                            int32_t *success_children, unsigned int child_count)
-{
-        uuid_t          *gfid = NULL;
-        int             i = 0;
-        int             child = 0;
-
-        for (i = 0; i < child_count; i++) {
-                child = success_children[i];
-                if (child == -1)
-                        break;
-                if ((!gfid) && (!uuid_is_null (bufs[child].ia_gfid))) {
-                        gfid = &bufs[child].ia_gfid;
-                } else if (gfid && (!uuid_is_null (bufs[child].ia_gfid))) {
-                        if (uuid_compare (*gfid, bufs[child].ia_gfid)) {
-                                GF_ASSERT (0);
-                                goto out;
-                        }
-                }
-        }
-        if (gfid && (!uuid_is_null (*gfid)))
-                uuid_copy (uuid, *gfid);
-out:
-        return;
+afr_reply_wipe(struct afr_reply *reply)
+{
+    if (reply->xdata) {
+        dict_unref(reply->xdata);
+        reply->xdata = NULL;
+    }
+
+    if (reply->xattr) {
+        dict_unref(reply->xattr);
+        reply->xattr = NULL;
+    }
 }
 
-static gf_boolean_t
-afr_lookup_conflicting_entries (afr_local_t *local, xlator_t *this)
+void
+afr_replies_wipe(struct afr_reply *replies, int count)
 {
-        afr_private_t           *priv = NULL;
-        gf_boolean_t            conflict = _gf_false;
+    int i = 0;
 
-        priv = this->private;
-        conflict =  afr_conflicting_iattrs (local->cont.lookup.bufs,
-                                            local->cont.lookup.success_children,
-                                            priv->child_count, local->loc.path,
-                                            this->name);
-        return conflict;
+    for (i = 0; i < count; i++) {
+        afr_reply_wipe(&replies[i]);
+    }
 }
 
-gf_boolean_t
-afr_open_only_data_self_heal (char *data_self_heal)
+void
+afr_local_replies_wipe(afr_local_t *local, afr_private_t *priv)
 {
-        return !strcmp (data_self_heal, "open");
+    if (!local->replies)
+        return;
+
+    afr_replies_wipe(local->replies, priv->child_count);
+
+    memset(local->replies, 0, sizeof(*local->replies) * priv->child_count);
 }
 
-gf_boolean_t
-afr_data_self_heal_enabled (char *data_self_heal)
+static gf_boolean_t
+afr_fop_lock_is_unlock(call_frame_t *frame)
+{
+    afr_local_t *local = frame->local;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            if ((F_UNLCK == local->cont.inodelk.in_flock.l_type) &&
+                (local->cont.inodelk.in_cmd == F_SETLKW ||
+                 local->cont.inodelk.in_cmd == F_SETLK))
+                return _gf_true;
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            if (ENTRYLK_UNLOCK == local->cont.entrylk.in_cmd)
+                return _gf_true;
+            break;
+        default:
+            return _gf_false;
+    }
+    return _gf_false;
+}
+
+static gf_boolean_t
+afr_lk_is_unlock(int32_t cmd, struct gf_flock *flock)
 {
-        gf_boolean_t    enabled = _gf_false;
+    switch (cmd) {
+        case F_RESLK_UNLCK:
+            return _gf_true;
+            break;
 
-        if (gf_string2boolean (data_self_heal, &enabled) == -1) {
-                enabled = !strcmp (data_self_heal, "open");
-                GF_ASSERT (enabled);
-        }
+#if F_SETLKW != F_SETLKW64
+        case F_SETLKW64:
+#endif
+        case F_SETLKW:
 
-        return enabled;
+#if F_SETLK != F_SETLK64
+        case F_SETLK64:
+#endif
+        case F_SETLK:
+            if (F_UNLCK == flock->l_type)
+                return _gf_true;
+            break;
+        default:
+            return _gf_false;
+    }
+    return _gf_false;
 }
 
-static void
-afr_lookup_set_self_heal_params (afr_local_t *local, xlator_t *this)
+void
+afr_handle_inconsistent_fop(call_frame_t *frame, int32_t *op_ret,
+                            int32_t *op_errno)
 {
-        int                     i = 0;
-        struct iatt             *bufs = NULL;
-        dict_t                  **xattr = NULL;
-        afr_private_t           *priv = NULL;
-        int32_t                 child1 = -1;
-        int32_t                 child2 = -1;
-        afr_self_heal_t         *sh = NULL;
-        gf_boolean_t            split_brain = _gf_false;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        priv  = this->private;
-        sh = &local->self_heal;
+    if (!frame || !frame->this || !frame->local || !frame->this->private)
+        return;
 
-        split_brain = afr_is_split_brain (this, local->cont.lookup.inode);
-        afr_detect_self_heal_by_lookup_status (local, this, split_brain);
+    if (*op_ret < 0)
+        return;
 
-        if (afr_lookup_gfid_missing_count (local, this))
-                local->self_heal.do_gfid_self_heal    = _gf_true;
+    /* Failing inodelk/entrylk/lk here is not a good idea because we
+     * need to cleanup the locks on the other bricks if we choose to fail
+     * the fop here. The brick may go down just after unwind happens as well
+     * so anyways the fop will fail when the next fop is sent so leaving
+     * it like this for now.*/
+    local = frame->local;
+    switch (local->op) {
+        case GF_FOP_LOOKUP:
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+        case GF_FOP_LK:
+            return;
+        default:
+            break;
+    }
 
-        if (_gf_true == afr_lookup_conflicting_entries (local, this))
-                local->self_heal.do_missing_entry_self_heal    = _gf_true;
-        else
-                afr_update_gfid_from_iatts (local->self_heal.sh_gfid_req,
-                                            local->cont.lookup.bufs,
-                                            local->cont.lookup.success_children,
-                                            priv->child_count);
+    priv = frame->this->private;
+    if (!priv->consistent_io)
+        return;
 
-        bufs = local->cont.lookup.bufs;
-        for (i = 1; i < local->success_count; i++) {
-                child1 = local->cont.lookup.success_children[i-1];
-                child2 = local->cont.lookup.success_children[i];
-                afr_detect_self_heal_by_iatt (local, this,
-                                              &bufs[child1], &bufs[child2]);
-        }
+    if (local->event_generation &&
+        (local->event_generation != priv->event_generation))
+        goto inconsistent;
 
-        xattr = local->cont.lookup.xattrs;
-        for (i = 0; i < local->success_count; i++) {
-                child1 = local->cont.lookup.success_children[i];
-                afr_lookup_set_self_heal_params_by_xattr (local, this,
-                                                          xattr[child1]);
-        }
-        if (afr_open_only_data_self_heal (priv->data_self_heal)
-            && !split_brain)
-                sh->do_data_self_heal = _gf_false;
+    return;
+inconsistent:
+    *op_ret = -1;
+    *op_errno = ENOTCONN;
 }
 
-int
-afr_self_heal_lookup_unwind (call_frame_t *frame, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             int32_t sh_failed)
+void
+afr_local_cleanup(afr_local_t *local, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        int         ret    = -1;
-        dict_t      *xattr = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
+    if (!local)
+        return;
 
-        if (op_ret == -1) {
-                local->op_ret = -1;
-                if (afr_error_more_important (local->op_errno, op_errno))
-                        local->op_errno = op_errno;
+    syncbarrier_destroy(&local->barrier);
 
-                goto out;
-        } else {
-                local->op_ret = 0;
-        }
+    afr_local_transaction_cleanup(local, this);
 
-        afr_lookup_done_success_action (frame, this, _gf_true);
-        xattr = local->cont.lookup.xattr;
-        if (xattr) {
-                ret = dict_set_int32 (xattr, "sh-failed", sh_failed);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
-                                "sh-failed to %d", local->loc.path, sh_failed);
-        }
-out:
-        AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->cont.lookup.inode, &local->cont.lookup.buf,
-                          local->cont.lookup.xattr,
-                          &local->cont.lookup.postparent);
+    priv = this->private;
 
-        return 0;
-}
+    loc_wipe(&local->loc);
+    loc_wipe(&local->newloc);
 
-//TODO: At the moment only lookup needs this, so not doing any checks, in the
-// future we will have to do fop specific operations
-void
-afr_post_gfid_sh_success (call_frame_t *sh_frame, xlator_t *this)
-{
-        afr_local_t             *local = NULL;
-        afr_local_t             *sh_local = NULL;
-        afr_private_t           *priv = NULL;
-        afr_self_heal_t         *sh = NULL;
-        int                     i = 0;
-        struct iatt             *lookup_bufs = NULL;
-        struct iatt             *lookup_parentbufs = NULL;
+    if (local->fd)
+        fd_unref(local->fd);
 
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        local = sh->orig_frame->local;
-        lookup_bufs = local->cont.lookup.bufs;
-        lookup_parentbufs = local->cont.lookup.postparents;
-        priv = this->private;
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
 
-        memcpy (lookup_bufs, sh->buf, priv->child_count * sizeof (*sh->buf));
-        memcpy (lookup_parentbufs, sh->parentbufs,
-                priv->child_count * sizeof (*sh->parentbufs));
+    if (local->xattr_rsp)
+        dict_unref(local->xattr_rsp);
 
-        afr_reset_xattr (local->cont.lookup.xattrs, priv->child_count);
-        if (local->cont.lookup.xattr) {
-                dict_unref (local->cont.lookup.xattr);
-                local->cont.lookup.xattr = NULL;
-        }
+    if (local->dict)
+        dict_unref(local->dict);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i])
-                        local->cont.lookup.xattrs[i] = dict_ref (sh->xattr[i]);
-        }
+    afr_local_replies_wipe(local, priv);
+    GF_FREE(local->replies);
 
-        afr_reset_children (local->cont.lookup.success_children,
-                            priv->child_count);
-        afr_children_copy (local->cont.lookup.success_children,
-                           sh->fresh_children, priv->child_count);
-}
+    GF_FREE(local->child_up);
 
-static void
-afr_lookup_perform_self_heal (call_frame_t *frame, xlator_t *this,
-                              gf_boolean_t *sh_launched)
-{
-        unsigned int         up_count = 0;
-        afr_private_t       *priv    = NULL;
-        afr_local_t         *local   = NULL;
-        char                *reason  = NULL;
-
-        GF_ASSERT (sh_launched);
-        *sh_launched = _gf_false;
-        priv         = this->private;
-        local        = frame->local;
-
-        up_count  = afr_up_children_count (local->child_up, priv->child_count);
-        if (up_count == 1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Only 1 child up - do not attempt to detect self heal");
-                goto out;
-        }
+    GF_FREE(local->read_attempted);
+
+    GF_FREE(local->readable);
+    GF_FREE(local->readable2);
+
+    if (local->inode)
+        inode_unref(local->inode);
 
-        afr_lookup_set_self_heal_params (local, this);
-        if (afr_can_self_heal_proceed (&local->self_heal, priv)) {
-                if  (afr_is_transaction_running (local))
-                        goto out;
+    if (local->parent)
+        inode_unref(local->parent);
 
-                reason = "lookup detected pending operations";
-                afr_launch_self_heal (frame, this, local->cont.lookup.inode,
-                                      _gf_true, local->cont.lookup.buf.ia_type,
-                                      reason, afr_post_gfid_sh_success,
-                                      afr_self_heal_lookup_unwind);
-                *sh_launched = _gf_true;
-        }
-out:
-        return;
-}
+    if (local->parent2)
+        inode_unref(local->parent2);
 
-void
-afr_get_fresh_children (int32_t *success_children, int32_t *sources,
-                        int32_t *fresh_children, unsigned int child_count)
-{
-        unsigned int i = 0;
-        unsigned int j = 0;
-
-        GF_ASSERT (success_children);
-        GF_ASSERT (sources);
-        GF_ASSERT (fresh_children);
-
-        afr_reset_children (fresh_children, child_count);
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                if (afr_is_read_child (success_children, sources, child_count,
-                                       success_children[i])) {
-                        fresh_children[j] = success_children[i];
-                        j++;
-                }
-        }
-}
+    if (local->refreshinode)
+        inode_unref(local->refreshinode);
 
-static int
-afr_lookup_set_read_ctx (afr_local_t *local, xlator_t *this, int32_t read_child)
-{
-        afr_private_t           *priv = NULL;
+    { /* getxattr */
+        GF_FREE(local->cont.getxattr.name);
+    }
 
-        GF_ASSERT (read_child >= 0);
+    { /* lk */
+        GF_FREE(local->cont.lk.locked_nodes);
+        GF_FREE(local->cont.lk.dom_locked_nodes);
+        GF_FREE(local->cont.lk.dom_lock_op_ret);
+        GF_FREE(local->cont.lk.dom_lock_op_errno);
+    }
 
-        priv = this->private;
-        afr_get_fresh_children (local->cont.lookup.success_children,
-                                local->cont.lookup.sources,
-                                local->fresh_children, priv->child_count);
-        afr_inode_set_read_ctx (this, local->cont.lookup.inode, read_child,
-                                local->fresh_children);
+    { /* create */
+        if (local->cont.create.fd)
+            fd_unref(local->cont.create.fd);
+        if (local->cont.create.params)
+            dict_unref(local->cont.create.params);
+    }
 
-        return 0;
+    { /* mknod */
+        if (local->cont.mknod.params)
+            dict_unref(local->cont.mknod.params);
+    }
+
+    { /* mkdir */
+        if (local->cont.mkdir.params)
+            dict_unref(local->cont.mkdir.params);
+    }
+
+    { /* symlink */
+        if (local->cont.symlink.params)
+            dict_unref(local->cont.symlink.params);
+    }
+
+    { /* writev */
+        GF_FREE(local->cont.writev.vector);
+        if (local->cont.writev.iobref)
+            iobref_unref(local->cont.writev.iobref);
+    }
+
+    { /* setxattr */
+        if (local->cont.setxattr.dict)
+            dict_unref(local->cont.setxattr.dict);
+    }
+
+    { /* fsetxattr */
+        if (local->cont.fsetxattr.dict)
+            dict_unref(local->cont.fsetxattr.dict);
+    }
+
+    { /* removexattr */
+        GF_FREE(local->cont.removexattr.name);
+    }
+    { /* xattrop */
+        if (local->cont.xattrop.xattr)
+            dict_unref(local->cont.xattrop.xattr);
+    }
+    { /* symlink */
+        GF_FREE(local->cont.symlink.linkpath);
+    }
+
+    { /* opendir */
+        GF_FREE(local->cont.opendir.checksum);
+    }
+
+    { /* open */
+        if (local->cont.open.fd)
+            fd_unref(local->cont.open.fd);
+    }
+
+    { /* readdirp */
+        if (local->cont.readdir.dict)
+            dict_unref(local->cont.readdir.dict);
+    }
+
+    { /* inodelk */
+        GF_FREE(local->cont.inodelk.volume);
+        if (local->cont.inodelk.xdata)
+            dict_unref(local->cont.inodelk.xdata);
+    }
+
+    { /* entrylk */
+        GF_FREE(local->cont.entrylk.volume);
+        GF_FREE(local->cont.entrylk.basename);
+        if (local->cont.entrylk.xdata)
+            dict_unref(local->cont.entrylk.xdata);
+    }
+
+    if (local->xdata_req)
+        dict_unref(local->xdata_req);
+
+    if (local->xdata_rsp)
+        dict_unref(local->xdata_rsp);
 }
 
 int
-afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
-                                gf_boolean_t fail_conflict)
+afr_frame_return(call_frame_t *frame)
 {
-        int32_t             read_child = -1;
-        int32_t             ret        = -1;
-        afr_local_t         *local     = NULL;
-        gf_boolean_t        fresh_lookup = _gf_false;
+    afr_local_t *local = NULL;
+    int call_count = 0;
 
-        local   = frame->local;
-        fresh_lookup = local->cont.lookup.fresh_lookup;
+    local = frame->local;
 
-        if (local->loc.parent == NULL)
-                fail_conflict = _gf_true;
-
-        if (afr_lookup_conflicting_entries (local, this)) {
-                if (fail_conflict == _gf_false)
-                        ret = 0;
-                goto out;
-        }
+    LOCK(&frame->lock);
+    {
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        ret = afr_lookup_select_read_child (local, this, &read_child);
-        if (!afr_is_transaction_running (local) || fresh_lookup) {
-                if (read_child < 0)
-                        goto out;
+    return call_count;
+}
 
-                ret = afr_lookup_set_read_ctx (local, this, read_child);
-                if (ret)
-                        goto out;
-        }
+static char *afr_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY, NULL};
 
-        ret = afr_lookup_build_response_params (local, this);
-        if (ret)
-                goto out;
-        afr_update_loc_gfids (&local->loc,
-                              &local->cont.lookup.buf,
-                              &local->cont.lookup.postparent);
+gf_boolean_t
+afr_is_xattr_ignorable(char *key)
+{
+    int i = 0;
 
-        ret = 0;
-out:
-        if (ret) {
-                local->op_ret = -1;
-                local->op_errno = EIO;
-        }
-        return ret;
+    if (!strncmp(key, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX)))
+        return _gf_true;
+    for (i = 0; afr_ignore_xattrs[i]; i++) {
+        if (!strcmp(key, afr_ignore_xattrs[i]))
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
-int
-afr_lookup_get_latest_subvol (afr_local_t *local, xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        int32_t       *success_children = NULL;
-        struct iatt   *bufs = NULL;
-        int           i = 0;
-        int           child = 0;
-        int           lsubvol = -1;
-
-        priv = this->private;
-        success_children = local->cont.lookup.success_children;
-        bufs = local->cont.lookup.bufs;
-        for (i = 0; i < priv->child_count; i++) {
-                child = success_children[i];
-                if (child == -1)
-                        break;
-                if (uuid_is_null (bufs[child].ia_gfid))
-                        continue;
-                if (lsubvol < 0) {
-                        lsubvol = child;
-                } else if (bufs[lsubvol].ia_ctime < bufs[child].ia_ctime) {
-                        lsubvol = child;
-                } else if ((bufs[lsubvol].ia_ctime == bufs[child].ia_ctime) &&
-                  (bufs[lsubvol].ia_ctime_nsec < bufs[child].ia_ctime_nsec)) {
-                        lsubvol = child;
-                }
-        }
-        return lsubvol;
+static gf_boolean_t
+afr_xattr_match_needed(dict_t *this, char *key1, data_t *value1, void *data)
+{
+    /* Ignore all non-disk (i.e. virtual) xattrs right away. */
+    if (!gf_is_valid_xattr_namespace(key1))
+        return _gf_false;
+
+    /* Ignore on-disk xattrs that AFR doesn't need to heal. */
+    if (!afr_is_xattr_ignorable(key1))
+        return _gf_true;
+
+    return _gf_false;
 }
 
-void
-afr_lookup_mark_other_entries_stale (afr_local_t *local, xlator_t *this,
-                                     int subvol)
-{
-        afr_private_t *priv = NULL;
-        int32_t       *success_children = NULL;
-        struct iatt   *bufs = NULL;
-        int           i = 0;
-        int           child = 0;
-
-        priv = this->private;
-        success_children = local->cont.lookup.success_children;
-        bufs = local->cont.lookup.bufs;
-        memcpy (local->fresh_children, success_children,
-                sizeof (*success_children) * priv->child_count);
-        for (i = 0; i < priv->child_count; i++) {
-                child = local->fresh_children[i];
-                if (child == -1)
-                        break;
-                if (child == subvol)
-                        continue;
-                if (uuid_is_null (bufs[child].ia_gfid) &&
-                    (bufs[child].ia_type == bufs[subvol].ia_type))
-                        continue;
-                afr_children_rm_child (success_children, child,
-                                       priv->child_count);
-                local->success_count--;
-        }
-        afr_reset_children (local->fresh_children, priv->child_count);
+gf_boolean_t
+afr_xattrs_are_equal(dict_t *dict1, dict_t *dict2)
+{
+    return are_dicts_equal(dict1, dict2, afr_xattr_match_needed, NULL);
 }
 
-void
-afr_succeed_lookup_on_latest_iatt (afr_local_t *local, xlator_t *this)
+static int
+afr_get_parent_read_subvol(xlator_t *this, inode_t *parent,
+                           struct afr_reply *replies, unsigned char *readable)
 {
-        int    lsubvol = 0;
+    int i = 0;
+    int par_read_subvol = -1;
+    int par_read_subvol_iter = -1;
+    afr_private_t *priv = NULL;
 
-        if (!afr_lookup_conflicting_entries (local, this))
-                goto out;
+    priv = this->private;
 
-        lsubvol = afr_lookup_get_latest_subvol (local, this);
-        if (lsubvol < 0)
-                goto out;
-        afr_lookup_mark_other_entries_stale (local, this, lsubvol);
-out:
-        return;
-}
+    if (parent)
+        par_read_subvol = afr_data_subvol_get(parent, this, NULL, NULL, NULL,
+                                              NULL);
 
-static void
-afr_lookup_done (call_frame_t *frame, xlator_t *this)
-{
-        int                 unwind = 1;
-        afr_private_t       *priv  = NULL;
-        afr_local_t         *local = NULL;
-        int                 ret = -1;
-        gf_boolean_t        sh_launched = _gf_false;
-        gf_boolean_t        fail_conflict = _gf_false;
-        int                 gfid_miss_count = 0;
-        int                 enotconn_count = 0;
-        int                 up_children_count = 0;
-
-        priv  = this->private;
-        local = frame->local;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-        if (local->op_ret < 0)
-                goto unwind;
-
-        if (local->cont.lookup.parent_entrylk && local->success_count > 1)
-                afr_succeed_lookup_on_latest_iatt (local, this);
-
-        gfid_miss_count = afr_lookup_gfid_missing_count (local, this);
-        up_children_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-        enotconn_count = priv->child_count - up_children_count;
-        if ((gfid_miss_count == local->success_count) &&
-            (enotconn_count > 0)) {
-                local->op_ret = -1;
-                local->op_errno = EIO;
-                gf_log (this->name, GF_LOG_ERROR, "Failing lookup for %s, "
-                        "LOOKUP on a file without gfid is not allowed when "
-                        "some of the children are down", local->loc.path);
-                goto unwind;
-        }
-
-        if ((gfid_miss_count == local->success_count) &&
-            uuid_is_null (local->cont.lookup.gfid_req)) {
-                local->op_ret = -1;
-                local->op_errno = ENODATA;
-                gf_log (this->name, GF_LOG_ERROR, "%s: No gfid present",
-                        local->loc.path);
-                goto unwind;
-        }
-
-        if (gfid_miss_count && uuid_is_null (local->cont.lookup.gfid_req))
-                fail_conflict = _gf_true;
-        ret = afr_lookup_done_success_action (frame, this, fail_conflict);
-        if (ret)
-                goto unwind;
-        uuid_copy (local->self_heal.sh_gfid_req, local->cont.lookup.gfid_req);
+        if (replies[i].op_ret < 0)
+            continue;
 
-        afr_lookup_perform_self_heal (frame, this, &sh_launched);
-        if (sh_launched) {
-                unwind = 0;
-                goto unwind;
+        if (par_read_subvol_iter == -1) {
+            par_read_subvol_iter = i;
+            continue;
         }
 
- unwind:
-         if (unwind) {
-                 AFR_STACK_UNWIND (lookup, frame, local->op_ret,
-                                   local->op_errno, local->cont.lookup.inode,
-                                   &local->cont.lookup.buf,
-                                   local->cont.lookup.xattr,
-                                   &local->cont.lookup.postparent);
-        }
-}
+        if ((par_read_subvol_iter != par_read_subvol) && readable[i])
+            par_read_subvol_iter = i;
 
-/*
- * During a lookup, some errors are more "important" than
- * others in that they must be given higher priority while
- * returning to the user.
- *
- * The hierarchy is ESTALE > ENOENT > others
- *
- */
+        if (i == par_read_subvol)
+            par_read_subvol_iter = i;
+    }
+    /* At the end of the for-loop, the only reason why @par_read_subvol_iter
+     * could be -1 is when this LOOKUP has failed on all sub-volumes.
+     * So it is okay to send an arbitrary subvolume (0 in this case)
+     * as parent read subvol.
+     */
+    if (par_read_subvol_iter == -1)
+        par_read_subvol_iter = 0;
 
-gf_boolean_t
-afr_error_more_important (int32_t old_errno, int32_t new_errno)
+    return par_read_subvol_iter;
+}
+
+int
+afr_read_subvol_decide(inode_t *inode, xlator_t *this,
+                       afr_read_subvol_args_t *args, unsigned char *readable)
 {
-        gf_boolean_t ret = _gf_true;
+    int event = 0;
+    afr_private_t *priv = NULL;
+    unsigned char *intersection = NULL;
 
-        /* Nothing should ever overwrite ESTALE */
-        if (old_errno == ESTALE)
-                ret = _gf_false;
+    priv = this->private;
+    intersection = alloca0(priv->child_count);
 
-        /* Nothing should overwrite ENOENT, except ESTALE/EIO*/
-        else if ((old_errno == ENOENT) && (new_errno != ESTALE)
-                 && (new_errno != EIO))
-                ret = _gf_false;
+    afr_readables_intersect_get(inode, this, &event, intersection);
 
-        return ret;
-}
+    if (AFR_COUNT(intersection, priv->child_count) <= 0) {
+        /* TODO: If we have one brick with valid data_readable and
+         * another with metadata_readable, try to send an iatt with
+         * valid bits from both.*/
+        return -1;
+    }
 
-int32_t
-afr_resultant_errno_get (int32_t *children,
-                         int *child_errno, unsigned int child_count)
-{
-        int     i = 0;
-        int32_t op_errno = 0;
-        int     child = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (children) {
-                        child = children[i];
-                        if (child == -1)
-                                break;
-                } else {
-                        child = i;
-                }
-                if (afr_error_more_important (op_errno, child_errno[child]))
-                                op_errno = child_errno[child];
-        }
-        return op_errno;
+    memcpy(readable, intersection, sizeof(*readable) * priv->child_count);
+
+    return afr_read_subvol_select_by_policy(inode, this, intersection, args);
 }
 
-static void
-afr_lookup_handle_error (afr_local_t *local, int32_t op_ret,  int32_t op_errno)
+static inline int
+afr_first_up_child(call_frame_t *frame, xlator_t *this)
 {
-        GF_ASSERT (local);
-        if (op_errno == ENOENT)
-                local->enoent_count++;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        if (afr_error_more_important (local->op_errno, op_errno))
-                local->op_errno = op_errno;
+    local = frame->local;
+    priv = this->private;
 
-        if (local->op_errno == ESTALE) {
-                local->op_ret = -1;
-        }
+    for (i = 0; i < priv->child_count; i++)
+        if (local->replies[i].valid && local->replies[i].op_ret == 0)
+            return i;
+    return -1;
 }
 
 static void
-afr_set_root_inode_on_first_lookup (afr_local_t *local, xlator_t *this,
-                                    inode_t *inode)
+afr_attempt_readsubvol_set(call_frame_t *frame, xlator_t *this,
+                           unsigned char *success_replies,
+                           unsigned char *data_readable, int *read_subvol)
 {
-        afr_private_t           *priv = NULL;
-        GF_ASSERT (inode);
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int spb_subvol = -1;
+    int child_count = -1;
 
-        if (!__is_root_gfid (inode->gfid))
-                goto out;
-        if (!afr_is_fresh_lookup (&local->loc, this))
-                goto out;
-        priv = this->private;
-        if ((priv->first_lookup)) {
-                gf_log (this->name, GF_LOG_INFO, "added root inode");
-                priv->root_inode = inode_ref (inode);
-                priv->first_lookup = 0;
-        }
-out:
+    if (*read_subvol != -1)
         return;
-}
 
-static void
-afr_lookup_cache_args (afr_local_t *local, int child_index, dict_t *xattr,
-                       struct iatt *buf, struct iatt *postparent)
-{
-        GF_ASSERT (child_index >= 0);
-        local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
-        local->cont.lookup.postparents[child_index] = *postparent;
-        local->cont.lookup.bufs[child_index] = *buf;
+    priv = this->private;
+    local = frame->local;
+    child_count = priv->child_count;
+
+    afr_split_brain_read_subvol_get(local->inode, this, frame, &spb_subvol);
+    if ((spb_subvol >= 0) &&
+        (AFR_COUNT(success_replies, child_count) == child_count)) {
+        *read_subvol = spb_subvol;
+    } else if (!priv->quorum_count ||
+               frame->root->pid == GF_CLIENT_PID_GLFS_HEAL) {
+        *read_subvol = afr_first_up_child(frame, this);
+    } else if (priv->quorum_count &&
+               afr_has_quorum(data_readable, this, NULL)) {
+        /* read_subvol is guaranteed to be valid if we hit this path. */
+        *read_subvol = afr_first_up_child(frame, this);
+    } else {
+        /* If quorum is enabled and we do not have a
+           readable yet, it means all good copies are down.
+        */
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
+               "no read "
+               "subvols for %s",
+               local->loc.path);
+    }
+    if (*read_subvol >= 0)
+        dict_del_sizen(local->replies[*read_subvol].xdata, GF_CONTENT_KEY);
 }
 
 static void
-afr_lookup_handle_first_success (afr_local_t *local, xlator_t *this,
-                                 inode_t *inode, struct iatt *buf)
-{
-        local->cont.lookup.inode      = inode_ref (inode);
-        local->cont.lookup.buf        = *buf;
-        afr_set_root_inode_on_first_lookup (local, this, inode);
-}
+afr_lookup_done(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = -1;
+    int op_errno = 0;
+    int read_subvol = 0;
+    int par_read_subvol = 0;
+    int ret = -1;
+    unsigned char *readable = NULL;
+    unsigned char *success_replies = NULL;
+    int event = 0;
+    struct afr_reply *replies = NULL;
+    uuid_t read_gfid = {
+        0,
+    };
+    gf_boolean_t locked_entry = _gf_false;
+    gf_boolean_t in_flight_create = _gf_false;
+    gf_boolean_t can_interpret = _gf_true;
+    inode_t *parent = NULL;
+    ia_type_t ia_type = IA_INVAL;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+    char *gfid_heal_msg = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+    parent = local->loc.parent;
+
+    locked_entry = afr_is_possibly_under_txn(AFR_ENTRY_TRANSACTION, local,
+                                             this);
+
+    readable = alloca0(priv->child_count);
+    success_replies = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(parent, this, readable, NULL, &event);
+    par_read_subvol = afr_get_parent_read_subvol(this, parent, replies,
+                                                 readable);
+
+    /* First, check if we have a gfid-change from somewhere,
+       If so, propagate that so that a fresh lookup can be
+       issued
+    */
+    if (local->cont.lookup.needs_fresh_lookup) {
+        local->op_ret = -1;
+        local->op_errno = ESTALE;
+        goto error;
+    }
 
-static int32_t
-afr_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *dict,
-                   dict_t *xdata)
-{
-        int              ret            = 0;
-        char            *pathinfo       = NULL;
-        gf_boolean_t     is_local        = _gf_false;
-        afr_private_t   *priv           = NULL;
-        int32_t          child_index    = -1;
+    op_errno = afr_final_errno(frame->local, this->private);
+    local->op_errno = op_errno;
 
-        if (op_ret != 0) {
-                goto out;
+    read_subvol = -1;
+    afr_fill_success_replies(local, priv, success_replies);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret == -1) {
+            if (locked_entry && replies[i].op_errno == ENOENT) {
+                in_flight_create = _gf_true;
+            }
+            continue;
         }
 
-        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
-        if (ret != 0) {
-                goto out;
+        if (read_subvol == -1 || !readable[read_subvol]) {
+            read_subvol = i;
+            gf_uuid_copy(read_gfid, replies[i].poststat.ia_gfid);
+            ia_type = replies[i].poststat.ia_type;
+            local->op_ret = 0;
         }
+    }
 
-        ret = afr_local_pathinfo (pathinfo, &is_local);
+    if (in_flight_create && !afr_has_quorum(success_replies, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = ENOENT;
+        goto error;
+    }
+
+    if (read_subvol == -1)
+        goto error;
+    /* We now have a read_subvol, which is readable[] (if there
+       were any). Next we look for GFID mismatches. We don't
+       consider a GFID mismatch as an error if read_subvol is
+       readable[] but the mismatching GFID subvol is not.
+    */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1) {
+            continue;
+        }
+
+        if (!gf_uuid_compare(replies[i].poststat.ia_gfid, read_gfid))
+            continue;
+
+        can_interpret = _gf_false;
+
+        if (locked_entry)
+            continue;
+
+        /* Now GFIDs mismatch. It's OK as long as this subvol
+           is not readable[] but read_subvol is */
+        if (readable[read_subvol] && !readable[i])
+            continue;
+
+        /* If we were called from glfsheal and there is still a gfid
+         * mismatch, succeed the lookup and let glfsheal print the
+         * response via gfid-heal-msg.*/
+        if (!dict_get_str_sizen(local->xattr_req, "gfid-heal-msg",
+                                &gfid_heal_msg))
+            goto cant_interpret;
+
+        /* LOG ERROR */
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        goto error;
+    }
+
+    /* Forth, for the finalized GFID, pick the best subvolume
+       to return stats from.
+    */
+    read_subvol = -1;
+    memset(readable, 0, sizeof(*readable) * priv->child_count);
+    if (can_interpret) {
+        if (!afr_has_quorum(success_replies, this, NULL))
+            goto cant_interpret;
+        /* It is safe to call afr_replies_interpret() because we have
+           a response from all the UP subvolumes and all of them resolved
+           to the same GFID
+        */
+        gf_uuid_copy(args.gfid, read_gfid);
+        args.ia_type = ia_type;
+        ret = afr_replies_interpret(frame, this, local->inode, NULL);
+        read_subvol = afr_read_subvol_decide(local->inode, this, &args,
+                                             readable);
+        if (read_subvol == -1)
+            goto cant_interpret;
         if (ret) {
-                goto out;
+            afr_inode_need_refresh_set(local->inode, this);
+            dict_del_sizen(local->replies[read_subvol].xdata, GF_CONTENT_KEY);
+        }
+    } else {
+    cant_interpret:
+        afr_attempt_readsubvol_set(frame, this, success_replies, readable,
+                                   &read_subvol);
+        if (read_subvol == -1) {
+            goto error;
         }
+    }
 
-        priv = this->private;
-        /*
-         * Note that one local subvolume will override another here.  The only
-         * way to avoid that would be to retain extra information about whether
-         * the previous read_child is local, and it's just not worth it.  Even
-         * the slowest local subvolume is far preferable to a remote one.
-         */
-        if (is_local) {
-                child_index = (int32_t)(long)cookie;
-                gf_log (this->name, GF_LOG_INFO,
-                        "selecting local read_child %s",
-                        priv->children[child_index]->name);
-                priv->read_child = child_index;
+    afr_handle_quota_size(frame, this);
+
+    afr_set_need_heal(this, local);
+    if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg_debug(this->name, 0,
+                     "Arbiter cannot be a read subvol "
+                     "for %s",
+                     local->loc.path);
+        goto error;
+    }
+
+    ret = dict_get_str_sizen(local->xattr_req, "gfid-heal-msg", &gfid_heal_msg);
+    if (!ret) {
+        ret = dict_set_str_sizen(local->replies[read_subvol].xdata,
+                                 "gfid-heal-msg", gfid_heal_msg);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting gfid-heal-msg dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
         }
+    }
 
-out:
-        STACK_DESTROY(frame->root);
-        return 0;
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->replies[read_subvol].poststat,
+                     local->replies[read_subvol].xdata,
+                     &local->replies[par_read_subvol].postparent);
+    return;
+
+error:
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL, NULL);
 }
 
-static void
-afr_attempt_local_discovery (xlator_t *this, int32_t child_index)
+/*
+ * During a lookup, some errors are more "important" than
+ * others in that they must be given higher priority while
+ * returning to the user.
+ *
+ * The hierarchy is ENODATA > ENOENT > ESTALE > ENOSPC others
+ */
+
+int
+afr_higher_errno(int32_t old_errno, int32_t new_errno)
 {
-        call_frame_t    *newframe = NULL;
-        loc_t            tmploc = {0,};
-        afr_private_t   *priv = this->private;
+    if (old_errno == ENODATA || new_errno == ENODATA)
+        return ENODATA;
+    if (old_errno == ENOENT || new_errno == ENOENT)
+        return ENOENT;
+    if (old_errno == ESTALE || new_errno == ESTALE)
+        return ESTALE;
+    if (old_errno == ENOSPC || new_errno == ENOSPC)
+        return ENOSPC;
 
-        newframe = create_frame(this,this->ctx->pool);
-        if (!newframe) {
-                return;
-        }
+    return new_errno;
+}
+
+int
+afr_final_errno(afr_local_t *local, afr_private_t *priv)
+{
+    int i = 0;
+    int op_errno = 0;
+    int tmp_errno = 0;
 
-        tmploc.gfid[sizeof(tmploc.gfid)-1] = 1;
-        STACK_WIND_COOKIE (newframe, afr_discovery_cbk,
-                           (void *)(long)child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->getxattr,
-                           &tmploc, GF_XATTR_PATHINFO_KEY, NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret >= 0)
+            continue;
+        tmp_errno = local->replies[i].op_errno;
+        op_errno = afr_higher_errno(op_errno, tmp_errno);
+    }
+
+    return op_errno;
+}
+
+static int32_t
+afr_local_discovery_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict,
+                        dict_t *xdata)
+{
+    int ret = 0;
+    char *pathinfo = NULL;
+    gf_boolean_t is_local = _gf_false;
+    afr_private_t *priv = NULL;
+    int32_t child_index = -1;
+
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    priv = this->private;
+    child_index = (int32_t)(long)cookie;
+
+    ret = dict_get_str_sizen(dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+    if (ret != 0) {
+        goto out;
+    }
+
+    ret = glusterfs_is_local_pathinfo(pathinfo, &is_local);
+    if (ret) {
+        goto out;
+    }
+
+    /*
+     * Note that one local subvolume will override another here.  The only
+     * way to avoid that would be to retain extra information about whether
+     * the previous read_child is local, and it's just not worth it.  Even
+     * the slowest local subvolume is far preferable to a remote one.
+     */
+    if (is_local) {
+        priv->local[child_index] = 1;
+        /* Don't set arbiter as read child. */
+        if (AFR_IS_ARBITER_BRICK(priv, child_index))
+            goto out;
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_LOCAL_CHILD,
+               "selecting local read_child %s",
+               priv->children[child_index]->name);
+
+        priv->read_child = child_index;
+    }
+out:
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 static void
-afr_lookup_handle_success (afr_local_t *local, xlator_t *this, int32_t child_index,
-                           int32_t op_ret, int32_t op_errno, inode_t *inode,
-                           struct iatt *buf, dict_t *xattr,
-                           struct iatt *postparent)
+afr_attempt_local_discovery(xlator_t *this, int32_t child_index)
 {
-        afr_private_t   *priv   = this->private;
+    call_frame_t *newframe = NULL;
+    loc_t tmploc = {
+        0,
+    };
+    afr_private_t *priv = this->private;
 
-        if (local->success_count == 0) {
-                if (local->op_errno != ESTALE) {
-                        local->op_ret = op_ret;
-                        local->op_errno = 0;
-                }
-                afr_lookup_handle_first_success (local, this, inode, buf);
-        }
-        afr_lookup_update_lk_counts (local, this,
-                                     child_index, xattr);
+    newframe = create_frame(this, this->ctx->pool);
+    if (!newframe) {
+        return;
+    }
 
-        afr_lookup_cache_args (local, child_index, xattr,
-                               buf, postparent);
+    tmploc.gfid[sizeof(tmploc.gfid) - 1] = 1;
+    STACK_WIND_COOKIE(newframe, afr_local_discovery_cbk,
+                      (void *)(long)child_index, priv->children[child_index],
+                      priv->children[child_index]->fops->getxattr, &tmploc,
+                      GF_XATTR_PATHINFO_KEY, NULL);
+}
 
-        if (local->do_discovery && (priv->read_child == (-1))) {
-                afr_attempt_local_discovery(this,child_index);
-        }
+int
+afr_lookup_sh_metadata_wrap(void *opaque)
+{
+    call_frame_t *frame = opaque;
+    afr_local_t *local = NULL;
+    xlator_t *this = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0, first = -1;
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+    replies = local->replies;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        first = i;
+        break;
+    }
+    if (first == -1)
+        goto out;
+
+    if (afr_selfheal_metadata_by_stbuf(this, &replies[first].poststat))
+        goto out;
+
+    afr_local_replies_wipe(local, this->private);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+    if (local->xattr_req) {
+        dict_copy(local->xattr_req, dict);
+    }
+
+    ret = dict_set_sizen_str_sizen(dict, "link-count", GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
+
+    if (loc_is_nameless(&local->loc)) {
+        ret = afr_selfheal_unlocked_discover_on(frame, local->inode,
+                                                local->loc.gfid, local->replies,
+                                                local->child_up, dict);
+    } else {
+        inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
+                                                local->loc.name, local->replies,
+                                                local->child_up, dict);
+    }
+    if (inode)
+        inode_unref(inode);
+out:
+    if (loc_is_nameless(&local->loc))
+        afr_discover_done(frame, this);
+    else
+        afr_lookup_done(frame, this);
 
-        local->cont.lookup.success_children[local->success_count] = child_index;
-        local->success_count++;
+    if (dict)
+        dict_unref(dict);
+
+    return 0;
 }
 
-int
-afr_lookup_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this,  int32_t op_ret,  int32_t op_errno,
-                inode_t *inode,   struct iatt *buf, dict_t *xattr,
-                struct iatt *postparent)
+gf_boolean_t
+afr_is_pending_set(xlator_t *this, dict_t *xdata, int type)
 {
-        afr_local_t *   local = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
+    int idx = -1;
+    afr_private_t *priv = NULL;
+    void *pending_raw = NULL;
+    int *pending_int = NULL;
+    int i = 0;
 
-         child_index = (long) cookie;
+    priv = this->private;
+    idx = afr_index_for_transaction_type(type);
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
+    if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw) == 0) {
+        if (pending_raw) {
+            pending_int = pending_raw;
 
-                if (op_ret == -1) {
-                        afr_lookup_handle_error (local, op_ret, op_errno);
-                        goto unlock;
-                }
-                afr_lookup_handle_success (local, this, child_index, op_ret,
-                                           op_errno, inode, buf, xattr,
-                                           postparent);
+            if (ntoh32(pending_int[idx]))
+                return _gf_true;
+        }
+    }
 
-         }
-unlock:
-        UNLOCK (&frame->lock);
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
+        if (!pending_raw)
+            continue;
+        pending_int = pending_raw;
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-               afr_lookup_done (frame, this);
-        }
+        if (ntoh32(pending_int[idx]))
+            return _gf_true;
+    }
 
-         return 0;
+    return _gf_false;
 }
 
-int
-afr_lookup_cont_init (afr_local_t *local, unsigned int child_count)
-{
-        int               ret            = -ENOMEM;
-        struct iatt       *iatts         = NULL;
-        int32_t           *success_children = NULL;
-        int32_t           *sources       = NULL;
-
-        GF_ASSERT (local);
-        local->cont.lookup.xattrs = GF_CALLOC (child_count,
-                                               sizeof (*local->cont.lookup.xattr),
-                                               gf_afr_mt_dict_t);
-        if (NULL == local->cont.lookup.xattrs)
-                goto out;
+static gf_boolean_t
+afr_can_start_metadata_self_heal(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0, first = -1;
+    gf_boolean_t start = _gf_false;
+    struct iatt stbuf = {
+        0,
+    };
+
+    local = frame->local;
+    replies = local->replies;
+    priv = this->private;
+
+    if (!priv->metadata_self_heal)
+        return _gf_false;
 
-        iatts = GF_CALLOC (child_count, sizeof (*iatts), gf_afr_mt_iatt);
-        if (NULL == iatts)
-                goto out;
-        local->cont.lookup.postparents = iatts;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (first == -1) {
+            first = i;
+            stbuf = replies[i].poststat;
+            continue;
+        }
 
-        iatts = GF_CALLOC (child_count, sizeof (*iatts), gf_afr_mt_iatt);
-        if (NULL == iatts)
-                goto out;
-        local->cont.lookup.bufs = iatts;
+        if (afr_is_pending_set(this, replies[i].xdata,
+                               AFR_METADATA_TRANSACTION)) {
+            /* Let shd do the heal so that lookup is not blocked
+             * on getting metadata lock/doing the heal */
+            start = _gf_false;
+            break;
+        }
 
-        success_children = afr_children_create (child_count);
-        if (NULL == success_children)
-                goto out;
-        local->cont.lookup.success_children = success_children;
+        if (gf_uuid_compare(stbuf.ia_gfid, replies[i].poststat.ia_gfid)) {
+            start = _gf_false;
+            break;
+        }
+        if (!IA_EQUAL(stbuf, replies[i].poststat, type)) {
+            start = _gf_false;
+            break;
+        }
 
-        local->fresh_children = afr_children_create (child_count);
-        if (NULL == local->fresh_children)
-                goto out;
+        /*Check if iattrs need heal*/
+        if ((!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, prot))) {
+            start = _gf_true;
+            continue;
+        }
 
-        sources = GF_CALLOC (sizeof (*sources), child_count, gf_afr_mt_int32_t);
-        if (NULL == sources)
-                goto out;
-        local->cont.lookup.sources = sources;
+        /*Check if xattrs need heal*/
+        if (!afr_xattrs_are_equal(replies[first].xdata, replies[i].xdata))
+            start = _gf_true;
+    }
 
-        ret = 0;
+    return start;
+}
+
+int
+afr_lookup_metadata_heal_check(call_frame_t *frame, xlator_t *this)
+
+{
+    call_frame_t *heal = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    if (!afr_can_start_metadata_self_heal(frame, this))
+        goto out;
+
+    heal = afr_frame_create(this, &ret);
+    if (!heal) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = synctask_new(this->ctx->env, afr_lookup_sh_metadata_wrap,
+                       afr_refresh_selfheal_done, heal, frame);
+    if (ret)
+        goto out;
+    return ret;
 out:
-        return ret;
+    if (loc_is_nameless(&local->loc))
+        afr_discover_done(frame, this);
+    else
+        afr_lookup_done(frame, this);
+    if (heal)
+        AFR_STACK_DESTROY(heal);
+    return ret;
 }
 
 int
-afr_lookup (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, dict_t *xattr_req)
+afr_lookup_selfheal_wrap(void *opaque)
 {
-        afr_private_t    *priv           = NULL;
-        afr_local_t      *local          = NULL;
-        void              *gfid_req      = NULL;
-        int               ret            = -1;
-        int               i              = 0;
-        int               call_count     = 0;
-        uint64_t          ctx            = 0;
-        int32_t           op_errno       = 0;
+    int ret = 0;
+    call_frame_t *frame = opaque;
+    afr_local_t *local = NULL;
+    xlator_t *this = NULL;
+    inode_t *inode = NULL;
+    uuid_t pargfid = {
+        0,
+    };
 
-        priv = this->private;
+    local = frame->local;
+    this = frame->this;
+    loc_pargfid(&local->loc, pargfid);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
+    ret = afr_selfheal_name(frame->this, pargfid, local->loc.name,
+                            &local->cont.lookup.gfid_req, local->xattr_req);
+    if (ret == -EIO)
+        goto unwind;
 
-        local->op_ret = -1;
+    afr_local_replies_wipe(local, this->private);
 
-        frame->local = local;
-        local->fop = GF_FOP_LOOKUP;
+    inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
+                                            local->loc.name, local->replies,
+                                            local->child_up, local->xattr_req);
+    if (inode)
+        inode_unref(inode);
 
-        loc_copy (&local->loc, loc);
-        ret = loc_path (&local->loc, NULL);
-        if (ret < 0) {
-                op_errno = EINVAL;
-                goto out;
-        }
+    afr_lookup_metadata_heal_check(frame, this);
+    return 0;
 
-        ret = inode_ctx_get (local->loc.inode, this, &ctx);
-        if (ret == 0) {
-                /* lookup is a revalidate */
+unwind:
+    AFR_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-                local->read_child_index = afr_inode_get_read_ctx (this,
-                                                               local->loc.inode,
-                                                               NULL);
+int
+afr_lookup_entry_heal(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    call_frame_t *heal = NULL;
+    int i = 0, first = -1;
+    gf_boolean_t name_state_mismatch = _gf_false;
+    struct afr_reply *replies = NULL;
+    int ret = 0;
+    unsigned char *par_readables = NULL;
+    unsigned char *success = NULL;
+    int32_t op_errno = 0;
+    uuid_t gfid = {0};
+
+    local = frame->local;
+    replies = local->replies;
+    priv = this->private;
+    par_readables = alloca0(priv->child_count);
+    success = alloca0(priv->child_count);
+
+    ret = afr_inode_read_subvol_get(local->loc.parent, this, par_readables,
+                                    NULL, NULL);
+    if (ret < 0 || AFR_COUNT(par_readables, priv->child_count) == 0) {
+        /* In this case set par_readables to all 1 so that name_heal
+         * need checks at the end of this function will flag missing
+         * entry when name state mismatches*/
+        memset(par_readables, 1, priv->child_count);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret == 0) {
+            if (gf_uuid_is_null(gfid)) {
+                gf_uuid_copy(gfid, replies[i].poststat.ia_gfid);
+            }
+            success[i] = 1;
         } else {
-                LOCK (&priv->read_child_lock);
-                {
-                        if (priv->hash_mode) {
-                                local->read_child_index = -1;
-                        }
-                        else {
-                                local->read_child_index =
-                                        (++priv->read_child_rr) %
-                                        (priv->child_count);
-                        }
-                }
-                UNLOCK (&priv->read_child_lock);
-                local->cont.lookup.fresh_lookup = _gf_true;
+            if ((replies[i].op_errno != ENOTCONN) &&
+                (replies[i].op_errno != ENOENT) &&
+                (replies[i].op_errno != ESTALE)) {
+                op_errno = replies[i].op_errno;
+            }
         }
 
-        local->child_up = memdup (priv->child_up,
-                                  sizeof (*local->child_up) * priv->child_count);
-        if (NULL == local->child_up) {
-                op_errno = ENOMEM;
-                goto out;
+        /*gfid is missing, needs heal*/
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA)) {
+            goto name_heal;
         }
 
-        ret = afr_lookup_cont_init (local, priv->child_count);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
+        if (first == -1) {
+            first = i;
+            continue;
         }
 
-        local->call_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-        call_count = local->call_count;
-        if (local->call_count == 0) {
-                ret      = -1;
-                op_errno = ENOTCONN;
-                goto out;
+        if (replies[i].op_ret != replies[first].op_ret) {
+            name_state_mismatch = _gf_true;
         }
 
-        /* By default assume ENOTCONN. On success it will be set to 0. */
-        local->op_errno = ENOTCONN;
-
-        ret = afr_lookup_xattr_req_prepare (local, this, xattr_req, &local->loc,
-                                            &gfid_req);
-        if (ret) {
-                local->op_errno = -ret;
-                goto out;
-        }
-        afr_lookup_save_gfid (local->cont.lookup.gfid_req, gfid_req,
-                              &local->loc);
-        local->fop = GF_FOP_LOOKUP;
-        if (priv->choose_local && !priv->did_discovery) {
-                if (gfid_req && __is_root_gfid(gfid_req)) {
-                        local->do_discovery = _gf_true;
-                        priv->did_discovery = _gf_true;
-                }
+        if (replies[i].op_ret == 0) {
+            /* Rename after this lookup may succeed if we don't do
+             * a name-heal and the destination may not have pending xattrs
+             * to indicate which name is good and which is bad so always do
+             * this heal*/
+            if (gf_uuid_compare(replies[i].poststat.ia_gfid, gfid)) {
+                goto name_heal;
+            }
         }
+    }
+
+    if (name_state_mismatch) {
+        if (!priv->quorum_count)
+            goto name_heal;
+        if (!afr_has_quorum(success, this, NULL))
+            goto name_heal;
+        if (op_errno)
+            goto name_heal;
         for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_lookup_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           &local->loc, local->xattr_req);
-                        if (!--call_count)
-                                break;
-                }
+            if (!replies[i].valid)
+                continue;
+            if (par_readables[i] && replies[i].op_ret < 0 &&
+                replies[i].op_errno != ENOTCONN) {
+                goto name_heal;
+            }
         }
+    }
 
-        ret = 0;
-out:
-        if (ret)
-                AFR_STACK_UNWIND (lookup, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+    goto metadata_heal;
 
-        return 0;
-}
+name_heal:
+    heal = afr_frame_create(this, NULL);
+    if (!heal)
+        goto metadata_heal;
 
+    ret = synctask_new(this->ctx->env, afr_lookup_selfheal_wrap,
+                       afr_refresh_selfheal_done, heal, frame);
+    if (ret) {
+        AFR_STACK_DESTROY(heal);
+        goto metadata_heal;
+    }
+    return ret;
 
-/* {{{ open */
+metadata_heal:
+    ret = afr_lookup_metadata_heal_check(frame, this);
 
-int
-__afr_fd_ctx_set (xlator_t *this, fd_t *fd)
-{
-        afr_private_t * priv   = NULL;
-        int             ret    = -1;
-        uint64_t        ctx    = 0;
-        afr_fd_ctx_t *  fd_ctx = NULL;
+    return ret;
+}
 
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (fd, out);
+int
+afr_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+               struct iatt *postparent)
+{
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
+    GF_UNUSED int ret = 0;
+    int8_t need_heal = 1;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    /*
+     * On revalidate lookup if the gfid-changed, afr should unwind the fop
+     * with ESTALE so that a fresh lookup will be sent by the top xlator.
+     * So remember it.
+     */
+    if (xdata && dict_get_sizen(xdata, "gfid-changed"))
+        local->cont.lookup.needs_fresh_lookup = _gf_true;
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        local->replies[child_index].need_heal = need_heal;
+    } else {
+        local->replies[child_index].need_heal = need_heal;
+    }
+    if (op_ret != -1) {
+        local->replies[child_index].poststat = *buf;
+        local->replies[child_index].postparent = *postparent;
+        if (xdata)
+            local->replies[child_index].xdata = dict_ref(xdata);
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        afr_lookup_entry_heal(frame, this);
+    }
+
+    return 0;
+}
 
-        priv = this->private;
+static void
+afr_discover_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int read_subvol = -1;
+    int ret = 0;
+    unsigned char *data_readable = NULL;
+    unsigned char *success_replies = NULL;
 
-        ret = __fd_ctx_get (fd, this, &ctx);
+    priv = this->private;
+    local = frame->local;
+    data_readable = alloca0(priv->child_count);
+    success_replies = alloca0(priv->child_count);
 
-        if (ret == 0)
-                goto out;
+    afr_fill_success_replies(local, priv, success_replies);
+    if (AFR_COUNT(success_replies, priv->child_count) > 0)
+        local->op_ret = 0;
 
-        fd_ctx = GF_CALLOC (1, sizeof (afr_fd_ctx_t),
-                            gf_afr_mt_afr_fd_ctx_t);
-        if (!fd_ctx) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    if (local->op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(frame->local, this->private);
+        goto error;
+    }
 
-        fd_ctx->pre_op_done = GF_CALLOC (sizeof (*fd_ctx->pre_op_done),
-                                         priv->child_count,
-                                         gf_afr_mt_char);
-        if (!fd_ctx->pre_op_done) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    if (!afr_has_quorum(success_replies, this, frame))
+        goto unwind;
 
-        fd_ctx->pre_op_piggyback = GF_CALLOC (sizeof (*fd_ctx->pre_op_piggyback),
-                                              priv->child_count,
-                                              gf_afr_mt_char);
-        if (!fd_ctx->pre_op_piggyback) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    ret = afr_replies_interpret(frame, this, local->inode, NULL);
+    if (ret) {
+        afr_inode_need_refresh_set(local->inode, this);
+    }
 
-        fd_ctx->opened_on = GF_CALLOC (sizeof (*fd_ctx->opened_on),
-                                       priv->child_count,
-                                       gf_afr_mt_int32_t);
-        if (!fd_ctx->opened_on) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    read_subvol = afr_read_subvol_decide(local->inode, this, NULL,
+                                         data_readable);
 
-        fd_ctx->lock_piggyback = GF_CALLOC (sizeof (*fd_ctx->lock_piggyback),
-                                            priv->child_count,
-                                            gf_afr_mt_char);
-        if (!fd_ctx->lock_piggyback) {
-                ret = -ENOMEM;
-                goto out;
-        }
+unwind:
+    afr_attempt_readsubvol_set(frame, this, success_replies, data_readable,
+                               &read_subvol);
+    if (read_subvol == -1)
+        goto error;
 
-        fd_ctx->lock_acquired = GF_CALLOC (sizeof (*fd_ctx->lock_acquired),
-                                           priv->child_count,
-                                           gf_afr_mt_char);
-        if (!fd_ctx->lock_acquired) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg_debug(this->name, 0,
+                     "Arbiter cannot be a read subvol "
+                     "for %s",
+                     local->loc.path);
+    }
 
-        fd_ctx->up_count   = priv->up_count;
-        fd_ctx->down_count = priv->down_count;
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->replies[read_subvol].poststat,
+                     local->replies[read_subvol].xdata,
+                     &local->replies[read_subvol].postparent);
+    return;
 
-        fd_ctx->locked_on = GF_CALLOC (sizeof (*fd_ctx->locked_on),
-                                       priv->child_count,
-                                       gf_afr_mt_char);
-        if (!fd_ctx->locked_on) {
-                ret = -ENOMEM;
-                goto out;
-        }
+error:
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL, NULL);
+}
 
-	pthread_mutex_init (&fd_ctx->delay_lock, NULL);
-        INIT_LIST_HEAD (&fd_ctx->paused_calls);
-        INIT_LIST_HEAD (&fd_ctx->entries);
+static int
+afr_ta_id_file_check(void *opaque)
+{
+    afr_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *dict = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    fd_t *fd = NULL;
+    int ret = 0;
+
+    this = opaque;
+    priv = this->private;
+
+    ret = afr_fill_ta_loc(this, &loc, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        goto out;
+    }
+
+    ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], &loc, &stbuf,
+                        0, 0, 0);
+    if (ret == 0) {
+        goto out;
+    } else if (ret == -ENOENT) {
+        fd = fd_create(loc.inode, getpid());
+        if (!fd)
+            goto out;
+        dict = dict_new();
+        if (!dict)
+            goto out;
+        gf_uuid_generate(gfid);
+        ret = dict_set_gfuuid(dict, "gfid-req", gfid, true);
+        ret = syncop_create(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                            O_RDWR, 0664, fd, &stbuf, dict, NULL);
+    }
 
-        ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set fd ctx (%p)", fd);
 out:
-        return ret;
+    if (ret == 0) {
+        gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to lookup/create thin-arbiter id file.");
+    }
+    if (dict)
+        dict_unref(dict);
+    if (fd)
+        fd_unref(fd);
+    loc_wipe(&loc);
+
+    return 0;
 }
 
+static int
+afr_ta_id_file_check_cbk(int ret, call_frame_t *ta_frame, void *opaque)
+{
+    return 0;
+}
 
-int
-afr_fd_ctx_set (xlator_t *this, fd_t *fd)
+static void
+afr_discover_done(call_frame_t *frame, xlator_t *this)
 {
-        int ret = -1;
+    int ret = 0;
+    afr_private_t *priv = NULL;
 
-        LOCK (&fd->lock);
-        {
-                ret = __afr_fd_ctx_set (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    priv = this->private;
+    if (!priv->thin_arbiter_count)
+        goto unwind;
+    if (!gf_uuid_is_null(priv->ta_gfid))
+        goto unwind;
 
-        return ret;
+    ret = synctask_new(this->ctx->env, afr_ta_id_file_check,
+                       afr_ta_id_file_check_cbk, NULL, this);
+    if (ret)
+        goto unwind;
+unwind:
+    afr_discover_unwind(frame, this);
 }
 
-/* {{{ flush */
-
 int
-afr_flush_unwind (call_frame_t *frame, xlator_t *this)
+afr_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+                 struct iatt *postparent)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
+    GF_UNUSED int ret = 0;
+    int8_t need_heal = 1;
 
-        local = frame->local;
+    child_index = (long)cookie;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+    local = frame->local;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (flush, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  NULL);
-        }
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (op_ret != -1) {
+        local->replies[child_index].poststat = *buf;
+        local->replies[child_index].postparent = *postparent;
+        if (xdata)
+            local->replies[child_index].xdata = dict_ref(xdata);
+    }
 
-        return 0;
+    if (local->do_discovery && (op_ret == 0))
+        afr_attempt_local_discovery(this, child_index);
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        local->replies[child_index].need_heal = need_heal;
+    } else {
+        local->replies[child_index].need_heal = need_heal;
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        afr_lookup_metadata_heal_check(frame, this);
+    }
+
+    return 0;
 }
 
+int
+afr_discover_do(call_frame_t *frame, xlator_t *this, int err)
+{
+    int ret = 0;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err) {
+        local->op_errno = err;
+        goto out;
+    }
+
+    call_count = local->call_count = AFR_COUNT(local->child_up,
+                                               priv->child_count);
+
+    ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
+                                       &local->loc);
+    if (ret) {
+        local->op_errno = -ret;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(
+                frame, afr_discover_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
+            if (!--call_count)
+                break;
+        }
+    }
+
+    return 0;
+out:
+    AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+    return 0;
+}
 
 int
-afr_flush_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_discover(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-        int need_unwind = 0;
+    int op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int event = 0;
 
-        local = frame->local;
-        priv  = this->private;
+    priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-                local->op_errno = op_errno;
+    if (!local->call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    if (__is_root_gfid(loc->inode->gfid)) {
+        if (!priv->root_inode)
+            priv->root_inode = inode_ref(loc->inode);
+
+        if (priv->choose_local && !priv->did_discovery) {
+            /* Logic to detect which subvolumes of AFR are
+               local, in order to prefer them for reads
+            */
+            local->do_discovery = _gf_true;
+            priv->did_discovery = _gf_true;
         }
-        UNLOCK (&frame->lock);
+    }
 
-        if (need_unwind)
-                afr_flush_unwind (frame, this);
+    local->op = GF_FOP_LOOKUP;
 
-        call_count = afr_frame_return (frame);
+    loc_copy(&local->loc, loc);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
+    local->inode = inode_ref(loc->inode);
+
+    if (xattr_req) {
+        /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+           allocate one for us */
+        local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
+        if (!local->xattr_req) {
+            op_errno = ENOMEM;
+            goto out;
         }
+    }
 
+    if (gf_uuid_is_null(loc->inode->gfid)) {
+        afr_discover_do(frame, this, 0);
         return 0;
+    }
+
+    afr_read_subvol_get(loc->inode, this, NULL, NULL, &event,
+                        AFR_DATA_TRANSACTION, NULL);
+
+    afr_discover_do(frame, this, 0);
+
+    return 0;
+out:
+    AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int
+afr_lookup_do(call_frame_t *frame, xlator_t *this, int err)
+{
+    int ret = 0;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err < 0) {
+        local->op_errno = err;
+        goto out;
+    }
+
+    call_count = local->call_count = AFR_COUNT(local->child_up,
+                                               priv->child_count);
+
+    ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
+                                       &local->loc);
+    if (ret) {
+        local->op_errno = -ret;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(
+                frame, afr_lookup_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
+            if (!--call_count)
+                break;
+        }
+    }
+    return 0;
+out:
+    AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+    return 0;
 }
 
+/*
+ * afr_lookup()
+ *
+ * The goal here is to figure out what the element getting looked up is.
+ * i.e what is the GFID, inode type and a conservative estimate of the
+ * inode attributes are.
+ *
+ * As we lookup, operations may be underway on the entry name and the
+ * inode. In lookup() we are primarily concerned only with the entry
+ * operations. If the entry is getting unlinked or renamed, we detect
+ * what operation is underway by querying for on-going transactions and
+ * pending self-healing on the entry through xdata.
+ *
+ * If the entry is a file/dir, it may need self-heal and/or in a
+ * split-brain condition. Lookup is not the place to worry about these
+ * conditions. Outcast marking will naturally handle them in the read
+ * paths.
+ *
+ * Here is a brief goal of what we are trying to achieve:
+ *
+ * - LOOKUP on all subvolumes concurrently, querying on-going transaction
+ *   and pending self-heal info from the servers.
+ *
+ * - If all servers reply the same inode type and GFID, the overall call
+ *   MUST be a success.
+ *
+ * - If inode types or GFIDs mismatch, and there IS either an on-going
+ *   transaction or pending self-heal, inspect what the nature of the
+ *   transaction or pending heal is, and select the appropriate subvolume's
+ *   reply as the winner.
+ *
+ * - If inode types or GFIDs mismatch, and there are no on-going transactions
+ *   or pending self-heal on the entry name on any of the servers, fail the
+ *   lookup with EIO. Something has gone wrong beyond reasonable action.
+ */
 
 int
-afr_flush_wind (call_frame_t *frame, xlator_t *this)
+afr_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int i = 0;
-        int call_count = -1;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    int event = 0;
+    int ret = 0;
 
-        local = frame->local;
-        priv = this->private;
+    if (loc_is_nameless(loc)) {
+        if (xattr_req)
+            dict_del_sizen(xattr_req, "gfid-req");
+        afr_discover(frame, this, loc, xattr_req);
+        return 0;
+    }
 
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
+    if (afr_is_private_directory(this->private, loc->parent->gfid, loc->name,
+                                 frame->root->pid)) {
+        op_errno = EPERM;
+        goto out;
+    }
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->call_count = call_count;
+    if (!local->call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_flush_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->flush,
-                                           local->fd, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    local->op = GF_FOP_LOOKUP;
 
-        return 0;
-}
+    loc_copy(&local->loc, loc);
 
+    local->inode = inode_ref(loc->inode);
 
-int
-afr_flush_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
+    if (xattr_req) {
+        /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+           allocate one for us */
+        local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
+        if (!local->xattr_req) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = dict_get_gfuuid(local->xattr_req, "gfid-req",
+                              &local->cont.lookup.gfid_req);
+        if (ret == 0) {
+            dict_del_sizen(local->xattr_req, "gfid-req");
+        }
+    }
 
-        local = frame->local;
+    afr_read_subvol_get(loc->parent, this, NULL, NULL, &event,
+                        AFR_DATA_TRANSACTION, NULL);
 
-        local->transaction.unwind (frame, this);
+    afr_lookup_do(frame, this, 0);
 
-        AFR_STACK_DESTROY (frame);
+    return 0;
+out:
+    AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+void
+_afr_cleanup_fd_ctx(xlator_t *this, afr_fd_ctx_t *fd_ctx)
+{
+    afr_private_t *priv = this->private;
+
+    if (fd_ctx->lk_heal_info) {
+        LOCK(&priv->lock);
+        {
+            list_del(&fd_ctx->lk_heal_info->pos);
+        }
+        afr_lk_heal_info_cleanup(fd_ctx->lk_heal_info);
+        fd_ctx->lk_heal_info = NULL;
+    }
+    GF_FREE(fd_ctx->opened_on);
+    GF_FREE(fd_ctx);
+    return;
+}
 
 int
-afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+afr_cleanup_fd_ctx(xlator_t *this, fd_t *fd)
 {
-        afr_private_t *priv  = NULL;
-        afr_local_t   *local = NULL;
-        call_frame_t  *transaction_frame = NULL;
-        int            ret        = -1;
-        int            op_errno   = 0;
+    uint64_t ctx = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    ret = fd_ctx_get(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        priv = this->private;
+    fd_ctx = (afr_fd_ctx_t *)(long)ctx;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
+    if (fd_ctx) {
+        _afr_cleanup_fd_ctx(this, fd_ctx);
+    }
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+out:
+    return 0;
+}
 
-        local->op = GF_FOP_FLUSH;
+int
+afr_release(xlator_t *this, fd_t *fd)
+{
+    afr_cleanup_fd_ctx(this, fd);
 
-        local->transaction.fop    = afr_flush_wind;
-        local->transaction.done   = afr_flush_done;
-        local->transaction.unwind = afr_flush_unwind;
+    return 0;
+}
 
-        local->fd                 = fd_ref (fd);
+afr_fd_ctx_t *
+__afr_fd_ctx_get(fd_t *fd, xlator_t *this)
+{
+    uint64_t ctx = 0;
+    int ret = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start  = 0;
-        local->transaction.len    = 0;
+    ret = __fd_ctx_get(fd, this, &ctx);
 
-        ret = afr_open_fd_fix (transaction_frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        ret = __afr_fd_ctx_set(this, fd);
+        if (ret < 0)
+            goto out;
 
+        ret = __fd_ctx_get(fd, this, &ctx);
+        if (ret < 0)
+            goto out;
+    }
 
-        ret = 0;
+    fd_ctx = (afr_fd_ctx_t *)(long)ctx;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-
-                AFR_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-        }
-
-        return 0;
+    return fd_ctx;
 }
 
-/* }}} */
+afr_fd_ctx_t *
+afr_fd_ctx_get(fd_t *fd, xlator_t *this)
+{
+    afr_fd_ctx_t *fd_ctx = NULL;
 
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(fd, this);
+    }
+    UNLOCK(&fd->lock);
+
+    return fd_ctx;
+}
 
 int
-afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
+__afr_fd_ctx_set(xlator_t *this, fd_t *fd)
 {
-        uint64_t        ctx = 0;
-        afr_fd_ctx_t    *fd_ctx = NULL;
-        int             ret = 0;
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
-
-        ret = fd_ctx_get (fd, this, &ctx);
-        if (ret < 0)
-                goto out;
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    uint64_t ctx = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int i = 0;
 
-        if (fd_ctx) {
-                GF_FREE (fd_ctx->pre_op_done);
+    VALIDATE_OR_GOTO(this->private, out);
+    VALIDATE_OR_GOTO(fd, out);
 
-                GF_FREE (fd_ctx->opened_on);
+    priv = this->private;
 
-                GF_FREE (fd_ctx->locked_on);
+    ret = __fd_ctx_get(fd, this, &ctx);
 
-                GF_FREE (fd_ctx->pre_op_piggyback);
-                list_for_each_entry_safe (paused_call, tmp, &fd_ctx->paused_calls,
-                                          call_list) {
-                        list_del_init (&paused_call->call_list);
-                        GF_FREE (paused_call);
-                }
+    if (ret == 0)
+        goto out;
 
-                GF_FREE (fd_ctx->lock_piggyback);
+    fd_ctx = GF_CALLOC(1, sizeof(afr_fd_ctx_t), gf_afr_mt_afr_fd_ctx_t);
+    if (!fd_ctx) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-                GF_FREE (fd_ctx->lock_acquired);
+    fd_ctx->opened_on = GF_CALLOC(sizeof(*fd_ctx->opened_on), priv->child_count,
+                                  gf_afr_mt_int32_t);
+    if (!fd_ctx->opened_on) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-		pthread_mutex_destroy (&fd_ctx->delay_lock);
+    for (i = 0; i < priv->child_count; i++) {
+        if (fd_is_anonymous(fd))
+            fd_ctx->opened_on[i] = AFR_FD_OPENED;
+        else
+            fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
+    }
 
-                GF_FREE (fd_ctx);
-        }
+    fd_ctx->readdir_subvol = -1;
+    fd_ctx->lk_heal_info = NULL;
 
+    ret = __fd_ctx_set(fd, this, (uint64_t)(long)fd_ctx);
+    if (ret)
+        gf_msg_debug(this->name, 0, "failed to set fd ctx (%p)", fd);
 out:
-        return 0;
+    if (ret && fd_ctx)
+        _afr_cleanup_fd_ctx(this, fd_ctx);
+    return ret;
 }
 
+/* {{{ flush */
 
 int
-afr_release (xlator_t *this, fd_t *fd)
+afr_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, dict_t *xdata)
 {
-        afr_locked_fd_t *locked_fd = NULL;
-        afr_locked_fd_t *tmp       = NULL;
-        afr_private_t   *priv      = NULL;
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-        priv = this->private;
+    local = frame->local;
 
-        afr_cleanup_fd_ctx (this, fd);
+    LOCK(&frame->lock);
+    {
+        if (op_ret != -1) {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        } else {
+            local->op_errno = op_errno;
+        }
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        list_for_each_entry_safe (locked_fd, tmp, &priv->saved_fds,
-                                  list) {
+    if (call_count == 0)
+        AFR_STACK_UNWIND(flush, frame, local->op_ret, local->op_errno,
+                         local->xdata_rsp);
 
-                if (locked_fd->fd == fd) {
-                        list_del_init (&locked_fd->list);
-                        GF_FREE (locked_fd);
-                }
+    return 0;
+}
 
+static int
+afr_flush_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+
+    priv = this->private;
+    local = frame->local;
+    call_count = local->call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, afr_flush_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->flush,
+                              local->fd, xdata);
+            if (!--call_count)
+                break;
         }
+    }
 
-        return 0;
+    return 0;
 }
 
+afr_local_t *
+afr_wakeup_same_fd_delayed_op(xlator_t *this, afr_lock_t *lock, fd_t *fd)
+{
+    afr_local_t *local = NULL;
+
+    if (lock->delay_timer) {
+        local = list_entry(lock->post_op.next, afr_local_t,
+                           transaction.owner_list);
+        if (fd == local->fd) {
+            if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+                local = NULL;
+            } else {
+                lock->delay_timer = NULL;
+            }
+        } else {
+            local = NULL;
+        }
+    }
+
+    return local;
+}
 
-/* {{{ fsync */
+void
+afr_delayed_changelog_wake_resume(xlator_t *this, inode_t *inode,
+                                  call_stub_t *stub)
+{
+    afr_inode_ctx_t *ctx = NULL;
+    afr_lock_t *lock = NULL;
+    afr_local_t *metadata_local = NULL;
+    afr_local_t *data_local = NULL;
+    LOCK(&inode->lock);
+    {
+        (void)__afr_inode_ctx_get(this, inode, &ctx);
+        lock = &ctx->lock[AFR_DATA_TRANSACTION];
+        data_local = afr_wakeup_same_fd_delayed_op(this, lock, stub->args.fd);
+        lock = &ctx->lock[AFR_METADATA_TRANSACTION];
+        metadata_local = afr_wakeup_same_fd_delayed_op(this, lock,
+                                                       stub->args.fd);
+    }
+    UNLOCK(&inode->lock);
+
+    if (data_local) {
+        data_local->transaction.resume_stub = stub;
+    } else if (metadata_local) {
+        metadata_local->transaction.resume_stub = stub;
+    } else {
+        call_resume(stub);
+    }
+    if (data_local) {
+        afr_delayed_changelog_wake_up_cbk(data_local);
+    }
+    if (metadata_local) {
+        afr_delayed_changelog_wake_up_cbk(metadata_local);
+    }
+}
 
 int
-afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
+afr_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = ENOMEM;
 
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+    local->op = GF_FOP_FLUSH;
+    if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
+        goto out;
 
-                if (op_ret == 0) {
-                        local->op_ret = 0;
+    local->fd = fd_ref(fd);
 
-                        if (local->success_count == 0) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
-                        }
+    stub = fop_flush_stub(frame, afr_flush_wrapper, fd, xdata);
+    if (!stub)
+        goto out;
 
-                        if (child_index == read_child) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
-                        }
+    afr_delayed_changelog_wake_resume(this, fd->inode, stub);
 
-                        local->success_count++;
-                }
+    return 0;
+out:
+    AFR_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+    return 0;
+}
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+int
+afr_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-        call_count = afr_frame_return (frame);
+    local = frame->local;
 
-        if (call_count == 0) {
-                AFR_STACK_UNWIND (fsync, frame, local->op_ret, local->op_errno,
-                                  &local->cont.fsync.prebuf,
-                                  &local->cont.fsync.postbuf,
-                                  NULL);
+    LOCK(&frame->lock);
+    {
+        if (op_ret == 0) {
+            local->op_ret = 0;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        } else {
+            local->op_errno = op_errno;
         }
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        return 0;
-}
+    if (call_count == 0)
+        AFR_STACK_UNWIND(fsyncdir, frame, local->op_ret, local->op_errno,
+                         local->xdata_rsp);
 
+    return 0;
+}
 
 int
-afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           int32_t datasync, dict_t *xdata)
+afr_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+             dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int ret = -1;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int32_t call_count = 0;
+    int32_t op_errno = ENOMEM;
 
-        priv = this->private;
+    priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    local->op = GF_FOP_FSYNCDIR;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
 
-        call_count = local->call_count;
-
-        local->fd             = fd_ref (fd);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsync_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsync,
-                                           fd, datasync, xdata);
-                        if (!--call_count)
-                                break;
-                }
+    call_count = local->call_count;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND(frame, afr_fsyncdir_cbk, priv->children[i],
+                       priv->children[i]->fops->fsyncdir, fd, datasync, xdata);
+            if (!--call_count)
+                break;
         }
+    }
 
-        ret = 0;
+    return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
+
+    return 0;
 }
 
 /* }}} */
 
-/* {{{ fsync */
+static int
+afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this);
 
-int32_t
-afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xdata)
+static gf_boolean_t
+afr_is_conflicting_lock_present(int32_t op_ret, int32_t op_errno)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
-
-        local = frame->local;
+    if (op_ret == -1 && op_errno == EAGAIN)
+        return _gf_true;
+    return _gf_false;
+}
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+static void
+afr_fop_lock_unwind(call_frame_t *frame, glusterfs_fop_t op, int32_t op_ret,
+                    int32_t op_errno, dict_t *xdata)
+{
+    switch (op) {
+        case GF_FOP_INODELK:
+            AFR_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_FINODELK:
+            AFR_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_ENTRYLK:
+            AFR_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_FENTRYLK:
+            AFR_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, xdata);
+            break;
+        default:
+            break;
+    }
+}
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+static void
+afr_fop_lock_wind(call_frame_t *frame, xlator_t *this, int child_index,
+                  int32_t (*lock_cbk)(call_frame_t *, void *, xlator_t *,
+                                      int32_t, int32_t, dict_t *))
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = child_index;
+
+    switch (local->op) {
+        case GF_FOP_INODELK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->inodelk,
+                (const char *)local->cont.inodelk.volume, &local->loc,
+                local->cont.inodelk.cmd, &local->cont.inodelk.flock,
+                local->cont.inodelk.xdata);
+            break;
+        case GF_FOP_FINODELK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->finodelk,
+                (const char *)local->cont.inodelk.volume, local->fd,
+                local->cont.inodelk.cmd, &local->cont.inodelk.flock,
+                local->cont.inodelk.xdata);
+            break;
+        case GF_FOP_ENTRYLK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->entrylk, local->cont.entrylk.volume,
+                &local->loc, local->cont.entrylk.basename,
+                local->cont.entrylk.cmd, local->cont.entrylk.type,
+                local->cont.entrylk.xdata);
+            break;
+        case GF_FOP_FENTRYLK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->fentrylk, local->cont.entrylk.volume,
+                local->fd, local->cont.entrylk.basename,
+                local->cont.entrylk.cmd, local->cont.entrylk.type,
+                local->cont.entrylk.xdata);
+            break;
+        default:
+            break;
+    }
+}
 
-        call_count = afr_frame_return (frame);
+void
+afr_fop_lock_proceed(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                  local->op_errno, xdata);
+    local = frame->local;
+    priv = frame->this->private;
 
-        return 0;
+    if (local->fop_lock_state != AFR_FOP_LOCK_PARALLEL) {
+        afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
+                            local->xdata_rsp);
+        return;
+    }
+    /* At least one child is up */
+    /*
+     * Non-blocking locks also need to be serialized.  Otherwise there is
+     * a chance that both the mounts which issued same non-blocking inodelk
+     * may endup not acquiring the lock on any-brick.
+     * Ex: Mount1 and Mount2
+     * request for full length lock on file f1.  Mount1 afr may acquire the
+     * partial lock on brick-1 and may not acquire the lock on brick-2
+     * because Mount2 already got the lock on brick-2, vice versa.  Since
+     * both the mounts only got partial locks, afr treats them as failure in
+     * gaining the locks and unwinds with EAGAIN errno.
+     */
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop_lock_state = AFR_FOP_LOCK_SERIAL;
+    afr_local_replies_wipe(local, priv);
+    if (local->xdata_rsp)
+        dict_unref(local->xdata_rsp);
+    local->xdata_rsp = NULL;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            local->cont.inodelk.cmd = local->cont.inodelk.in_cmd;
+            local->cont.inodelk.flock = local->cont.inodelk.in_flock;
+            if (local->cont.inodelk.xdata)
+                dict_unref(local->cont.inodelk.xdata);
+            local->cont.inodelk.xdata = NULL;
+            if (local->xdata_req)
+                local->cont.inodelk.xdata = dict_ref(local->xdata_req);
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            local->cont.entrylk.cmd = local->cont.entrylk.in_cmd;
+            if (local->cont.entrylk.xdata)
+                dict_unref(local->cont.entrylk.xdata);
+            local->cont.entrylk.xdata = NULL;
+            if (local->xdata_req)
+                local->cont.entrylk.xdata = dict_ref(local->xdata_req);
+            break;
+        default:
+            break;
+    }
+    afr_serialized_lock_wind(frame, frame->this);
 }
 
+static int32_t
+afr_unlock_partial_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-int32_t
-afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int32_t datasync, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int ret = -1;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+    int child_index = (long)cookie;
+    uuid_t gfid = {0};
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    local = frame->local;
+    priv = this->private;
 
-        priv = this->private;
+    if (op_ret < 0 && op_errno != ENOTCONN) {
+        if (local->fd)
+            gf_uuid_copy(gfid, local->fd->inode->gfid);
+        else
+            loc_gfid(&local->loc, gfid);
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "%s: Failed to unlock %s on %s "
+               "with lk_owner: %s",
+               uuid_utoa(gfid), gf_fop_list[local->op],
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        afr_fop_lock_proceed(frame);
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    return 0;
+}
 
-        call_count = local->call_count;
+static int32_t
+afr_unlock_locks_and_proceed(call_frame_t *frame, xlator_t *this,
+                             int call_count)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    if (call_count == 0) {
+        afr_fop_lock_proceed(frame);
+        goto out;
+    }
+
+    local = frame->local;
+    priv = this->private;
+    local->call_count = call_count;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            local->cont.inodelk.flock.l_type = F_UNLCK;
+            local->cont.inodelk.cmd = F_SETLK;
+            if (local->cont.inodelk.xdata)
+                dict_unref(local->cont.inodelk.xdata);
+            local->cont.inodelk.xdata = NULL;
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            local->cont.entrylk.cmd = ENTRYLK_UNLOCK;
+            if (local->cont.entrylk.xdata)
+                dict_unref(local->cont.entrylk.xdata);
+            local->cont.entrylk.xdata = NULL;
+            break;
+        default:
+            break;
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fsyncdir_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fsyncdir,
-                                    fd, datasync, xdata);
-                        if (!--call_count)
-                                break;
-                }
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
-        return 0;
-}
+        if (local->replies[i].op_ret == -1)
+            continue;
 
-/* }}} */
+        afr_fop_lock_wind(frame, this, i, afr_unlock_partial_lock_cbk);
 
-/* {{{ xattrop */
+        if (!--call_count)
+            break;
+    }
+
+out:
+    return 0;
+}
 
 int32_t
-afr_xattrop_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 dict_t *xattr, dict_t *xdata)
+afr_fop_lock_done(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    int i = 0;
+    int lock_count = 0;
+    unsigned char *success = NULL;
 
-        local = frame->local;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        if (!local->cont.xattrop.xattr)
-                                local->cont.xattrop.xattr = dict_ref (xattr);
-                        local->op_ret = 0;
-                }
+    local = frame->local;
+    priv = this->private;
+    success = alloca0(priv->child_count);
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-        call_count = afr_frame_return (frame);
+        if (local->replies[i].op_ret == 0) {
+            lock_count++;
+            success[i] = 1;
+        }
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (xattrop, frame, local->op_ret, local->op_errno,
-                local->cont.xattrop.xattr, xdata);
+        if (local->op_ret == -1 && local->op_errno == EAGAIN)
+            continue;
 
-        return 0;
-}
+        if ((local->replies[i].op_ret == -1) &&
+            (local->replies[i].op_errno == EAGAIN)) {
+            local->op_ret = -1;
+            local->op_errno = EAGAIN;
+            continue;
+        }
 
+        if (local->replies[i].op_ret == 0)
+            local->op_ret = 0;
 
-int32_t
-afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
-{
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int ret = -1;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
+        local->op_errno = local->replies[i].op_errno;
+    }
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    if (afr_fop_lock_is_unlock(frame))
+        goto unwind;
 
-        priv = this->private;
+    if (afr_is_conflicting_lock_present(local->op_ret, local->op_errno)) {
+        afr_unlock_locks_and_proceed(frame, this, lock_count);
+    } else if (priv->quorum_count && !afr_has_quorum(success, this, NULL)) {
+        local->fop_lock_state = AFR_FOP_LOCK_QUORUM_FAILED;
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
+        if (local->op_errno == 0)
+            local->op_errno = afr_quorum_errno(priv);
+        afr_unlock_locks_and_proceed(frame, this, lock_count);
+    } else {
+        goto unwind;
+    }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    return 0;
+unwind:
+    afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
+                        local->xdata_rsp);
+    return 0;
+}
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+static int
+afr_common_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
 
-        call_count = local->call_count;
+    local = frame->local;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_xattrop_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->xattrop,
-                                    loc, optype, xattr, xdata);
-                        if (!--call_count)
-                                break;
-                }
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (op_ret == 0 && xdata) {
+        local->replies[child_index].xdata = dict_ref(xdata);
+        LOCK(&frame->lock);
+        {
+            if (!local->xdata_rsp)
+                local->xdata_rsp = dict_ref(xdata);
         }
-
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
-        return 0;
+        UNLOCK(&frame->lock);
+    }
+    return 0;
 }
 
-/* }}} */
-
-/* {{{ fxattrop */
+static int32_t
+afr_serialized_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-int32_t
-afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = (long)cookie;
+    int next_child = 0;
 
-        int call_count = -1;
+    local = frame->local;
+    priv = this->private;
 
-        local = frame->local;
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        if (!local->cont.fxattrop.xattr)
-                                local->cont.fxattrop.xattr = dict_ref (xattr);
+    for (next_child = child_index + 1; next_child < priv->child_count;
+         next_child++) {
+        if (local->child_up[next_child])
+            break;
+    }
 
-                        local->op_ret = 0;
-                }
+    if (afr_is_conflicting_lock_present(op_ret, op_errno) ||
+        (next_child == priv->child_count)) {
+        afr_fop_lock_done(frame, this);
+    } else {
+        afr_fop_lock_wind(frame, this, next_child, afr_serialized_lock_cbk);
+    }
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fxattrop, frame, local->op_ret, local->op_errno,
-                                  local->cont.fxattrop.xattr, xdata);
-
-        return 0;
+    return 0;
 }
 
-
-int32_t
-afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+static int
+afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int ret = -1;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    priv = this->private;
+    local = frame->local;
 
-        priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            afr_fop_lock_wind(frame, this, i, afr_serialized_lock_cbk);
+            break;
+        }
+    }
+    return 0;
+}
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+static int32_t
+afr_parallel_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+{
+    int call_count = 0;
 
-        call_count = local->call_count;
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fxattrop_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fxattrop,
-                                    fd, optype, xattr, xdata);
-                        if (!--call_count)
-                                break;
-                }
-        }
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        afr_fop_lock_done(frame, this);
 
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    return 0;
 }
 
-/* }}} */
+static int
+afr_parallel_lock_wind(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    int i = 0;
 
+    priv = this->private;
+    local = frame->local;
+    call_count = local->call_count;
 
-int32_t
-afr_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+        afr_fop_lock_wind(frame, this, i, afr_parallel_lock_cbk);
+        if (!--call_count)
+            break;
+    }
+    return 0;
+}
 
+static int
+afr_fop_handle_lock(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    afr_local_t *local = frame->local;
+    int op_errno = 0;
 
-        local = frame->local;
+    if (!afr_fop_lock_is_unlock(frame)) {
+        if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
+            goto out;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+        switch (local->op) {
+            case GF_FOP_INODELK:
+            case GF_FOP_FINODELK:
+                local->cont.inodelk.cmd = F_SETLK;
+                break;
+            case GF_FOP_ENTRYLK:
+            case GF_FOP_FENTRYLK:
+                local->cont.entrylk.cmd = ENTRYLK_LOCK_NB;
+                break;
+            default:
+                break;
+        }
+    }
 
-                local->op_errno = op_errno;
+    if (local->xdata_req) {
+        switch (local->op) {
+            case GF_FOP_INODELK:
+            case GF_FOP_FINODELK:
+                local->cont.inodelk.xdata = dict_ref(local->xdata_req);
+                break;
+            case GF_FOP_ENTRYLK:
+            case GF_FOP_FENTRYLK:
+                local->cont.entrylk.xdata = dict_ref(local->xdata_req);
+                break;
+            default:
+                break;
         }
-        UNLOCK (&frame->lock);
+    }
 
-        call_count = afr_frame_return (frame);
+    local->fop_lock_state = AFR_FOP_LOCK_PARALLEL;
+    afr_parallel_lock_wind(frame, this);
+out:
+    return -op_errno;
+}
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (inodelk, frame, local->op_ret,
-                                  local->op_errno, xdata);
+static int32_t
+afr_handle_inodelk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                   const char *volume, loc_t *loc, fd_t *fd, int32_t cmd,
+                   struct gf_flock *flock, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = fop;
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd && (flock->l_type != F_UNLCK)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        local->fd = fd_ref(fd);
+    }
+
+    local->cont.inodelk.volume = gf_strdup(volume);
+    if (!local->cont.inodelk.volume) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->cont.inodelk.in_cmd = cmd;
+    local->cont.inodelk.cmd = cmd;
+    local->cont.inodelk.in_flock = *flock;
+    local->cont.inodelk.flock = *flock;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    op_errno = -afr_fop_handle_lock(frame, frame->this);
+    if (op_errno)
+        goto out;
+    return 0;
+out:
+    afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
+int32_t
+afr_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+    afr_handle_inodelk(frame, this, GF_FOP_INODELK, volume, loc, NULL, cmd,
+                       flock, xdata);
+    return 0;
+}
 
 int32_t
-afr_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd,
-             struct gf_flock *flock, dict_t *xdata)
+afr_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int ret = -1;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
+    afr_handle_inodelk(frame, this, GF_FOP_FINODELK, volume, NULL, fd, cmd,
+                       flock, xdata);
+    return 0;
+}
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+static int
+afr_handle_entrylk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                   const char *volume, loc_t *loc, fd_t *fd,
+                   const char *basename, entrylk_cmd cmd, entrylk_type type,
+                   dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = fop;
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd && (cmd != ENTRYLK_UNLOCK)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        local->fd = fd_ref(fd);
+    }
+    local->cont.entrylk.cmd = cmd;
+    local->cont.entrylk.in_cmd = cmd;
+    local->cont.entrylk.type = type;
+    local->cont.entrylk.volume = gf_strdup(volume);
+    local->cont.entrylk.basename = gf_strdup(basename);
+    if (!local->cont.entrylk.volume || !local->cont.entrylk.basename) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+    op_errno = -afr_fop_handle_lock(frame, frame->this);
+    if (op_errno)
+        goto out;
+
+    return 0;
+out:
+    afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
+    return 0;
+}
 
-        priv = this->private;
+int
+afr_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    afr_handle_entrylk(frame, this, GF_FOP_ENTRYLK, volume, loc, NULL, basename,
+                       cmd, type, xdata);
+    return 0;
+}
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+int
+afr_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
+{
+    afr_handle_entrylk(frame, this, GF_FOP_FENTRYLK, volume, NULL, fd, basename,
+                       cmd, type, xdata);
+    return 0;
+}
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+int
+afr_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct statvfs *statvfs, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    struct statvfs *buf = NULL;
 
-        call_count = local->call_count;
+    local = frame->local;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_inodelk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->inodelk,
-                                    volume, loc, cmd, flock, xdata);
-
-                        if (!--call_count)
-                                break;
+    LOCK(&frame->lock);
+    {
+        if (op_ret != 0) {
+            local->op_errno = op_errno;
+            goto unlock;
+        }
+
+        local->op_ret = op_ret;
+
+        buf = &local->cont.statfs.buf;
+        if (local->cont.statfs.buf_set) {
+            if (statvfs->f_bavail < buf->f_bavail) {
+                *buf = *statvfs;
+                if (xdata) {
+                    if (local->xdata_rsp)
+                        dict_unref(local->xdata_rsp);
+                    local->xdata_rsp = dict_ref(xdata);
                 }
+            }
+        } else {
+            *buf = *statvfs;
+            local->cont.statfs.buf_set = 1;
+            if (xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
+    }
+unlock:
+    call_count = --local->call_count;
+    UNLOCK(&frame->lock);
 
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
-        return 0;
+    if (call_count == 0)
+        AFR_STACK_UNWIND(statfs, frame, local->op_ret, local->op_errno,
+                         &local->cont.statfs.buf, local->xdata_rsp);
+
+    return 0;
 }
 
+int
+afr_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_STATFS;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    if (priv->arbiter_count == 1 && local->child_up[ARBITER_BRICK_INDEX])
+        local->call_count--;
+    call_count = local->call_count;
+    if (!call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            if (AFR_IS_ARBITER_BRICK(priv, i))
+                continue;
+            STACK_WIND(frame, afr_statfs_cbk, priv->children[i],
+                       priv->children[i]->fops->statfs, loc, xdata);
+            if (!--call_count)
+                break;
+        }
+    }
+
+    return 0;
+out:
+    AFR_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
+
+    return 0;
+}
 
 int32_t
-afr_finodelk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+afr_lk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
                   dict_t *xdata)
-
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
+    int call_count = -1;
+    int child_index = (long)cookie;
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    local = frame->local;
 
-        call_count = afr_frame_return (frame);
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "gfid=%s: unlock failed on subvolume %s "
+               "with lock owner %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (finodelk, frame, local->op_ret,
-                                  local->op_errno, xdata);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
+                         local->xdata_rsp);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
-              dict_t *xdata)
+afr_lk_unlock(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int ret = -1;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    local = frame->local;
+    priv = this->private;
 
-        priv = this->private;
+    call_count = afr_locked_nodes_count(local->cont.lk.locked_nodes,
+                                        priv->child_count);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
+                         local->xdata_rsp);
+        return 0;
+    }
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    local->call_count = call_count;
 
-        call_count = local->call_count;
+    local->cont.lk.user_flock.l_type = F_UNLCK;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_finodelk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->finodelk,
-                                    volume, fd, cmd, flock, xdata);
-
-                        if (!--call_count)
-                                break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->cont.lk.locked_nodes[i]) {
+            STACK_WIND_COOKIE(frame, afr_lk_unlock_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->lk,
+                              local->fd, F_SETLK, &local->cont.lk.user_flock,
+                              NULL);
+
+            if (!--call_count)
+                break;
         }
+    }
 
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+           int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = -1;
 
-        local = frame->local;
+    local = frame->local;
+    priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+    child_index = (long)cookie;
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret < 0 && op_errno == EAGAIN) {
+        local->op_ret = -1;
+        local->op_errno = EAGAIN;
 
-        call_count = afr_frame_return (frame);
+        afr_lk_unlock(frame, this);
+        return 0;
+    }
+
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lk.locked_nodes[child_index] = 1;
+        local->cont.lk.ret_flock = *lock;
+    }
+
+    child_index++;
+
+    if (child_index < priv->child_count) {
+        STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)child_index,
+                          priv->children[child_index],
+                          priv->children[child_index]->fops->lk, local->fd,
+                          local->cont.lk.cmd, &local->cont.lk.user_flock,
+                          local->xdata_req);
+    } else if (priv->quorum_count &&
+               !afr_has_quorum(local->cont.lk.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (entrylk, frame, local->op_ret,
-                                  local->op_errno, xdata);
+        afr_lk_unlock(frame, this);
+    } else {
+        if (local->op_ret < 0)
+            local->op_errno = afr_final_errno(local, priv);
 
-        return 0;
-}
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
+                         &local->cont.lk.ret_flock, local->xdata_rsp);
+    }
 
+    return 0;
+}
 
-int32_t
-afr_entrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc,
-             const char *basename, entrylk_cmd cmd, entrylk_type type,
-             dict_t *xdata)
+int
+afr_lk_transaction_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int ret = -1;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    return 0;
+}
 
-        priv = this->private;
+int
+afr_lk_txn_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                    dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = -1;
+
+    local = frame->local;
+    child_index = (long)cookie;
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lk.locked_nodes[child_index] = 1;
+        local->cont.lk.ret_flock = *lock;
+    }
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+int
+afr_lk_txn_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int child_index = (long)cookie;
+
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "gfid=%s: unlock failed on subvolume %s "
+               "with lock owner %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
+    return 0;
+}
+int
+afr_lk_transaction(void *opaque)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    char *wind_on = NULL;
+    int op_errno = 0;
+    int i = 0;
+    int ret = 0;
+
+    frame = (call_frame_t *)opaque;
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+    wind_on = alloca0(priv->child_count);
+
+    if (priv->arbiter_count || priv->child_count != 3) {
+        op_errno = ENOTSUP;
+        gf_msg(frame->this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Lock healing supported only for replica 3 volumes.",
+               uuid_utoa(local->fd->inode->gfid));
+        goto err;
+    }
+
+    op_errno = -afr_dom_lock_acquire(frame);  // Released during
+                                              // AFR_STACK_UNWIND
+    if (op_errno != 0) {
+        goto err;
+    }
+    if (priv->quorum_count &&
+        !afr_has_quorum(local->cont.lk.dom_locked_nodes, this, NULL)) {
+        op_errno = afr_final_errno(local, priv);
+        goto err;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->child_up[i] && local->cont.lk.dom_locked_nodes[i])
+            wind_on[i] = 1;
+    }
+    AFR_ONLIST(wind_on, frame, afr_lk_txn_wind_cbk, lk, local->fd,
+               local->cont.lk.cmd, &local->cont.lk.user_flock,
+               local->xdata_req);
+
+    if (priv->quorum_count &&
+        !afr_has_quorum(local->cont.lk.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
+        goto unlock;
+    } else {
+        if (local->cont.lk.user_flock.l_type == F_UNLCK)
+            ret = afr_remove_lock_from_saved_locks(local, this);
+        else
+            ret = afr_add_lock_to_saved_locks(frame, this);
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = -ret;
+            goto unlock;
+        }
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
+                         &local->cont.lk.ret_flock, local->xdata_rsp);
+    }
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    return 0;
 
-        call_count = local->call_count;
+unlock:
+    local->cont.lk.user_flock.l_type = F_UNLCK;
+    AFR_ONLIST(local->cont.lk.locked_nodes, frame, afr_lk_txn_unlock_cbk, lk,
+               local->fd, F_SETLK, &local->cont.lk.user_flock, NULL);
+err:
+    AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
+    return -1;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_entrylk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->entrylk,
-                                    volume, loc, basename, cmd, type, xdata);
-
-                        if (!--call_count)
-                                break;
-                }
+int
+afr_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *flock, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+    int i = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_LK;
+    if (!afr_lk_is_unlock(cmd, flock)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+            goto out;
+    }
+
+    local->cont.lk.locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.locked_nodes),
+        gf_afr_mt_char);
+
+    if (!local->cont.lk.locked_nodes) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->fd = fd_ref(fd);
+    local->cont.lk.cmd = cmd;
+    local->cont.lk.user_flock = *flock;
+    local->cont.lk.ret_flock = *flock;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    if (afr_is_lock_mode_mandatory(xdata)) {
+        ret = synctask_new(this->ctx->env, afr_lk_transaction,
+                           afr_lk_transaction_cbk, frame, frame);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto out;
         }
-
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
         return 0;
-}
+    }
 
+    STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)0, priv->children[i],
+                      priv->children[i]->fops->lk, fd, cmd, flock,
+                      local->xdata_req);
 
+    return 0;
+out:
+    AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
 
-int32_t
-afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+    return 0;
+}
 
+int32_t
+afr_lease_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
+                     dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    local = frame->local;
+    call_count = afr_frame_return(frame);
 
-        call_count = afr_frame_return (frame);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno, lease,
+                         xdata);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fentrylk, frame, local->op_ret,
-                                  local->op_errno, xdata);
-
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_fentrylk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd,
-              const char *basename, entrylk_cmd cmd,
-              entrylk_type type, dict_t *xdata)
+afr_lease_unlock(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int ret = -1;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    local = frame->local;
+    priv = this->private;
 
-        priv = this->private;
+    call_count = afr_locked_nodes_count(local->cont.lease.locked_nodes,
+                                        priv->child_count);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
+                         &local->cont.lease.ret_lease, NULL);
+        return 0;
+    }
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    local->call_count = call_count;
 
-        call_count = local->call_count;
+    local->cont.lease.user_lease.cmd = GF_UNLK_LEASE;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fentrylk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fentrylk,
-                                    volume, fd, basename, cmd, type, xdata);
-
-                        if (!--call_count)
-                                break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->cont.lease.locked_nodes[i]) {
+            STACK_WIND(frame, afr_lease_unlock_cbk, priv->children[i],
+                       priv->children[i]->fops->lease, &local->loc,
+                       &local->cont.lease.user_lease, NULL);
+
+            if (!--call_count)
+                break;
         }
+    }
 
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_statfs_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno,
-                struct statvfs *statvfs, dict_t *xdata)
+afr_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_lease *lease, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = -1;
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
-
-                if (op_ret == 0) {
-                        local->op_ret   = op_ret;
-
-                        if (local->cont.statfs.buf_set) {
-                                if (statvfs->f_bavail < local->cont.statfs.buf.f_bavail)
-                                        local->cont.statfs.buf = *statvfs;
-                        } else {
-                                local->cont.statfs.buf = *statvfs;
-                                local->cont.statfs.buf_set = 1;
-                        }
-                }
+    local = frame->local;
+    priv = this->private;
 
-                if (op_ret == -1)
-                        local->op_errno = op_errno;
+    child_index = (long)cookie;
 
-        }
-        UNLOCK (&frame->lock);
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret < 0 && op_errno == EAGAIN) {
+        local->op_ret = -1;
+        local->op_errno = EAGAIN;
 
-        call_count = afr_frame_return (frame);
+        afr_lease_unlock(frame, this);
+        return 0;
+    }
+
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lease.locked_nodes[child_index] = 1;
+        local->cont.lease.ret_lease = *lease;
+    }
+
+    child_index++;
+    if (child_index < priv->child_count) {
+        STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)child_index,
+                          priv->children[child_index],
+                          priv->children[child_index]->fops->lease, &local->loc,
+                          &local->cont.lease.user_lease, xdata);
+    } else if (priv->quorum_count &&
+               !afr_has_quorum(local->cont.lease.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->cont.statfs.buf, xdata);
+        afr_lease_unlock(frame, this);
+    } else {
+        if (local->op_ret < 0)
+            local->op_errno = afr_final_errno(local, priv);
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
+                         &local->cont.lease.ret_lease, NULL);
+    }
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-afr_statfs (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, dict_t *xdata)
+int
+afr_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata)
 {
-        afr_private_t *  priv        = NULL;
-        int              child_count = 0;
-        afr_local_t   *  local       = NULL;
-        int              i           = 0;
-        int              ret = -1;
-        int              call_count = 0;
-        int32_t          op_errno    = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
 
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
+    priv = this->private;
 
-        priv = this->private;
-        child_count = priv->child_count;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    local->op = GF_FOP_LEASE;
+    local->cont.lease.locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lease.locked_nodes),
+        gf_afr_mt_char);
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    if (!local->cont.lease.locked_nodes) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        call_count = local->call_count;
+    loc_copy(&local->loc, loc);
+    local->cont.lease.user_lease = *lease;
+    local->cont.lease.ret_lease = *lease;
 
-        for (i = 0; i < child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_statfs_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->statfs,
-                                    loc, xdata);
-                        if (!--call_count)
-                                break;
-                }
-        }
+    STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)0, priv->children[0],
+                      priv->children[0]->fops->lease, loc, lease, xdata);
 
-        ret = 0;
+    return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
+    AFR_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
 
-int32_t
-afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                   dict_t *xdata)
-{
-        afr_local_t * local = NULL;
-        int call_count = -1;
+    return 0;
+}
 
-        local = frame->local;
-        call_count = afr_frame_return (frame);
+int
+afr_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = 0;
+    gf_boolean_t failed = _gf_false;
+    gf_boolean_t succeeded = _gf_false;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    call_count = afr_frame_return(frame);
+    if (call_count)
+        goto out;
+    /* If any of the subvolumes failed with other than ENOTCONN
+     * return error else return success unless all the subvolumes
+     * failed.
+     * TODO: In case of failure, we need to unregister the xattrs
+     * from the other subvolumes where it succeeded (once upcall
+     * fixes the Bz-1371622)*/
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0 &&
+            local->replies[i].op_errno != ENOTCONN) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+            if (local->xdata_rsp)
+                dict_unref(local->xdata_rsp);
+            local->xdata_rsp = NULL;
+            if (local->replies[i].xdata) {
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+            failed = _gf_true;
+            break;
+        }
+        if (local->replies[i].op_ret == 0) {
+            succeeded = _gf_true;
+            local->op_ret = 0;
+            local->op_errno = 0;
+            if (!local->xdata_rsp && local->replies[i].xdata) {
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+        }
+    }
+
+    if (!succeeded && !failed) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+    }
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  lock, xdata);
+    AFR_STACK_UNWIND(ipc, frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
 
-        return 0;
+out:
+    return 0;
 }
 
-
-int32_t
-afr_lk_unlock (call_frame_t *frame, xlator_t *this)
+int
+afr_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        afr_local_t   * local = NULL;
-        afr_private_t * priv  = NULL;
-        int i = 0;
-        int call_count = 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_cnt = -1;
 
-        local = frame->local;
-        priv  = this->private;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
 
-        call_count = afr_locked_nodes_count (local->cont.lk.locked_nodes,
-                                             priv->child_count);
+    if (op != GF_IPC_TARGET_UPCALL)
+        goto wind_default;
 
-        if (call_count == 0) {
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock, NULL);
-                return 0;
-        }
+    VALIDATE_OR_GOTO(this->private, err);
+    priv = this->private;
 
-        local->call_count = call_count;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto err;
 
-        local->cont.lk.user_flock.l_type = F_UNLCK;
+    call_cnt = local->call_count;
 
+    if (xdata) {
         for (i = 0; i < priv->child_count; i++) {
-                if (local->cont.lk.locked_nodes[i]) {
-                        STACK_WIND (frame, afr_lk_unlock_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->lk,
-                                    local->fd, F_SETLK,
-                                    &local->cont.lk.user_flock, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+            if (dict_set_int8(xdata, priv->pending_key[i], 0) < 0)
+                goto err;
         }
+    }
 
-        return 0;
-}
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
 
+        STACK_WIND_COOKIE(frame, afr_ipc_cbk, (void *)(long)i,
+                          priv->children[i], priv->children[i]->fops->ipc, op,
+                          xdata);
+        if (!--call_cnt)
+            break;
+    }
+    return 0;
 
-int32_t
-afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+err:
+    if (op_errno == -1)
+        op_errno = errno;
+    AFR_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
+
+    return 0;
+
+wind_default:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
+}
+
+int
+afr_forget(xlator_t *this, inode_t *inode)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int child_index = -1;
-/*        int            ret  = 0; */
+    uint64_t ctx_int = 0;
+    afr_inode_ctx_t *ctx = NULL;
+
+    afr_spb_choice_timeout_cancel(this, inode);
+    inode_ctx_del(inode, this, &ctx_int);
+    if (!ctx_int)
+        return 0;
 
+    ctx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
+    afr_inode_ctx_destroy(ctx);
+    return 0;
+}
 
-        local = frame->local;
-        priv  = this->private;
+int
+afr_priv_dump(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
+    gf_proc_dump_add_section("%s", key_prefix);
+    gf_proc_dump_write("child_count", "%u", priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        sprintf(key, "child_up[%d]", i);
+        gf_proc_dump_write(key, "%d", priv->child_up[i]);
+        sprintf(key, "pending_key[%d]", i);
+        gf_proc_dump_write(key, "%s", priv->pending_key[i]);
+        sprintf(key, "pending_reads[%d]", i);
+        gf_proc_dump_write(key, "%" PRId64,
+                           GF_ATOMIC_GET(priv->pending_reads[i]));
+        sprintf(key, "child_latency[%d]", i);
+        gf_proc_dump_write(key, "%" PRId64, priv->child_latency[i]);
+        sprintf(key, "halo_child_up[%d]", i);
+        gf_proc_dump_write(key, "%d", priv->halo_child_up[i]);
+    }
+    gf_proc_dump_write("data_self_heal", "%d", priv->data_self_heal);
+    gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
+    gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
+    gf_proc_dump_write("read_child", "%d", priv->read_child);
+    gf_proc_dump_write("wait_count", "%u", priv->wait_count);
+    gf_proc_dump_write("heal-wait-queue-length", "%d", priv->heal_wait_qlen);
+    gf_proc_dump_write("heal-waiters", "%d", priv->heal_waiters);
+    gf_proc_dump_write("background-self-heal-count", "%d",
+                       priv->background_self_heal_count);
+    gf_proc_dump_write("healers", "%d", priv->healers);
+    gf_proc_dump_write("read-hash-mode", "%d", priv->hash_mode);
+    gf_proc_dump_write("use-anonymous-inode", "%d", priv->use_anon_inode);
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        gf_proc_dump_write("quorum-type", "auto");
+    } else if (priv->quorum_count == 0) {
+        gf_proc_dump_write("quorum-type", "none");
+    } else {
+        gf_proc_dump_write("quorum-type", "fixed");
+        gf_proc_dump_write("quorum-count", "%d", priv->quorum_count);
+    }
+    gf_proc_dump_write("up", "%u", afr_has_quorum(priv->child_up, this, NULL));
+    if (priv->thin_arbiter_count) {
+        gf_proc_dump_write("ta_child_up", "%d", priv->ta_child_up);
+        gf_proc_dump_write("ta_bad_child_index", "%d",
+                           priv->ta_bad_child_index);
+        gf_proc_dump_write("ta_notify_dom_lock_offset", "%" PRId64,
+                           priv->ta_notify_dom_lock_offset);
+    }
+
+    return 0;
+}
 
-        child_index = (long) cookie;
+/**
+ * find_child_index - find the child's index in the array of subvolumes
+ * @this: AFR
+ * @child: child
+ */
 
-        if (!child_went_down (op_ret, op_errno) && (op_ret == -1)) {
-                local->op_ret   = -1;
-                local->op_errno = op_errno;
+static int
+afr_find_child_index(xlator_t *this, xlator_t *child)
+{
+    afr_private_t *priv = NULL;
+    int child_count = -1;
+    int i = -1;
 
-                afr_lk_unlock (frame, this);
-                return 0;
-        }
+    priv = this->private;
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        child_count++;
+    }
 
-        if (op_ret == 0) {
-                local->op_ret        = 0;
-                local->op_errno      = 0;
-                local->cont.lk.locked_nodes[child_index] = 1;
-                local->cont.lk.ret_flock = *lock;
-        }
+    for (i = 0; i < child_count; i++) {
+        if ((xlator_t *)child == priv->children[i])
+            break;
+    }
 
-        child_index++;
+    return i;
+}
 
-        if (child_index < priv->child_count) {
-                STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) child_index,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->lk,
-                                   local->fd, local->cont.lk.cmd,
-                                   &local->cont.lk.user_flock, xdata);
-        } else if (local->op_ret == -1) {
-                /* all nodes have gone down */
+int
+__afr_get_up_children_count(afr_private_t *priv)
+{
+    int up_children = 0;
+    int i = 0;
 
-                AFR_STACK_UNWIND (lk, frame, -1, ENOTCONN,
-                                  &local->cont.lk.ret_flock, NULL);
-        } else {
-                /* locking has succeeded on all nodes that are up */
+    for (i = 0; i < priv->child_count; i++)
+        if (priv->child_up[i] == 1)
+            up_children++;
 
-                /* temporarily
-                   ret = afr_mark_locked_nodes (this, local->fd,
-                   local->cont.lk.locked_nodes);
-                   if (ret)
-                   gf_log (this->name, GF_LOG_DEBUG,
-                   "Could not save locked nodes info in fdctx");
+    return up_children;
+}
 
-                   ret = afr_save_locked_fd (this, local->fd);
-                   if (ret)
-                   gf_log (this->name, GF_LOG_DEBUG,
-                   "Could not save locked fd");
+static int
+__get_heard_from_all_status(xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    int i;
 
-                */
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock, NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!priv->last_event[i]) {
+            return 0;
         }
-
+    }
+    if (priv->thin_arbiter_count && !priv->ta_child_up) {
         return 0;
+    }
+    return 1;
 }
 
-
-int
-afr_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+glusterfs_event_t
+__afr_transform_event_from_state(xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
-        int32_t op_errno = 0;
-        int     ret      = -1;
+    int i = 0;
+    int up_children = 0;
+    afr_private_t *priv = this->private;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    if (__get_heard_from_all_status(this))
+        /* have_heard_from_all. Let afr_notify() do the propagation. */
+        return GF_EVENT_MAXVAL;
 
-        priv = this->private;
+    up_children = __afr_get_up_children_count(priv);
+    /* Treat the children with pending notification, as having sent a
+     * GF_EVENT_CHILD_DOWN. i.e. set the event as GF_EVENT_SOME_DESCENDENT_DOWN,
+     * as done in afr_notify() */
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->last_event[i])
+            continue;
+        priv->last_event[i] = GF_EVENT_SOME_DESCENDENT_DOWN;
+        priv->child_up[i] = 0;
+    }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    if (up_children)
+        /* We received at least one child up */
+        return GF_EVENT_CHILD_UP;
+    else
+        return GF_EVENT_CHILD_DOWN;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    return GF_EVENT_MAXVAL;
+}
 
-        local->cont.lk.locked_nodes = GF_CALLOC (priv->child_count,
-                                                 sizeof (*local->cont.lk.locked_nodes),
-                                                 gf_afr_mt_char);
+static void
+afr_notify_cbk(void *data)
+{
+    xlator_t *this = data;
+    afr_private_t *priv = this->private;
+    glusterfs_event_t event = GF_EVENT_MAXVAL;
+    gf_boolean_t propagate = _gf_false;
+
+    LOCK(&priv->lock);
+    {
+        if (!priv->timer) {
+            /*
+             * Either child_up/child_down is already sent to parent.
+             * This is a spurious wake up.
+             */
+            goto unlock;
+        }
+        priv->timer = NULL;
+        event = __afr_transform_event_from_state(this);
+        if (event != GF_EVENT_MAXVAL)
+            propagate = _gf_true;
+    }
+unlock:
+    UNLOCK(&priv->lock);
+    if (propagate)
+        default_notify(this, event, NULL);
+}
 
-        if (!local->cont.lk.locked_nodes) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+static void
+__afr_launch_notify_timer(xlator_t *this, afr_private_t *priv)
+{
+    struct timespec delay = {
+        0,
+    };
 
-        local->fd            = fd_ref (fd);
-        local->cont.lk.cmd   = cmd;
-        local->cont.lk.user_flock = *flock;
-        local->cont.lk.ret_flock = *flock;
+    gf_msg_debug(this->name, 0, "Initiating child-down timer");
+    delay.tv_sec = 10;
+    delay.tv_nsec = 0;
+    priv->timer = gf_timer_call_after(this->ctx, delay, afr_notify_cbk, this);
+    if (priv->timer == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_TIMER_CREATE_FAIL,
+               "Cannot create timer for delayed initialization");
+    }
+}
 
-        STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) 0,
-                           priv->children[i],
-                           priv->children[i]->fops->lk,
-                           fd, cmd, flock, xdata);
+static int
+find_best_down_child(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int i = -1;
+    int32_t best_child = -1;
+    int64_t best_latency = INT64_MAX;
 
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!priv->child_up[i] && priv->child_latency[i] >= 0 &&
+            priv->child_latency[i] < best_latency) {
+            best_child = i;
+            best_latency = priv->child_latency[i];
+        }
+    }
+    if (best_child >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "Found best down child (%d) @ %" PRId64 " ms latency",
+                     best_child, best_latency);
+    }
+    return best_child;
 }
 
 int
-afr_forget (xlator_t *this, inode_t *inode)
+find_worst_up_child(xlator_t *this)
 {
-        uint64_t        ctx_addr = 0;
-        afr_inode_ctx_t *ctx     = NULL;
+    afr_private_t *priv = NULL;
+    int i = -1;
+    int32_t worst_child = -1;
+    int64_t worst_latency = INT64_MIN;
 
-        inode_ctx_get (inode, this, &ctx_addr);
+    priv = this->private;
 
-        if (!ctx_addr)
-                goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->child_up[i] && priv->child_latency[i] >= 0 &&
+            priv->child_latency[i] > worst_latency) {
+            worst_child = i;
+            worst_latency = priv->child_latency[i];
+        }
+    }
+    if (worst_child >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "Found worst up child (%d) @ %" PRId64 " ms latency",
+                     worst_child, worst_latency);
+    }
+    return worst_child;
+}
 
-        ctx = (afr_inode_ctx_t *)(long)ctx_addr;
-        GF_FREE (ctx->fresh_children);
-        GF_FREE (ctx);
-out:
-        return 0;
+void
+__afr_handle_ping_event(xlator_t *this, xlator_t *child_xlator, const int idx,
+                        int64_t halo_max_latency_msec, int32_t *event,
+                        int64_t child_latency_msec)
+{
+    afr_private_t *priv = NULL;
+    int up_children = 0;
+
+    priv = this->private;
+
+    priv->child_latency[idx] = child_latency_msec;
+    gf_msg_debug(child_xlator->name, 0, "Client ping @ %" PRId64 " ms",
+                 child_latency_msec);
+    if (priv->shd.iamshd)
+        return;
+
+    up_children = __afr_get_up_children_count(priv);
+
+    if (child_latency_msec > halo_max_latency_msec &&
+        priv->child_up[idx] == 1 && up_children > priv->halo_min_replicas) {
+        if ((up_children - 1) < priv->halo_min_replicas) {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Overriding halo threshold, "
+                   "min replicas: %d",
+                   priv->halo_min_replicas);
+        } else {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Child latency (%" PRId64
+                   " ms) "
+                   "exceeds halo threshold (%" PRId64
+                   "), "
+                   "marking child down.",
+                   child_latency_msec, halo_max_latency_msec);
+            if (priv->halo_child_up[idx]) {
+                *event = GF_EVENT_CHILD_DOWN;
+            }
+        }
+    } else if (child_latency_msec < halo_max_latency_msec &&
+               priv->child_up[idx] == 0) {
+        if (up_children < priv->halo_max_replicas) {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Child latency (%" PRId64
+                   " ms) "
+                   "below halo threshold (%" PRId64
+                   "), "
+                   "marking child up.",
+                   child_latency_msec, halo_max_latency_msec);
+            if (priv->halo_child_up[idx]) {
+                *event = GF_EVENT_CHILD_UP;
+            }
+        } else {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Not marking child %d up, "
+                   "max replicas (%d) reached.",
+                   idx, priv->halo_max_replicas);
+        }
+    }
 }
 
-int
-afr_priv_dump (xlator_t *this)
+static int64_t
+afr_get_halo_latency(xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        char  key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char  key[GF_DUMP_MAX_BUF_LEN];
-        int   i = 0;
+    afr_private_t *priv = NULL;
+    int64_t halo_max_latency_msec = 0;
 
+    priv = this->private;
 
-        GF_ASSERT (this);
-        priv = this->private;
+    if (priv->shd.iamshd) {
+        halo_max_latency_msec = priv->shd.halo_max_latency_msec;
+    } else if (priv->nfsd.iamnfsd) {
+        halo_max_latency_msec = priv->nfsd.halo_max_latency_msec;
+    } else {
+        halo_max_latency_msec = priv->halo_max_latency_msec;
+    }
+    gf_msg_debug(this->name, 0, "Using halo latency %" PRId64,
+                 halo_max_latency_msec);
+    return halo_max_latency_msec;
+}
 
-        GF_ASSERT (priv);
-        snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
-        gf_proc_dump_add_section(key_prefix);
-        gf_proc_dump_write("child_count", "%u", priv->child_count);
-        gf_proc_dump_write("read_child_rr", "%u", priv->read_child_rr);
-        for (i = 0; i < priv->child_count; i++) {
-                sprintf (key, "child_up[%d]", i);
-                gf_proc_dump_write(key, "%d", priv->child_up[i]);
-                sprintf (key, "pending_key[%d]", i);
-                gf_proc_dump_write(key, "%s", priv->pending_key[i]);
-        }
-        gf_proc_dump_write("data_self_heal", "%s", priv->data_self_heal);
-        gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
-        gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
-        gf_proc_dump_write("data_change_log", "%d", priv->data_change_log);
-        gf_proc_dump_write("metadata_change_log", "%d", priv->metadata_change_log);
-        gf_proc_dump_write("entry-change_log", "%d", priv->entry_change_log);
-        gf_proc_dump_write("read_child", "%d", priv->read_child);
-        gf_proc_dump_write("favorite_child", "%d", priv->favorite_child);
-        gf_proc_dump_write("wait_count", "%u", priv->wait_count);
+void
+__afr_handle_child_up_event(xlator_t *this, xlator_t *child_xlator,
+                            const int idx, int64_t child_latency_msec,
+                            int32_t *event, int32_t *call_psh,
+                            int32_t *up_child)
+{
+    afr_private_t *priv = NULL;
+    int up_children = 0;
+    int worst_up_child = -1;
+    int64_t halo_max_latency_msec = afr_get_halo_latency(this);
+
+    priv = this->private;
+
+    /*
+     * This only really counts if the child was never up
+     * (value = -1) or had been down (value = 0).  See
+     * comment at GF_EVENT_CHILD_DOWN for a more detailed
+     * explanation.
+     */
+    if (priv->child_up[idx] != 1) {
+        priv->event_generation++;
+    }
+    priv->child_up[idx] = 1;
+
+    *call_psh = 1;
+    *up_child = idx;
+    up_children = __afr_get_up_children_count(priv);
+    /*
+     * If this is an _actual_ CHILD_UP event, we
+     * want to set the child_latency to MAX to indicate
+     * the child needs ping data to be available before doing child-up
+     */
+    if (!priv->halo_enabled)
+        goto out;
+
+    if (child_latency_msec < 0) {
+        /*set to INT64_MAX-1 so that it is found for best_down_child*/
+        priv->halo_child_up[idx] = 1;
+        if (priv->child_latency[idx] < 0) {
+            priv->child_latency[idx] = AFR_HALO_MAX_LATENCY;
+        }
+    }
+
+    /*
+     * Handle the edge case where we exceed
+     * halo_min_replicas and we've got a child which is
+     * marked up as it was helping to satisfy the
+     * halo_min_replicas even though it's latency exceeds
+     * halo_max_latency_msec.
+     */
+    if (up_children > priv->halo_min_replicas) {
+        worst_up_child = find_worst_up_child(this);
+        if (worst_up_child >= 0 &&
+            priv->child_latency[worst_up_child] > halo_max_latency_msec) {
+            gf_msg_debug(this->name, 0,
+                         "Marking child %d down, "
+                         "doesn't meet halo threshold (%" PRId64
+                         "), and > "
+                         "halo_min_replicas (%d)",
+                         worst_up_child, halo_max_latency_msec,
+                         priv->halo_min_replicas);
+            priv->child_up[worst_up_child] = 0;
+            up_children--;
+        }
+    }
+
+    if (up_children > priv->halo_max_replicas && !priv->shd.iamshd) {
+        worst_up_child = find_worst_up_child(this);
+        if (worst_up_child < 0) {
+            worst_up_child = idx;
+        }
+        priv->child_up[worst_up_child] = 0;
+        up_children--;
+        gf_msg_debug(this->name, 0,
+                     "Marking child %d down, "
+                     "up_children (%d) > halo_max_replicas (%d)",
+                     worst_up_child, up_children, priv->halo_max_replicas);
+    }
+out:
+    if (up_children == 1) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOL_UP,
+               "Subvolume '%s' came back up; "
+               "going online.",
+               child_xlator->name);
+        gf_event(EVENT_AFR_SUBVOL_UP, "client-pid=%d; subvol=%s",
+                 this->ctx->cmd_args.client_pid, this->name);
+    } else {
+        *event = GF_EVENT_SOME_DESCENDENT_UP;
+    }
 
-        return 0;
+    priv->last_event[idx] = *event;
 }
 
+void
+__afr_handle_child_down_event(xlator_t *this, xlator_t *child_xlator, int idx,
+                              int64_t child_latency_msec, int32_t *event,
+                              int32_t *call_psh, int32_t *up_child)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int up_children = 0;
+    int down_children = 0;
+    int best_down_child = -1;
+
+    priv = this->private;
+
+    /*
+     * If a brick is down when we start, we'll get a
+     * CHILD_DOWN to indicate its initial state.  There
+     * was never a CHILD_UP in this case, so if we
+     * increment "down_count" the difference between than
+     * and "up_count" will no longer be the number of
+     * children that are currently up.  This has serious
+     * implications e.g. for quorum enforcement, so we
+     * don't increment these values unless the event
+     * represents an actual state transition between "up"
+     * (value = 1) and anything else.
+     */
+    if (priv->child_up[idx] == 1) {
+        priv->event_generation++;
+    }
+
+    /*
+     * If this is an _actual_ CHILD_DOWN event, we
+     * want to set the child_latency to < 0 to indicate
+     * the child is really disconnected.
+     */
+    if (child_latency_msec < 0) {
+        priv->child_latency[idx] = child_latency_msec;
+        priv->halo_child_up[idx] = 0;
+    }
+    priv->child_up[idx] = 0;
+
+    up_children = __afr_get_up_children_count(priv);
+    /*
+     * Handle the edge case where we need to find the
+     * next best child (to mark up) as marking this child
+     * down would cause us to fall below halo_min_replicas.
+     * We will also force the SHD to heal this child _now_
+     * as we want it to be up to date if we are going to
+     * begin using it synchronously.
+     */
+    if (priv->halo_enabled && up_children < priv->halo_min_replicas) {
+        best_down_child = find_best_down_child(this);
+        if (best_down_child >= 0) {
+            gf_msg_debug(this->name, 0,
+                         "Swapping out child %d for "
+                         "child %d to satisfy halo_min_replicas (%d).",
+                         idx, best_down_child, priv->halo_min_replicas);
+            priv->child_up[best_down_child] = 1;
+            *call_psh = 1;
+            *up_child = best_down_child;
+        }
+    }
+    for (i = 0; i < priv->child_count; i++)
+        if (priv->child_up[i] == 0)
+            down_children++;
+    if (down_children == priv->child_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SUBVOLS_DOWN,
+               "All subvolumes are down. Going "
+               "offline until at least one of them "
+               "comes back up.");
+        gf_event(EVENT_AFR_SUBVOLS_DOWN, "client-pid=%d; subvol=%s",
+                 this->ctx->cmd_args.client_pid, this->name);
+    } else {
+        *event = GF_EVENT_SOME_DESCENDENT_DOWN;
+    }
+    priv->last_event[idx] = *event;
+}
 
-/**
- * find_child_index - find the child's index in the array of subvolumes
- * @this: AFR
- * @child: child
- */
+void
+afr_ta_lock_release_synctask(xlator_t *this)
+{
+    call_frame_t *ta_frame = NULL;
+    int ret = 0;
 
-static int
-find_child_index (xlator_t *this, xlator_t *child)
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        return;
+    }
+
+    ret = synctask_new(this->ctx->env, afr_release_notify_lock_for_ta,
+                       afr_ta_lock_release_done, ta_frame, this);
+    if (ret) {
+        STACK_DESTROY(ta_frame->root);
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to release "
+               "AFR_TA_DOM_NOTIFY lock.");
+    }
+}
+
+static void
+afr_handle_inodelk_contention(xlator_t *this, struct gf_upcall *upcall)
 {
-        afr_private_t *priv = NULL;
-        int i = -1;
+    struct gf_upcall_inodelk_contention *lc = NULL;
+    unsigned int inmem_count = 0;
+    unsigned int onwire_count = 0;
+    afr_private_t *priv = this->private;
 
-        priv = this->private;
+    lc = upcall->data;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if ((xlator_t *) child == priv->children[i])
-                        break;
-        }
+    if (strcmp(lc->domain, AFR_TA_DOM_NOTIFY) != 0)
+        return;
+
+    if (priv->shd.iamshd) {
+        /* shd should ignore AFR_TA_DOM_NOTIFY release requests. */
+        return;
+    }
+    LOCK(&priv->lock);
+    {
+        if (priv->release_ta_notify_dom_lock == _gf_true) {
+            /* Ignore multiple release requests from shds.*/
+            UNLOCK(&priv->lock);
+            return;
+        }
+        priv->release_ta_notify_dom_lock = _gf_true;
+        inmem_count = priv->ta_in_mem_txn_count;
+        onwire_count = priv->ta_on_wire_txn_count;
+    }
+    UNLOCK(&priv->lock);
+    if (inmem_count || onwire_count)
+        /* lock release will happen in txn code path after
+         * in-memory or on-wire txns are over.*/
+        return;
 
-        return i;
+    afr_ta_lock_release_synctask(this);
+}
+
+static void
+afr_handle_upcall_event(xlator_t *this, struct gf_upcall *upcall)
+{
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+    afr_private_t *priv = this->private;
+    inode_t *inode = NULL;
+    inode_table_t *itable = NULL;
+    int i = 0;
+
+    switch (upcall->event_type) {
+        case GF_UPCALL_INODELK_CONTENTION:
+            afr_handle_inodelk_contention(this, upcall);
+            break;
+        case GF_UPCALL_CACHE_INVALIDATION:
+            up_ci = (struct gf_upcall_cache_invalidation *)upcall->data;
+
+            /* Since md-cache will be aggressively filtering
+             * lookups, the stale read issue will be more
+             * pronounced. Hence when a pending xattr is set notify
+             * all the md-cache clients to invalidate the existing
+             * stat cache and send the lookup next time */
+            if (!up_ci->dict)
+                break;
+            for (i = 0; i < priv->child_count; i++) {
+                if (!dict_get(up_ci->dict, priv->pending_key[i]))
+                    continue;
+                up_ci->flags |= UP_INVAL_ATTR;
+                itable = ((xlator_t *)this->graph->top)->itable;
+                /*Internal processes may not have itable for
+                 *top xlator*/
+                if (itable)
+                    inode = inode_find(itable, upcall->gfid);
+                if (inode)
+                    afr_inode_need_refresh_set(inode, this);
+                break;
+            }
+            break;
+        default:
+            break;
+    }
 }
 
 int32_t
-afr_notify (xlator_t *this, int32_t event,
-            void *data, void *data2)
-{
-        afr_private_t   *priv               = NULL;
-        int             i                   = -1;
-        int             up_children         = 0;
-        int             down_children       = 0;
-        int             propagate           = 0;
-        int             had_heard_from_all  = 0;
-        int             have_heard_from_all = 0;
-        int             idx                 = -1;
-        int             ret                 = -1;
-        int             call_psh            = 0;
-        int             up_child            = AFR_ALL_CHILDREN;
-        dict_t          *input              = NULL;
-        dict_t          *output             = NULL;
-
-        priv = this->private;
-
-        if (!priv)
-                return 0;
-
-        /*
-         * We need to reset this in case children come up in "staggered"
-         * fashion, so that we discover a late-arriving local subvolume.  Note
-         * that we could end up issuing N lookups to the first subvolume, and
-         * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
-         */
-        priv->did_discovery = _gf_false;
+afr_notify(xlator_t *this, int32_t event, void *data, void *data2)
+{
+    afr_private_t *priv = NULL;
+    xlator_t *child_xlator = NULL;
+    int i = -1;
+    int propagate = 0;
+    int had_heard_from_all = 0;
+    int have_heard_from_all = 0;
+    int idx = -1;
+    int ret = -1;
+    int call_psh = 0;
+    int up_child = -1;
+    dict_t *input = NULL;
+    dict_t *output = NULL;
+    gf_boolean_t had_quorum = _gf_false;
+    gf_boolean_t has_quorum = _gf_false;
+    int64_t halo_max_latency_msec = 0;
+    int64_t child_latency_msec = -1;
+
+    child_xlator = (xlator_t *)data;
+
+    priv = this->private;
+
+    if (!priv)
+        return 0;
 
-        had_heard_from_all = 1;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!priv->last_event[i]) {
-                        had_heard_from_all = 0;
-                }
+    /*
+     * We need to reset this in case children come up in "staggered"
+     * fashion, so that we discover a late-arriving local subvolume.  Note
+     * that we could end up issuing N lookups to the first subvolume, and
+     * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+     */
+    priv->did_discovery = _gf_false;
+
+    /* parent xlators don't need to know about every child_up, child_down
+     * because of afr ha. If all subvolumes go down, child_down has
+     * to be triggered. In that state when 1 subvolume comes up child_up
+     * needs to be triggered. dht optimizes revalidate lookup by sending
+     * it only to one of its subvolumes. When child up/down happens
+     * for afr's subvolumes dht should be notified by child_modified. The
+     * subsequent revalidate lookup happens on all the dht's subvolumes
+     * which triggers afr self-heals if any.
+     */
+    idx = afr_find_child_index(this, child_xlator);
+    if (idx < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_CHILD_UP,
+               "Received child_up from invalid subvolume");
+        goto out;
+    }
+
+    had_quorum = priv->quorum_count &&
+                 afr_has_quorum(priv->child_up, this, NULL);
+    if (event == GF_EVENT_CHILD_PING) {
+        child_latency_msec = (int64_t)(uintptr_t)data2;
+        if (priv->halo_enabled) {
+            halo_max_latency_msec = afr_get_halo_latency(this);
+
+            /* Calculates the child latency and sets event
+             */
+            LOCK(&priv->lock);
+            {
+                __afr_handle_ping_event(this, child_xlator, idx,
+                                        halo_max_latency_msec, &event,
+                                        child_latency_msec);
+            }
+            UNLOCK(&priv->lock);
+        } else {
+            LOCK(&priv->lock);
+            {
+                priv->child_latency[idx] = child_latency_msec;
+            }
+            UNLOCK(&priv->lock);
         }
+    }
 
-        /* parent xlators dont need to know about every child_up, child_down
-         * because of afr ha. If all subvolumes go down, child_down has
-         * to be triggered. In that state when 1 subvolume comes up child_up
-         * needs to be triggered. dht optimizes revalidate lookup by sending
-         * it only to one of its subvolumes. When child up/down happens
-         * for afr's subvolumes dht should be notified by child_modified. The
-         * subsequent revalidate lookup happens on all the dht's subvolumes
-         * which triggers afr self-heals if any.
+    if (event == GF_EVENT_CHILD_PING) {
+        /* This is the only xlator that handles PING, no reason to
+         * propagate.
          */
-        idx = find_child_index (this, data);
-        if (idx < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Received child_up "
-                        "from invalid subvolume");
-                goto out;
+        goto out;
+    }
+
+    if (event == GF_EVENT_TRANSLATOR_OP) {
+        LOCK(&priv->lock);
+        {
+            had_heard_from_all = __get_heard_from_all_status(this);
+        }
+        UNLOCK(&priv->lock);
+
+        if (!had_heard_from_all) {
+            ret = -1;
+        } else {
+            input = data;
+            output = data2;
+            ret = afr_xl_op(this, input, output);
         }
+        goto out;
+    }
+
+    if (event == GF_EVENT_UPCALL) {
+        afr_handle_upcall_event(this, data);
+    }
 
+    LOCK(&priv->lock);
+    {
+        had_heard_from_all = __get_heard_from_all_status(this);
         switch (event) {
-        case GF_EVENT_CHILD_UP:
-                LOCK (&priv->lock);
-                {
-                        /*
-                         * This only really counts if the child was never up
-                         * (value = -1) or had been down (value = 0).  See
-                         * comment at GF_EVENT_CHILD_DOWN for a more detailed
-                         * explanation.
-                         */
-                        if (priv->child_up[idx] != 1) {
-                                priv->up_count++;
-                        }
-                        priv->child_up[idx] = 1;
-
-                        call_psh = 1;
-                        up_child = idx;
-                        for (i = 0; i < priv->child_count; i++)
-                                if (priv->child_up[i] == 1)
-                                        up_children++;
-                        if (up_children == 1) {
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "Subvolume '%s' came back up; "
-                                        "going online.", ((xlator_t *)data)->name);
-                        } else {
-                                event = GF_EVENT_CHILD_MODIFIED;
-                        }
-
-                        priv->last_event[idx] = event;
+            case GF_EVENT_PARENT_UP:
+                __afr_launch_notify_timer(this, priv);
+                propagate = 1;
+                break;
+            case GF_EVENT_CHILD_UP:
+                if (priv->thin_arbiter_count &&
+                    (idx == AFR_CHILD_THIN_ARBITER)) {
+                    priv->ta_child_up = 1;
+                    priv->ta_event_gen++;
+                    break;
                 }
-                UNLOCK (&priv->lock);
-
+                __afr_handle_child_up_event(this, child_xlator, idx,
+                                            child_latency_msec, &event,
+                                            &call_psh, &up_child);
+                __afr_lock_heal_synctask(this, priv, idx);
                 break;
 
-        case GF_EVENT_CHILD_DOWN:
-                LOCK (&priv->lock);
-                {
-                        /*
-                         * If a brick is down when we start, we'll get a
-                         * CHILD_DOWN to indicate its initial state.  There
-                         * was never a CHILD_UP in this case, so if we
-                         * increment "down_count" the difference between than
-                         * and "up_count" will no longer be the number of
-                         * children that are currently up.  This has serious
-                         * implications e.g. for quorum enforcement, so we
-                         * don't increment these values unless the event
-                         * represents an actual state transition between "up"
-                         * (value = 1) and anything else.
-                         */
-                        if (priv->child_up[idx] == 1) {
-                                priv->down_count++;
-                        }
-                        priv->child_up[idx] = 0;
-
-                        for (i = 0; i < priv->child_count; i++)
-                                if (priv->child_up[i] == 0)
-                                        down_children++;
-                        if (down_children == priv->child_count) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "All subvolumes are down. Going offline "
-                                        "until atleast one of them comes back up.");
-                        } else {
-                                event = GF_EVENT_CHILD_MODIFIED;
-                        }
-
-                        priv->last_event[idx] = event;
+            case GF_EVENT_CHILD_DOWN:
+                if (priv->thin_arbiter_count &&
+                    (idx == AFR_CHILD_THIN_ARBITER)) {
+                    priv->ta_child_up = 0;
+                    priv->ta_event_gen++;
+                    afr_ta_locked_priv_invalidate(priv);
+                    break;
                 }
-                UNLOCK (&priv->lock);
-
+                __afr_handle_child_down_event(this, child_xlator, idx,
+                                              child_latency_msec, &event,
+                                              &call_psh, &up_child);
+                __afr_mark_pending_lk_heal(this, priv, idx);
                 break;
 
-        case GF_EVENT_CHILD_CONNECTING:
-                LOCK (&priv->lock);
-                {
-                        priv->last_event[idx] = event;
-                }
-                UNLOCK (&priv->lock);
+            case GF_EVENT_CHILD_CONNECTING:
+                priv->last_event[idx] = event;
 
                 break;
 
-        case GF_EVENT_TRANSLATOR_OP:
-                input = data;
-                output = data2;
-                ret = afr_xl_op (this, input, output);
-                goto out;
+            case GF_EVENT_SOME_DESCENDENT_DOWN:
+                priv->last_event[idx] = event;
                 break;
-
-        default:
+            default:
                 propagate = 1;
                 break;
         }
+        have_heard_from_all = __get_heard_from_all_status(this);
+        if (!had_heard_from_all && have_heard_from_all) {
+            if (priv->timer) {
+                gf_timer_call_cancel(this->ctx, priv->timer);
+                priv->timer = NULL;
+            }
+            /* This is the first event which completes aggregation
+               of events from all subvolumes. If at least one subvol
+               had come up, propagate CHILD_UP, but only this time
+            */
+            event = GF_EVENT_CHILD_DOWN;
+            for (i = 0; i < priv->child_count; i++) {
+                if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
+                    event = GF_EVENT_CHILD_UP;
+                    break;
+                }
 
-        /* have all subvolumes reported status once by now? */
-        have_heard_from_all = 1;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!priv->last_event[i])
-                        have_heard_from_all = 0;
+                if (priv->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
+                    event = GF_EVENT_CHILD_CONNECTING;
+                    /* continue to check other events for CHILD_UP */
+                }
+            }
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (priv->quorum_count) {
+        has_quorum = afr_has_quorum(priv->child_up, this, NULL);
+        if (!had_quorum && has_quorum) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_QUORUM_MET,
+                   "Client-quorum is met");
+            gf_event(EVENT_AFR_QUORUM_MET, "client-pid=%d; subvol=%s",
+                     this->ctx->cmd_args.client_pid, this->name);
+        }
+        if (had_quorum && !has_quorum) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
+                   "Client-quorum is not met");
+            gf_event(EVENT_AFR_QUORUM_FAIL, "client-pid=%d; subvol=%s",
+                     this->ctx->cmd_args.client_pid, this->name);
+        }
+    }
+
+    /* if all subvols have reported status, no need to hide anything
+       or wait for anything else. Just propagate blindly */
+    if (have_heard_from_all)
+        propagate = 1;
+
+    ret = 0;
+    if (propagate)
+        ret = default_notify(this, event, data);
+
+    if ((!had_heard_from_all) || call_psh) {
+        /* Launch self-heal on all local subvolumes if:
+         * a) We have_heard_from_all for the first time
+         * b) Already heard from everyone, but we now got a child-up
+         *    event.
+         */
+        if (have_heard_from_all) {
+            afr_selfheal_childup(this, priv);
         }
+    }
+out:
+    return ret;
+}
 
-        /* if all subvols have reported status, no need to hide anything
-           or wait for anything else. Just propagate blindly */
-        if (have_heard_from_all)
-                propagate = 1;
+int
+afr_local_init(afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
+{
+    int __ret = -1;
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+
+    __ret = syncbarrier_init(&local->barrier);
+    if (__ret) {
+        if (op_errno)
+            *op_errno = __ret;
+        goto out;
+    }
+
+    local->child_up = GF_MALLOC(priv->child_count * sizeof(*local->child_up),
+                                gf_afr_mt_char);
+    if (!local->child_up) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    memcpy(local->child_up, priv->child_up,
+           sizeof(*local->child_up) * priv->child_count);
+    local->call_count = AFR_COUNT(local->child_up, priv->child_count);
+    if (local->call_count == 0) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
+               "no subvolumes up");
+        if (op_errno)
+            *op_errno = ENOTCONN;
+        goto out;
+    }
+
+    local->event_generation = priv->event_generation;
+
+    local->read_attempted = GF_CALLOC(priv->child_count, sizeof(char),
+                                      gf_afr_mt_char);
+    if (!local->read_attempted) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->readable = GF_CALLOC(priv->child_count, sizeof(char),
+                                gf_afr_mt_char);
+    if (!local->readable) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->readable2 = GF_CALLOC(priv->child_count, sizeof(char),
+                                 gf_afr_mt_char);
+    if (!local->readable2) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->read_subvol = -1;
+
+    local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
+                               gf_afr_mt_reply_t);
+    if (!local->replies) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->need_full_crawl = _gf_false;
+    if (priv->thin_arbiter_count) {
+        local->ta_child_up = priv->ta_child_up;
+        local->ta_failed_subvol = AFR_CHILD_UNKNOWN;
+        local->read_txn_query_child = AFR_CHILD_UNKNOWN;
+        local->ta_event_gen = priv->ta_event_gen;
+        local->fop_state = TA_SUCCESS;
+    }
+    local->is_new_entry = _gf_false;
+
+    INIT_LIST_HEAD(&local->healer);
+    return 0;
+out:
+    return -1;
+}
 
-        if (!had_heard_from_all && have_heard_from_all) {
-                /* This is the first event which completes aggregation
-                   of events from all subvolumes. If at least one subvol
-                   had come up, propagate CHILD_UP, but only this time
-                */
-                event = GF_EVENT_CHILD_DOWN;
-
-                LOCK (&priv->lock);
-                {
-                        up_children = afr_up_children_count (priv->child_up,
-                                                             priv->child_count);
-                        for (i = 0; i < priv->child_count; i++) {
-                                if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
-                                        event = GF_EVENT_CHILD_UP;
-                                        break;
-                                }
-
-                                if (priv->last_event[i] ==
-                                                GF_EVENT_CHILD_CONNECTING) {
-                                        event = GF_EVENT_CHILD_CONNECTING;
-                                        /* continue to check other events for CHILD_UP */
-                                }
-                        }
-                }
-                UNLOCK (&priv->lock);
-        }
+int
+afr_internal_lock_init(afr_internal_lock_t *lk, size_t child_count)
+{
+    int ret = -ENOMEM;
 
-        ret = 0;
-        if (propagate)
-                ret = default_notify (this, event, data);
-        if (call_psh && priv->shd.iamshd)
-                afr_proactive_self_heal ((void*) (long) up_child);
+    lk->lower_locked_nodes = GF_CALLOC(sizeof(*lk->lower_locked_nodes),
+                                       child_count, gf_afr_mt_char);
+    if (NULL == lk->lower_locked_nodes)
+        goto out;
 
+    lk->lock_op_ret = -1;
+    lk->lock_op_errno = EUCLEAN;
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-int
-afr_first_up_child (unsigned char *child_up, size_t child_count)
+void
+afr_matrix_cleanup(int32_t **matrix, unsigned int m)
 {
-        int         ret      = -1;
-        int         i        = 0;
+    int i = 0;
 
-        GF_ASSERT (child_up);
+    if (!matrix)
+        goto out;
+    for (i = 0; i < m; i++) {
+        GF_FREE(matrix[i]);
+    }
 
-        for (i = 0; i < child_count; i++) {
-                if (child_up[i]) {
-                        ret = i;
-                        break;
-                }
-        }
+    GF_FREE(matrix);
+out:
+    return;
+}
 
-        return ret;
+int32_t **
+afr_matrix_create(unsigned int m, unsigned int n)
+{
+    int32_t **matrix = NULL;
+    int i = 0;
+
+    matrix = GF_CALLOC(sizeof(*matrix), m, gf_afr_mt_int32_t);
+    if (!matrix)
+        goto out;
+
+    for (i = 0; i < m; i++) {
+        matrix[i] = GF_CALLOC(sizeof(*matrix[i]), n, gf_afr_mt_int32_t);
+        if (!matrix[i])
+            goto out;
+    }
+    return matrix;
+out:
+    afr_matrix_cleanup(matrix, m);
+    return NULL;
 }
 
 int
-afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
+afr_transaction_local_init(afr_local_t *local, xlator_t *this)
+{
+    int ret = -ENOMEM;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    INIT_LIST_HEAD(&local->transaction.wait_list);
+    INIT_LIST_HEAD(&local->transaction.owner_list);
+    INIT_LIST_HEAD(&local->ta_waitq);
+    INIT_LIST_HEAD(&local->ta_onwireq);
+    ret = afr_internal_lock_init(&local->internal_lock, priv->child_count);
+    if (ret < 0)
+        goto out;
+
+    ret = -ENOMEM;
+    local->pre_op_compat = priv->pre_op_compat;
+
+    local->transaction.pre_op = GF_CALLOC(sizeof(*local->transaction.pre_op),
+                                          priv->child_count, gf_afr_mt_char);
+    if (!local->transaction.pre_op)
+        goto out;
+
+    local->transaction.changelog_xdata = GF_CALLOC(
+        sizeof(*local->transaction.changelog_xdata), priv->child_count,
+        gf_afr_mt_dict_t);
+    if (!local->transaction.changelog_xdata)
+        goto out;
+
+    if (priv->arbiter_count == 1) {
+        local->transaction.pre_op_sources = GF_CALLOC(
+            sizeof(*local->transaction.pre_op_sources), priv->child_count,
+            gf_afr_mt_char);
+        if (!local->transaction.pre_op_sources)
+            goto out;
+    }
+
+    local->transaction.failed_subvols = GF_CALLOC(
+        sizeof(*local->transaction.failed_subvols), priv->child_count,
+        gf_afr_mt_char);
+    if (!local->transaction.failed_subvols)
+        goto out;
+
+    local->pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!local->pending)
+        goto out;
+
+    ret = 0;
+out:
+    return ret;
+}
+
+void
+afr_set_low_priority(call_frame_t *frame)
 {
-        int     ret = -1;
+    frame->root->pid = LOW_PRIO_PROC_PID;
+}
 
-        local->op_ret = -1;
-        local->op_errno = EUCLEAN;
-
-        local->child_up = GF_CALLOC (priv->child_count,
-                                     sizeof (*local->child_up),
-                                     gf_afr_mt_char);
-        if (!local->child_up) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
+void
+afr_priv_destroy(afr_private_t *priv)
+{
+    int i = 0;
+    int child_count = -1;
 
-        memcpy (local->child_up, priv->child_up,
-                sizeof (*local->child_up) * priv->child_count);
-        local->call_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-        if (local->call_count == 0) {
-                gf_log (THIS->name, GF_LOG_INFO, "no subvolumes up");
-                if (op_errno)
-                        *op_errno = ENOTCONN;
-                goto out;
-        }
+    if (!priv)
+        goto out;
 
-        local->child_errno = GF_CALLOC (priv->child_count,
-                                        sizeof (*local->child_errno),
-                                        gf_afr_mt_int32_t);
-        if (!local->child_errno) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
+    GF_FREE(priv->sh_domain);
+    GF_FREE(priv->last_event);
 
-        ret = 0;
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        child_count++;
+    }
+    if (priv->pending_key) {
+        for (i = 0; i < child_count; i++)
+            GF_FREE(priv->pending_key[i]);
+    }
+
+    GF_FREE(priv->pending_reads);
+    GF_FREE(priv->local);
+    GF_FREE(priv->pending_key);
+    GF_FREE(priv->children);
+    GF_FREE(priv->anon_inode);
+    GF_FREE(priv->child_up);
+    GF_FREE(priv->halo_child_up);
+    GF_FREE(priv->child_latency);
+    LOCK_DESTROY(&priv->lock);
+
+    GF_FREE(priv);
 out:
-        return ret;
+    return;
 }
 
-int
-afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
-                        transaction_lk_type_t lk_type)
+int **
+afr_mark_pending_changelog(afr_private_t *priv, unsigned char *pending,
+                           dict_t *xattr, ia_type_t iat)
 {
-        int             ret = -ENOMEM;
+    int i = 0;
+    int **changelog = NULL;
+    int idx = -1;
+    int m_idx = 0;
+    int d_idx = 0;
+    int ret = 0;
 
-        lk->inode_locked_nodes = GF_CALLOC (sizeof (*lk->inode_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->inode_locked_nodes)
-                goto out;
+    m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
 
-        lk->entry_locked_nodes = GF_CALLOC (sizeof (*lk->entry_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->entry_locked_nodes)
-                goto out;
+    idx = afr_index_from_ia_type(iat);
 
-        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
-                                      child_count, gf_afr_mt_char);
-        if (NULL == lk->locked_nodes)
-                goto out;
+    changelog = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!changelog)
+        goto out;
 
-        lk->lower_locked_nodes = GF_CALLOC (sizeof (*lk->lower_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->lower_locked_nodes)
-                goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!pending[i])
+            continue;
 
-        lk->lock_op_ret   = -1;
-        lk->lock_op_errno = EUCLEAN;
-        lk->transaction_lk_type = lk_type;
+        changelog[i][m_idx] = hton32(1);
+        if (idx != -1)
+            changelog[i][idx] = hton32(1);
+        /* If the newentry marking is on a newly created directory,
+         * then mark it with the full-heal indicator.
+         */
+        if ((IA_ISDIR(iat)) && (priv->esh_granular))
+            changelog[i][d_idx] = hton32(1);
+    }
+    ret = afr_set_pending_dict(priv, xattr, changelog);
+    if (ret < 0) {
+        afr_matrix_cleanup(changelog, priv->child_count);
+        return NULL;
+    }
+out:
+    return changelog;
+}
 
-        ret = 0;
+static dict_t *
+afr_set_heal_info(char *status)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_sizen(dict, "heal-info", status);
+    if (ret)
+        gf_msg("", GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Failed to set heal-info key to "
+               "%s",
+               status);
 out:
-        return ret;
+    /* Any error other than EINVAL, dict_set_dynstr frees status */
+    if (ret == -ENOMEM || ret == -EINVAL) {
+        GF_FREE(status);
+    }
+
+    if (ret && dict) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+    return dict;
 }
 
-void
-afr_matrix_cleanup (int32_t **matrix, unsigned int m)
+static gf_boolean_t
+afr_is_dirty_count_non_unary_for_txn(xlator_t *this, struct afr_reply *replies,
+                                     afr_transaction_type type)
 {
-        int             i         = 0;
+    afr_private_t *priv = this->private;
+    int *dirty = alloca0(priv->child_count * sizeof(int));
+    int i = 0;
 
-        if (!matrix)
-                goto out;
-        for (i = 0; i < m; i++) {
-                GF_FREE (matrix[i]);
+    afr_selfheal_extract_xattr(this, replies, type, dirty, NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (dirty[i] > 1)
+            return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static gf_boolean_t
+afr_is_dirty_count_non_unary(xlator_t *this, struct afr_reply *replies,
+                             ia_type_t ia_type)
+{
+    gf_boolean_t data_chk = _gf_false;
+    gf_boolean_t mdata_chk = _gf_false;
+    gf_boolean_t entry_chk = _gf_false;
+
+    switch (ia_type) {
+        case IA_IFDIR:
+            mdata_chk = _gf_true;
+            entry_chk = _gf_true;
+            break;
+        case IA_IFREG:
+            mdata_chk = _gf_true;
+            data_chk = _gf_true;
+            break;
+        default:
+            /*IA_IFBLK, IA_IFCHR, IA_IFLNK, IA_IFIFO, IA_IFSOCK*/
+            mdata_chk = _gf_true;
+            break;
+    }
+
+    if (data_chk && afr_is_dirty_count_non_unary_for_txn(
+                        this, replies, AFR_DATA_TRANSACTION)) {
+        return _gf_true;
+    } else if (mdata_chk && afr_is_dirty_count_non_unary_for_txn(
+                                this, replies, AFR_METADATA_TRANSACTION)) {
+        return _gf_true;
+    } else if (entry_chk && afr_is_dirty_count_non_unary_for_txn(
+                                this, replies, AFR_ENTRY_TRANSACTION)) {
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static int
+afr_update_heal_status(xlator_t *this, struct afr_reply *replies,
+                       ia_type_t ia_type, gf_boolean_t *esh, gf_boolean_t *dsh,
+                       gf_boolean_t *msh, unsigned char pending)
+{
+    int ret = -1;
+    GF_UNUSED int ret1 = 0;
+    int i = 0;
+    int io_domain_lk_count = 0;
+    int shd_domain_lk_count = 0;
+    afr_private_t *priv = NULL;
+    char *key1 = NULL;
+    char *key2 = NULL;
+
+    priv = this->private;
+    key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(this->name));
+    key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(priv->sh_domain));
+    sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
+    sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if ((replies[i].valid != 1) || (replies[i].op_ret != 0))
+            continue;
+        if (!io_domain_lk_count) {
+            ret1 = dict_get_int32(replies[i].xdata, key1, &io_domain_lk_count);
+        }
+        if (!shd_domain_lk_count) {
+            ret1 = dict_get_int32(replies[i].xdata, key2, &shd_domain_lk_count);
+        }
+    }
+
+    if (!pending) {
+        if ((afr_is_dirty_count_non_unary(this, replies, ia_type)) ||
+            (!io_domain_lk_count)) {
+            /* Needs heal. */
+            ret = 0;
+        } else {
+            /* No heal needed. */
+            *dsh = *esh = *msh = 0;
         }
+    } else {
+        if (shd_domain_lk_count) {
+            ret = -EAGAIN; /*For 'possibly-healing'. */
+        } else {
+            ret = 0; /*needs heal. Just set a non -ve value so that it is
+                       assumed as the source index.*/
+        }
+    }
+    return ret;
+}
 
-        GF_FREE (matrix);
+/*return EIO, EAGAIN or pending*/
+int
+afr_lockless_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                     inode_t **inode, gf_boolean_t *entry_selfheal,
+                     gf_boolean_t *data_selfheal,
+                     gf_boolean_t *metadata_selfheal, unsigned char *pending)
+{
+    int ret = -1;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    gf_boolean_t dsh = _gf_false;
+    gf_boolean_t msh = _gf_false;
+    gf_boolean_t esh = _gf_false;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *valid_on = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+    sources = alloca0(sizeof(*sources) * priv->child_count);
+    sinks = alloca0(sizeof(*sinks) * priv->child_count);
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    valid_on = alloca0(sizeof(*valid_on) * priv->child_count);
+
+    ret = afr_selfheal_unlocked_inspect(frame, this, gfid, inode, &dsh, &msh,
+                                        &esh, replies);
+    if (ret)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            valid_on[i] = 1;
+        }
+    }
+    if (msh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_METADATA_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
+    if (dsh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_DATA_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
+    if (esh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_ENTRY_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
+
+    ret = afr_update_heal_status(this, replies, (*inode)->ia_type, &esh, &dsh,
+                                 &msh, *pending);
 out:
-        return;
+    *data_selfheal = dsh;
+    *entry_selfheal = esh;
+    *metadata_selfheal = msh;
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    return ret;
 }
 
-int32_t**
-afr_matrix_create (unsigned int m, unsigned int n)
-{
-        int32_t         **matrix = NULL;
-        int             i       = 0;
-
-        matrix = GF_CALLOC (sizeof (*matrix), m, gf_afr_mt_int32_t);
-        if (!matrix)
+int
+afr_get_heal_info(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    gf_boolean_t data_selfheal = _gf_false;
+    gf_boolean_t metadata_selfheal = _gf_false;
+    gf_boolean_t entry_selfheal = _gf_false;
+    unsigned char pending = 0;
+    dict_t *dict = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+    inode_t *inode = NULL;
+    char *substr = NULL;
+    char *status = NULL;
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *heal_local = NULL;
+
+    /*Use frame with lk-owner set*/
+    heal_frame = afr_frame_create(frame->this, &op_errno);
+    if (!heal_frame) {
+        ret = -1;
+        goto out;
+    }
+    heal_local = heal_frame->local;
+    heal_frame->local = frame->local;
+
+    ret = afr_lockless_inspect(heal_frame, this, loc->gfid, &inode,
+                               &entry_selfheal, &data_selfheal,
+                               &metadata_selfheal, &pending);
+
+    if (ret == -ENOMEM) {
+        ret = -1;
+        goto out;
+    }
+
+    if (pending & PFLAG_PENDING) {
+        gf_asprintf(&substr, "-pending");
+        if (!substr)
+            goto out;
+    }
+
+    if (ret == -EIO) {
+        ret = gf_asprintf(&status, "split-brain%s", substr ? substr : "");
+        if (ret < 0) {
+            goto out;
+        }
+        dict = afr_set_heal_info(status);
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+    } else if (ret == -EAGAIN) {
+        ret = gf_asprintf(&status, "possibly-healing%s", substr ? substr : "");
+        if (ret < 0) {
+            goto out;
+        }
+        dict = afr_set_heal_info(status);
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+    } else if (ret >= 0) {
+        /* value of ret = source index
+         * so ret >= 0 and at least one of the 3 booleans set to
+         * true means a source is identified; heal is required.
+         */
+        if (!data_selfheal && !entry_selfheal && !metadata_selfheal) {
+            status = gf_strdup("no-heal");
+            if (!status) {
+                ret = -1;
                 goto out;
-
-        for (i = 0; i < m; i++) {
-                matrix[i] = GF_CALLOC (sizeof (*matrix[i]), n,
-                                       gf_afr_mt_int32_t);
-                if (!matrix[i])
-                        goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
+        } else {
+            ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
+            if (ret < 0) {
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
+        }
+    } else if (ret < 0) {
+        /* Apart from above checked -ve ret values, there are
+         * other possible ret values like ENOTCONN
+         * (returned when number of valid replies received are
+         * less than 2)
+         * in which case heal is required when one of the
+         * selfheal booleans is set.
+         */
+        if (data_selfheal || entry_selfheal || metadata_selfheal) {
+            ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
+            if (ret < 0) {
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
         }
-        return matrix;
+    }
+
+    ret = 0;
+    op_errno = 0;
+
 out:
-        afr_matrix_cleanup (matrix, m);
-        return NULL;
+    if (heal_frame) {
+        heal_frame->local = heal_local;
+        AFR_STACK_DESTROY(heal_frame);
+    }
+    AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    if (dict)
+        dict_unref(dict);
+    if (inode)
+        inode_unref(inode);
+    GF_FREE(substr);
+    return ret;
+}
+
+int
+_afr_is_split_brain(call_frame_t *frame, xlator_t *this,
+                    struct afr_reply *replies, afr_transaction_type type,
+                    gf_boolean_t *spb)
+{
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    int sources_count = 0;
+    int ret = 0;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    witness = alloca0(priv->child_count * sizeof(*witness));
+
+    ret = afr_selfheal_find_direction(frame, this, replies, type,
+                                      priv->child_up, sources, sinks, witness,
+                                      NULL);
+    if (ret)
+        return ret;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+    if (!sources_count)
+        *spb = _gf_true;
+
+    return ret;
 }
 
 int
-afr_transaction_local_init (afr_local_t *local, xlator_t *this)
+afr_is_split_brain(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb)
 {
-        int            child_up_count = 0;
-        int            ret = -ENOMEM;
-        afr_private_t *priv = NULL;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
 
-        priv = this->private;
-        ret = afr_internal_lock_init (&local->internal_lock, priv->child_count,
-                                      AFR_TRANSACTION_LK);
-        if (ret < 0)
-                goto out;
+    priv = this->private;
 
-        ret = -ENOMEM;
-        child_up_count = afr_up_children_count (local->child_up,
-                                                priv->child_count);
-        if (priv->optimistic_change_log && child_up_count == priv->child_count)
-                local->optimistic_change_log = 1;
+    replies = alloca0(sizeof(*replies) * priv->child_count);
 
-        local->first_up_child = afr_first_up_child (local->child_up,
-                                                    priv->child_count);
+    ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
+    if (ret)
+        goto out;
 
-        local->transaction.eager_lock =
-                GF_CALLOC (sizeof (*local->transaction.eager_lock),
-                           priv->child_count,
-                           gf_afr_mt_int32_t);
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        ret = -EAGAIN;
+        goto out;
+    }
 
-        if (!local->transaction.eager_lock)
-                goto out;
+    ret = _afr_is_split_brain(frame, this, replies, AFR_DATA_TRANSACTION,
+                              d_spb);
+    if (ret)
+        goto out;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children)
-                goto out;
+    ret = _afr_is_split_brain(frame, this, replies, AFR_METADATA_TRANSACTION,
+                              m_spb);
+out:
+    if (replies) {
+        afr_replies_wipe(replies, priv->child_count);
+        replies = NULL;
+    }
+    return ret;
+}
 
-        if (local->fd) {
-                local->fd_open_on = GF_CALLOC (sizeof (*local->fd_open_on),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-                if (!local->fd_open_on)
-                        goto out;
+int
+afr_get_split_brain_status_cbk(int ret, call_frame_t *frame, void *opaque)
+{
+    GF_FREE(opaque);
+    return 0;
+}
+
+int
+afr_get_split_brain_status(void *opaque)
+{
+    gf_boolean_t d_spb = _gf_false;
+    gf_boolean_t m_spb = _gf_false;
+    int ret = -1;
+    int op_errno = 0;
+    int i = 0;
+    char *choices = NULL;
+    char *status = NULL;
+    dict_t *dict = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    afr_spb_status_t *data = NULL;
+
+    data = opaque;
+    frame = data->frame;
+    this = frame->this;
+    loc = data->loc;
+    priv = this->private;
+    children = priv->children;
+
+    inode = afr_inode_find(this, loc->gfid);
+    if (!inode)
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* Calculation for string length :
+     * (child_count X length of child-name) + SLEN("    Choices :")
+     * child-name consists of :
+     * a) 251 = max characters for volname according to GD_VOLUME_NAME_MAX
+     * b) strlen("-client-00,") assuming 16 replicas
+     */
+    choices = alloca0(priv->child_count * (256 + SLEN("-client-00,")) +
+                      SLEN("    Choices:"));
+
+    ret = afr_is_split_brain(frame, this, inode, loc->gfid, &d_spb, &m_spb);
+    if (ret) {
+        op_errno = -ret;
+        if (ret == -EAGAIN) {
+            ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
+                                           SBRAIN_HEAL_NO_GO_MSG);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, -ret,
+                       AFR_MSG_DICT_SET_FAILED,
+                       "Failed to set GF_AFR_SBRAIN_STATUS in dict");
+            }
+        }
+        ret = -1;
+        goto out;
+    }
+
+    if (d_spb || m_spb) {
+        sprintf(choices, "    Choices:");
+        for (i = 0; i < priv->child_count; i++) {
+            strcat(choices, children[i]->name);
+            strcat(choices, ",");
         }
+        choices[strlen(choices) - 1] = '\0';
 
-        local->transaction.pre_op = GF_CALLOC (sizeof (*local->transaction.pre_op),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-        if (!local->transaction.pre_op)
-                goto out;
+        ret = gf_asprintf(&status,
+                          "data-split-brain:%s    "
+                          "metadata-split-brain:%s%s",
+                          (d_spb) ? "yes" : "no", (m_spb) ? "yes" : "no",
+                          choices);
 
-        local->pending = afr_matrix_create (priv->child_count,
-                                            AFR_NUM_CHANGE_LOGS);
-        if (!local->pending)
-                goto out;
+        if (-1 == ret) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = dict_set_dynstr_sizen(dict, GF_AFR_SBRAIN_STATUS, status);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    } else {
+        ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
+                                       SFILE_NOT_UNDER_DATA);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    }
 
-        local->transaction.txn_changelog = afr_matrix_create (priv->child_count,
-                                                           AFR_NUM_CHANGE_LOGS);
-        if (!local->transaction.txn_changelog)
-                goto out;
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    if (dict)
+        dict_unref(dict);
+    if (inode)
+        inode_unref(inode);
+    return ret;
 }
 
-void
-afr_reset_children (int32_t *fresh_children, int32_t child_count)
-{
-        unsigned int i = 0;
-        for (i = 0; i < child_count; i++)
-                fresh_children[i] = -1;
+int32_t
+afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int ret = 0;
+    int op_errno = 0;
+    dict_t *dict = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+
+    local = frame->local;
+    dict = dict_new();
+    if (!dict) {
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    heal_frame = afr_frame_create(this, &op_errno);
+    if (!heal_frame) {
+        ret = -1;
+        goto out;
+    }
+    heal_local = heal_frame->local;
+    heal_frame->local = frame->local;
+    /*Initiate heal with heal_frame with lk-owner set so that inodelk/entrylk
+     * work correctly*/
+    ret = afr_selfheal_do(heal_frame, this, loc->gfid);
+
+    if (ret == 1 || ret == 2) {
+        ret = dict_set_sizen_str_sizen(dict, "sh-fail-msg",
+                                       SFILE_NOT_IN_SPLIT_BRAIN);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Failed to set sh-fail-msg in dict");
+        ret = 0;
+        goto out;
+    } else {
+        if (local->xdata_rsp) {
+            /* 'sh-fail-msg' has been set in the dict during self-heal.*/
+            dict_copy(local->xdata_rsp, dict);
+            ret = 0;
+        } else if (ret < 0) {
+            op_errno = -ret;
+            ret = -1;
+        }
+    }
+
+out:
+    if (heal_frame) {
+        heal_frame->local = heal_local;
+        AFR_STACK_DESTROY(heal_frame);
+    }
+    if (local->op == GF_FOP_GETXATTR)
+        AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    else if (local->op == GF_FOP_SETXATTR)
+        AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-int32_t*
-afr_children_create (int32_t child_count)
+int
+afr_get_child_index_from_name(xlator_t *this, char *name)
 {
-        int32_t           *children = NULL;
-        int               i               = 0;
-
-        GF_ASSERT (child_count > 0);
+    afr_private_t *priv = this->private;
+    int index = -1;
 
-        children = GF_CALLOC (child_count, sizeof (*children),
-                              gf_afr_mt_int32_t);
-        if (NULL == children)
-                goto out;
-        for (i = 0; i < child_count; i++)
-                children[i] = -1;
+    for (index = 0; index < priv->child_count; index++) {
+        if (!strcmp(priv->children[index]->name, name))
+            goto out;
+    }
+    index = -1;
 out:
-        return children;
+    return index;
 }
 
 void
-afr_children_add_child (int32_t *children, int32_t child,
-                        int32_t child_count)
-{
-        gf_boolean_t child_found = _gf_false;
-        int          i               = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (children[i] == -1)
-                        break;
-                if (children[i] == child) {
-                        child_found = _gf_true;
-                        break;
-                }
-        }
-
-        if (!child_found) {
-                GF_ASSERT (i < child_count);
-                children[i] = child;
-        }
+afr_priv_need_heal_set(afr_private_t *priv, gf_boolean_t need_heal)
+{
+    LOCK(&priv->lock);
+    {
+        priv->need_heal = need_heal;
+    }
+    UNLOCK(&priv->lock);
 }
 
 void
-afr_children_rm_child (int32_t *children, int32_t child, int32_t child_count)
-{
-        int          i = 0;
-
-        GF_ASSERT ((child >= 0) && (child < child_count));
-        for (i = 0; i < child_count; i++) {
-                if (children[i] == -1)
-                        break;
-                if (children[i] == child) {
-                        if (i != (child_count - 1))
-                                memmove (children + i, children + i + 1,
-                                         sizeof (*children)*(child_count - i - 1));
-                        children[child_count - 1] = -1;
-                        break;
-                }
+afr_set_need_heal(xlator_t *this, afr_local_t *local)
+{
+    int i = 0;
+    afr_private_t *priv = this->private;
+    gf_boolean_t need_heal = _gf_false;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].need_heal) {
+            need_heal = _gf_true;
+            break;
         }
+    }
+    afr_priv_need_heal_set(priv, need_heal);
+    return;
 }
 
-int
-afr_get_children_count (int32_t *children, unsigned int child_count)
+gf_boolean_t
+afr_get_need_heal(xlator_t *this)
 {
-        int count = 0;
-        int i = 0;
+    afr_private_t *priv = this->private;
+    gf_boolean_t need_heal = _gf_true;
 
-        for (i = 0; i < child_count; i++) {
-                if (children[i] == -1)
-                        break;
-                count++;
-        }
-        return count;
+    LOCK(&priv->lock);
+    {
+        need_heal = priv->need_heal;
+    }
+    UNLOCK(&priv->lock);
+    return need_heal;
 }
 
-void
-afr_set_low_priority (call_frame_t *frame)
+int
+afr_get_msg_id(char *op_type)
 {
-        frame->root->pid = LOW_PRIO_PROC_PID;
+    if (!strcmp(op_type, GF_AFR_REPLACE_BRICK))
+        return AFR_MSG_REPLACE_BRICK_STATUS;
+    else if (!strcmp(op_type, GF_AFR_ADD_BRICK))
+        return AFR_MSG_ADD_BRICK_STATUS;
+    return -1;
 }
 
 int
-afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags)
+afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *heal_frame,
+                                    void *opaque)
 {
-        int             ret = 0;
-        uint64_t        ctx = 0;
-        afr_fd_ctx_t    *fd_ctx      = NULL;
+    call_frame_t *txn_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *heal_local = NULL;
+    xlator_t *this = NULL;
 
-        GF_ASSERT (fd && fd->inode);
-        ret = afr_fd_ctx_set (this, fd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not set fd ctx for fd=%p", fd);
-                goto out;
+    heal_local = heal_frame->local;
+    txn_frame = heal_local->heal_frame;
+    local = txn_frame->local;
+    this = txn_frame->this;
+
+    /* Refresh the inode agan and proceed with the transaction.*/
+    afr_inode_refresh(txn_frame, this, local->inode, NULL, local->refreshfn);
+
+    AFR_STACK_DESTROY(heal_frame);
+
+    return 0;
+}
+
+int
+afr_fav_child_reset_sink_xattrs(void *opaque)
+{
+    call_frame_t *heal_frame = NULL;
+    call_frame_t *txn_frame = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t d_spb = _gf_false;
+    gf_boolean_t m_spb = _gf_false;
+    afr_local_t *heal_local = NULL;
+    afr_local_t *txn_local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    unsigned char *locked_on = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    int ret = 0;
+
+    heal_frame = (call_frame_t *)opaque;
+    heal_local = heal_frame->local;
+    txn_frame = heal_local->heal_frame;
+    txn_local = txn_frame->local;
+    this = txn_frame->this;
+    inode = txn_local->inode;
+    priv = this->private;
+    locked_on = alloca0(priv->child_count);
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
+                              AFR_DATA_TRANSACTION, &d_spb);
+
+    ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
+                              AFR_METADATA_TRANSACTION, &m_spb);
+
+    /* Take appropriate locks and reset sink xattrs. */
+    if (d_spb) {
+        ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name, 0, 0,
+                                   locked_on);
+        {
+            if (ret < priv->child_count)
+                goto data_unlock;
+            ret = __afr_selfheal_data_prepare(
+                heal_frame, this, inode, locked_on, sources, sinks,
+                healed_sinks, undid_pending, locked_replies, NULL);
+        }
+    data_unlock:
+        afr_selfheal_uninodelk(heal_frame, this, inode, this->name, 0, 0,
+                               locked_on);
+    }
+
+    if (m_spb) {
+        memset(locked_on, 0, sizeof(*locked_on) * priv->child_count);
+        memset(undid_pending, 0, sizeof(*undid_pending) * priv->child_count);
+        ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name,
+                                   LLONG_MAX - 1, 0, locked_on);
+        {
+            if (ret < priv->child_count)
+                goto mdata_unlock;
+            ret = __afr_selfheal_metadata_prepare(
+                heal_frame, this, inode, locked_on, sources, sinks,
+                healed_sinks, undid_pending, locked_replies, NULL);
         }
+    mdata_unlock:
+        afr_selfheal_uninodelk(heal_frame, this, inode, this->name,
+                               LLONG_MAX - 1, 0, locked_on);
+    }
 
-        ret = fd_ctx_get (fd, this, &ctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not get fd ctx for fd=%p", fd);
+    return ret;
+}
+
+/*
+ * Concatenates the xattrs in local->replies separated by a delimiter.
+ */
+int
+afr_serialize_xattrs_with_delimiter(call_frame_t *frame, xlator_t *this,
+                                    char *buf, const char *default_str,
+                                    int32_t *serz_len, char delimiter)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    char *xattr = NULL;
+    int i = 0;
+    int len = 0;
+    int keylen = 0;
+    size_t str_len = 0;
+    int ret = -1;
+
+    priv = this->private;
+    local = frame->local;
+
+    keylen = strlen(local->cont.getxattr.name);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid || local->replies[i].op_ret) {
+            str_len = strlen(default_str);
+            buf = strncat(buf, default_str, str_len);
+            len += str_len;
+            buf[len++] = delimiter;
+            buf[len] = '\0';
+        } else {
+            ret = dict_get_strn(local->replies[i].xattr,
+                                local->cont.getxattr.name, keylen, &xattr);
+            if (ret) {
+                gf_msg("TEST", GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                       "Failed to get the node_uuid of brick "
+                       "%d",
+                       i);
                 goto out;
-        }
+            }
+            str_len = strlen(xattr);
+            buf = strncat(buf, xattr, str_len);
+            len += str_len;
+            buf[len++] = delimiter;
+            buf[len] = '\0';
+        }
+    }
+    buf[--len] = '\0'; /*remove the last delimiter*/
+    if (serz_len)
+        *serz_len = ++len;
+    ret = 0;
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-        fd_ctx->opened_on[child] = AFR_FD_OPENED;
-        if (!IA_ISDIR (fd->inode->ia_type)) {
-                fd_ctx->flags            = flags;
-        }
-        ret = 0;
 out:
-        return ret;
+    return ret;
+}
+
+uint64_t
+afr_write_subvol_get(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    uint64_t write_subvol = 0;
+
+    local = frame->local;
+    LOCK(&local->inode->lock);
+    write_subvol = local->inode_ctx->write_subvol;
+    UNLOCK(&local->inode->lock);
+
+    return write_subvol;
+}
+
+int
+afr_write_subvol_set(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *data_accused = NULL;
+    unsigned char *metadata_accused = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint64_t val = 0;
+    int event = 0;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+    data_accused = alloca0(priv->child_count);
+    metadata_accused = alloca0(priv->child_count);
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+    event = local->event_generation;
+
+    afr_readables_fill(frame, this, local->inode, data_accused,
+                       metadata_accused, data_readable, metadata_readable,
+                       NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_readable[i])
+            datamap |= (1 << i);
+        if (metadata_readable[i])
+            metadatamap |= (1 << i);
+    }
+
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
+
+    LOCK(&local->inode->lock);
+    {
+        if (local->inode_ctx->write_subvol == 0 &&
+            local->transaction.type == AFR_DATA_TRANSACTION) {
+            local->inode_ctx->write_subvol = val;
+        }
+    }
+    UNLOCK(&local->inode->lock);
+
+    return 0;
+}
+
+int
+afr_write_subvol_reset(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+
+    local = frame->local;
+    LOCK(&local->inode->lock);
+    {
+        GF_ASSERT(local->inode_ctx->lock_count > 0);
+        local->inode_ctx->lock_count--;
+
+        if (!local->inode_ctx->lock_count)
+            local->inode_ctx->write_subvol = 0;
+    }
+    UNLOCK(&local->inode->lock);
+
+    return 0;
+}
+
+int
+afr_set_inode_local(xlator_t *this, afr_local_t *local, inode_t *inode)
+{
+    int ret = 0;
+
+    local->inode = inode_ref(inode);
+    LOCK(&local->inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, local->inode, &local->inode_ctx);
+    }
+    UNLOCK(&local->inode->lock);
+    if (ret < 0) {
+        gf_msg_callingfn(
+            this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_INODE_CTX_GET_FAILED,
+            "Error getting inode ctx %s", uuid_utoa(local->inode->gfid));
+    }
+    return ret;
 }
 
 gf_boolean_t
-afr_have_quorum (char *logname, afr_private_t *priv)
+afr_ta_is_fop_called_from_synctask(xlator_t *this)
 {
-        unsigned int        quorum = 0;
+    struct synctask *task = NULL;
+    gf_lkowner_t tmp_owner = {
+        0,
+    };
 
-        GF_VALIDATE_OR_GOTO(logname,priv,out);
+    task = synctask_get();
+    if (!task)
+        return _gf_false;
 
-        quorum = priv->quorum_count;
-        if (quorum != AFR_QUORUM_AUTO) {
-                return (priv->up_count >= (priv->down_count + quorum));
-        }
+    set_lk_owner_from_ptr(&tmp_owner, (void *)this);
 
-        quorum = priv->child_count / 2 + 1;
-        if (priv->up_count >= (priv->down_count + quorum)) {
-                return _gf_true;
-        }
+    if (!is_same_lkowner(&tmp_owner, &task->frame->root->lk_owner))
+        return _gf_false;
 
-        /*
-         * Special case for even numbers of nodes: if we have exactly half
-         * and that includes the first ("senior-most") node, then that counts
-         * as quorum even if it wouldn't otherwise.  This supports e.g. N=2
-         * while preserving the critical property that there can only be one
-         * such group.
-         */
-        if ((priv->child_count % 2) == 0) {
-                quorum = priv->child_count / 2;
-                if (priv->up_count >= (priv->down_count + quorum)) {
-                        if (priv->child_up[0]) {
-                                return _gf_true;
-                        }
-                }
-        }
+    return _gf_true;
+}
 
+int
+afr_ta_post_op_lock(xlator_t *this, loc_t *loc)
+{
+    int ret = 0;
+    uuid_t gfid = {
+        0,
+    };
+    afr_private_t *priv = this->private;
+    gf_boolean_t locked = _gf_false;
+    struct gf_flock flock1 = {
+        0,
+    };
+    struct gf_flock flock2 = {
+        0,
+    };
+    int32_t cmd = 0;
+
+    /* Clients must take AFR_TA_DOM_NOTIFY lock only when the previous lock
+     * has been released in afr_notify due to upcall notification from shd.
+     */
+    GF_ASSERT(priv->ta_notify_dom_lock_offset == 0);
+
+    if (!priv->shd.iamshd)
+        GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
+    flock1.l_type = F_WRLCK;
+
+    while (!locked) {
+        if (priv->shd.iamshd) {
+            cmd = F_SETLKW;
+            flock1.l_start = 0;
+            flock1.l_len = 0;
+        } else {
+            cmd = F_SETLK;
+            gf_uuid_generate(gfid);
+            flock1.l_start = gfid_to_ino(gfid);
+            if (flock1.l_start < 0)
+                flock1.l_start = -flock1.l_start;
+            flock1.l_len = 1;
+        }
+        ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                             AFR_TA_DOM_NOTIFY, loc, cmd, &flock1, NULL, NULL);
+        if (!ret) {
+            locked = _gf_true;
+            priv->ta_notify_dom_lock_offset = flock1.l_start;
+        } else if (ret == -EAGAIN) {
+            continue;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+                   "Failed to get "
+                   "AFR_TA_DOM_NOTIFY lock on %s.",
+                   loc->name);
+            goto out;
+        }
+    }
+
+    flock2.l_type = F_WRLCK;
+    flock2.l_start = 0;
+    flock2.l_len = 0;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, loc, F_SETLKW, &flock2, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to get AFR_TA_DOM_MODIFY lock on %s.", loc->name);
+        flock1.l_type = F_UNLCK;
+        ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                             AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock1, NULL,
+                             NULL);
+    }
+out:
+    return ret;
+}
+
+int
+afr_ta_post_op_unlock(xlator_t *this, loc_t *loc)
+{
+    afr_private_t *priv = this->private;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = 0;
+
+    if (!priv->shd.iamshd)
+        GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
+    flock.l_type = F_UNLCK;
+    flock.l_start = 0;
+    flock.l_len = 0;
+
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_MODIFY lock.");
+        goto out;
+    }
+
+    if (!priv->shd.iamshd)
+        /* Mounts (clients) will not release the AFR_TA_DOM_NOTIFY lock
+         * in post-op as they use it as a notification mechanism. When
+         * shd sends a lock request on TA during heal, the clients will
+         * receive a lock-contention upcall notification upon which they
+         * will release the AFR_TA_DOM_NOTIFY lock after completing the
+         * in flight I/O.*/
+        goto out;
+
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_NOTIFY lock.");
+    }
 out:
+    return ret;
+}
+
+call_frame_t *
+afr_ta_frame_create(xlator_t *this)
+{
+    call_frame_t *frame = NULL;
+    void *lk_owner = NULL;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        return NULL;
+    lk_owner = (void *)this;
+    afr_set_lk_owner(frame, this, lk_owner);
+    return frame;
+}
+
+gf_boolean_t
+afr_ta_has_quorum(afr_private_t *priv, afr_local_t *local)
+{
+    int data_count = 0;
+
+    data_count = AFR_COUNT(local->child_up, priv->child_count);
+    if (data_count == 2) {
+        return _gf_true;
+    } else if (data_count == 1 && local->ta_child_up) {
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static gf_boolean_t
+afr_is_add_replica_mount_lookup_on_root(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+
+    if (frame->root->pid != GF_CLIENT_PID_ADD_REPLICA_MOUNT)
         return _gf_false;
+
+    local = frame->local;
+
+    if (local->op != GF_FOP_LOOKUP)
+        /* TODO:If the replica count is being increased on a plain distribute
+         * volume that was never mounted, we need to allow setxattr on '/' with
+         * GF_CLIENT_PID_NO_ROOT_SQUASH to accomodate for DHT layout setting */
+        return _gf_false;
+
+    if (local->inode == NULL)
+        return _gf_false;
+
+    if (!__is_root_gfid(local->inode->gfid))
+        return _gf_false;
+
+    return _gf_true;
 }
 
-void
-afr_priv_destroy (afr_private_t *priv)
+gf_boolean_t
+afr_lookup_has_quorum(call_frame_t *frame, const unsigned int up_children_count)
 {
-        int            i           = 0;
+    if (frame && (up_children_count > 0) &&
+        afr_is_add_replica_mount_lookup_on_root(frame))
+        return _gf_true;
 
-        if (!priv)
-                goto out;
-        inode_unref (priv->root_inode);
-        GF_FREE (priv->shd.pos);
-        GF_FREE (priv->shd.pending);
-        GF_FREE (priv->shd.inprogress);
-//        for (i = 0; i < priv->child_count; i++)
-//                if (priv->shd.timer && priv->shd.timer[i])
-//                        gf_timer_call_cancel (this->ctx, priv->shd.timer[i]);
-        GF_FREE (priv->shd.timer);
-
-        if (priv->shd.healed)
-                eh_destroy (priv->shd.healed);
-
-        if (priv->shd.heal_failed)
-                eh_destroy (priv->shd.heal_failed);
-
-        if (priv->shd.split_brain)
-                eh_destroy (priv->shd.split_brain);
-
-        GF_FREE (priv->last_event);
-        if (priv->pending_key) {
-                for (i = 0; i < priv->child_count; i++)
-                        GF_FREE (priv->pending_key[i]);
-        }
-        GF_FREE (priv->pending_key);
-        GF_FREE (priv->children);
-        GF_FREE (priv->child_up);
-        LOCK_DESTROY (&priv->lock);
-        LOCK_DESTROY (&priv->read_child_lock);
-        pthread_mutex_destroy (&priv->mutex);
-        GF_FREE (priv);
-out:
-        return;
+    return _gf_false;
 }
 
-int
-xlator_subvolume_count (xlator_t *this)
+void
+afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this)
 {
-        int i = 0;
-        xlator_list_t *list = NULL;
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    unsigned char *success_replies = NULL;
+
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+
+    if (priv->quorum_count && !afr_has_quorum(success_replies, this, NULL)) {
+        local->op_errno = afr_final_errno(local, priv);
+        if (!local->op_errno)
+            local->op_errno = afr_quorum_errno(priv);
+        local->op_ret = -1;
+    }
+}
+
+gf_boolean_t
+afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv, int child)
+{
+    int *pending = NULL;
+    int ret = 0;
+    int i = 0;
+
+    ret = dict_get_ptr(dict, priv->pending_key[child], (void *)&pending);
+    if (ret == 0) {
+        for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+            /* Not doing a ntoh32(pending) as we just want to check
+             * if it is non-zero or not. */
+            if (pending[i]) {
+                return _gf_true;
+            }
+        }
+    }
 
-        for (list = this->children; list; list = list->next)
-                i++;
-        return i;
+    return _gf_false;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index ce91ffba729..f8bf8340dab 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -8,704 +8,339 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 #include <string.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "checksum.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 
 #include "afr.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-int
-afr_examine_dir_sh_unwind (call_frame_t *frame, xlator_t *this, int32_t op_ret,
-                           int32_t op_errno, int32_t sh_failed)
-{
-        afr_local_t *local  = NULL;
-
-        local = frame->local;
-
-        afr_set_opendir_done (this, local->fd->inode);
-
-        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd, NULL);
-
-        return 0;
-}
-
-
-gf_boolean_t
-__checksums_differ (uint32_t *checksum, int child_count,
-                    unsigned char *child_up)
-{
-        int          ret            = _gf_false;
-        int          i              = 0;
-        uint32_t     cksum          = 0;
-        gf_boolean_t activate_check = _gf_false;
-
-        for (i = 0; i < child_count; i++) {
-                if (!child_up[i])
-                        continue;
-                if (_gf_false == activate_check) {
-                        cksum          = checksum[i];
-                        activate_check = _gf_true;
-                        continue;
-                }
-
-                if (cksum != checksum[i]) {
-                        ret = _gf_true;
-                        break;
-                }
-
-                cksum = checksum[i];
-        }
-
-        return ret;
-}
-
+#include "afr-transaction.h"
 
 int32_t
-afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             gf_dirent_t *entries, dict_t *xdata)
+afr_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        afr_private_t *   priv        = NULL;
-        afr_local_t *     local       = NULL;
-        afr_self_heal_t * sh          = NULL;
-        gf_dirent_t *     entry       = NULL;
-        gf_dirent_t *     tmp         = NULL;
-        char              *reason     = NULL;
-        int               child_index = 0;
-        uint32_t          entry_cksum = 0;
-        int               call_count  = 0;
-        off_t             last_offset = 0;
-        inode_t           *inode      = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-        inode = local->fd->inode;
-
-        child_index = (long) cookie;
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int32_t child_index = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: failed to do opendir on %s",
-                        local->loc.path, priv->children[child_index]->name);
-                local->op_ret = -1;
-                local->op_ret = op_errno;
-                goto out;
-        }
-
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%s: no entries found in %s",
-                        local->loc.path, priv->children[child_index]->name);
-                goto out;
-        }
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+    child_index = (long)cookie;
 
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                entry_cksum = gf_rsync_weak_checksum ((unsigned char *)entry->d_name,
-                                                      strlen (entry->d_name));
-                local->cont.opendir.checksum[child_index] ^= entry_cksum;
-        }
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
 
-        list_for_each_entry (entry, &entries->list, list) {
-                last_offset = entry->d_off;
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+        } else {
+            local->op_ret = op_ret;
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        /* read more entries */
-
-        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->readdir,
-                           local->fd, 131072, last_offset, NULL);
-
-        return 0;
-
-out:
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (__checksums_differ (local->cont.opendir.checksum,
-                                        priv->child_count,
-                                        local->child_up)) {
-
-                        sh->do_entry_self_heal  = _gf_true;
-                        sh->forced_merge          = _gf_true;
-
-                        reason = "checksums of directory differ";
-                        afr_launch_self_heal (frame, this, inode, _gf_false,
-                                              inode->ia_type, reason, NULL,
-                                              afr_examine_dir_sh_unwind);
-                } else {
-                        afr_set_opendir_done (this, inode);
-
-                        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                                          local->op_errno, local->fd, NULL);
-                }
-        }
+    if (call_count == 0) {
+        afr_handle_replies_quorum(frame, this);
+        AFR_STACK_UNWIND(opendir, frame, local->op_ret, local->op_errno,
+                         local->fd, NULL);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_examine_dir (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv       = NULL;
-        afr_local_t *   local      = NULL;
-        int             i          = 0;
-        int             call_count = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        local->cont.opendir.checksum = GF_CALLOC (priv->child_count,
-                                                  sizeof (*local->cont.opendir.checksum),
-                                                  gf_afr_mt_int32_t);
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->readdir,
-                                           local->fd, 131072, 0, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
-int32_t
-afr_opendir_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 fd_t *fd, dict_t *xdata)
+afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
 {
-        afr_private_t *priv              = NULL;
-        afr_local_t   *local             = NULL;
-        int32_t        up_children_count = 0;
-        int            ret               = -1;
-        int            call_count        = -1;
-        int32_t        child_index       = 0;
-
-        priv  = this->private;
-        local = frame->local;
-        child_index = (long) cookie;
-
-        up_children_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret >= 0) {
-                        local->op_ret = op_ret;
-                        ret = afr_child_fd_ctx_set (this, fd, child_index, 0);
-                        if (ret) {
-                                local->op_ret = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (local->op_ret != 0)
-                        goto out;
-
-                if (!afr_is_opendir_done (this, local->fd->inode) &&
-                    up_children_count > 1) {
-
-                        /*
-                         * This is the first opendir on this inode. We need
-                         * to check if the directory's entries are the same
-                         * on all subvolumes. This is needed in addition
-                         * to regular entry self-heal because the readdir
-                         * call is sent only to the first subvolume, and
-                         * thus files that exist only there will never be healed
-                         * otherwise (assuming changelog shows no anomalies).
-                         */
-
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "reading contents of directory %s looking for mismatch",
-                                local->loc.path);
-
-                        afr_examine_dir (frame, this);
-
-                } else {
-                        /* do the unwind */
-                        goto out;
-                }
-        }
-
-        return 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = -1;
+    int32_t op_errno = ENOMEM;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-out:
-        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd, NULL);
-
-        return 0;
-}
-
-
-int32_t
-afr_opendir (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, fd_t *fd)
-{
-        afr_private_t * priv        = NULL;
-        afr_local_t   * local       = NULL;
-        int             child_count = 0;
-        int             i           = 0;
-        int             ret         = -1;
-        int             call_count  = -1;
-        int32_t         op_errno    = 0;
+    priv = this->private;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        priv = this->private;
+    local->op = GF_FOP_OPENDIR;
 
-        child_count = priv->child_count;
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        op_errno = afr_quorum_errno(priv);
+        goto out;
+    }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        goto out;
 
-        loc_copy (&local->loc, loc);
+    loc_copy(&local->loc, loc);
 
-        local->fd    = fd_ref (fd);
+    local->fd = fd_ref(fd);
+    local->fd_ctx = fd_ctx;
 
-        call_count = local->call_count;
+    call_count = local->call_count;
 
-        for (i = 0; i < child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_opendir_cbk,
-                                           (void*) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->opendir,
-                                           loc, fd, NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, afr_opendir_cbk, (void *)(long)i,
+                              priv->children[i],
+                              priv->children[i]->fops->opendir, loc, fd, NULL);
 
-                        if (!--call_count)
-                                break;
-                }
+            if (!--call_count)
+                break;
         }
+    }
 
-        ret = 0;
+    return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL);
-
-        return 0;
+    AFR_STACK_UNWIND(opendir, frame, -1, op_errno, fd, NULL);
+    return 0;
 }
 
-
-/**
- * Common algorithm for directory read calls:
- *
- * - Try the fop on the first child that is up
- * - if we have failed due to ENOTCONN:
- *     try the next child
- *
- * Applicable to: readdir
- */
-
-
-struct entry_name {
-        char *name;
-        struct list_head list;
-};
-
-
-static gf_boolean_t
-remembered_name (const char *name, struct list_head *entries)
+static int
+afr_validate_read_subvol(inode_t *inode, xlator_t *this, int par_read_subvol)
 {
-        struct entry_name *e   = NULL;
-        gf_boolean_t       ret = _gf_false;
-
-        list_for_each_entry (e, entries, list) {
-                if (!strcmp (name, e->name)) {
-                        ret = _gf_true;
-                        goto out;
-                }
-        }
+    int gen = 0;
+    int entry_read_subvol = 0;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(inode, this, data_readable, metadata_readable,
+                              &gen);
+
+    if (gen != priv->event_generation || !data_readable[par_read_subvol] ||
+        !metadata_readable[par_read_subvol])
+        return -1;
+
+    /* Once the control reaches the following statement, it means that the
+     * parent's read subvol is perfectly readable. So calling
+     * either afr_data_subvol_get() or afr_metadata_subvol_get() would
+     * yield the same result. Hence, choosing afr_data_subvol_get() below.
+     */
+
+    if (!priv->consistent_metadata)
+        return 0;
 
-out:
-        return ret;
+    /* For an inode fetched through readdirp which is yet to be linked,
+     * inode ctx would not be initialised (yet). So this function returns
+     * -1 above due to gen being 0, which is why it is OK to pass NULL for
+     *  read_subvol_args here.
+     */
+    entry_read_subvol = afr_data_subvol_get(inode, this, NULL, NULL, NULL,
+                                            NULL);
+    if (entry_read_subvol != par_read_subvol)
+        return -1;
+
+    return 0;
 }
 
-
 static void
-afr_remember_entries (gf_dirent_t *entries, fd_t *fd)
+afr_readdir_transform_entries(call_frame_t *frame, gf_dirent_t *subvol_entries,
+                              int subvol, gf_dirent_t *entries, fd_t *fd)
 {
-        struct entry_name *n      = NULL;
-        gf_dirent_t       *entry  = NULL;
-        int                ret    = 0;
-        uint64_t           ctx    = 0;
-        afr_fd_ctx_t      *fd_ctx = NULL;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return;
+    int ret = -1;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t need_heal = _gf_false;
+    gf_boolean_t validate_subvol = _gf_false;
+
+    this = THIS;
+    priv = this->private;
+
+    need_heal = afr_get_need_heal(this);
+    validate_subvol = need_heal | priv->consistent_metadata;
+
+    list_for_each_entry_safe(entry, tmp, &subvol_entries->list, list)
+    {
+        if (afr_is_private_directory(priv, fd->inode->gfid, entry->d_name,
+                                     frame->root->pid)) {
+            continue;
         }
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        list_del_init(&entry->list);
+        list_add_tail(&entry->list, &entries->list);
 
-        list_for_each_entry (entry, &entries->list, list) {
-                n = GF_CALLOC (1, sizeof (*n), gf_afr_mt_entry_name);
-                n->name = gf_strdup (entry->d_name);
-                INIT_LIST_HEAD (&n->list);
+        if (!validate_subvol)
+            continue;
 
-                list_add (&n->list, &fd_ctx->entries);
+        if (entry->inode) {
+            ret = afr_validate_read_subvol(entry->inode, this, subvol);
+            if (ret == -1) {
+                inode_unref(entry->inode);
+                entry->inode = NULL;
+                continue;
+            }
         }
+    }
 }
 
-
-static off_t
-afr_filter_entries (gf_dirent_t *entries, fd_t *fd)
+int32_t
+afr_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *subvol_entries,
+                dict_t *xdata)
 {
-        gf_dirent_t  *entry  = NULL;
-        gf_dirent_t  *tmp    = NULL;
-        int           ret    = 0;
-        uint64_t      ctx    = 0;
-        afr_fd_ctx_t *fd_ctx = NULL;
-        off_t         offset = 0;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return -1;
-        }
+    afr_local_t *local = NULL;
+    gf_dirent_t entries;
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+    INIT_LIST_HEAD(&entries.list);
 
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                offset = entry->d_off;
+    local = frame->local;
 
-                if (remembered_name (entry->d_name, &fd_ctx->entries)) {
-                        list_del (&entry->list);
-                        GF_FREE (entry);
-                }
-        }
+    if (op_ret < 0 && !local->cont.readdir.offset) {
+        /* failover only if this was first readdir, detected
+           by offset == 0 */
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-        return offset;
-}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
+    if (op_ret >= 0)
+        afr_readdir_transform_entries(frame, subvol_entries, (long)cookie,
+                                      &entries, local->fd);
 
-static void
-afr_forget_entries (fd_t *fd)
-{
-        struct entry_name *entry  = NULL;
-        struct entry_name *tmp    = NULL;
-        int                ret    = 0;
-        uint64_t           ctx    = 0;
-        afr_fd_ctx_t      *fd_ctx = NULL;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return;
-        }
+    AFR_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, xdata);
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+    gf_dirent_free(&entries);
 
-        list_for_each_entry_safe (entry, tmp, &fd_ctx->entries, list) {
-                GF_FREE (entry->name);
-                list_del (&entry->list);
-                GF_FREE (entry);
-        }
+    return 0;
 }
 
-
-int32_t
-afr_readdir_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 gf_dirent_t *entries, dict_t *xdata)
+int
+afr_readdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries, NULL);
-
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    fd_ctx = afr_fd_ctx_get(local->fd, this);
+    if (!fd_ctx) {
+        local->op_errno = EINVAL;
+        local->op_ret = -1;
+    }
+
+    if (subvol == -1 || !fd_ctx) {
+        AFR_STACK_UNWIND(readdir, frame, local->op_ret, local->op_errno, 0, 0);
         return 0;
+    }
+
+    fd_ctx->readdir_subvol = subvol;
+
+    if (local->op == GF_FOP_READDIR)
+        STACK_WIND_COOKIE(frame, afr_readdir_cbk, (void *)(long)subvol,
+                          priv->children[subvol],
+                          priv->children[subvol]->fops->readdir, local->fd,
+                          local->cont.readdir.size, local->cont.readdir.offset,
+                          local->xdata_req);
+    else
+        STACK_WIND_COOKIE(frame, afr_readdir_cbk, (void *)(long)subvol,
+                          priv->children[subvol],
+                          priv->children[subvol]->fops->readdirp, local->fd,
+                          local->cont.readdir.size, local->cont.readdir.offset,
+                          local->xdata_req);
+    return 0;
 }
 
-
-int32_t
-afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                  dict_t *xdata)
+int
+afr_do_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, int whichop, dict_t *dict)
 {
-        afr_private_t *  priv            = NULL;
-        afr_local_t *    local           = NULL;
-        xlator_t **      children        = NULL;
-        int32_t          next_call_child = -1;
-        int              ret             = 0;
-        int32_t          *last_index     = NULL;
-        int32_t          read_child      = -1;
-        int32_t         *fresh_children   = NULL;
-        uint64_t         ctx             = 0;
-        afr_fd_ctx_t    *fd_ctx          = NULL;
-        off_t            offset          = 0;
-        int32_t         call_child       = -1;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
-
-        read_child = (long) cookie;
-        last_index = &local->cont.readdir.last_index;
-        fresh_children = local->fresh_children;
-
-        /* the value of the last_index changes if afr_next_call_child is
-         * called. So to find the call_child of this callback use last_index
-         * before the next_call_child call.
-         */
-        if (*last_index == -1)
-                call_child = read_child;
-        else
-                call_child = fresh_children[*last_index];
-
-        if (priv->strict_readdir) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not get fd ctx for fd=%p", local->fd);
-                        op_ret   = -1;
-                        op_errno = -ret;
-                        goto out;
-                }
-
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                if (op_ret == -1) {
-                        next_call_child = afr_next_call_child (fresh_children,
-                                                               local->child_up,
-                                                               priv->child_count,
-                                                               last_index,
-                                                               read_child);
-                        if (next_call_child < 0)
-                                goto out;
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "starting readdir afresh on child %d, offset %"PRId64,
-                                next_call_child, (uint64_t) 0);
-
-                        fd_ctx->failed_over = _gf_true;
-
-                        STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                           (void *) (long) read_child,
-                                           children[next_call_child],
-                                           children[next_call_child]->fops->readdirp,
-                                           local->fd,
-                                           local->cont.readdir.size, 0,
-                                           local->cont.readdir.dict);
-                        return 0;
-                }
-        }
-
-        if (priv->strict_readdir) {
-                if (fd_ctx->failed_over) {
-                        if (list_empty (&entries->list)) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "no entries found");
-                                goto out;
-                        }
-
-                        offset = afr_filter_entries (entries, local->fd);
-
-                        afr_remember_entries (entries, local->fd);
-
-                        if (list_empty (&entries->list)) {
-                                /* All the entries we got were duplicate. We
-                                   shouldn't send an empty list now, because
-                                   that will make the application stop reading. So
-                                   try to get more entries */
-
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "trying to fetch non-duplicate entries "
-                                        "from offset %"PRId64", child %s",
-                                        offset, children[call_child]->name);
-
-                                STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                                   (void *) (long) read_child,
-                                                   children[call_child],
-                                                   children[call_child]->fops->readdirp,
-                                                   local->fd, local->cont.readdir.size, offset,
-                                                   local->cont.readdir.dict);
-                                return 0;
-                        }
-                } else {
-                        afr_remember_entries (entries, local->fd);
-                }
-        }
-
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    int subvol = -1;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    local->op = whichop;
+    local->fd = fd_ref(fd);
+    local->cont.readdir.size = size;
+    local->cont.readdir.offset = offset;
+    local->xdata_req = (dict) ? dict_ref(dict) : NULL;
+
+    subvol = fd_ctx->readdir_subvol;
+
+    if (offset == 0 || subvol == -1) {
+        /* First readdir has option of failing over and selecting
+           an appropriate read subvolume */
+        afr_read_txn(frame, this, fd->inode, afr_readdir_wind,
+                     AFR_DATA_TRANSACTION);
+    } else {
+        /* But continued readdirs MUST stick to the same subvolume
+           without an option to failover */
+        afr_readdir_wind(frame, this, subvol);
+    }
+
+    return 0;
 out:
-        AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, NULL);
-
-        return 0;
+    AFR_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 int32_t
-afr_do_readdir (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, size_t size, off_t offset, int whichop, dict_t *dict)
+afr_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
 {
-        afr_private_t *  priv       = NULL;
-        xlator_t **      children   = NULL;
-        int              call_child = 0;
-        afr_local_t     *local      = NULL;
-        uint64_t         ctx        = 0;
-        afr_fd_ctx_t    *fd_ctx     = NULL;
-        int              ret        = -1;
-        int32_t          op_errno   = 0;
-        uint64_t         read_child = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        read_child = afr_inode_get_read_ctx (this, fd->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.readdir.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-
-        local->fd                  = fd_ref (fd);
-        local->cont.readdir.size   = size;
-        local->cont.readdir.dict   = (dict)? dict_ref (dict) : NULL;
-
-        if (priv->strict_readdir) {
-                ret = fd_ctx_get (fd, this, &ctx);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not get fd ctx for fd=%p", fd);
-                        op_errno = -ret;
-                        goto out;
-                }
-
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                if (fd_ctx->last_tried != call_child) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "first up child has changed from %d to %d, "
-                                "restarting readdir from offset 0",
-                                fd_ctx->last_tried, call_child);
+    afr_do_readdir(frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
 
-                        fd_ctx->failed_over = _gf_true;
-                        offset = 0;
-                }
-
-                fd_ctx->last_tried = call_child;
-        }
-
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
-                                   (void *) (long) call_child,
-                                   children[call_child],
-                                   children[call_child]->fops->readdir, fd,
-                                   size, offset, dict);
-        else
-                STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                   (void *) (long) call_child,
-                                   children[call_child],
-                                   children[call_child]->fops->readdirp, fd,
-                                   size, offset, dict);
-
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, dict_t *xdata)
+afr_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
-        return 0;
-}
-
+    afr_do_readdir(frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
 
-int32_t
-afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, dict_t *dict)
-{
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_releasedir (xlator_t *this, fd_t *fd)
+afr_releasedir(xlator_t *this, fd_t *fd)
 {
-        afr_forget_entries (fd);
-        afr_cleanup_fd_ctx (this, fd);
+    afr_cleanup_fd_ctx(this, fd);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.h b/xlators/cluster/afr/src/afr-dir-read.h
index 09456d15949..773e925ec6c 100644
--- a/xlators/cluster/afr/src/afr-dir-read.h
+++ b/xlators/cluster/afr/src/afr-dir-read.h
@@ -11,26 +11,23 @@
 #ifndef __DIR_READ_H__
 #define __DIR_READ_H__
 
-
 int32_t
-afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd, dict_t *xdata);
+afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata);
 
 int32_t
-afr_releasedir (xlator_t *this, fd_t *fd);
+afr_releasedir(xlator_t *this, fd_t *fd);
 
 int32_t
-afr_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset, dict_t *xdata);
-
+afr_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata);
 
 int32_t
-afr_readdirp (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, size_t size, off_t offset, dict_t *dict);
+afr_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict);
 
 int32_t
-afr_checksum (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, int32_t flags, dict_t *xdata);
-
+afr_checksum(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             dict_t *xdata);
 
 #endif /* __DIR_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index 4d86fa4805e..b7cceb79158 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -8,1248 +8,930 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
 
+void
+afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this);
+
 int
-afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
+afr_build_parent_loc(loc_t *parent, loc_t *child, int32_t *op_errno)
 {
-        int     ret = -1;
-        char    *child_path = NULL;
-
-        if (!child->parent) {
-                if (op_errno)
-                        *op_errno = EINVAL;
-                goto out;
-        }
-
-        child_path = gf_strdup (child->path);
-        if (!child_path) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
-        parent->path = gf_strdup( dirname (child_path) );
-	if (!parent->path) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
-        parent->inode  = inode_ref (child->parent);
-        uuid_copy (parent->gfid, child->pargfid);
-
-        ret = 0;
+    int ret = -1;
+    char *child_path = NULL;
+
+    if (!child->parent) {
+        if (op_errno)
+            *op_errno = EINVAL;
+        goto out;
+    }
+
+    child_path = gf_strdup(child->path);
+    if (!child_path) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    parent->path = gf_strdup(dirname(child_path));
+    if (!parent->path) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    parent->inode = inode_ref(child->parent);
+    gf_uuid_copy(parent->gfid, child->pargfid);
+
+    ret = 0;
 out:
-	GF_FREE(child_path);
+    GF_FREE(child_path);
 
-        return ret;
+    return ret;
 }
 
-/* {{{ create */
-
-int
-afr_create_unwind (call_frame_t *frame, xlator_t *this)
+static void
+__afr_dir_write_finalize(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-        struct iatt  *unwind_buf = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.create.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.create.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.create.buf;
-                }
-
-                AFR_STACK_UNWIND (create, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.create.fd,
-                                  local->cont.create.inode,
-                                  unwind_buf, &local->cont.create.preparent,
-                                  &local->cont.create.postparent,
-                                  local->xdata_rsp);
-        }
-
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int inode_read_subvol = -1;
+    int parent_read_subvol = -1;
+    int parent2_read_subvol = -1;
+    int i = 0;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret == -1)
+            continue;
+        gf_uuid_copy(args.gfid, local->replies[i].poststat.ia_gfid);
+        args.ia_type = local->replies[i].poststat.ia_type;
+        break;
+    }
+
+    if (local->inode) {
+        if (local->op != GF_FOP_RENAME && local->op != GF_FOP_LINK)
+            afr_replies_interpret(frame, this, local->inode, NULL);
+
+        inode_read_subvol = afr_data_subvol_get(local->inode, this, NULL, NULL,
+                                                NULL, &args);
+    }
+
+    if (local->parent)
+        parent_read_subvol = afr_data_subvol_get(local->parent, this, NULL,
+                                                 local->readable, NULL, NULL);
+
+    if (local->parent2)
+        parent2_read_subvol = afr_data_subvol_get(local->parent2, this, NULL,
+                                                  local->readable2, NULL, NULL);
+
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    afr_pick_error_xdata(local, priv, local->parent, local->readable,
+                         local->parent2, local->readable2);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0) {
+            if (local->inode)
+                afr_inode_need_refresh_set(local->inode, this);
+            if (local->parent)
+                afr_inode_need_refresh_set(local->parent, this);
+            if (local->parent2)
+                afr_inode_need_refresh_set(local->parent2, this);
+            continue;
+        }
+
+        if (local->op_ret == -1) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+
+            local->cont.dir_fop.buf = local->replies[i].poststat;
+            local->cont.dir_fop.preparent = local->replies[i].preparent;
+            local->cont.dir_fop.postparent = local->replies[i].postparent;
+            local->cont.dir_fop.prenewparent = local->replies[i].preparent2;
+            local->cont.dir_fop.postnewparent = local->replies[i].postparent2;
+            if (local->xdata_rsp) {
+                dict_unref(local->xdata_rsp);
+                local->xdata_rsp = NULL;
+            }
+
+            if (local->replies[i].xdata)
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            continue;
+        }
+
+        if (i == inode_read_subvol) {
+            local->cont.dir_fop.buf = local->replies[i].poststat;
+            if (local->replies[i].xdata) {
+                if (local->xdata_rsp)
+                    dict_unref(local->xdata_rsp);
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+        }
+
+        if (i == parent_read_subvol) {
+            local->cont.dir_fop.preparent = local->replies[i].preparent;
+            local->cont.dir_fop.postparent = local->replies[i].postparent;
+        }
+
+        if (i == parent2_read_subvol) {
+            local->cont.dir_fop.prenewparent = local->replies[i].preparent2;
+            local->cont.dir_fop.postnewparent = local->replies[i].postparent2;
+        }
+    }
 }
 
-
-int
-afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     fd_t *fd, inode_t *inode, struct iatt *buf,
+static void
+__afr_dir_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                     int op_ret, int op_errno, struct iatt *poststat,
                      struct iatt *preparent, struct iatt *postparent,
+                     struct iatt *preparent2, struct iatt *postparent2,
                      dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv  = NULL;
-        uint64_t        ctx = 0;
-        afr_fd_ctx_t    *fd_ctx = NULL;
-        int             ret = 0;
-        int             call_count = -1;
-        int             child_index = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv  = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret = op_ret;
-
-                        ret = afr_fd_ctx_set (this, fd);
-
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not set ctx on fd=%p", fd);
-
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-
-                        ret = fd_ctx_get (fd, this, &ctx);
-
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not get fd ctx for fd=%p", fd);
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-
-                        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                        fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-                        fd_ctx->flags                  = local->cont.create.flags;
-
-                        if (local->success_count == 0) {
-                                local->cont.create.buf        = *buf;
-				if (xdata)
-					local->xdata_rsp = dict_ref(xdata);
-			}
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.create.read_child_buf = *buf;
-                                local->cont.create.preparent      = *preparent;
-                                local->cont.create.postparent     = *postparent;
-                        }
-
-                        local->cont.create.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-
-unlock:
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child,
-                                              local->cont.create.buf.ia_gfid);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+    afr_local_t *local = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    if (op_ret >= 0) {
+        if (poststat)
+            local->replies[child_index].poststat = *poststat;
+        if (preparent)
+            local->replies[child_index].preparent = *preparent;
+        if (postparent)
+            local->replies[child_index].postparent = *postparent;
+        if (preparent2)
+            local->replies[child_index].preparent2 = *preparent2;
+        if (postparent2)
+            local->replies[child_index].postparent2 = *postparent2;
+        if (fd_ctx)
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+    } else {
+        if (op_errno != ENOTEMPTY)
+            afr_transaction_fop_failed(frame, this, child_index);
+        if (fd_ctx)
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+    }
+
+    return;
 }
 
-
-int
-afr_create_wind (call_frame_t *frame, xlator_t *this)
+static int
+__afr_dir_write_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *buf,
+                    struct iatt *preparent, struct iatt *postparent,
+                    struct iatt *preparent2, struct iatt *postparent2,
+                    dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    priv = this->private;
+    local = frame->local;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+    LOCK(&frame->lock);
+    {
+        __afr_dir_write_fill(frame, this, child_index, op_ret, op_errno, buf,
+                             preparent, postparent, preparent2, postparent2,
+                             xdata);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+    if (call_count == 0) {
+        __afr_dir_write_finalize(frame, this);
 
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_create_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->create,
-                                           &local->loc,
-                                           local->cont.create.flags,
-                                           local->cont.create.mode,
-                                           local->umask,
-                                           local->cont.create.fd,
-                                           local->xdata_req);
-                        if (!--call_count)
-                                break;
-                }
+        if (afr_txn_nothing_failed(frame, this)) {
+            /*if it did pre-op, it will do post-op changing ctime*/
+            if (priv->consistent_metadata && afr_needs_changelog_update(local))
+                afr_zero_fill_stat(local);
+            local->transaction.unwind(frame, this);
         }
 
-        return 0;
-}
-
-
-int
-afr_create_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
+        afr_mark_entry_pending_changelog(frame, this);
 
-        AFR_STACK_DESTROY (frame);
+        afr_transaction_resume(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *params)
+afr_mark_new_entry_changelog_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int op_ret, int op_errno,
+                                 dict_t *xattr, dict_t *xdata)
 {
-        afr_private_t  *priv  = NULL;
-        afr_local_t    *local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int             ret = -1;
-        int             op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        QUORUM_CHECK(create,out);
+    int call_count = 0;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
-
-        local->cont.create.flags = flags;
-        local->cont.create.mode  = mode;
-        local->cont.create.fd    = fd_ref (fd);
-        local->umask  = umask;
-        if (params)
-                local->xdata_req = dict_ref (params);
+    call_count = afr_frame_return(frame);
 
-        local->transaction.fop    = afr_create_wind;
-        local->transaction.done   = afr_create_done;
-        local->transaction.unwind = afr_create_unwind;
+    if (call_count == 0)
+        AFR_STACK_DESTROY(frame);
 
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-        ret = 0;
-out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (create, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL, NULL);
-        }
-
-        return 0;
+    return 0;
 }
 
-/* }}} */
-
-/* {{{ mknod */
-
-int
-afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
+void
+afr_mark_new_entry_changelog(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.mknod.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mknod.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mknod.buf;
-                }
-
-                AFR_STACK_UNWIND (mknod, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.mknod.inode,
-                                  unwind_buf, &local->cont.mknod.preparent,
-                                  &local->cont.mknod.postparent,
-                                  NULL);
-        }
-
-        return 0;
+    call_frame_t *new_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *new_local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int32_t **changelog = NULL;
+    int i = 0;
+    int op_errno = ENOMEM;
+    unsigned char *pending = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame)
+        goto out;
+
+    new_local = AFR_FRAME_INIT(new_frame, op_errno);
+    if (!new_local)
+        goto out;
+
+    xattr = dict_new();
+    if (!xattr)
+        goto out;
+
+    pending = alloca0(priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] &&
+            !local->transaction.failed_subvols[i]) {
+            call_count++;
+            continue;
+        }
+        pending[i] = 1;
+    }
+
+    changelog = afr_mark_pending_changelog(priv, pending, xattr,
+                                           local->cont.dir_fop.buf.ia_type);
+    if (!changelog)
+        goto out;
+
+    new_local->pending = changelog;
+    gf_uuid_copy(new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
+    new_local->loc.inode = inode_ref(local->inode);
+
+    new_local->call_count = call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (pending[i])
+            continue;
+
+        STACK_WIND_COOKIE(new_frame, afr_mark_new_entry_changelog_cbk,
+                          (void *)(long)i, priv->children[i],
+                          priv->children[i]->fops->xattrop, &new_local->loc,
+                          GF_XATTROP_ADD_ARRAY, xattr, NULL);
+        if (!--call_count)
+            break;
+    }
+
+    new_frame = NULL;
+out:
+    if (new_frame)
+        AFR_STACK_DESTROY(new_frame);
+    if (xattr)
+        dict_unref(xattr);
+    return;
 }
 
-
-int
-afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+void
+afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int pre_op_count = 0;
+    int failed_count = 0;
+    unsigned char *success_replies = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        child_index = (long) cookie;
+    if (local->op_ret < 0)
+        return;
 
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+    if (local->op != GF_FOP_CREATE && local->op != GF_FOP_MKNOD &&
+        local->op != GF_FOP_MKDIR)
+        return;
 
-                if (op_ret != -1) {
-                        local->op_ret = op_ret;
+    pre_op_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
+    failed_count = AFR_COUNT(local->transaction.failed_subvols,
+                             priv->child_count);
 
-                        if (local->success_count == 0)
-                                local->cont.mknod.buf   = *buf;
+    /* FOP succeeded on all bricks. */
+    if (pre_op_count == priv->child_count && !failed_count)
+        return;
 
-                        if (child_index == local->read_child_index) {
-                                local->cont.mknod.read_child_buf = *buf;
-                                local->cont.mknod.preparent      = *preparent;
-                                local->cont.mknod.postparent     = *postparent;
-                        }
+    /* FOP did not suceed on quorum no. of bricks. */
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+    if (!afr_has_quorum(success_replies, this, NULL))
+        return;
 
-                        local->cont.mknod.inode = inode;
+    if (priv->thin_arbiter_count) {
+        /*Mark new entry using ta file*/
+        local->is_new_entry = _gf_true;
+        return;
+    }
 
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
+    afr_mark_new_entry_changelog(frame, this);
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child,
-                                              local->cont.mknod.buf.ia_gfid);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+    return;
 }
 
+/* {{{ create */
 
-int32_t
-afr_mknod_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_create_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->mknod,
-                                           &local->loc, local->cont.mknod.mode,
-                                           local->cont.mknod.dev,
-                                           local->umask,
-                                           local->xdata_req);
-                        if (!--call_count)
-                                break;
-                }
-        }
+    main_frame = afr_transaction_detach_fop_frame(frame);
 
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(create, main_frame, local->op_ret, local->op_errno,
+                     local->cont.create.fd, local->inode,
+                     &local->cont.dir_fop.buf, &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_mknod_done (call_frame_t *frame, xlator_t *this)
+afr_create_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t * local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t dev, mode_t umask, dict_t *params)
+afr_create_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        QUORUM_CHECK(mknod,out);
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
-
-        local->cont.mknod.mode  = mode;
-        local->cont.mknod.dev   = dev;
-        local->umask = umask;
-        if (params)
-                local->xdata_req = dict_ref (params);
-
-        local->transaction.fop    = afr_mknod_wind;
-        local->transaction.done   = afr_mknod_done;
-        local->transaction.unwind = afr_mknod_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-        ret = 0;
-out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (mknod, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
-        }
-
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_create_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->create, &local->loc,
+                      local->cont.create.flags, local->cont.create.mode,
+                      local->umask, local->cont.create.fd, local->xdata_req);
+    return 0;
 }
 
-/* }}} */
-
-/* {{{ mkdir */
-
-
 int
-afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
+afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.mkdir.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mkdir.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mkdir.buf;
-                }
-
-                AFR_STACK_UNWIND (mkdir, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.mkdir.inode,
-                                  unwind_buf, &local->cont.mkdir.preparent,
-                                  &local->cont.mkdir.postparent,
-                                  NULL);
-        }
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+
+    local->fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!local->fd_ctx)
+        goto out;
+
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->op = GF_FOP_CREATE;
+    local->cont.create.flags = flags;
+    local->fd_ctx->flags = flags;
+    local->cont.create.mode = mode;
+    local->cont.create.fd = fd_ref(fd);
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_create_wind;
+    local->transaction.unwind = afr_create_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
+/* }}} */
+
+/* {{{ mknod */
 
 int
-afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+afr_mknod_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret           = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.mkdir.buf   = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.mkdir.read_child_buf = *buf;
-                                local->cont.mkdir.preparent      = *preparent;
-                                local->cont.mkdir.postparent     = *postparent;
-                        }
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-                        local->cont.mkdir.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno         = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child,
-                                              local->cont.mkdir.buf.ia_gfid);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(mknod, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_mkdir_wind (call_frame_t *frame, xlator_t *this)
+afr_mknod_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->mkdir,
-                                           &local->loc, local->cont.mkdir.mode,
-                                           local->umask,
-                                           local->xdata_req);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_mkdir_done (call_frame_t *frame, xlator_t *this)
+afr_mknod_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t * local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_mknod_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->mknod, &local->loc,
+                      local->cont.mknod.mode, local->cont.mknod.dev,
+                      local->umask, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
+afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t dev, mode_t umask, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        QUORUM_CHECK(mkdir,out);
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
-
-        local->cont.mkdir.mode  = mode;
-        local->umask = umask;
-        if (params)
-                local->xdata_req = dict_ref (params);
-
-        local->transaction.fop    = afr_mkdir_wind;
-        local->transaction.done   = afr_mkdir_done;
-        local->transaction.unwind = afr_mkdir_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-        ret = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->op = GF_FOP_MKNOD;
+    local->cont.mknod.mode = mode;
+    local->cont.mknod.dev = dev;
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_mknod_wind;
+    local->transaction.unwind = afr_mknod_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-
-                AFR_STACK_UNWIND (mkdir, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
-/* {{{ link */
-
+/* {{{ mkdir */
 
 int
-afr_link_unwind (call_frame_t *frame, xlator_t *this)
+afr_mkdir_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.link.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.link.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.link.buf;
-                }
-
-                AFR_STACK_UNWIND (link, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.link.inode,
-                                  unwind_buf, &local->cont.link.preparent,
-                                  &local->cont.link.postparent,
-                                  NULL);
-        }
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
+    local = frame->local;
+
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(mkdir, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+afr_mkdir_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
                    struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret   = op_ret;
-
-                        if (local->success_count == 0) {
-                                local->cont.link.buf        = *buf;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.link.read_child_buf = *buf;
-                                local->cont.link.preparent      = *preparent;
-                                local->cont.link.postparent     = *postparent;
-                        }
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-		local->cont.link.inode = inode;
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
+}
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child,
-                                              local->cont.link.buf.ia_gfid);
-                local->transaction.unwind (frame, this);
+int
+afr_mkdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-                local->transaction.resume (frame, this);
-        }
+    local = frame->local;
+    priv = this->private;
 
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_mkdir_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->mkdir, &local->loc,
+                      local->cont.mkdir.mode, local->umask, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_link_wind (call_frame_t *frame, xlator_t *this)
+afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv  = this->private;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.mkdir.mode = mode;
+    local->umask = umask;
+
+    if (!xdata || !dict_get_sizen(xdata, "gfid-req")) {
+        op_errno = EPERM;
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, op_errno,
+                         AFR_MSG_GFID_NULL,
+                         "mkdir: %s is received "
+                         "without gfid-req %p",
+                         loc->path, xdata);
+        goto out;
+    }
+
+    local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    if (!local->xdata_req) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->op = GF_FOP_MKDIR;
+    local->transaction.wind = afr_mkdir_wind;
+    local->transaction.unwind = afr_mkdir_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+    AFR_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+/* }}} */
 
-        local->call_count = call_count;
+/* {{{ link */
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->link,
-                                           &local->loc,
-                                           &local->newloc, local->xdata_req);
+int
+afr_link_unwind(call_frame_t *frame, xlator_t *this)
+{
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-                        if (!--call_count)
-                                break;
-                }
-        }
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(link, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_link_done (call_frame_t *frame, xlator_t *this)
+afr_link_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t * local = frame->local;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
+}
 
-        local->transaction.unwind (frame, this);
+int
+afr_link_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        AFR_STACK_DESTROY (frame);
+    local = frame->local;
+    priv = this->private;
 
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_link_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->link, &local->loc,
+                      &local->newloc, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        priv = this->private;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        QUORUM_CHECK(link,out);
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->newloc, newloc);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
+    local->inode = inode_ref(oldloc->inode);
+    local->parent = inode_ref(newloc->parent);
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        loc_copy (&local->loc,    oldloc);
-        loc_copy (&local->newloc, newloc);
-        if (xdata)
-                local->xdata_req = dict_ref (xdata);
+    if (!local->xdata_req)
+        goto out;
 
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
+    local->op = GF_FOP_LINK;
 
-        local->transaction.fop    = afr_link_wind;
-        local->transaction.done   = afr_link_done;
-        local->transaction.unwind = afr_link_unwind;
+    local->transaction.wind = afr_link_wind;
+    local->transaction.unwind = afr_link_unwind;
 
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, newloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, newloc,
+                               &op_errno);
+    if (ret)
+        goto out;
 
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (newloc->path);
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(newloc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-        ret = 0;
+    return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (link, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ symlink */
 
-
-int
-afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
-{
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.symlink.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.symlink.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.symlink.buf;
-                }
-
-                AFR_STACK_UNWIND (symlink, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.symlink.inode,
-                                  unwind_buf, &local->cont.symlink.preparent,
-                                  &local->cont.symlink.postparent,
-                                  NULL);
-        }
-
-        return 0;
-}
-
-
 int
-afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, inode_t *inode,
-                      struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+afr_symlink_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret   = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.symlink.buf        = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.symlink.read_child_buf = *buf;
-                                local->cont.symlink.preparent      = *preparent;
-                                local->cont.symlink.postparent     = *postparent;
-                        }
-
-                        local->cont.symlink.inode    = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child,
-                                              local->cont.symlink.buf.ia_gfid);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(symlink, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_symlink_wind (call_frame_t *frame, xlator_t *this)
+afr_symlink_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->symlink,
-                                           local->cont.symlink.linkpath,
-                                           &local->loc,
-                                           local->umask,
-                                           local->xdata_req);
-
-                        if (!--call_count)
-                                break;
-
-                }
-        }
-
-        return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_symlink_done (call_frame_t *frame, xlator_t *this)
+afr_symlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_symlink_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->symlink,
+                      local->cont.symlink.linkpath, &local->loc, local->umask,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkpath, loc_t *loc, mode_t umask, dict_t *params)
+afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        QUORUM_CHECK(symlink,out);
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
-
-        local->cont.symlink.linkpath = gf_strdup (linkpath);
-        local->umask = umask;
-        if (params)
-                local->xdata_req = dict_ref (params);
-
-        local->transaction.fop    = afr_symlink_wind;
-        local->transaction.done   = afr_symlink_done;
-        local->transaction.unwind = afr_symlink_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (loc->path);
-
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-        ret = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.symlink.linkpath = gf_strdup(linkpath);
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_SYMLINK;
+    local->transaction.wind = afr_symlink_wind;
+    local->transaction.unwind = afr_symlink_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-        if (ret  < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (symlink, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -1257,217 +939,118 @@ out:
 /* {{{ rename */
 
 int
-afr_rename_unwind (call_frame_t *frame, xlator_t *this)
+afr_rename_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.rename.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.rename.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.rename.buf;
-                }
-
-                AFR_STACK_UNWIND (rename, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  unwind_buf,
-                                  &local->cont.rename.preoldparent,
-                                  &local->cont.rename.postoldparent,
-                                  &local->cont.rename.prenewparent,
-                                  &local->cont.rename.postnewparent,
-                                  NULL);
-        }
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
+
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(rename, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.buf, &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent,
+                     &local->cont.dir_fop.prenewparent,
+                     &local->cont.dir_fop.postnewparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                     struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent,
-                     dict_t *xdata)
+afr_rename_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        int call_count = -1;
-        int child_index = -1;
-
-        local = frame->local;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno) && op_errno != ENOTEMPTY)
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-
-                                if (buf) {
-                                        local->cont.rename.buf = *buf;
-                                }
-
-                                local->success_count++;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.rename.read_child_buf = *buf;
-
-                                local->cont.rename.preoldparent  = *preoldparent;
-                                local->cont.rename.postoldparent = *postoldparent;
-                                local->cont.rename.prenewparent  = *prenewparent;
-                                local->cont.rename.postnewparent = *postnewparent;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preoldparent, postoldparent, prenewparent,
+                               postnewparent, xdata);
 }
 
-
-int32_t
-afr_rename_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_rename_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->rename,
-                                           &local->loc,
-                                           &local->newloc, NULL);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
 int
-afr_rename_done (call_frame_t *frame, xlator_t *this)
+afr_rename_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        AFR_STACK_DESTROY (frame);
+    local = frame->local;
+    priv = this->private;
 
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_rename_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->rename, &local->loc,
+                      &local->newloc, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        QUORUM_CHECK(rename,out);
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        loc_copy (&local->loc,    oldloc);
-        loc_copy (&local->newloc, newloc);
-
-        local->read_child_index = afr_inode_get_read_ctx (this, oldloc->inode, NULL);
-
-        local->transaction.fop    = afr_rename_wind;
-        local->transaction.done   = afr_rename_done;
-        local->transaction.unwind = afr_rename_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, oldloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-        ret = afr_build_parent_loc (&local->transaction.new_parent_loc, newloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (oldloc->path);
-        local->transaction.new_basename = AFR_BASENAME (newloc->path);
-
-        afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
-
-        ret = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->newloc, newloc);
+
+    local->inode = inode_ref(oldloc->inode);
+    local->parent = inode_ref(oldloc->parent);
+    local->parent2 = inode_ref(newloc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_RENAME;
+    local->transaction.wind = afr_rename_wind;
+    local->transaction.unwind = afr_rename_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, oldloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+    ret = afr_build_parent_loc(&local->transaction.new_parent_loc, newloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(oldloc->path);
+    local->transaction.new_basename = AFR_BASENAME(newloc->path);
+    ret = afr_transaction(transaction_frame, this,
+                          AFR_ENTRY_RENAME_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-
-                AFR_STACK_UNWIND (rename, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -1475,395 +1058,205 @@ out:
 /* {{{ unlink */
 
 int
-afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
+afr_unlink_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (unlink, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.unlink.preparent,
-                                  &local->cont.unlink.postparent,
-                                  NULL);
-        }
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
-
-
-int
-afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
-{
-        afr_local_t *   local = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == local->read_child_index) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret   = op_ret;
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
-
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+    AFR_STACK_UNWIND(unlink, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
 }
 
-
-int32_t
-afr_unlink_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_unlink_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->unlink,
-                                           &local->loc, local->xflag,
-                                           local->xdata_req);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
-int32_t
-afr_unlink_done (call_frame_t *frame, xlator_t *this)
+int
+afr_unlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        AFR_STACK_DESTROY (frame);
+    local = frame->local;
+    priv = this->private;
 
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_unlink_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->unlink, &local->loc,
+                      local->xflag, local->xdata_req);
+    return 0;
 }
 
-
-int32_t
-afr_unlink (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int xflag, dict_t *xdata)
+int
+afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        QUORUM_CHECK(unlink,out);
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        loc_copy (&local->loc, loc);
-        local->xflag = xflag;
-        if (xdata)
-                local->xdata_req = dict_ref (xdata);
-
-        local->transaction.fop    = afr_unlink_wind;
-        local->transaction.done   = afr_unlink_done;
-        local->transaction.unwind = afr_unlink_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-        ret = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->xflag = xflag;
+
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_UNLINK;
+    local->transaction.wind = afr_unlink_wind;
+    local->transaction.unwind = afr_unlink_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (unlink, frame, -1, op_errno,
-                                  NULL, NULL, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ rmdir */
 
-
-
 int
-afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
+afr_rmdir_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (rmdir, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.rmdir.preparent,
-                                  &local->cont.rmdir.postparent,
-                                  NULL);
-        }
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
-
-
-int
-afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
-{
-        afr_local_t *   local = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno) && (op_errno != ENOTEMPTY))
-                        afr_transaction_fop_failed (frame, this, child_index);
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-                        }
-
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+    AFR_STACK_UNWIND(rmdir, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
 }
 
-
 int
-afr_rmdir_wind (call_frame_t *frame, xlator_t *this)
+afr_rmdir_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->rmdir,
-                                           &local->loc, local->cont.rmdir.flags,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_rmdir_done (call_frame_t *frame, xlator_t *this)
+afr_rmdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t * local = frame->local;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local->transaction.unwind (frame, this);
+    local = frame->local;
+    priv = this->private;
 
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_rmdir_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->rmdir, &local->loc,
+                      local->cont.rmdir.flags, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_rmdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int flags, dict_t *xdata)
+afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        QUORUM_CHECK(rmdir,out);
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->cont.rmdir.flags = flags;
-        loc_copy (&local->loc, loc);
-
-        local->transaction.fop    = afr_rmdir_wind;
-        local->transaction.done   = afr_rmdir_done;
-        local->transaction.unwind = afr_rmdir_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-        ret = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.rmdir.flags = flags;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_RMDIR;
+    local->transaction.wind = afr_rmdir_wind;
+    local->transaction.unwind = afr_rmdir_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-dir-write.h b/xlators/cluster/afr/src/afr-dir-write.h
index 02f0a3682d9..1d88c3b9b26 100644
--- a/xlators/cluster/afr/src/afr-dir-write.h
+++ b/xlators/cluster/afr/src/afr-dir-write.h
@@ -12,36 +12,35 @@
 #define __DIR_WRITE_H__
 
 int32_t
-afr_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *xdata);
+afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
-afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev, mode_t umask, dict_t *xdata);
+afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t dev, mode_t umask, dict_t *xdata);
 
 int32_t
-afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
+afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata);
 
 int32_t
-afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int xflag, dict_t *xdata);
+afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
 
 int32_t
-afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int flags, dict_t *xdata);
+afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
 
 int32_t
-afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata);
 
 int32_t
-afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata);
 
 int
-afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *oldloc, mode_t umask, dict_t *params);
+afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *oldloc, mode_t umask, dict_t *params);
 
 #endif /* __DIR_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index 40e5abd3a03..c5521704de2 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
 #include <fnmatch.h>
@@ -16,1351 +15,1880 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/quota-common-utils.h>
+
+#include "afr-transaction.h"
+#include "afr-messages.h"
 
-/**
- * Common algorithm for inode read calls:
- *
- * - Try the fop on the first child that is up
- * - if we have failed due to ENOTCONN:
- *     try the next child
- *
- * Applicable to: access, stat, fstat, readlink, getxattr
- */
+/*
+ * Quota size xattrs are not maintained by afr. There is a
+ * possibility that they differ even when both the directory changelog xattrs
+ * suggest everything is fine. So if there is at least one 'source' check among
+ * the sources which has the maximum quota size. Otherwise check among all the
+ * available ones for maximum quota size. This way if there is a source and
+ * stale copies it always votes for the 'source'.
+ * */
+
+int
+afr_handle_quota_size(call_frame_t *frame, xlator_t *this)
+{
+    unsigned char *readable = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0;
+    int ret = 0;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t max_size = {
+        0,
+    };
+    int readable_cnt = 0;
+    int read_subvol = -1;
+
+    local = frame->local;
+    priv = this->private;
+    replies = local->replies;
+
+    readable = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(local->inode, this, readable, 0, 0);
+
+    readable_cnt = AFR_COUNT(readable, priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (readable_cnt && !readable[i])
+            continue;
+        if (!replies[i].xdata)
+            continue;
+        ret = quota_dict_get_meta(replies[i].xdata, QUOTA_SIZE_KEY,
+                                  SLEN(QUOTA_SIZE_KEY), &size);
+        if (ret == -1)
+            continue;
+        if (read_subvol == -1)
+            read_subvol = i;
+        if (size.size > max_size.size ||
+            (size.file_count + size.dir_count) >
+                (max_size.file_count + max_size.dir_count))
+            read_subvol = i;
+
+        if (size.size > max_size.size)
+            max_size.size = size.size;
+        if (size.file_count > max_size.file_count)
+            max_size.file_count = size.file_count;
+        if (size.dir_count > max_size.dir_count)
+            max_size.dir_count = size.dir_count;
+    }
+
+    if (max_size.size == 0 && max_size.file_count == 0 &&
+        max_size.dir_count == 0)
+        return read_subvol;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (readable_cnt && !readable[i])
+            continue;
+        if (!replies[i].xdata)
+            continue;
+        quota_dict_set_meta(replies[i].xdata, QUOTA_SIZE_KEY, &max_size,
+                            IA_IFDIR);
+    }
+
+    return read_subvol;
+}
 
 /* {{{ access */
 
-int32_t
-afr_access_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int
+afr_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
 
-        read_child = (long) cookie;
-
-        if (op_ret == -1) {
-                last_index = &local->cont.access.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
-
-                STACK_WIND_COOKIE (frame, afr_access_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->access,
-                                   &local->loc, local->cont.access.mask,
-                                   NULL);
-        }
+    local = frame->local;
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
-        }
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
+        afr_read_txn_continue(frame, this, (long)cookie);
         return 0;
-}
-
+    }
 
-int32_t
-afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-            dict_t *xdata)
-{
-        afr_private_t   *priv      = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        afr_local_t     *local     = NULL;
-        int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
+    AFR_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    return 0;
+}
 
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
+int
+afr_access_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        children = priv->children;
+    priv = this->private;
+    local = frame->local;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(access, frame, local->op_ret, local->op_errno, 0);
+        return 0;
+    }
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    STACK_WIND_COOKIE(frame, afr_access_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->access, &local->loc,
+                      local->cont.access.mask, local->xdata_req);
+    return 0;
+}
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+int
+afr_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int mask,
+           dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        read_child = afr_inode_get_read_ctx (this, loc->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.access.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    local->op = GF_FOP_ACCESS;
+    loc_copy(&local->loc, loc);
+    local->cont.access.mask = mask;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        loc_copy (&local->loc, loc);
-        local->cont.access.mask = mask;
+    afr_read_txn(frame, this, loc->inode, afr_access_wind,
+                 AFR_METADATA_TRANSACTION);
 
-        STACK_WIND_COOKIE (frame, afr_access_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->access,
-                           loc, mask, xdata);
-
-        ret = 0;
+    return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (access, frame, -1, op_errno, NULL);
-        return 0;
-}
+    AFR_STACK_UNWIND(access, frame, -1, op_errno, NULL);
 
+    return 0;
+}
 
 /* }}} */
 
 /* {{{ stat */
 
-int32_t
-afr_stat_cbk (call_frame_t *frame, void *cookie,
-              xlator_t *this, int32_t op_ret, int32_t op_errno,
-              struct iatt *buf, dict_t *xdata)
+int
+afr_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
+    afr_local_t *local = NULL;
 
-        priv     = this->private;
-        children = priv->children;
+    local = frame->local;
 
-        read_child = (long) cookie;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-        local = frame->local;
-
-        if (op_ret == -1) {
-                last_index = &local->cont.stat.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
-
-                STACK_WIND_COOKIE (frame, afr_stat_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->stat,
-                                   &local->loc, NULL);
-        }
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
-        }
+    AFR_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+int
+afr_stat_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t   *priv      = NULL;
-        afr_local_t     *local     = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    priv = this->private;
+    local = frame->local;
 
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
-
-        children = priv->children;
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(stat, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    STACK_WIND_COOKIE(
+        frame, afr_stat_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->stat, &local->loc, local->xdata_req);
+    return 0;
+}
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+int
+afr_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        read_child = afr_inode_get_read_ctx (this, loc->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.stat.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-        loc_copy (&local->loc, loc);
+    local->op = GF_FOP_STAT;
+    loc_copy(&local->loc, loc);
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->stat,
-                           loc, xdata);
+    afr_read_txn(frame, this, loc->inode, afr_stat_wind, AFR_DATA_TRANSACTION);
 
-        ret = 0;
+    return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ fstat */
 
-int32_t
-afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf,
-               dict_t *xdata)
+int
+afr_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        afr_private_t   *priv           = NULL;
-        afr_local_t     *local          = NULL;
-        xlator_t        **children      = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
 
-        read_child = (long) cookie;
-
-        if (op_ret == -1) {
-                last_index = &local->cont.fstat.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
-
-                STACK_WIND_COOKIE (frame, afr_fstat_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->fstat,
-                                   local->fd, NULL);
-        }
+    local = frame->local;
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
-        }
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
+        afr_read_txn_continue(frame, this, (long)cookie);
         return 0;
-}
-
-
-int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, dict_t *xdata)
-{
-        afr_private_t   *priv      = NULL;
-        afr_local_t     *local     = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        int32_t         op_errno   = 0;
-        int32_t         read_child = 0;
-        int             ret        = -1;
+    }
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (fd, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    AFR_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
 
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
-
-        children = priv->children;
-
-        VALIDATE_OR_GOTO (fd->inode, out);
+    return 0;
+}
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+int
+afr_fstat_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    priv = this->private;
+    local = frame->local;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(fstat, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
 
-        read_child = afr_inode_get_read_ctx (this, fd->inode,
-                                             local->fresh_children);
+    STACK_WIND_COOKIE(
+        frame, afr_fstat_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->fstat, local->fd, local->xdata_req);
+    return 0;
+}
 
+int32_t
+afr_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.fstat.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    local->op = GF_FOP_FSTAT;
+    local->fd = fd_ref(fd);
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        local->fd = fd_ref (fd);
+    afr_fix_open(fd, this);
 
-        ret = afr_open_fd_fix (frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-        STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->fstat,
-                           fd, xdata);
+    afr_read_txn(frame, this, fd->inode, afr_fstat_wind, AFR_DATA_TRANSACTION);
 
-        ret = 0;
+    return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ readlink */
 
-int32_t
-afr_readlink_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  const char *buf, struct iatt *sbuf, dict_t *xdata)
+int
+afr_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, const char *buf,
+                 struct iatt *sbuf, dict_t *xdata)
 {
-        afr_private_t * priv                  = NULL;
-        afr_local_t *   local                 = NULL;
-        xlator_t **     children              = NULL;
-        int             unwind                = 1;
-        int32_t         *last_index           = NULL;
-        int32_t         next_call_child       = -1;
-        int32_t         read_child            = -1;
-        int32_t         *fresh_children        = NULL;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
 
-        read_child = (long) cookie;
-
-        if (op_ret == -1) {
-                last_index = &local->cont.readlink.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->readlink,
-                                   &local->loc,
-                                   local->cont.readlink.size, NULL);
-        }
+    local = frame->local;
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, sbuf,
-                                  xdata);
-        }
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
+        afr_read_txn_continue(frame, this, (long)cookie);
         return 0;
-}
+    }
 
+    AFR_STACK_UNWIND(readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
+    return 0;
+}
 
-int32_t
-afr_readlink (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, size_t size, dict_t *xdata)
+int
+afr_readlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t   *priv      = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        afr_local_t     *local     = NULL;
-        int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    local = frame->local;
+    priv = this->private;
 
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
-
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(readlink, frame, local->op_ret, local->op_errno, 0, 0,
+                         0);
+        return 0;
+    }
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    STACK_WIND_COOKIE(frame, afr_readlink_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->readlink, &local->loc,
+                      local->cont.readlink.size, local->xdata_req);
+    return 0;
+}
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-        read_child = afr_inode_get_read_ctx (this, loc->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.readlink.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+int
+afr_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-        loc_copy (&local->loc, loc);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.readlink.size       = size;
+    local->op = GF_FOP_READLINK;
+    loc_copy(&local->loc, loc);
+    local->cont.readlink.size = size;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->readlink,
-                           loc, size, xdata);
+    afr_read_txn(frame, this, loc->inode, afr_readlink_wind,
+                 AFR_DATA_TRANSACTION);
 
-        ret = 0;
+    return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
+    AFR_STACK_UNWIND(readlink, frame, -1, op_errno, 0, 0, 0);
 
+    return 0;
+}
 
 /* }}} */
 
 /* {{{ getxattr */
 
 struct _xattr_key {
-        char *key;
-        struct list_head list;
+    char *key;
+    struct list_head list;
 };
 
-
-void
-__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
-                     void *data)
+int
+__gather_xattr_keys(dict_t *dict, char *key, data_t *value, void *data)
 {
-        struct list_head *  list  = data;
-        struct _xattr_key * xkey  = NULL;
+    struct list_head *list = data;
+    struct _xattr_key *xkey = NULL;
 
-        if (!strncmp (key, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
+    if (!strncmp(key, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX))) {
+        xkey = GF_MALLOC(sizeof(*xkey), gf_afr_mt_xattr_key);
+        if (!xkey)
+            return -1;
 
-                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
-                if (!xkey)
-                        return;
+        xkey->key = key;
+        INIT_LIST_HEAD(&xkey->list);
 
-                xkey->key = key;
-                INIT_LIST_HEAD (&xkey->list);
-
-                list_add_tail (&xkey->list, list);
-        }
+        list_add_tail(&xkey->list, list);
+    }
+    return 0;
 }
 
-
 void
-__filter_xattrs (dict_t *dict)
+afr_filter_xattrs(dict_t *dict)
 {
-        struct list_head   keys = {0,};
-        struct _xattr_key *key  = NULL;
-        struct _xattr_key *tmp  = NULL;
+    struct list_head keys = {
+        0,
+    };
+    struct _xattr_key *key = NULL;
+    struct _xattr_key *tmp = NULL;
 
-        INIT_LIST_HEAD (&keys);
+    INIT_LIST_HEAD(&keys);
 
-        dict_foreach (dict, __gather_xattr_keys,
-                      (void *) &keys);
+    dict_foreach(dict, __gather_xattr_keys, (void *)&keys);
 
-        list_for_each_entry_safe (key, tmp, &keys, list) {
-                dict_del (dict, key->key);
+    list_for_each_entry_safe(key, tmp, &keys, list)
+    {
+        dict_del(dict, key->key);
 
-                list_del_init (&key->list);
+        list_del_init(&key->list);
 
-                GF_FREE (key);
-        }
+        GF_FREE(key);
+    }
 }
 
+static gf_boolean_t
+afr_getxattr_ignorable_errnos(int32_t op_errno)
+{
+    if (op_errno == ENODATA || op_errno == ENOTSUP || op_errno == ERANGE ||
+        op_errno == ENAMETOOLONG)
+        return _gf_true;
 
-
-int32_t
-afr_getxattr_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *dict, dict_t *xdata)
+    return _gf_false;
+}
+int
+afr_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        read_child = (long) cookie;
-
-        if (op_ret == -1) {
-                last_index = &local->cont.getxattr.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->getxattr,
-                                   &local->loc,
-                                   local->cont.getxattr.name,
-                                   NULL);
-        }
+    if (op_ret < 0 && !afr_getxattr_ignorable_errnos(op_errno)) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-out:
-        if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        }
+    if (dict)
+        afr_filter_xattrs(dict);
+
+    AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+
+    return 0;
+}
 
+int
+afr_getxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno, NULL,
+                         NULL);
         return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_getxattr_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->getxattr, &local->loc,
+                      local->cont.getxattr.name, local->xdata_req);
+    return 0;
 }
 
 int32_t
-afr_getxattr_unwind (call_frame_t *frame, int op_ret, int op_errno,
-                     dict_t *dict, dict_t *xdata)
+afr_getxattr_unwind(call_frame_t *frame, int op_ret, int op_errno, dict_t *dict,
+                    dict_t *xdata)
 
 {
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int32_t
-afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this, int32_t op_ret, int32_t op_errno,
-                        dict_t *dict, dict_t *xdata)
+afr_fgetxattr_clrlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict,
+                        dict_t *xdata)
 {
-        afr_local_t     *local                  = NULL;
-        afr_private_t   *priv                   = NULL;
-        xlator_t        **children              = NULL;
-        dict_t          *xattr                  = NULL;
-        char            *tmp_report             = NULL;
-        char            lk_summary[1024]        = {0,};
-        int             serz_len                = 0;
-        int32_t         callcnt                 = 0;
-        long int        cky                     = 0;
-        int             ret                     = 0;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret == -1)
-                        local->child_errno[cky] = op_errno;
-
-                if (!local->dict)
-                        local->dict = dict_new ();
-                if (local->dict) {
-                        ret = dict_get_str (dict, local->cont.getxattr.name,
-                                            &tmp_report);
-                        if (ret)
-                                goto unlock;
-                        ret = dict_set_dynstr (local->dict,
-                                               children[cky]->name,
-                                               gf_strdup (tmp_report));
-                        if (ret)
-                                goto unlock;
-                }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    dict_t *xattr = NULL;
+    char *tmp_report = NULL;
+    char lk_summary[1024] = {
+        0,
+    };
+    int serz_len = 0;
+    int32_t callcnt = 0;
+    long int cky = 0;
+    int ret = 0;
+    int keylen = 0;
+    int children_keylen = 0;
+
+    priv = this->private;
+    children = priv->children;
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    children_keylen = strlen(children[cky]->name);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        if (op_ret == -1)
+            local->replies[cky].op_errno = op_errno;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (local->dict) {
+            ret = dict_get_strn(dict, local->cont.getxattr.name, keylen,
+                                &tmp_report);
+            if (ret)
+                goto unlock;
+            ret = dict_set_dynstrn(local->dict, children[cky]->name,
+                                   children_keylen, gf_strdup(tmp_report));
+            if (ret)
+                goto unlock;
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                xattr = dict_new ();
-                if (!xattr) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       lk_summary,
-                                                       &serz_len, '\n');
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error serializing dictionary");
-                        goto unwind;
-                }
-                if (serz_len == -1)
-                        snprintf (lk_summary, sizeof (lk_summary),
-                                  "No locks cleared.");
-                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
-                                       gf_strdup (lk_summary));
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error setting dictionary");
-                        goto unwind;
-                }
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        xattr = dict_new();
+        if (!xattr) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        ret = dict_serialize_value_with_delim(local->dict, lk_summary,
+                                              &serz_len, '\n');
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        if (serz_len == -1)
+            snprintf(lk_summary, sizeof(lk_summary), "No locks cleared.");
+        ret = dict_set_dynstrn(xattr, local->cont.getxattr.name, keylen,
+                               gf_strdup(lk_summary));
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting dictionary");
+            goto unwind;
+        }
+
+        op_errno = afr_final_errno(local, priv);
+
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+        if (xattr)
+            dict_unref(xattr);
+    }
 
-        unwind:
-                // Updating child_errno with more recent 'events'
-                local->child_errno[cky] = op_errno;
-                op_errno = afr_resultant_errno_get (NULL, local->child_errno,
-                                                    priv->child_count);
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+    return ret;
+}
 
-                if (xattr)
-                        dict_unref (xattr);
+int32_t
+afr_getxattr_clrlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    dict_t *xattr = NULL;
+    char *tmp_report = NULL;
+    char lk_summary[1024] = {
+        0,
+    };
+    int serz_len = 0;
+    int32_t callcnt = 0;
+    long int cky = 0;
+    int ret = 0;
+    int keylen = 0;
+    int children_keylen = 0;
+
+    priv = this->private;
+    children = priv->children;
+
+    local = frame->local;
+    cky = (long)cookie;
+
+    keylen = strlen(local->cont.getxattr.name);
+    children_keylen = strlen(children[cky]->name);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        if (op_ret == -1)
+            local->replies[cky].op_errno = op_errno;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (local->dict) {
+            ret = dict_get_strn(dict, local->cont.getxattr.name, keylen,
+                                &tmp_report);
+            if (ret)
+                goto unlock;
+            ret = dict_set_dynstrn(local->dict, children[cky]->name,
+                                   children_keylen, gf_strdup(tmp_report));
+            if (ret)
+                goto unlock;
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        xattr = dict_new();
+        if (!xattr) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        ret = dict_serialize_value_with_delim(local->dict, lk_summary,
+                                              &serz_len, '\n');
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        if (serz_len == -1)
+            snprintf(lk_summary, sizeof(lk_summary), "No locks cleared.");
+        ret = dict_set_dynstrn(xattr, local->cont.getxattr.name, keylen,
+                               gf_strdup(lk_summary));
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting dictionary");
+            goto unwind;
         }
 
-        return ret;
+        op_errno = afr_final_errno(local, priv);
+
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
+
+        if (xattr)
+            dict_unref(xattr);
+    }
+
+    return ret;
 }
 
 /**
  * node-uuid cbk uses next child querying mechanism
  */
 int32_t
-afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict, dict_t *xdata)
+afr_getxattr_node_uuid_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        afr_private_t  *priv            = NULL;
-        afr_local_t    *local           = NULL;
-        xlator_t      **children        = NULL;
-        int             unwind          = 1;
-        int             curr_call_child = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t **children = NULL;
+    int unwind = 1;
+    int curr_call_child = 0;
 
-        priv = this->private;
-        children = priv->children;
+    priv = this->private;
+    children = priv->children;
 
-        local = frame->local;
+    local = frame->local;
+
+    if (op_ret == -1) { /** query the _next_ child */
+
+        /**
+         * _current_ becomes _next_
+         * If done with all children and yet no success; give up !
+         */
+        curr_call_child = (int)((long)cookie);
+        if (++curr_call_child == priv->child_count)
+            goto unwind;
+
+        gf_msg_debug(this->name, op_errno,
+                     "op_ret (-1): Re-querying afr-child (%d/%d)",
+                     curr_call_child, priv->child_count);
+
+        unwind = 0;
+        STACK_WIND_COOKIE(
+            frame, afr_getxattr_node_uuid_cbk, (void *)(long)curr_call_child,
+            children[curr_call_child],
+            children[curr_call_child]->fops->getxattr, &local->loc,
+            local->cont.getxattr.name, local->xdata_req);
+    }
+
+unwind:
+    if (unwind)
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+
+    return 0;
+}
+
+/**
+ * list-node-uuids cbk returns the list of node_uuids for the subvolume.
+ */
+int32_t
+afr_getxattr_list_node_uuids_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr_serz = NULL;
+    long cky = 0;
+    int32_t tlen = 0;
+
+    local = frame->local;
+    priv = this->private;
+    cky = (long)cookie;
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        local->replies[cky].valid = 1;
+        local->replies[cky].op_ret = op_ret;
+        local->replies[cky].op_errno = op_errno;
+
+        if (op_ret < 0)
+            goto unlock;
+
+        local->op_ret = 0;
+
+        if (!local->xdata_rsp && xdata)
+            local->xdata_rsp = dict_ref(xdata);
+        local->replies[cky].xattr = dict_ref(dict);
+    }
+
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (op_ret == -1) { /** query the _next_ child */
-
-                /**
-                 * _current_ becomes _next_
-                 * If done with all childs and yet no success; give up !
-                 */
-                curr_call_child = (int) ((long)cookie);
-                if (++curr_call_child == priv->child_count)
-                        goto unwind;
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "op_ret (-1): Re-querying afr-child (%d/%d)",
-                        curr_call_child, priv->child_count);
-
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
-                                   (void *) (long) curr_call_child,
-                                   children[curr_call_child],
-                                   children[curr_call_child]->fops->getxattr,
-                                   &local->loc,
-                                   local->cont.getxattr.name,
-                                   NULL);
+    if (!callcnt) {
+        if (local->op_ret != 0) {
+            /* All bricks gave an error. */
+            local->op_errno = afr_final_errno(local, priv);
+            goto unwind;
         }
 
- unwind:
-        if (unwind)
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
+        /*Since we store the UUID0_STR as node uuid for down bricks and
+         *for non zero op_ret, assigning length to  priv->child_count
+         *number of uuids*/
+        local->cont.getxattr.xattr_len = (SLEN(UUID0_STR) + 2) *
+                                         priv->child_count;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (!local->dict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-        return 0;
+        xattr_serz = GF_CALLOC(local->cont.getxattr.xattr_len, sizeof(char),
+                               gf_common_mt_char);
+
+        if (!xattr_serz) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
+
+        ret = afr_serialize_xattrs_with_delimiter(frame, this, xattr_serz,
+                                                  UUID0_STR, &tlen, ' ');
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
+        ret = dict_set_dynstr_sizen(local->dict, GF_XATTR_LIST_NODE_UUIDS_KEY,
+                                    xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set node_uuid key in dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
+        } else {
+            local->op_ret = local->cont.getxattr.xattr_len - 1;
+            local->op_errno = 0;
+        }
+
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->dict, local->xdata_rsp);
+    }
+
+    return ret;
 }
 
 int32_t
-afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
-                           xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           dict_t *dict, dict_t *xdata)
+afr_getxattr_quota_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *dict,
+                            dict_t *xdata)
 {
-        afr_local_t *local          = NULL;
-        int32_t      callcnt        = 0;
-        int          ret            = 0;
-        char        *xattr          = NULL;
-        char        *xattr_serz     = NULL;
-        char        xattr_cky[1024] = {0,};
-        dict_t      *nxattr         = NULL;
-        long         cky            = 0;
-        int32_t      padding        = 0;
-        int32_t      tlen           = 0;
-
-        if (!frame || !frame->local || !this) {
-                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
-                goto out;
+    int idx = (long)cookie;
+    int call_count = 0;
+    afr_local_t *local = frame->local;
+    int read_subvol = -1;
+
+    local->replies[idx].valid = 1;
+    local->replies[idx].op_ret = op_ret;
+    local->replies[idx].op_errno = op_errno;
+    if (dict)
+        local->replies[idx].xdata = dict_ref(dict);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        local->inode = inode_ref(local->loc.inode);
+        read_subvol = afr_handle_quota_size(frame, this);
+        if (read_subvol != -1) {
+            op_ret = local->replies[read_subvol].op_ret;
+            op_errno = local->replies[read_subvol].op_errno;
+            dict = local->replies[read_subvol].xdata;
         }
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    }
+
+    return 0;
+}
+
+int32_t
+afr_getxattr_lockinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *dict,
+                          dict_t *xdata)
+{
+    int call_cnt = 0, len = 0;
+    char *lockinfo_buf = NULL;
+    dict_t *lockinfo = NULL, *newdict = NULL;
+    afr_local_t *local = NULL;
 
+    LOCK(&frame->lock);
+    {
         local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-                {
-                        callcnt = --local->call_count;
-
-                        if (!dict || (op_ret < 0))
-                                goto out;
-
-                        if (!local->dict)
-                                local->dict = dict_new ();
-
-                        if (local->dict) {
-                                ret = dict_get_str (dict,
-                                                    local->cont.getxattr.name,
-                                                    &xattr);
-                                if (ret)
-                                        goto out;
-
-                                xattr = gf_strdup (xattr);
-
-                                (void)snprintf (xattr_cky, 1024, "%s-%ld",
-                                                local->cont.getxattr.name, cky);
-                                ret = dict_set_dynstr (local->dict,
-                                                       xattr_cky, xattr);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Cannot set xattr cookie key");
-                                        goto out;
-                                }
-
-                                local->cont.getxattr.xattr_len += strlen (xattr) + 1;
-                        }
-                }
- out:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (!local->cont.getxattr.xattr_len)
-                        goto unwind;
-
-                nxattr = dict_new ();
-                if (!nxattr)
-                        goto unwind;
-
-                /* extra bytes for decorations (brackets and <>'s) */
-                padding += strlen (this->name) + strlen (AFR_PATHINFO_HEADER) + 4;
-                local->cont.getxattr.xattr_len += (padding + 2);
-
-                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
-                                        sizeof (char), gf_common_mt_char);
-
-                if (!xattr_serz)
-                        goto unwind;
-
-                /* the xlator info */
-                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
-                                this->name);
-
-                /* actual series of pathinfo */
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       xattr_serz + strlen (xattr_serz),
-                                                       &tlen, ' ');
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
-                                " dictionary");
-                        goto unwind;
+
+        call_cnt = --local->call_count;
+
+        if ((op_ret < 0) || (!dict && !xdata)) {
+            goto unlock;
+        }
+
+        if (xdata) {
+            if (!local->xdata_rsp) {
+                local->xdata_rsp = dict_new();
+                if (!local->xdata_rsp) {
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto unlock;
                 }
+            }
+        }
+
+        if (!dict) {
+            goto unlock;
+        }
 
-                /* closing part */
-                *(xattr_serz + padding + tlen) = ')';
-                *(xattr_serz + padding + tlen + 1) = '\0';
+        op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                      (void **)&lockinfo_buf, &len);
 
-                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
-                                       xattr_serz);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
-                                " key in dict");
+        if (!lockinfo_buf) {
+            goto unlock;
+        }
+
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto unlock;
+            }
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    if (lockinfo_buf != NULL) {
+        lockinfo = dict_new();
+        if (lockinfo == NULL) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        } else {
+            op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
+
+            if (lockinfo && local->dict) {
+                dict_copy(lockinfo, local->dict);
+            }
+        }
+    }
+
+    if (xdata && local->xdata_rsp) {
+        dict_copy(xdata, local->xdata_rsp);
+    }
+
+    if (!call_cnt) {
+        newdict = dict_new();
+        if (!newdict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, nxattr,
-                                  xdata);
+        op_ret = dict_allocate_and_serialize(
+            local->dict, (char **)&lockinfo_buf, (unsigned int *)&len);
+        if (op_ret != 0) {
+            local->op_ret = -1;
+            goto unwind;
+        }
 
-                if (nxattr)
-                        dict_unref (nxattr);
+        op_ret = dict_set_dynptr(newdict, GF_XATTR_LOCKINFO_KEY,
+                                 (void *)lockinfo_buf, len);
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -op_ret;
+            goto unwind;
         }
 
-        return ret;
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, newdict,
+                         local->xdata_rsp);
+    }
+
+    dict_unref(lockinfo);
+
+    return 0;
 }
 
-static gf_boolean_t
-afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk)
+int32_t
+afr_fgetxattr_lockinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        gf_boolean_t    is_spl = _gf_true;
+    int call_cnt = 0, len = 0;
+    char *lockinfo_buf = NULL;
+    dict_t *lockinfo = NULL, *newdict = NULL;
+    afr_local_t *local = NULL;
 
-        GF_ASSERT (cbk);
-        if (!cbk) {
-                is_spl = _gf_false;
-                goto out;
+    LOCK(&frame->lock);
+    {
+        local = frame->local;
+
+        call_cnt = --local->call_count;
+
+        if ((op_ret < 0) || (!dict && !xdata)) {
+            goto unlock;
         }
 
-        if (!strcmp (name, GF_XATTR_PATHINFO_KEY))
-                *cbk = afr_getxattr_pathinfo_cbk;
+        if (xdata) {
+            if (!local->xdata_rsp) {
+                local->xdata_rsp = dict_new();
+                if (!local->xdata_rsp) {
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto unlock;
+                }
+            }
+        }
 
-        else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
-                           strlen (GF_XATTR_CLRLK_CMD)))
-                *cbk = afr_getxattr_clrlk_cbk;
-        else
-                is_spl = _gf_false;
+        if (!dict) {
+            goto unlock;
+        }
 
-out:
-        return is_spl;
-}
+        op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                      (void **)&lockinfo_buf, &len);
 
-static void
-afr_getxattr_frm_all_children (xlator_t *this, call_frame_t *frame,
-                               const char *name, loc_t *loc,
-                               fop_getxattr_cbk_t cbk)
-{
-        afr_private_t   *priv           = NULL;
-        afr_local_t     *local          = NULL;
-        xlator_t        **children      = NULL;
-        int             i               = 0;
+        if (!lockinfo_buf) {
+            goto unlock;
+        }
 
-        priv     = this->private;
-        children = priv->children;
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto unlock;
+            }
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    if (lockinfo_buf != NULL) {
+        lockinfo = dict_new();
+        if (lockinfo == NULL) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        } else {
+            op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
+
+            if (lockinfo && local->dict) {
+                dict_copy(lockinfo, local->dict);
+            }
+        }
+    }
+
+    if (xdata && local->xdata_rsp) {
+        dict_copy(xdata, local->xdata_rsp);
+    }
+
+    if (!call_cnt) {
+        newdict = dict_new();
+        if (!newdict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-        local = frame->local;
-        local->call_count = priv->child_count;
+        op_ret = dict_allocate_and_serialize(
+            local->dict, (char **)&lockinfo_buf, (unsigned int *)&len);
+        if (op_ret != 0) {
+            local->op_ret = -1;
+            goto unwind;
+        }
 
-        for (i = 0; i < priv->child_count; i++) {
-                STACK_WIND_COOKIE (frame, cbk,
-                                   (void *) (long) i,
-                                   children[i], children[i]->fops->getxattr,
-                                   loc, name, NULL);
+        op_ret = dict_set_dynptr(newdict, GF_XATTR_LOCKINFO_KEY,
+                                 (void *)lockinfo_buf, len);
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -op_ret;
+            goto unwind;
         }
-        return;
+
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, newdict,
+                         local->xdata_rsp);
+    }
+
+    dict_unref(lockinfo);
+
+    return 0;
 }
 
 int32_t
-afr_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *name, dict_t *xdata)
+afr_fgetxattr_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        afr_private_t           *priv         = NULL;
-        xlator_t                **children    = NULL;
-        int                     call_child    = 0;
-        afr_local_t             *local        = NULL;
-        xlator_list_t           *trav         = NULL;
-        xlator_t                **sub_volumes = NULL;
-        int                     i             = 0;
-        int32_t                 op_errno      = 0;
-        int32_t                 read_child    = -1;
-        int                     ret           = -1;
-        fop_getxattr_cbk_t      cbk           = NULL;
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr = NULL;
+    char *xattr_serz = NULL;
+    int keylen = 0;
+    char xattr_cky[1024] = {
+        0,
+    };
+    int xattr_cky_len = 0;
+    dict_t *nxattr = NULL;
+    long cky = 0;
+    int32_t padding = 0;
+    int32_t tlen = 0;
+
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    xattr_cky_len = snprintf(xattr_cky, sizeof(xattr_cky), "%s-%ld",
+                             local->cont.getxattr.name, cky);
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
 
+        if (op_ret < 0) {
+            local->op_errno = op_errno;
+        } else {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        }
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        if (!dict || (op_ret < 0))
+            goto unlock;
 
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict)
+                goto unlock;
+        }
+        ret = dict_get_strn(dict, local->cont.getxattr.name, keylen, &xattr);
+        if (ret)
+            goto unlock;
 
-        children = priv->children;
+        xattr = gf_strdup(xattr);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+        ret = dict_set_dynstrn(local->dict, xattr_cky, xattr_cky_len, xattr);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set xattr cookie key");
+            goto post_unlock;
+        }
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+        local->cont.getxattr.xattr_len += strlen(xattr) + 1;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!callcnt) {
+        if (!local->cont.getxattr.xattr_len)
+            goto unwind;
 
-        loc_copy (&local->loc, loc);
-        if (!name)
-                goto no_name;
+        nxattr = dict_new();
+        if (!nxattr)
+            goto unwind;
 
-        local->cont.getxattr.name = gf_strdup (name);
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding += strlen(this->name) + SLEN(AFR_PATHINFO_HEADER) + 4;
+        local->cont.getxattr.xattr_len += (padding + 2);
 
-        if (!strncmp (name, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: no data present for key %s",
-                        loc->path, name);
-                op_errno = ENODATA;
-                goto out;
-        }
-        if ((strcmp (GF_XATTR_MARKER_KEY, name) == 0)
-            && (-1 == frame->root->pid)) {
+        xattr_serz = GF_MALLOC(local->cont.getxattr.xattr_len,
+                               gf_common_mt_char);
 
-                local->marker.call_count = priv->child_count;
+        if (!xattr_serz)
+            goto unwind;
 
-                sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
-                for (i = 0, trav = this->children; trav ;
-                     trav = trav->next, i++) {
+        /* the xlator info */
+        int xattr_serz_len = sprintf(
+            xattr_serz, "(<" AFR_PATHINFO_HEADER "%s> ", this->name);
 
-                        *(sub_volumes + i)  = trav->xlator;
-                }
+        /* actual series of pathinfo */
+        ret = dict_serialize_value_with_delim(
+            local->dict, xattr_serz + xattr_serz_len, &tlen, ' ');
+        if (ret) {
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
 
-                if (cluster_getmarkerattr (frame, this, loc, name,
-                                           local, afr_getxattr_unwind,
-                                           sub_volumes,
-                                           priv->child_count,
-                                           MARKER_UUID_TYPE,
-                                           priv->vol_uuid)) {
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%s: failed to get marker attr (%s)",
-                                loc->path, name);
-                        op_errno = EINVAL;
-                        goto out;
-                }
+        /* closing part */
+        *(xattr_serz + padding + tlen) = ')';
+        *(xattr_serz + padding + tlen + 1) = '\0';
 
-                return 0;
+        ret = dict_set_dynstrn(nxattr, local->cont.getxattr.name, keylen,
+                               xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set pathinfo key in dict");
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
         }
 
-        /*
-         * if we are doing getxattr with pathinfo as the key then we
-         * collect information from all childs
-         */
-        if (afr_is_special_xattr (name, &cbk)) {
-                afr_getxattr_frm_all_children (this, frame, name,
-                                               loc, cbk);
-                return 0;
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, local->op_ret, local->op_errno,
+                         nxattr, local->xdata_rsp);
+
+        if (nxattr)
+            dict_unref(nxattr);
+    }
+
+out:
+    return ret;
+}
+
+int32_t
+afr_getxattr_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *dict,
+                          dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr = NULL;
+    char *xattr_serz = NULL;
+    char xattr_cky[1024] = {
+        0,
+    };
+    int keylen = 0;
+    int xattr_cky_len = 0;
+    dict_t *nxattr = NULL;
+    long cky = 0;
+    int32_t padding = 0;
+    int32_t tlen = 0;
+
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    xattr_cky_len = snprintf(xattr_cky, sizeof(xattr_cky), "%s-%ld",
+                             local->cont.getxattr.name, cky);
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+
+        if (op_ret < 0) {
+            local->op_errno = op_errno;
+        } else {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
 
-        if (XATTR_IS_NODE_UUID (name)) {
-                i = 0;
-                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
-                                   (void *) (long) i,
-                                   children[i],
-                                   children[i]->fops->getxattr,
-                                   loc, name, xdata);
-                return 0;
+        if (!dict || (op_ret < 0))
+            goto unlock;
+
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict)
+                goto unlock;
         }
+        ret = dict_get_strn(dict, local->cont.getxattr.name, keylen, &xattr);
+        if (ret)
+            goto unlock;
 
-        if (*priv->vol_uuid) {
-                if ((match_uuid_local (name, priv->vol_uuid) == 0)
-                    && (-1 == frame->root->pid)) {
-                        local->marker.call_count = priv->child_count;
-
-                        sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
-                        for (i = 0, trav = this->children; trav ;
-                             trav = trav->next, i++) {
-
-                                *(sub_volumes + i)  = trav->xlator;
-
-                        }
-
-                        if (cluster_getmarkerattr (frame, this, loc,
-                                                   name, local,
-                                                   afr_getxattr_unwind,
-                                                   sub_volumes,
-                                                   priv->child_count,
-                                                   MARKER_XTIME_TYPE,
-                                                   priv->vol_uuid)) {
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "%s: failed to get marker attr (%s)",
-                                        loc->path, name);
-                                op_errno = EINVAL;
-                                goto out;
-                        }
-
-                        return 0;
-                }
+        xattr = gf_strdup(xattr);
+
+        ret = dict_set_dynstrn(local->dict, xattr_cky, xattr_cky_len, xattr);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set xattr cookie key");
+            goto post_unlock;
         }
 
-no_name:
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
+        local->cont.getxattr.xattr_len += strlen(xattr) + 1;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!callcnt) {
+        if (!local->cont.getxattr.xattr_len)
+            goto unwind;
+
+        nxattr = dict_new();
+        if (!nxattr)
+            goto unwind;
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding += strlen(this->name) + SLEN(AFR_PATHINFO_HEADER) + 4;
+        local->cont.getxattr.xattr_len += (padding + 2);
+
+        xattr_serz = GF_MALLOC(local->cont.getxattr.xattr_len,
+                               gf_common_mt_char);
+
+        if (!xattr_serz)
+            goto unwind;
+
+        /* the xlator info */
+        int xattr_serz_len = sprintf(
+            xattr_serz, "(<" AFR_PATHINFO_HEADER "%s> ", this->name);
+
+        /* actual series of pathinfo */
+        ret = dict_serialize_value_with_delim(
+            local->dict, xattr_serz + xattr_serz_len, &tlen, ' ');
+        if (ret) {
+            GF_FREE(xattr_serz);
+            goto unwind;
         }
 
-        read_child = afr_inode_get_read_ctx (this, loc->inode, local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.getxattr.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
+        /* closing part */
+        *(xattr_serz + padding + tlen) = ')';
+        *(xattr_serz + padding + tlen + 1) = '\0';
+
+        ret = dict_set_dynstrn(nxattr, local->cont.getxattr.name, keylen,
+                               xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set pathinfo key in dict");
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
         }
 
-        STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->getxattr,
-                           loc, name, xdata);
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         nxattr, local->xdata_rsp);
+
+        if (nxattr)
+            dict_unref(nxattr);
+    }
 
-        ret = 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    return ret;
 }
 
-/* {{{ fgetxattr */
+static int
+afr_aggregate_stime_xattr(dict_t *this, char *key, data_t *value, void *data)
+{
+    int ret = 0;
+
+    if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
+        ret = gf_get_max_stime(THIS, data, key, value);
 
+    return ret;
+}
 
 int32_t
-afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
-                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                   dict_t *dict, dict_t *xdata)
+afr_common_getxattr_stime_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *dict,
+                              dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
 
-        local = frame->local;
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
+
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+
+        if (!dict || (op_ret < 0)) {
+            local->op_errno = op_errno;
+            goto cleanup;
+        }
+
+        if (!local->dict)
+            local->dict = dict_copy_with_ref(dict, NULL);
+        else
+            dict_foreach(dict, afr_aggregate_stime_xattr, local->dict);
+        local->op_ret = 0;
+    }
+
+cleanup:
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->dict, xdata);
+    }
+
+out:
+    return 0;
+}
 
-        read_child = (long) cookie;
-
-        if (op_ret == -1) {
-                last_index = &local->cont.getxattr.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->fgetxattr,
-                                   local->fd,
-                                   local->cont.getxattr.name,
-                                   NULL);
+static gf_boolean_t
+afr_is_special_xattr(const char *name, fop_getxattr_cbk_t *cbk,
+                     gf_boolean_t is_fgetxattr)
+{
+    gf_boolean_t is_spl = _gf_true;
+
+    GF_ASSERT(cbk);
+    if (!cbk || !name) {
+        is_spl = _gf_false;
+        goto out;
+    }
+
+    if (!strcmp(name, GF_XATTR_PATHINFO_KEY) ||
+        !strcmp(name, GF_XATTR_USER_PATHINFO_KEY)) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_pathinfo_cbk;
+        } else {
+            *cbk = afr_getxattr_pathinfo_cbk;
+        }
+    } else if (!strncmp(name, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD))) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_clrlk_cbk;
+        } else {
+            *cbk = afr_getxattr_clrlk_cbk;
         }
+    } else if (!strncmp(name, GF_XATTR_LOCKINFO_KEY,
+                        SLEN(GF_XATTR_LOCKINFO_KEY))) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_lockinfo_cbk;
+        } else {
+            *cbk = afr_getxattr_lockinfo_cbk;
+        }
+    } else if (fnmatch(GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
+        *cbk = afr_common_getxattr_stime_cbk;
+    } else if (strcmp(name, QUOTA_SIZE_KEY) == 0) {
+        *cbk = afr_getxattr_quota_size_cbk;
+    } else if (!strcmp(name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
+        *cbk = afr_getxattr_list_node_uuids_cbk;
+    } else {
+        is_spl = _gf_false;
+    }
 
 out:
-        if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
+    return is_spl;
+}
 
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict,
-                                  xdata);
+static void
+afr_getxattr_all_subvols(xlator_t *this, call_frame_t *frame, const char *name,
+                         loc_t *loc, fop_getxattr_cbk_t cbk)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+
+    priv = this->private;
+
+    local = frame->local;
+    // local->call_count set in afr_local_init
+    call_count = local->call_count;
+
+    if (!strcmp(name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
+        GF_FREE(local->cont.getxattr.name);
+        local->cont.getxattr.name = gf_strdup(GF_XATTR_NODE_UUID_KEY);
+    }
+
+    // If up-children count is 0, afr_local_init would have failed already
+    // and the call would have unwound so not handling it here.
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, cbk, (void *)(long)i, priv->children[i],
+                              priv->children[i]->fops->getxattr, loc,
+                              local->cont.getxattr.name, NULL);
+            if (!--call_count)
+                break;
         }
+    }
+    return;
+}
 
-        return 0;
+int
+afr_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                         xlator_t **subvols)
+{
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+
+    memcpy(subvols, priv->children, sizeof(*subvols) * priv->child_count);
+
+    if (type == MARKER_XTIME_TYPE) {
+        /*Don't error out on ENOENT/ENOTCONN */
+        gauge[MCNT_NOTFOUND] = 0;
+        gauge[MCNT_ENOTCONN] = 0;
+    }
+    return priv->child_count;
 }
 
-int32_t
-afr_fgetxattr_unwind (call_frame_t *frame,
-                      int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+static int
+afr_handle_heal_xattrs(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       const char *heal_op)
+{
+    int ret = -1;
+    afr_spb_status_t *data = NULL;
+
+    if (!strcmp(heal_op, GF_HEAL_INFO)) {
+        afr_get_heal_info(frame, this, loc);
+        ret = 0;
+        goto out;
+    }
+
+    if (!strcmp(heal_op, GF_AFR_HEAL_SBRAIN)) {
+        afr_heal_splitbrain_file(frame, this, loc);
+        ret = 0;
+        goto out;
+    }
+
+    if (!strcmp(heal_op, GF_AFR_SBRAIN_STATUS)) {
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_spb_status_t);
+        if (!data) {
+            ret = 1;
+            goto out;
+        }
+        data->frame = frame;
+        data->loc = loc;
+        ret = synctask_new(this->ctx->env, afr_get_split_brain_status,
+                           afr_get_split_brain_status_cbk, NULL, data);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                   "Failed to create"
+                   " synctask. Unable to fetch split-brain status"
+                   " for %s.",
+                   loc->name);
+            ret = 1;
+            goto out;
+        }
+        goto out;
+    }
+
+out:
+    if (ret == 1) {
+        AFR_STACK_UNWIND(getxattr, frame, -1, ENOMEM, NULL, NULL);
+        if (data)
+            GF_FREE(data);
+        ret = 0;
+    }
+    return ret;
+}
 
+int32_t
+afr_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata)
 {
-        AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t **children = NULL;
+    int i = 0;
+    int32_t op_errno = 0;
+    int ret = -1;
+    fop_getxattr_cbk_t cbk = NULL;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    priv = this->private;
+
+    children = priv->children;
+
+    loc_copy(&local->loc, loc);
+
+    local->op = GF_FOP_GETXATTR;
+
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    if (!name)
+        goto no_name;
+
+    local->cont.getxattr.name = gf_strdup(name);
+
+    if (!local->cont.getxattr.name) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (!strncmp(name, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX))) {
+        op_errno = ENODATA;
+        goto out;
+    }
+
+    if (cluster_handle_marker_getxattr(frame, loc, name, priv->vol_uuid,
+                                       afr_getxattr_unwind,
+                                       afr_marker_populate_args) == 0)
+        return 0;
+
+    ret = afr_handle_heal_xattrs(frame, this, &local->loc, name);
+    if (ret == 0)
+        return 0;
+
+    /*
+     * Heal daemons don't have IO threads ... and as a result they
+     * send this getxattr down and eventually crash :(
+     */
+    op_errno = -1;
+    GF_CHECK_XATTR_KEY_AND_GOTO(name, IO_THREADS_QUEUE_SIZE_KEY, op_errno, out);
+
+    /*
+     * Special xattrs which need responses from all subvols
+     */
+    if (afr_is_special_xattr(name, &cbk, 0)) {
+        afr_getxattr_all_subvols(this, frame, name, loc, cbk);
+        return 0;
+    }
+
+    if (XATTR_IS_NODE_UUID(name)) {
+        i = 0;
+        STACK_WIND_COOKIE(frame, afr_getxattr_node_uuid_cbk, (void *)(long)i,
+                          children[i], children[i]->fops->getxattr, loc, name,
+                          xdata);
         return 0;
+    }
+
+no_name:
+
+    afr_read_txn(frame, this, local->loc.inode, afr_getxattr_wind,
+                 AFR_METADATA_TRANSACTION);
+
+    ret = 0;
+out:
+    if (ret < 0)
+        AFR_STACK_UNWIND(getxattr, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
+/* {{{ fgetxattr */
+
 int32_t
-afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name, dict_t *xdata)
+afr_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        afr_private_t   *priv         = NULL;
-        xlator_t        **children    = NULL;
-        int             call_child    = 0;
-        afr_local_t     *local        = NULL;
-        int32_t         op_ret        = -1;
-        int32_t         op_errno      = 0;
-        int32_t         read_child    = -1;
+    afr_local_t *local = NULL;
 
+    local = frame->local;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-        children = priv->children;
+    if (dict)
+        afr_filter_xattrs(dict);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
-        frame->local = local;
+    AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
-        op_ret = afr_local_init (local, priv, &op_errno);
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                goto out;
-        }
+    return 0;
+}
+
+int
+afr_fgetxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local->fd = fd_ref (fd);
-        if (name)
-                local->cont.getxattr.name = gf_strdup (name);
+    local = frame->local;
+    priv = this->private;
 
-        /* pathinfo gets handled only in getxattr() */
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(fgetxattr, frame, local->op_ret, local->op_errno, NULL,
+                         NULL);
+        return 0;
+    }
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    STACK_WIND_COOKIE(frame, afr_fgetxattr_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fgetxattr, local->fd,
+                      local->cont.getxattr.name, local->xdata_req);
+    return 0;
+}
 
-        read_child = afr_inode_get_read_ctx (this, fd->inode, local->fresh_children);
-        op_ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.getxattr.last_index);
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                op_ret = -1;
-                goto out;
+static void
+afr_fgetxattr_all_subvols(xlator_t *this, call_frame_t *frame,
+                          fop_fgetxattr_cbk_t cbk)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+
+    priv = this->private;
+
+    local = frame->local;
+    // local->call_count set in afr_local_init
+    call_count = local->call_count;
+
+    // If up-children count is 0, afr_local_init would have failed already
+    // and the call would have unwound so not handling it here.
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, cbk, (void *)(long)i, priv->children[i],
+                              priv->children[i]->fops->fgetxattr, local->fd,
+                              local->cont.getxattr.name, NULL);
+            if (!--call_count)
+                break;
         }
+    }
 
-        STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->fgetxattr,
-                           fd, name, xdata);
+    return;
+}
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, NULL, NULL);
+int
+afr_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    fop_fgetxattr_cbk_t cbk = NULL;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_FGETXATTR;
+    local->fd = fd_ref(fd);
+    if (name) {
+        local->cont.getxattr.name = gf_strdup(name);
+        if (!local->cont.getxattr.name) {
+            op_errno = ENOMEM;
+            goto out;
         }
+    }
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    /* pathinfo gets handled only in getxattr(), but we need to handle
+     * lockinfo.
+     * If we are doing fgetxattr with lockinfo as the key then we
+     * collect information from all children.
+     */
+    if (afr_is_special_xattr(name, &cbk, 1)) {
+        afr_fgetxattr_all_subvols(this, frame, cbk);
         return 0;
-}
+    }
+
+    afr_fix_open(fd, this);
 
+    afr_read_txn(frame, this, fd->inode, afr_fgetxattr_wind,
+                 AFR_METADATA_TRANSACTION);
+
+    return 0;
+out:
+    AFR_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
+
+    return 0;
+}
 
 /* }}} */
 
 /* {{{ readv */
 
-/**
- * read algorithm:
- *
- * if the user has specified a read subvolume, use it
- * otherwise -
- *   use the inode number to hash it to one of the subvolumes, and
- *   read from there (to balance read load)
- *
- * if any of the above read's fail, try the children in sequence
- * beginning at the beginning
- */
+int
+afr_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iovec *vector, int32_t count,
+              struct iatt *buf, struct iobref *iobref, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
 
-int32_t
-afr_readv_cbk (call_frame_t *frame, void *cookie,
-               xlator_t *this, int32_t op_ret, int32_t op_errno,
-               struct iovec *vector, int32_t count, struct iatt *buf,
-               struct iobref *iobref, dict_t *xdata)
+    local = frame->local;
+
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
+
+    AFR_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, buf, iobref,
+                     xdata);
+    return 0;
+}
+
+int
+afr_readv_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         *fresh_children  = NULL;
-        int32_t         read_child      = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    local = frame->local;
+    priv = this->private;
 
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(readv, frame, local->op_ret, local->op_errno, 0, 0, 0,
+                         0, 0);
+        return 0;
+    }
 
-        children = priv->children;
+    STACK_WIND_COOKIE(
+        frame, afr_readv_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->readv, local->fd, local->cont.readv.size,
+        local->cont.readv.offset, local->cont.readv.flags, local->xdata_req);
+    return 0;
+}
 
-        local = frame->local;
+int
+afr_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-        read_child = (long) cookie;
-
-        if (op_ret == -1) {
-                last_index = &local->cont.readv.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
-
-                STACK_WIND_COOKIE (frame, afr_readv_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->readv,
-                                   local->fd, local->cont.readv.size,
-                                   local->cont.readv.offset,
-                                   local->cont.readv.flags,
-                                   NULL);
-        }
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_READ;
+    local->fd = fd_ref(fd);
+    local->cont.readv.size = size;
+    local->cont.readv.offset = offset;
+    local->cont.readv.flags = flags;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    afr_fix_open(fd, this);
 
+    afr_read_txn(frame, this, fd->inode, afr_readv_wind, AFR_DATA_TRANSACTION);
+
+    return 0;
 out:
-        if (unwind) {
-                AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
-                                  vector, count, buf, iobref, xdata);
-        }
+    AFR_STACK_UNWIND(readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
 
-        return 0;
+    return 0;
 }
 
+/* }}} */
 
-int32_t
-afr_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+/* {{{ seek */
+
+int
+afr_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, off_t offset, dict_t *xdata)
 {
-        afr_private_t * priv       = NULL;
-        afr_local_t   * local      = NULL;
-        xlator_t **     children   = NULL;
-        int             call_child = 0;
-        int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (fd, out);
-
-        priv     = this->private;
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    afr_local_t *local = NULL;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    local = frame->local;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-        read_child = afr_inode_get_read_ctx (this, fd->inode, local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.readv.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-        local->fd                    = fd_ref (fd);
+    AFR_STACK_UNWIND(seek, frame, op_ret, op_errno, offset, xdata);
+    return 0;
+}
 
-        local->cont.readv.size       = size;
-        local->cont.readv.offset     = offset;
-        local->cont.readv.flags      = flags;
+int
+afr_seek_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        ret = afr_open_fd_fix (frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-        STACK_WIND_COOKIE (frame, afr_readv_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->readv,
-                           fd, size, offset, flags, xdata);
+    local = frame->local;
+    priv = this->private;
 
-        ret = 0;
-out:
-        if (ret < 0) {
-                AFR_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL,
-                                  NULL, NULL);
-        }
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(seek, frame, local->op_ret, local->op_errno, 0, NULL);
         return 0;
+    }
+
+    STACK_WIND_COOKIE(
+        frame, afr_seek_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->seek, local->fd, local->cont.seek.offset,
+        local->cont.seek.what, local->xdata_req);
+    return 0;
 }
 
+int
+afr_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_SEEK;
+    local->fd = fd_ref(fd);
+    local->cont.seek.offset = offset;
+    local->cont.seek.what = what;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    afr_fix_open(fd, this);
+
+    afr_read_txn(frame, this, fd->inode, afr_seek_wind, AFR_DATA_TRANSACTION);
+
+    return 0;
+out:
+    AFR_STACK_UNWIND(seek, frame, -1, op_errno, 0, NULL);
+
+    return 0;
+}
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-inode-read.h b/xlators/cluster/afr/src/afr-inode-read.h
index e4091a793e0..8c982bc7e6f 100644
--- a/xlators/cluster/afr/src/afr-inode-read.h
+++ b/xlators/cluster/afr/src/afr-inode-read.h
@@ -12,31 +12,34 @@
 #define __INODE_READ_H__
 
 int32_t
-afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask, dict_t *xdata);
+afr_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata);
 
 int32_t
-afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc, dict_t *xdata);
+afr_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
 
 int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, dict_t *xdata);
+afr_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
 
 int32_t
-afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size, dict_t *xdata);
+afr_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata);
 
 int32_t
-afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata);
+afr_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata);
 
 int32_t
-afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name, dict_t *xdata);
+afr_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata);
 
 int32_t
-afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name, dict_t *xdata);
-
+afr_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata);
+
+int
+afr_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata);
+int
+afr_handle_quota_size(call_frame_t *frame, xlator_t *this);
 #endif /* __INODE_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 6ea288a5ee4..1d6e4f3570a 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -8,2100 +8,2558 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include "protocol-common.h"
+#include <glusterfs/byte-order.h>
 #include "afr-transaction.h"
-#include "afr-self-heal-common.h"
+#include "afr-self-heal.h"
+#include "afr-messages.h"
 
-/* {{{ writev */
-
-int
-afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+static void
+__afr_inode_write_finalize(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (writev, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.writev.prebuf,
-                                  &local->cont.writev.postbuf,
-                                  NULL);
-        }
-        return 0;
+    int i = 0;
+    int ret = 0;
+    int read_subvol = 0;
+    struct iatt *stbuf = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, local->inode, out);
+
+    /*This code needs to stay till DHT sends fops on linked
+     * inodes*/
+    if (!inode_is_linked(local->inode)) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->replies[i].valid)
+                continue;
+            if (local->replies[i].op_ret == -1)
+                continue;
+            if (!gf_uuid_is_null(local->replies[i].poststat.ia_gfid)) {
+                gf_uuid_copy(args.gfid, local->replies[i].poststat.ia_gfid);
+                args.ia_type = local->replies[i].poststat.ia_type;
+                break;
+            } else {
+                ret = dict_get_bin(local->replies[i].xdata,
+                                   DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+                if (ret)
+                    continue;
+                gf_uuid_copy(args.gfid, stbuf->ia_gfid);
+                args.ia_type = stbuf->ia_type;
+                break;
+            }
+        }
+    }
+
+    if (local->transaction.type == AFR_METADATA_TRANSACTION) {
+        read_subvol = afr_metadata_subvol_get(local->inode, this, NULL,
+                                              local->readable, NULL, &args);
+    } else {
+        read_subvol = afr_data_subvol_get(local->inode, this, NULL,
+                                          local->readable, NULL, &args);
+    }
+
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    afr_pick_error_xdata(local, priv, local->inode, local->readable, NULL,
+                         NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0)
+            continue;
+
+        /* Order of checks in the compound conditional
+           below is important.
+
+           - Highest precedence: largest op_ret
+           - Next precedence: if all op_rets are equal, read subvol
+           - Least precedence: any succeeded subvol
+        */
+        if ((local->op_ret < local->replies[i].op_ret) ||
+            ((local->op_ret == local->replies[i].op_ret) &&
+             (i == read_subvol))) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+
+            local->cont.inode_wfop.prebuf = local->replies[i].prestat;
+            local->cont.inode_wfop.postbuf = local->replies[i].poststat;
+
+            if (local->replies[i].xdata) {
+                if (local->xdata_rsp)
+                    dict_unref(local->xdata_rsp);
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+            if (local->replies[i].xattr) {
+                if (local->xattr_rsp)
+                    dict_unref(local->xattr_rsp);
+                local->xattr_rsp = dict_ref(local->replies[i].xattr);
+            }
+        }
+    }
+
+    afr_set_in_flight_sb_status(this, frame, local->inode);
+out:
+    return;
 }
 
-
-int
-afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+static void
+__afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                       int op_ret, int op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int read_child  = 0;
-
-        local = frame->local;
-
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->replies[child_index].valid = 1;
+
+    if (AFR_IS_ARBITER_BRICK(priv, child_index) && op_ret == 1)
+        op_ret = iov_length(local->cont.writev.vector,
+                            local->cont.writev.count);
+
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    if (op_ret >= 0) {
+        if (prebuf)
+            local->replies[child_index].prestat = *prebuf;
+        if (postbuf)
+            local->replies[child_index].poststat = *postbuf;
+        if (xattr)
+            local->replies[child_index].xattr = dict_ref(xattr);
+    } else {
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
+
+    return;
+}
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+static int
+__afr_inode_write_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_private_t *priv = NULL;
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+    priv = this->private;
+    local = frame->local;
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret              = op_ret;
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
+    LOCK(&frame->lock);
+    {
+        __afr_inode_write_fill(frame, this, child_index, op_ret, op_errno,
+                               prebuf, postbuf, xattr, xdata);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-                        if (child_index == read_child) {
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
-                }
+    if (call_count == 0) {
+        __afr_inode_write_finalize(frame, this);
 
-                local->op_errno = op_errno;
+        if (afr_txn_nothing_failed(frame, this)) {
+            /*if it did pre-op, it will do post-op changing ctime*/
+            if (priv->consistent_metadata && afr_needs_changelog_update(local))
+                afr_zero_fill_stat(local);
+            local->transaction.unwind(frame, this);
         }
-        UNLOCK (&frame->lock);
 
-        call_count = afr_frame_return (frame);
+        afr_transaction_resume(frame, this);
+    }
 
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-        return 0;
+    return 0;
 }
 
-int
-afr_writev_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int i = 0;
-        int call_count = -1;
-
-        local = frame->local;
-        priv = this->private;
+/* {{{ writev */
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+void
+afr_writev_copy_outvars(call_frame_t *src_frame, call_frame_t *dst_frame)
+{
+    afr_local_t *src_local = NULL;
+    afr_local_t *dst_local = NULL;
+
+    src_local = src_frame->local;
+    dst_local = dst_frame->local;
+
+    dst_local->op_ret = src_local->op_ret;
+    dst_local->op_errno = src_local->op_errno;
+    dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+    dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
+    if (src_local->xdata_rsp)
+        dst_local->xdata_rsp = dict_ref(src_local->xdata_rsp);
+}
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+void
+afr_writev_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
 
-        local->call_count = call_count;
+    local = frame->local;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_writev_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->writev,
-                                           local->fd,
-                                           local->cont.writev.vector,
-                                           local->cont.writev.count,
-                                           local->cont.writev.offset,
-                                           local->cont.writev.flags,
-                                           local->cont.writev.iobref,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    if (priv->consistent_metadata)
+        afr_zero_fill_stat(local);
 
-        return 0;
+    AFR_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
 }
 
-
 int
-afr_writev_done (call_frame_t *frame, xlator_t *this)
+afr_transaction_writev_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
+    call_frame_t *fop_frame = NULL;
 
-        local = frame->local;
+    fop_frame = afr_transaction_detach_fop_frame(frame);
 
-        iobref_unref (local->cont.writev.iobref);
-        local->cont.writev.iobref = NULL;
+    if (fop_frame) {
+        afr_writev_copy_outvars(frame, fop_frame);
+        afr_writev_unwind(fop_frame, this);
+    }
+    return 0;
+}
 
-        local->transaction.unwind (frame, this);
+static void
+afr_writev_handle_short_writes(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+    /*
+     * We already have the best case result of the writev calls staged
+     * as the return value. Any writev that returns some value less
+     * than the best case is now out of sync, so mark the fop as
+     * failed. Note that fops that have returned with errors have
+     * already been marked as failed.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if ((!local->replies[i].valid) || (local->replies[i].op_ret == -1))
+            continue;
+
+        if (local->replies[i].op_ret < local->op_ret)
+            afr_transaction_fop_failed(frame, this, i);
+    }
+}
 
-        AFR_STACK_DESTROY (frame);
+void
+afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    int ret = 0;
+    afr_local_t *local = frame->local;
+    uint32_t open_fd_count = 0;
+    uint32_t write_is_append = 0;
+    int32_t num_inodelks = 0;
+
+    LOCK(&frame->lock);
+    {
+        __afr_inode_write_fill(frame, this, child_index, op_ret, op_errno,
+                               prebuf, postbuf, NULL, xdata);
+        if (op_ret == -1 || !xdata)
+            goto unlock;
+
+        write_is_append = 0;
+        ret = dict_get_uint32(xdata, GLUSTERFS_WRITE_IS_APPEND,
+                              &write_is_append);
+        if (ret || !write_is_append)
+            local->append_write = _gf_false;
+
+        ret = dict_get_uint32(xdata, GLUSTERFS_ACTIVE_FD_COUNT, &open_fd_count);
+        if (ret < 0)
+            goto unlock;
+        if (open_fd_count > local->open_fd_count) {
+            local->open_fd_count = open_fd_count;
+            local->update_open_fd_count = _gf_true;
+        }
 
-        return 0;
+        ret = dict_get_int32_sizen(xdata, GLUSTERFS_INODELK_COUNT,
+                                   &num_inodelks);
+        if (ret < 0)
+            goto unlock;
+        if (num_inodelks > local->num_inodelks) {
+            local->num_inodelks = num_inodelks;
+            local->update_num_inodelks = _gf_true;
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
 }
 
+void
+afr_process_post_writev(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+
+    if (!local->stable_write && !local->append_write)
+        /* An appended write removes the necessity to
+           fsync() the file. This is because self-heal
+           has the logic to check for larger file when
+           the xattrs are not reliably pointing at
+           a stale file.
+        */
+        afr_fd_report_unstable_write(this, local);
+
+    __afr_inode_write_finalize(frame, this);
+
+    afr_writev_handle_short_writes(frame, this);
+
+    if (local->update_open_fd_count)
+        local->inode_ctx->open_fd_count = local->open_fd_count;
+    if (local->update_num_inodelks &&
+        local->transaction.type == AFR_DATA_TRANSACTION) {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        lock->num_inodelks = local->num_inodelks;
+    }
+}
 
 int
-afr_do_writev (call_frame_t *frame, xlator_t *this)
+afr_writev_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        call_frame_t    *transaction_frame = NULL;
-        afr_local_t     *local             = NULL;
-        int             op_ret   = -1;
-        int             op_errno = 0;
-
-        local = frame->local;
+    call_frame_t *fop_frame = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    afr_inode_write_fill(frame, this, child_index, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
 
-        transaction_frame->local = local;
-        frame->local = NULL;
+    call_count = afr_frame_return(frame);
 
-        local->op = GF_FOP_WRITE;
+    if (call_count == 0) {
+        afr_process_post_writev(frame, this);
 
-        local->success_count      = 0;
-
-        local->transaction.fop    = afr_writev_wind;
-        local->transaction.done   = afr_writev_done;
-        local->transaction.unwind = afr_writev_unwind;
-
-        local->transaction.main_frame = frame;
-        if (local->fd->flags & O_APPEND) {
-                local->transaction.start   = 0;
-                local->transaction.len     = 0;
+        if (!afr_txn_nothing_failed(frame, this)) {
+            // Don't unwind until post-op is complete
+            afr_transaction_resume(frame, this);
         } else {
-                local->transaction.start   = local->cont.writev.offset;
-                local->transaction.len     = iov_length (local->cont.writev.vector,
-                                                         local->cont.writev.count);
-        }
-
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL, NULL);
-        }
-
-        return 0;
+            /*
+             * Generally inode-write fops do transaction.unwind then
+             * transaction.resume, but writev needs to make sure that
+             * delayed post-op frame is placed in fdctx before unwind
+             * happens. This prevents the race of flush doing the
+             * changelog wakeup first in fuse thread and then this
+             * writev placing its delayed post-op frame in fdctx.
+             * This helps flush make sure all the delayed post-ops are
+             * completed.
+             */
+
+            fop_frame = afr_transaction_detach_fop_frame(frame);
+            afr_writev_copy_outvars(frame, fop_frame);
+            afr_transaction_resume(frame, this);
+            afr_writev_unwind(fop_frame, this);
+        }
+    }
+    return 0;
 }
 
 static int
-afr_prepare_loc (call_frame_t *frame, fd_t *fd)
-{
-        afr_local_t    *local = NULL;
-        char           *name = NULL;
-        char           *path = NULL;
-        int             ret = 0;
-
-        if ((!fd) || (!fd->inode))
-                return -1;
-
-        local = frame->local;
-        ret = inode_path (fd->inode, NULL, (char **)&path);
-        if (ret <= 0) {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "Unable to get path for gfid: %s",
-                        uuid_utoa (fd->inode->gfid));
-                return -1;
-        }
-
-        if (local->loc.path) {
-                if (strcmp (path, local->loc.path))
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "overwriting old loc->path %s with %s",
-                                local->loc.path, path);
-                GF_FREE ((char *)local->loc.path);
-        }
-        local->loc.path = path;
-
-        name = strrchr (local->loc.path, '/');
-        if (name)
-                name++;
-        local->loc.name = name;
-
-        if (local->loc.inode) {
-                inode_unref (local->loc.inode);
-        }
-        local->loc.inode = inode_ref (fd->inode);
-
-        if (local->loc.parent) {
-                inode_unref (local->loc.parent);
-        }
-
-        local->loc.parent = inode_parent (local->loc.inode, 0, NULL);
-
-        return 0;
+afr_arbiter_writev_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    static char byte = 0xFF;
+    static struct iovec vector = {&byte, 1};
+    int32_t count = 1;
+
+    STACK_WIND_COOKIE(
+        frame, afr_writev_wind_cbk, (void *)(long)subvol,
+        priv->children[subvol], priv->children[subvol]->fops->writev, local->fd,
+        &vector, count, local->cont.writev.offset, local->cont.writev.flags,
+        local->cont.writev.iobref, local->xdata_req);
+
+    return 0;
 }
 
-afr_fd_paused_call_t*
-afr_paused_call_create (call_frame_t *frame)
+int
+afr_writev_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t             *local = NULL;
-        afr_fd_paused_call_t   *paused_call = NULL;
-
-        local = frame->local;
-        GF_ASSERT (local->fop_call_continue);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        paused_call = GF_CALLOC (1, sizeof (*paused_call),
-                                  gf_afr_fd_paused_call_t);
-        if (paused_call) {
-                INIT_LIST_HEAD (&paused_call->call_list);
-                paused_call->frame = frame;
-        }
+    local = frame->local;
+    priv = this->private;
 
-        return paused_call;
+    if (AFR_IS_ARBITER_BRICK(priv, subvol)) {
+        afr_arbiter_writev_wind(frame, this, subvol);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_writev_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->writev, local->fd,
+                      local->cont.writev.vector, local->cont.writev.count,
+                      local->cont.writev.offset, local->cont.writev.flags,
+                      local->cont.writev.iobref, local->xdata_req);
+    return 0;
 }
 
-static int
-afr_pause_fd_fop (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx)
+int
+afr_do_writev(call_frame_t *frame, xlator_t *this)
 {
-        afr_fd_paused_call_t *paused_call = NULL;
-        int                    ret = 0;
-
-        paused_call = afr_paused_call_create (frame);
-        if (paused_call)
-                list_add (&paused_call->call_list, &fd_ctx->paused_calls);
-        else
-                ret = -ENOMEM;
+    call_frame_t *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = frame->local;
+    transaction_frame->local = local;
+    frame->local = NULL;
+
+    if (!AFR_FRAME_INIT(frame, op_errno))
+        goto out;
+
+    local->op = GF_FOP_WRITE;
+
+    local->transaction.wind = afr_writev_wind;
+    local->transaction.unwind = afr_transaction_writev_unwind;
+
+    local->transaction.main_frame = frame;
+
+    if (local->fd->flags & O_APPEND) {
+        /*
+         * Backend vfs ignores the 'offset' for append mode fd so
+         * locking just the region provided for the writev does not
+         * give consistency guarantee. The actual write may happen at a
+         * completely different range than the one provided by the
+         * offset, len in the fop. So lock the entire file.
+         */
+        local->transaction.start = 0;
+        local->transaction.len = 0;
+    } else {
+        local->transaction.start = local->cont.writev.offset;
+        local->transaction.len = iov_length(local->cont.writev.vector,
+                                            local->cont.writev.count);
+    }
+
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return ret;
+    AFR_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-static void
-afr_trigger_open_fd_self_heal (call_frame_t *frame, xlator_t *this)
+int
+afr_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        afr_local_t             *local = NULL;
-        afr_self_heal_t         *sh = NULL;
-        inode_t                 *inode = NULL;
-        char                    *reason = NULL;
+    afr_local_t *local = NULL;
+    int op_errno = ENOMEM;
+    int ret = -1;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->cont.writev.vector = iov_dup(vector, count);
+    if (!local->cont.writev.vector)
+        goto out;
+    local->cont.writev.count = count;
+    local->cont.writev.offset = offset;
+    local->cont.writev.flags = flags;
+    local->cont.writev.iobref = iobref_ref(iobref);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
+
+    if (dict_set_uint32(local->xdata_req, GLUSTERFS_ACTIVE_FD_COUNT, 4)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (dict_set_str_sizen(local->xdata_req, GLUSTERFS_INODELK_DOM_COUNT,
+                           this->name)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (dict_set_uint32(local->xdata_req, GLUSTERFS_WRITE_IS_APPEND, 4)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    /* Set append_write to be true speculatively. If on any
+       server it turns not be true, we unset it in the
+       callback.
+    */
+    local->append_write = _gf_true;
+
+    /* detect here, but set it in writev_wind_cbk *after* the unstable
+       write is performed
+    */
+    local->stable_write = !!((fd->flags | flags) & (O_SYNC | O_DSYNC));
+
+    afr_fix_open(fd, this);
+
+    afr_do_writev(frame, this);
+
+    return 0;
+out:
+    AFR_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        local = frame->local;
-        sh    = &local->self_heal;
-        inode = local->fd->inode;
+    return 0;
+}
 
-        sh->do_missing_entry_self_heal = _gf_true;
-        sh->do_gfid_self_heal = _gf_true;
-        sh->do_data_self_heal = _gf_true;
+/* }}} */
 
-        reason = "subvolume came online";
-        afr_launch_self_heal (frame, this, inode, _gf_true, inode->ia_type,
-                              reason, NULL, NULL);
-}
+/* {{{ truncate */
 
 int
-afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop)
+afr_truncate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        int                     ret = 0;
-        int                     i   = 0;
-        afr_fd_ctx_t            *fd_ctx = NULL;
-        inode_t                 *inode = NULL;
-        gf_boolean_t            need_self_heal = _gf_false;
-        int                     *need_open = NULL;
-        int                     need_open_count = 0;
-        afr_local_t             *local = NULL;
-        afr_private_t           *priv = NULL;
-        gf_boolean_t            fop_continue = _gf_true;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
 
-        GF_ASSERT (local->fd);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-        inode = local->fd->inode;
-        //gfid is not set in rebalance, that case needs to be handled.
-        if (fd_is_anonymous (local->fd) ||
-            !inode || uuid_is_null (inode->gfid)) {
-                fop_continue = _gf_true;
-                goto out;
-        }
+    AFR_STACK_UNWIND(truncate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
-        if (pause_fop)
-                GF_ASSERT (local->fop_call_continue);
+int
+afr_truncate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
 
-        ret = afr_prepare_loc (frame, local->fd);
-        if (ret < 0) {
-                //File does not exist we cant open it.
-                ret = 0;
-                goto out;
-        }
+    local = frame->local;
 
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-        if (!fd_ctx) {
-                ret = -EINVAL;
-                goto out;
-        }
+    if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+        local->stable_write = _gf_false;
 
-        LOCK (&local->fd->lock);
-        {
-                if (fd_ctx->up_count < priv->up_count) {
-                        need_self_heal = _gf_true;
-                        fd_ctx->up_count   = priv->up_count;
-                        fd_ctx->down_count = priv->down_count;
-                }
-
-                for (i = 0; i < priv->child_count; i++) {
-                        if ((fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED) &&
-                            local->child_up[i]) {
-                                fd_ctx->opened_on[i] = AFR_FD_OPENING;
-                                if (!need_open)
-                                        need_open = GF_CALLOC (priv->child_count,
-                                                               sizeof (*need_open),
-                                                               gf_afr_mt_int32_t);
-                                need_open[i] = 1;
-                                need_open_count++;
-                        } else if (pause_fop && local->child_up[i] &&
-                                   (fd_ctx->opened_on[i] == AFR_FD_OPENING)) {
-                                local->fop_paused = _gf_true;
-                        }
-                }
-
-                if (local->fop_paused) {
-                        GF_ASSERT (pause_fop);
-                        gf_log (this->name, GF_LOG_INFO, "Pause fd %p",
-                                local->fd);
-                        ret = afr_pause_fd_fop (frame, this, fd_ctx);
-                        if (ret)
-                                goto unlock;
-                        fop_continue = _gf_false;
-                }
-        }
-unlock:
-        UNLOCK (&local->fd->lock);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to fix fd for %s",
-                        local->loc.path);
-                fop_continue = _gf_false;
-                goto out;
-        }
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
 
-        if (need_self_heal)
-                afr_trigger_open_fd_self_heal (frame, this);
+int
+afr_truncate_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        if (!need_open_count)
-                goto out;
+    local = frame->local;
+    priv = this->private;
 
-        gf_log (this->name, GF_LOG_INFO, "Opening fd %p", local->fd);
-        afr_fix_open (frame, this, fd_ctx, need_open_count, need_open);
-        fop_continue = _gf_false;
-out:
-        GF_FREE (need_open);
-        if (fop_continue && local->fop_call_continue)
-                local->fop_call_continue (frame, this);
-        return ret;
+    STACK_WIND_COOKIE(frame, afr_truncate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->truncate, &local->loc,
+                      local->cont.truncate.offset, local->xdata_req);
+    return 0;
 }
 
 int
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref, dict_t *xdata)
+afr_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        int ret = -1;
-        int op_errno = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        priv = this->private;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        QUORUM_CHECK(writev,out);
+    local->cont.truncate.offset = offset;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    if (!local->xdata_req)
+        goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    local->transaction.wind = afr_truncate_wind;
+    local->transaction.unwind = afr_truncate_unwind;
 
-        local->cont.writev.vector     = iov_dup (vector, count);
-        local->cont.writev.count      = count;
-        local->cont.writev.offset     = offset;
-        local->cont.writev.flags      = flags;
-        local->cont.writev.iobref     = iobref_ref (iobref);
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        local->fd                = fd_ref (fd);
-        local->fop_call_continue = afr_do_writev;
+    local->op = GF_FOP_TRUNCATE;
 
-        ret = afr_open_fd_fix (frame, this, _gf_true);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+    local->transaction.main_frame = frame;
+    local->transaction.start = offset;
+    local->transaction.len = 0;
 
-        ret = 0;
+    /* Set it true speculatively, will get reset in afr_truncate_wind_cbk
+       if truncate was not a NOP */
+    local->stable_write = _gf_true;
+
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* }}} */
 
-/* {{{ truncate */
+/* {{{ ftruncate */
 
 int
-afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
+afr_ftruncate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (truncate, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.truncate.prebuf,
-                                  &local->cont.truncate.postbuf,
-                                  NULL);
-        }
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(ftruncate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+afr_ftruncate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                        struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->loc.inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno) && op_errno != EFBIG)
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
-}
-
-
-int32_t
-afr_truncate_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
+    afr_local_t *local = NULL;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+    local = frame->local;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->truncate,
-                                           &local->loc,
-                                           local->cont.truncate.offset,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+        local->stable_write = _gf_false;
 
-        return 0;
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_truncate_done (call_frame_t *frame, xlator_t *this)
+afr_ftruncate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        AFR_STACK_DESTROY (frame);
+    local = frame->local;
+    priv = this->private;
 
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_ftruncate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->ftruncate, local->fd,
+                      local->cont.ftruncate.offset, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_truncate (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, off_t offset, dict_t *xdata)
+afr_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        priv = this->private;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        QUORUM_CHECK(truncate,out);
+    local->cont.ftruncate.offset = offset;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (!local->xdata_req)
+        goto out;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    local->op = GF_FOP_FTRUNCATE;
 
-        local->cont.truncate.offset  = offset;
+    local->transaction.wind = afr_ftruncate_wind;
+    local->transaction.unwind = afr_ftruncate_unwind;
 
-        local->transaction.fop    = afr_truncate_wind;
-        local->transaction.done   = afr_truncate_done;
-        local->transaction.unwind = afr_truncate_unwind;
+    local->transaction.main_frame = frame;
 
-        loc_copy (&local->loc, loc);
+    local->transaction.start = local->cont.ftruncate.offset;
+    local->transaction.len = 0;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = offset;
-        local->transaction.len     = 0;
+    afr_fix_open(fd, this);
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+    /* Set it true speculatively, will get reset in afr_ftruncate_wind_cbk
+       if truncate was not a NOP */
+    local->stable_write = _gf_true;
 
-        ret = 0;
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        }
+    AFR_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
-/* {{{ ftruncate */
-
+/* {{{ setattr */
 
 int
-afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
+afr_setattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.ftruncate.prebuf,
-                                  &local->cont.ftruncate.postbuf,
-                                  NULL);
-        }
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(setattr, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+afr_setattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *preop,
+                     struct iatt *postop, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int need_unwind = 0;
-        int read_child  = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, preop,
+                                 postop, NULL, xdata);
 }
 
-
 int
-afr_ftruncate_wind (call_frame_t *frame, xlator_t *this)
+afr_setattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->ftruncate,
-                                           local->fd,
-                                           local->cont.ftruncate.offset,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_setattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->setattr, &local->loc,
+                      &local->cont.setattr.in_buf, local->cont.setattr.valid,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_ftruncate_done (call_frame_t *frame, xlator_t *this)
+afr_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+            int32_t valid, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->cont.setattr.in_buf = *buf;
+    local->cont.setattr.valid = valid;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_setattr_wind;
+    local->transaction.unwind = afr_setattr_unwind;
+
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
+
+    local->op = GF_FOP_SETATTR;
+
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
+
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
+/* {{{ fsetattr */
 
 int
-afr_do_ftruncate (call_frame_t *frame, xlator_t *this)
+afr_fsetattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t * transaction_frame = NULL;
-        afr_local_t *  local             = NULL;
-        int op_ret   = -1;
-        int op_errno = 0;
-
-        local = frame->local;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                goto out;
-        }
-
-        transaction_frame->local = local;
-        frame->local = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local->op = GF_FOP_FTRUNCATE;
+    local = frame->local;
 
-        local->transaction.fop    = afr_ftruncate_wind;
-        local->transaction.done   = afr_ftruncate_done;
-        local->transaction.unwind = afr_ftruncate_unwind;
-
-        local->transaction.main_frame = frame;
-
-        local->transaction.start   = local->cont.ftruncate.offset;
-        local->transaction.len     = 0;
-
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL,
-                                  NULL, NULL);
-        }
+    AFR_STACK_UNWIND(fsetattr, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
-        return 0;
+int
+afr_fsetattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                      struct iatt *postop, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, preop,
+                                 postop, NULL, xdata);
 }
 
+int
+afr_fsetattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fsetattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsetattr, local->fd,
+                      &local->cont.fsetattr.in_buf, local->cont.fsetattr.valid,
+                      local->xdata_req);
+    return 0;
+}
 
 int
-afr_ftruncate (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, off_t offset, dict_t *xdata)
+afr_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+             int32_t valid, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        priv = this->private;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        QUORUM_CHECK(ftruncate,out);
+    local->cont.fsetattr.in_buf = *buf;
+    local->cont.fsetattr.valid = valid;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    if (!local->xdata_req)
+        goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    local->transaction.wind = afr_fsetattr_wind;
+    local->transaction.unwind = afr_fsetattr_unwind;
 
-        local->cont.ftruncate.offset  = offset;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        local->fd = fd_ref (fd);
-        local->fop_call_continue = afr_do_ftruncate;
+    local->op = GF_FOP_FSETATTR;
 
-        ret = afr_open_fd_fix (frame, this, _gf_true);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+    afr_fix_open(fd, this);
 
-        ret = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
+
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-/* }}} */
-
-/* {{{ setattr */
+/* {{{ setxattr */
 
 int
-afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_setxattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (setattr, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.setattr.preop_buf,
-                                  &local->cont.setattr.postop_buf,
-                                  NULL);
-        }
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(setxattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
 }
 
+int
+afr_setxattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
+}
 
 int
-afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
+afr_setxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_setxattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->setxattr, &local->loc,
+                      local->cont.setxattr.dict, local->cont.setxattr.flags,
+                      local->xdata_req);
+    return 0;
+}
 
-        local = frame->local;
-        priv  = this->private;
+int
+afr_emptyb_set_pending_changelog_cbk(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int op_ret, int op_errno,
+                                     dict_t *xattr, dict_t *xdata)
 
-        read_child = afr_inode_get_read_ctx (this, local->loc.inode, NULL);
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i, ret = 0;
+    char *op_type = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+    local = frame->local;
+    priv = this->private;
+    i = (long)cookie;
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
+    ret = dict_get_str_sizen(local->xdata_req, "replicate-brick-op", &op_type);
+    if (ret)
+        goto out;
 
-                        if (child_index == read_child) {
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
+    gf_smsg(this->name, op_ret ? GF_LOG_ERROR : GF_LOG_INFO,
+            op_ret ? op_errno : 0, AFR_MSG_SET_PEND_XATTR, "name=%s",
+            priv->children[i]->name, "op_ret=%s",
+            op_ret ? "failed" : "succeeded", NULL);
 
-                        local->success_count++;
+out:
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+int
+afr_emptyb_set_pending_changelog(call_frame_t *frame, xlator_t *this,
+                                 unsigned char *locked_nodes)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int ret = 0, i = 0;
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+    local = frame->local;
+    priv = this->private;
 
-        call_count = afr_frame_return (frame);
+    AFR_ONLIST(locked_nodes, frame, afr_emptyb_set_pending_changelog_cbk,
+               xattrop, &local->loc, GF_XATTROP_ADD_ARRAY, local->xattr_req,
+               NULL);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+    /* It is sufficient if xattrop was successful on one child */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-        return 0;
+        if (local->replies[i].op_ret == 0) {
+            ret = 0;
+            goto out;
+        } else {
+            ret = afr_higher_errno(ret, local->replies[i].op_errno);
+        }
+    }
+out:
+    return -ret;
 }
 
-
-int32_t
-afr_setattr_wind (call_frame_t *frame, xlator_t *this)
+static int
+_afr_handle_empty_brick_type(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                             int empty_index, afr_transaction_type type,
+                             char *op_type, const int op_type_len)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+    int count = 0;
+    int ret = -ENOMEM;
+    int idx = -1;
+    int d_idx = -1;
+    unsigned char *locked_nodes = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    locked_nodes = alloca0(priv->child_count);
+
+    idx = afr_index_for_transaction_type(type);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    local->pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!local->pending)
+        goto out;
+
+    local->pending[empty_index][idx] = hton32(1);
+
+    if ((priv->esh_granular) && (type == AFR_ENTRY_TRANSACTION))
+        local->pending[empty_index][d_idx] = hton32(1);
+
+    local->xdata_req = dict_new();
+    if (!local->xdata_req)
+        goto out;
+
+    ret = dict_set_nstrn(local->xdata_req, "replicate-brick-op",
+                         SLEN("replicate-brick-op"), op_type, op_type_len);
+    if (ret)
+        goto out;
+
+    local->xattr_req = dict_new();
+    if (!local->xattr_req)
+        goto out;
+
+    ret = afr_set_pending_dict(priv, local->xattr_req, local->pending);
+    if (ret < 0)
+        goto out;
+
+    if (AFR_ENTRY_TRANSACTION == type) {
+        count = afr_selfheal_entrylk(frame, this, loc->inode, this->name, NULL,
+                                     locked_nodes);
+    } else {
+        count = afr_selfheal_inodelk(frame, this, loc->inode, this->name,
+                                     LLONG_MAX - 1, 0, locked_nodes);
+    }
+
+    if (!count) {
+        gf_smsg(this->name, GF_LOG_ERROR, EAGAIN, AFR_MSG_REPLACE_BRICK_STATUS,
+                NULL);
+        ret = -EAGAIN;
+        goto unlock;
+    }
+
+    ret = afr_emptyb_set_pending_changelog(frame, this, locked_nodes);
+    if (ret)
+        goto unlock;
+    ret = 0;
+unlock:
+    if (AFR_ENTRY_TRANSACTION == type) {
+        afr_selfheal_unentrylk(frame, this, loc->inode, this->name, NULL,
+                               locked_nodes, NULL);
+    } else {
+        afr_selfheal_uninodelk(frame, this, loc->inode, this->name,
+                               LLONG_MAX - 1, 0, locked_nodes);
+    }
+out:
+    return ret;
+}
 
-        local->call_count = call_count;
+void
+afr_brick_args_cleanup(void *opaque)
+{
+    afr_empty_brick_args_t *data = NULL;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->setattr,
-                                           &local->loc,
-                                           &local->cont.setattr.in_buf,
-                                           local->cont.setattr.valid,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    data = opaque;
+    loc_wipe(&data->loc);
+    GF_FREE(data);
+}
 
-        return 0;
+int
+_afr_handle_empty_brick_cbk(int ret, call_frame_t *frame, void *opaque)
+{
+    afr_brick_args_cleanup(opaque);
+    return 0;
 }
 
+int
+_afr_handle_empty_brick(void *opaque)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int empty_index = -1;
+    int ret = -1;
+    int op_errno = ENOMEM;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    char *op_type = NULL;
+    int op_type_len = 0;
+    afr_empty_brick_args_t *data = NULL;
+    call_frame_t *op_frame = NULL;
+
+    data = opaque;
+    frame = data->frame;
+    empty_index = data->empty_index;
+    if (!data->op_type)
+        goto out;
+
+    op_frame = copy_frame(frame);
+    if (!op_frame) {
+        ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_type = data->op_type;
+    op_type_len = strlen(op_type);
+    this = op_frame->this;
+    priv = this->private;
+
+    afr_set_lk_owner(op_frame, this, op_frame->root);
+    local = AFR_FRAME_INIT(op_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, &data->loc);
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, AFR_MSG_NEW_BRICK, "name=%s",
+            priv->children[empty_index]->name, NULL);
+
+    ret = _afr_handle_empty_brick_type(this, op_frame, &local->loc, empty_index,
+                                       AFR_METADATA_TRANSACTION, op_type,
+                                       op_type_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    dict_unref(local->xdata_req);
+    dict_unref(local->xattr_req);
+    afr_matrix_cleanup(local->pending, priv->child_count);
+    local->pending = NULL;
+    local->xattr_req = NULL;
+    local->xdata_req = NULL;
+
+    ret = _afr_handle_empty_brick_type(this, op_frame, &local->loc, empty_index,
+                                       AFR_ENTRY_TRANSACTION, op_type,
+                                       op_type_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+out:
+    if (op_frame) {
+        AFR_STACK_DESTROY(op_frame);
+    }
+    AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    return 0;
+}
 
 int
-afr_setattr_done (call_frame_t *frame, xlator_t *this)
+afr_split_brain_resolve_do(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                           char *data)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    local->xdata_req = dict_new();
+
+    if (!local->xdata_req) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(local->xdata_req, "heal-op",
+                               GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str_sizen(local->xdata_req, "child-name", data);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    /* set spb choice to -1 whether heal succeeds or not:
+     * If heal succeeds : spb-choice should be set to -1 as
+     *                    it is no longer valid; file is not
+     *                    in split-brain anymore.
+     * If heal doesn't succeed:
+     *                    spb-choice should be set to -1
+     *                    otherwise reads will be served
+     *                    from spb-choice which is misleading.
+     */
+    ret = afr_inode_split_brain_choice_set(loc->inode, this, -1);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN_SET_FAILED,
+                NULL);
+    afr_heal_splitbrain_file(frame, this, loc);
+    ret = 0;
+out:
+    if (ret < 0)
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+    return 0;
+}
 
-        local = frame->local;
+int
+afr_get_split_brain_child_index(xlator_t *this, void *value, size_t len)
+{
+    int spb_child_index = -1;
+    char *spb_child_str = NULL;
 
-        local->transaction.unwind (frame, this);
+    spb_child_str = alloca0(len + 1);
+    memcpy(spb_child_str, value, len);
 
-        AFR_STACK_DESTROY (frame);
+    if (!strcmp(spb_child_str, "none"))
+        return -2;
 
-        return 0;
+    spb_child_index = afr_get_child_index_from_name(this, spb_child_str);
+    if (spb_child_index < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                "subvol=%s", spb_child_str, NULL);
+    }
+    return spb_child_index;
 }
 
-
 int
-afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata)
+afr_can_set_split_brain_choice(void *opaque)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        QUORUM_CHECK(setattr,out);
+    afr_spbc_timeout_t *data = opaque;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    int ret = -1;
+
+    frame = data->frame;
+    loc = data->loc;
+    this = frame->this;
+
+    ret = afr_is_split_brain(frame, this, loc->inode, loc->gfid, &data->d_spb,
+                             &data->m_spb);
+
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED, "gfid=%s",
+                uuid_utoa(loc->gfid), NULL);
+    return ret;
+}
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+int
+afr_handle_split_brain_commands(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                                dict_t *dict)
+{
+    void *choice_value = NULL;
+    void *resolve_value = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_spbc_timeout_t *data = NULL;
+    int len = 0;
+    int spb_child_index = -1;
+    int ret = -1;
+    int op_errno = EINVAL;
+
+    priv = this->private;
+
+    ret = dict_get_ptr_and_len(dict, GF_AFR_SBRAIN_CHOICE, &choice_value, &len);
+    ret = dict_get_ptr_and_len(dict, GF_AFR_SBRAIN_RESOLVE, &resolve_value,
+                               &len);
+    if (!choice_value && !resolve_value) {
+        ret = -1;
+        goto out;
+    }
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local) {
+        ret = 1;
+        goto out;
+    }
+
+    local->op = GF_FOP_SETXATTR;
+
+    if (choice_value) {
+        spb_child_index = afr_get_split_brain_child_index(this, choice_value,
+                                                          len);
+        if (spb_child_index < 0) {
+            /* Case where value was "none" */
+            if (spb_child_index == -2)
+                spb_child_index = -1;
+            else {
+                ret = 1;
+                op_errno = EINVAL;
                 goto out;
+            }
         }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->cont.setattr.in_buf = *buf;
-        local->cont.setattr.valid  = valid;
-
-        local->transaction.fop    = afr_setattr_wind;
-        local->transaction.done   = afr_setattr_done;
-        local->transaction.unwind = afr_setattr_unwind;
-
-        loc_copy (&local->loc, loc);
-
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_spbc_timeout_t);
+        if (!data) {
+            ret = 1;
+            goto out;
+        }
+        data->spb_child_index = spb_child_index;
+        data->frame = frame;
+        loc_copy(&local->loc, loc);
+        data->loc = &local->loc;
+        ret = synctask_new(this->ctx->env, afr_can_set_split_brain_choice,
+                           afr_set_split_brain_choice, NULL, data);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                    "name=%s", loc->name, NULL);
+            ret = 1;
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = 0;
+        goto out;
+    }
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (resolve_value) {
+        spb_child_index = afr_get_split_brain_child_index(this, resolve_value,
+                                                          len);
+        if (spb_child_index < 0) {
+            ret = 1;
+            goto out;
+        }
 
+        afr_split_brain_resolve_do(frame, this, loc,
+                                   priv->children[spb_child_index]->name);
         ret = 0;
+    }
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
-        }
-
-        return 0;
+    /* key was correct but value was invalid when ret == 1 */
+    if (ret == 1) {
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+        if (data)
+            GF_FREE(data);
+        ret = 0;
+    }
+    return ret;
 }
 
-/* {{{ fsetattr */
-
 int
-afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_handle_spb_choice_timeout(xlator_t *this, call_frame_t *frame, dict_t *dict)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    int ret = -1;
+    int op_errno = 0;
+    uint64_t timeout = 0;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
+    priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.fsetattr.preop_buf,
-                                  &local->cont.fsetattr.postop_buf,
-                                  NULL);
-        }
+    ret = dict_get_uint64(dict, GF_AFR_SPB_CHOICE_TIMEOUT, &timeout);
+    if (!ret) {
+        priv->spb_choice_timeout = timeout * 60;
+        AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    }
 
-        return 0;
+    return ret;
 }
 
-
 int
-afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
+afr_handle_empty_brick(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                       dict_t *dict)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
+    int ret = -1;
+    int ab_ret = -1;
+    int empty_index = -1;
+    int op_errno = EPERM;
+    char *empty_brick = NULL;
+    char *op_type = NULL;
+    afr_empty_brick_args_t *data = NULL;
+
+    ret = dict_get_str_sizen(dict, GF_AFR_REPLACE_BRICK, &empty_brick);
+    if (!ret)
+        op_type = GF_AFR_REPLACE_BRICK;
+
+    ab_ret = dict_get_str_sizen(dict, GF_AFR_ADD_BRICK, &empty_brick);
+    if (!ab_ret)
+        op_type = GF_AFR_ADD_BRICK;
+
+    if (ret && ab_ret)
+        goto out;
+
+    if (frame->root->pid != GF_CLIENT_PID_ADD_REPLICA_MOUNT) {
+        gf_smsg(this->name, GF_LOG_ERROR, EPERM, AFR_MSG_INTERNAL_ATTR,
+                "op_type=%s", op_type, NULL);
+        ret = 1;
+        goto out;
+    }
+    empty_index = afr_get_child_index_from_name(this, empty_brick);
+
+    if (empty_index < 0) {
+        /* Didn't belong to this replica pair
+         * Just do a no-op
+         */
+        AFR_STACK_UNWIND(setxattr, frame, 0, 0, NULL);
+        return 0;
+    } else {
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_empty_brick_t);
+        if (!data) {
+            ret = 1;
+            op_errno = ENOMEM;
+            goto out;
+        }
+        data->frame = frame;
+        loc_copy(&data->loc, loc);
+        data->empty_index = empty_index;
+        data->op_type = op_type;
+        ret = synctask_new(this->ctx->env, _afr_handle_empty_brick,
+                           _afr_handle_empty_brick_cbk, NULL, data);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                    NULL);
+            ret = 1;
+            op_errno = ENOMEM;
+            afr_brick_args_cleanup(data);
+            goto out;
+        }
+    }
+    ret = 0;
+out:
+    if (ret == 1) {
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+        ret = 0;
+    }
+    return ret;
+}
 
-        local = frame->local;
-        priv  = this->private;
+static int
+afr_handle_special_xattr(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                         dict_t *dict)
+{
+    int ret = -1;
 
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+    ret = afr_handle_split_brain_commands(this, frame, loc, dict);
+    if (ret == 0)
+        goto out;
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+    ret = afr_handle_spb_choice_timeout(this, frame, dict);
+    if (ret == 0)
+        goto out;
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+    /* Applicable for replace-brick and add-brick commands */
+    ret = afr_handle_empty_brick(this, frame, loc, dict);
+out:
+    return ret;
+}
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
+int
+afr_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
 
-                        if (child_index == read_child) {
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.afr.*", dict, op_errno, out);
 
-                        local->success_count++;
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.afr.*", dict, op_errno, out);
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    ret = afr_handle_special_xattr(this, frame, loc, dict);
+    if (ret == 0)
+        return 0;
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        call_count = afr_frame_return (frame);
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+    local->cont.setxattr.dict = dict_ref(dict);
+    local->cont.setxattr.flags = flags;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        return 0;
-}
+    if (!local->xdata_req)
+        goto out;
 
+    local->transaction.wind = afr_setxattr_wind;
+    local->transaction.unwind = afr_setxattr_unwind;
 
-int32_t
-afr_fsetattr_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        local = frame->local;
-        priv = this->private;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+    local->op = GF_FOP_SETXATTR;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        local->call_count = call_count;
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsetattr,
-                                           local->fd,
-                                           &local->cont.fsetattr.in_buf,
-                                           local->cont.fsetattr.valid,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
+/* {{{ fsetxattr */
 
 int
-afr_fsetattr_done (call_frame_t *frame, xlator_t *this)
+afr_fsetxattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->transaction.unwind (frame, this);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-        AFR_STACK_DESTROY (frame);
+    AFR_STACK_UNWIND(fsetxattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
-        return 0;
+int
+afr_fsetxattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
 int
-afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata)
+afr_fsetxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fsetxattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsetxattr, local->fd,
+                      local->cont.fsetxattr.dict, local->cont.fsetxattr.flags,
+                      local->xdata_req);
+    return 0;
+}
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+int
+afr_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        priv = this->private;
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.afr.*", dict, op_errno, out);
 
-        QUORUM_CHECK(fsetattr,out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.afr.*", dict, op_errno, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    local->cont.fsetxattr.dict = dict_ref(dict);
+    local->cont.fsetxattr.flags = flags;
 
-        local->cont.fsetattr.in_buf = *buf;
-        local->cont.fsetattr.valid  = valid;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        local->transaction.fop    = afr_fsetattr_wind;
-        local->transaction.done   = afr_fsetattr_done;
-        local->transaction.unwind = afr_fsetattr_unwind;
+    if (!local->xdata_req)
+        goto out;
 
-        local->fd                 = fd_ref (fd);
+    local->transaction.wind = afr_fsetxattr_wind;
+    local->transaction.unwind = afr_fsetxattr_unwind;
 
-        ret = afr_open_fd_fix (transaction_frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->op = GF_FOP_FSETXATTR;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = 0;
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
+/* }}} */
 
-/* {{{ setxattr */
-
+/* {{{ removexattr */
 
 int
-afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_removexattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+    local = frame->local;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  NULL);
-        }
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
-
-int
-afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_local_t      *local         = NULL;
-        afr_private_t    *priv          = NULL;
-        int               call_count    = -1;
-        int               need_unwind   = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+    AFR_STACK_UNWIND(removexattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
 }
 
-
 int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
+afr_removexattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t      *local         = NULL;
-        afr_private_t    *priv          = NULL;
-        int               call_count    = -1;
-        int               i             = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->setxattr,
-                                           &local->loc,
-                                           local->cont.setxattr.dict,
-                                           local->cont.setxattr.flags,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_setxattr_done (call_frame_t *frame, xlator_t *this)
+afr_removexattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t   *local    = frame->local;
-
-        local->transaction.unwind (frame, this);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        AFR_STACK_DESTROY (frame);
+    local = frame->local;
+    priv = this->private;
 
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_removexattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->removexattr, &local->loc,
+                      local->cont.removexattr.name, local->xdata_req);
+    return 0;
 }
 
 int
-afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+afr_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
 {
-        afr_private_t  *priv              = NULL;
-        afr_local_t    *local             = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        data_pair_t    *trav              = NULL;
-        int             ret               = -1;
-        int             op_errno          = EINVAL;
-
-        VALIDATE_OR_GOTO (this, out);
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.afr.*", name, op_errno, out);
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.afr.*", name, op_errno, out);
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        priv = this->private;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        QUORUM_CHECK(setxattr,out);
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    local->cont.removexattr.name = gf_strdup(name);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->cont.setxattr.dict  = dict_ref (dict);
-        local->cont.setxattr.flags = flags;
+    local->transaction.wind = afr_removexattr_wind;
+    local->transaction.unwind = afr_removexattr_unwind;
 
-        local->transaction.fop    = afr_setxattr_wind;
-        local->transaction.done   = afr_setxattr_done;
-        local->transaction.unwind = afr_setxattr_unwind;
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        loc_copy (&local->loc, loc);
+    local->op = GF_FOP_REMOVEXATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        ret = 0;
+    return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-/* {{{ fsetxattr */
-
-
+/* ffremovexattr */
 int
-afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fremovexattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t    *local         = NULL;
-        call_frame_t   *main_frame    = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (fsetxattr, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  NULL);
-        }
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(fremovexattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
 }
 
+int
+afr_fremovexattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
+}
 
 int
-afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_fremovexattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t     *local       = NULL;
-        afr_private_t   *priv        = NULL;
-        int              call_count  = -1;
-        int              need_unwind = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
+    STACK_WIND_COOKIE(frame, afr_fremovexattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fremovexattr, local->fd,
+                      local->cont.removexattr.name, local->xdata_req);
+    return 0;
+}
 
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
-                }
+int
+afr_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.afr.*", name, op_errno, out);
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.afr.*", name, op_errno, out);
 
-        call_count = afr_frame_return (frame);
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        return 0;
-}
+    local->cont.removexattr.name = gf_strdup(name);
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
+    if (!local->xdata_req)
+        goto out;
 
-int
-afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t        *local       = NULL;
-        afr_private_t      *priv        = NULL;
-        int                 call_count  = -1;
-        int                 i           = 0;
+    local->transaction.wind = afr_fremovexattr_wind;
+    local->transaction.unwind = afr_fremovexattr_unwind;
 
-        local = frame->local;
-        priv = this->private;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+    local->op = GF_FOP_FREMOVEXATTR;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        local->call_count = call_count;
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsetxattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsetxattr,
-                                           local->fd,
-                                           local->cont.fsetxattr.dict,
-                                           local->cont.fsetxattr.flags,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
-}
+    AFR_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
 
+    return 0;
+}
 
 int
-afr_fsetxattr_done (call_frame_t *frame, xlator_t *this)
+afr_fallocate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local   = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(fallocate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
 }
 
 int
-afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
+afr_fallocate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        afr_private_t    *priv              = NULL;
-        afr_local_t      *local             = NULL;
-        call_frame_t     *transaction_frame = NULL;
-        int               ret               = -1;
-        int               op_errno          = EINVAL;
-        data_pair_t      *trav              = NULL;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
+int
+afr_fallocate_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fallocate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fallocate, local->fd,
+                      local->cont.fallocate.mode, local->cont.fallocate.offset,
+                      local->cont.fallocate.len, local->xdata_req);
+    return 0;
+}
 
-        if (ret)
-                goto out;
+int
+afr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata)
+{
+    call_frame_t *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        priv = this->private;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        QUORUM_CHECK(fsetxattr,out);
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
+    local->cont.fallocate.mode = mode;
+    local->cont.fallocate.offset = offset;
+    local->cont.fallocate.len = len;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                goto out;
-        }
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        transaction_frame->local = local;
+    if (!local->xdata_req)
+        goto out;
 
-        local->op_ret = -1;
+    local->op = GF_FOP_FALLOCATE;
 
-        local->cont.fsetxattr.dict  = dict_ref (dict);
-        local->cont.fsetxattr.flags = flags;
+    local->transaction.wind = afr_fallocate_wind;
+    local->transaction.unwind = afr_fallocate_unwind;
 
-        local->transaction.fop    = afr_fsetxattr_wind;
-        local->transaction.done   = afr_fsetxattr_done;
-        local->transaction.unwind = afr_fsetxattr_unwind;
+    local->transaction.main_frame = frame;
 
-        local->fd                 = fd_ref (fd);
+    local->transaction.start = local->cont.fallocate.offset;
+    local->transaction.len = 0;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start  = LLONG_MAX - 1;
-        local->transaction.len    = 0;
+    afr_fix_open(fd, this);
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        ret = 0;
+    return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
-
-/* {{{ removexattr */
-
+/* {{{ discard */
 
 int
-afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_discard_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+    local = frame->local;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (removexattr, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  NULL);
-        }
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(discard, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_discard_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
 
-        local = frame->local;
-        priv = this->private;
+int
+afr_discard_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_discard_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->discard, local->fd,
+                      local->cont.discard.offset, local->cont.discard.len,
+                      local->xdata_req);
+    return 0;
+}
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
+int
+afr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+    local->cont.discard.offset = offset;
+    local->cont.discard.len = len;
 
-        call_count = afr_frame_return (frame);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        return 0;
-}
+    if (!local->xdata_req)
+        goto out;
 
+    local->op = GF_FOP_DISCARD;
 
-int32_t
-afr_removexattr_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
+    local->transaction.wind = afr_discard_wind;
+    local->transaction.unwind = afr_discard_unwind;
 
-        local = frame->local;
-        priv = this->private;
+    local->transaction.main_frame = frame;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+    local->transaction.start = local->cont.discard.offset;
+    local->transaction.len = 0;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+    afr_fix_open(fd, this);
 
-        local->call_count = call_count;
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->removexattr,
-                                           &local->loc,
-                                           local->cont.removexattr.name,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
+/* {{{ zerofill */
 
 int
-afr_removexattr_done (call_frame_t *frame, xlator_t *this)
+afr_zerofill_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        AFR_STACK_DESTROY (frame);
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(discard, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name, dict_t *xdata)
+afr_zerofill_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        afr_private_t   *priv              = NULL;
-        afr_local_t     *local             = NULL;
-        call_frame_t    *transaction_frame = NULL;
-        int              ret               = -1;
-        int              op_errno          = 0;
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
 
-        VALIDATE_OR_GOTO (this, out);
+int
+afr_zerofill_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_zerofill_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->zerofill, local->fd,
+                      local->cont.zerofill.offset, local->cont.zerofill.len,
+                      local->xdata_req);
+    return 0;
+}
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
-                                 name, op_errno, out);
+int
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
-                                 name, op_errno, out);
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        priv = this->private;
+    local->cont.zerofill.offset = offset;
+    local->cont.zerofill.len = len;
 
-        QUORUM_CHECK(removexattr,out);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
+    if (!local->xdata_req)
+        goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    local->op = GF_FOP_ZEROFILL;
 
-        local->cont.removexattr.name = gf_strdup (name);
+    local->transaction.wind = afr_zerofill_wind;
+    local->transaction.unwind = afr_zerofill_unwind;
 
-        local->transaction.fop    = afr_removexattr_wind;
-        local->transaction.done   = afr_removexattr_done;
-        local->transaction.unwind = afr_removexattr_unwind;
+    local->transaction.main_frame = frame;
 
-        loc_copy (&local->loc, loc);
+    local->transaction.start = local->cont.zerofill.offset;
+    local->transaction.len = len;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    afr_fix_open(fd, this);
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        ret = 0;
+    return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-/* ffremovexattr */
-int
-afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+/* }}} */
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (fremovexattr, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  NULL);
-        }
-        return 0;
+int32_t
+afr_xattrop_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                     dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, xattr, xdata);
 }
 
-
 int
-afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_xattrop_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_xattrop_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->xattrop, &local->loc,
+                      local->cont.xattrop.optype, local->cont.xattrop.xattr,
+                      local->xdata_req);
+    return 0;
+}
 
-        call_count = afr_frame_return (frame);
+int
+afr_xattrop_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(xattrop, main_frame, local->op_ret, local->op_errno,
+                     local->xattr_rsp, local->xdata_rsp);
+    return 0;
+}
 
 int32_t
-afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this)
+afr_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->call_count = call_count;
+    local->cont.xattrop.xattr = dict_ref(xattr);
+    local->cont.xattrop.optype = optype;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fremovexattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fremovexattr,
-                                           local->fd,
-                                           local->cont.removexattr.name,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    local->transaction.wind = afr_xattrop_wind;
+    local->transaction.unwind = afr_xattrop_unwind;
 
-        return 0;
-}
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
+    local->op = GF_FOP_XATTROP;
 
-int
-afr_fremovexattr_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = frame->local;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        local->transaction.unwind (frame, this);
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        AFR_STACK_DESTROY (frame);
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
+int32_t
+afr_fxattrop_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                      dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, xattr, xdata);
+}
 
 int
-afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name, dict_t *xdata)
+afr_fxattrop_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_ret   = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (this, out);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fxattrop_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fxattrop, local->fd,
+                      local->cont.xattrop.optype, local->cont.xattrop.xattr,
+                      local->xdata_req);
+    return 0;
+}
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
-                                 name, op_errno, out);
+int
+afr_fxattrop_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
-                                 name, op_errno, out);
+    local = frame->local;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-        priv = this->private;
+    AFR_STACK_UNWIND(fxattrop, main_frame, local->op_ret, local->op_errno,
+                     local->xattr_rsp, local->xdata_rsp);
+    return 0;
+}
 
-        QUORUM_CHECK(fremovexattr, out);
+int32_t
+afr_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->cont.xattrop.xattr = dict_ref(xattr);
+    local->cont.xattrop.optype = optype;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    local->transaction.wind = afr_fxattrop_wind;
+    local->transaction.unwind = afr_fxattrop_unwind;
+
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
+
+    local->op = GF_FOP_FXATTROP;
+
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
+
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                goto out;
-        }
+    AFR_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
 
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
+int
+afr_fsync_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    local = frame->local;
 
-        transaction_frame->local = local;
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-        local->op_ret = -1;
+    AFR_STACK_UNWIND(fsync, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
 
-        local->cont.removexattr.name = gf_strdup (name);
+    return 0;
+}
 
-        local->transaction.fop    = afr_fremovexattr_wind;
-        local->transaction.done   = afr_fremovexattr_done;
-        local->transaction.unwind = afr_fremovexattr_unwind;
+int
+afr_fsync_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
 
-        local->fd = fd_ref (fd);
+int
+afr_fsync_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local = frame->local;
+    priv = this->private;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+    STACK_WIND_COOKIE(frame, afr_fsync_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsync, local->fd,
+                      local->cont.fsync.datasync, local->xdata_req);
+    return 0;
+}
 
-        op_ret = 0;
+int
+afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int32_t op_errno = ENOMEM;
+    int8_t last_fsync = 0;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    if (xdata) {
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+        if (dict_get_int8(xdata, "last-fsync", &last_fsync) == 0) {
+            if (last_fsync) {
+                local->transaction.disable_delayed_post_op = _gf_true;
+            }
+        }
+    } else {
+        local->xdata_req = dict_new();
+    }
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
+
+    local->op = GF_FOP_FSYNC;
+    local->cont.fsync.datasync = datasync;
+
+    if (afr_fd_has_witnessed_unstable_write(this, fd->inode)) {
+        /* don't care. we only wanted to CLEAR the bit */
+    }
+
+    local->transaction.wind = afr_fsync_wind;
+    local->transaction.unwind = afr_fsync_unwind;
+
+    local->transaction.main_frame = frame;
+
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index ed11079fd1b..a787069b7a1 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -12,60 +12,83 @@
 #define __INODE_WRITE_H__
 
 int32_t
-afr_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *xdata);
+afr_chmod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dict_t *xdata);
 
 int32_t
-afr_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid, dict_t *xdata);
+afr_chown(call_frame_t *frame, xlator_t *this, loc_t *loc, uid_t uid, gid_t gid,
+          dict_t *xdata);
 
 int
-afr_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid, dict_t *xdata);
+afr_fchown(call_frame_t *frame, xlator_t *this, fd_t *fd, uid_t uid, gid_t gid,
+           dict_t *xdata);
 
 int32_t
-afr_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode, dict_t *xdata);
+afr_fchmod(call_frame_t *frame, xlator_t *this, fd_t *fd, mode_t mode,
+           dict_t *xdata);
 
 int32_t
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	    struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref, dict_t *xdata);
+afr_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata);
 
 int32_t
-afr_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset, dict_t *xdata);
+afr_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata);
 
 int32_t
-afr_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset, dict_t *xdata);
+afr_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata);
 
 int32_t
-afr_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2], dict_t *xdata);
+afr_utimens(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct timespec tv[2], dict_t *xdata);
 
 int
-afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata);
+afr_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+            int32_t valid, dict_t *xdata);
 
 int
-afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata);
+afr_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+             int32_t valid, dict_t *xdata);
 
 int32_t
-afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata);
+afr_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata);
 
 int32_t
-afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata);
+afr_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata);
 
 int32_t
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *name, dict_t *xdata);
+afr_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata);
 
 int32_t
-afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name, dict_t *xdata);
+afr_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata);
 
+int
+afr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata);
+
+int
+afr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata);
+
+int
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata);
+
+int32_t
+afr_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+int32_t
+afr_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+int
+afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index 5e61be4d4df..bc8eabe0f43 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -8,2270 +8,784 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "dict.h"
-#include "byte-order.h"
-#include "common-utils.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
+#include "afr-messages.h"
 
 #include <signal.h>
 
-
-#define LOCKED_NO       0x0        /* no lock held */
-#define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path */
-#define LOCKED_LOWER    0x2        /* for lower path */
-
-#define AFR_TRACE_INODELK_IN(frame, this, params ...)           \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->inodelk_trace)                      \
-                        break;                                  \
-                afr_trace_inodelk_in (frame, this, params);     \
-        } while (0);
-
-#define AFR_TRACE_INODELK_OUT(frame, this, params ...)          \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->inodelk_trace)                      \
-                        break;                                  \
-                afr_trace_inodelk_out (frame, this, params);    \
-        } while (0);
-
-#define AFR_TRACE_ENTRYLK_IN(frame, this, params ...)           \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->entrylk_trace)                      \
-                        break;                                  \
-                afr_trace_entrylk_in (frame, this, params);     \
-        } while (0);
-
-#define AFR_TRACE_ENTRYLK_OUT(frame, this, params ...)          \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->entrylk_trace)                      \
-                        break;                                  \
-                afr_trace_entrylk_out (frame, this, params);    \
-        } while (0);
-
-int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
-
-static int
-afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this);
-
-static uint64_t afr_lock_number = 1;
-
-static uint64_t
-get_afr_lock_number ()
-{
-        return (++afr_lock_number);
-}
-
-int
-afr_set_lock_number (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_number = get_afr_lock_number ();
-
-        return 0;
-}
+#define LOCKED_NO 0x0    /* no lock held */
+#define LOCKED_YES 0x1   /* for DATA, METADATA, ENTRY and higher_path */
+#define LOCKED_LOWER 0x2 /* for lower path */
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner)
+afr_lockee_cleanup(afr_lockee_t *lockee)
 {
-        gf_log (this->name, GF_LOG_TRACE,
-                "Setting lk-owner=%llu",
-                (unsigned long long) (unsigned long)lk_owner);
+    if (lockee->fd) {
+        fd_unref(lockee->fd);
+        lockee->fd = NULL;
+    } else {
+        loc_wipe(&lockee->loc);
+    }
 
-        set_lk_owner_from_ptr (&frame->root->lk_owner, lk_owner);
-}
+    GF_FREE(lockee->basename);
+    lockee->basename = NULL;
+    GF_FREE(lockee->locked_nodes);
+    lockee->locked_nodes = NULL;
 
-static int
-is_afr_lock_selfheal (afr_local_t *local)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        int                  ret      = -1;
-
-        int_lock = &local->internal_lock;
-
-        switch (int_lock->selfheal_lk_type) {
-        case AFR_DATA_SELF_HEAL_LK:
-        case AFR_METADATA_SELF_HEAL_LK:
-                ret = 1;
-                break;
-        case AFR_ENTRY_SELF_HEAL_LK:
-                ret = 0;
-                break;
-        }
-
-        return ret;
-
-}
-
-int32_t
-internal_lock_count (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int32_t call_count = 0;
-        int i = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        if (local->fd) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && local->fd_open_on[i])
-                                ++call_count;
-                }
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i])
-                                ++call_count;
-                }
-        }
-
-        return call_count;
-}
-
-static void
-afr_print_inodelk (char *str, int size, int cmd,
-                   struct gf_flock *flock, gf_lkowner_t *owner)
-{
-        char *cmd_str = NULL;
-        char *type_str = NULL;
-
-        switch (cmd) {
-#if F_GETLK != F_GETLK64
-        case F_GETLK64:
-#endif
-        case F_GETLK:
-                cmd_str = "GETLK";
-                break;
-
-#if F_SETLK != F_SETLK64
-        case F_SETLK64:
-#endif
-        case F_SETLK:
-                cmd_str = "SETLK";
-                break;
-
-#if F_SETLKW != F_SETLKW64
-        case F_SETLKW64:
-#endif
-        case F_SETLKW:
-                cmd_str = "SETLKW";
-                break;
-
-        default:
-                cmd_str = "<null>";
-                break;
-        }
-
-        switch (flock->l_type) {
-        case F_RDLCK:
-                type_str = "READ";
-                break;
-        case F_WRLCK:
-                type_str = "WRITE";
-                break;
-        case F_UNLCK:
-                type_str = "UNLOCK";
-                break;
-        default:
-                type_str = "UNKNOWN";
-                break;
-        }
-
-        snprintf (str, size, "lock=INODELK, cmd=%s, type=%s, "
-                  "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
-                  cmd_str, type_str, (unsigned long long) flock->l_start,
-                  (unsigned long long) flock->l_len,
-                  (unsigned long long) flock->l_pid,
-                  lkowner_utoa (owner));
-
-}
-
-static void
-afr_print_lockee (char *str, int size, loc_t *loc, fd_t *fd,
-                  int child_index)
-{
-        snprintf (str, size, "path=%s, fd=%p, child=%d",
-                  loc->path ? loc->path : "<nul>",
-                  fd ? fd : NULL,
-                  child_index);
+    return;
 }
 
 void
-afr_print_entrylk (char *str, int size, const char *basename,
-                   gf_lkowner_t *owner)
-{
-        snprintf (str, size, "Basename=%s, lk-owner=%s",
-                  basename ? basename : "<nul>",
-                  lkowner_utoa (owner));
-}
-
-static void
-afr_print_verdict (int op_ret, int op_errno, char *str)
-{
-        if (op_ret < 0) {
-                if (op_errno == EAGAIN)
-                        strcpy (str, "EAGAIN");
-                else
-                        strcpy (str, "FAILED");
-        }
-        else
-                strcpy (str, "GRANTED");
-}
-
-static void
-afr_set_lock_call_type (afr_lock_call_type_t lock_call_type,
-                        char *lock_call_type_str,
-                        afr_internal_lock_t *int_lock)
-{
-        switch (lock_call_type) {
-        case AFR_INODELK_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_INODELK_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_INODELK_SELFHEAL");
-                break;
-        case AFR_INODELK_NB_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_INODELK_NB_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_INODELK_NB_SELFHEAL");
-                break;
-        case AFR_ENTRYLK_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_SELFHEAL");
-                break;
-        case AFR_ENTRYLK_NB_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_NB_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_NB_SELFHEAL");
-                break;
-        default:
-                strcpy (lock_call_type_str, "UNKNOWN");
-                break;
-        }
-
-}
-
-static void
-afr_trace_inodelk_out (call_frame_t *frame, xlator_t *this,
-                       afr_lock_call_type_t lock_call_type,
-                       afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
-                       int op_ret, int op_errno, int32_t child_index)
+afr_lockees_cleanup(afr_internal_lock_t *int_lock)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        char lockee[256];
-        char lock_call_type_str[256];
-        char verdict[16];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+    int i = 0;
 
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        afr_print_verdict (op_ret, op_errno, verdict);
-
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] [%s] lk-owner=%s Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict, lkowner_utoa (&frame->root->lk_owner), lockee,
-                (unsigned long long) int_lock->lock_number);
+    for (i = 0; i < int_lock->lockee_count; i++) {
+        afr_lockee_cleanup(&int_lock->lockee[i]);
+    }
 
+    return;
 }
-
-static void
-afr_trace_inodelk_in (call_frame_t *frame, xlator_t *this,
-                      afr_lock_call_type_t lock_call_type,
-                      afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
-                      int32_t cmd, int32_t child_index)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        afr_print_inodelk (lock, 256, cmd, flock, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number);
-
+int
+afr_entry_lockee_cmp(const void *l1, const void *l2)
+{
+    const afr_lockee_t *r1 = l1;
+    const afr_lockee_t *r2 = l2;
+    int ret = 0;
+    uuid_t gfid1 = {0};
+    uuid_t gfid2 = {0};
+
+    loc_gfid((loc_t *)&r1->loc, gfid1);
+    loc_gfid((loc_t *)&r2->loc, gfid2);
+    ret = gf_uuid_compare(gfid1, gfid2);
+    /*Entrylks with NULL basename are the 'smallest'*/
+    if (ret == 0) {
+        if (!r1->basename)
+            return -1;
+        if (!r2->basename)
+            return 1;
+        ret = strcmp(r1->basename, r2->basename);
+    }
+
+    if (ret <= 0)
+        return -1;
+    else
+        return 1;
 }
 
-static void
-afr_trace_entrylk_in (call_frame_t *frame, xlator_t *this,
-                      afr_lock_call_type_t lock_call_type,
-                      afr_lock_op_type_t lk_op_type, const char *basename,
-                      int32_t child_index)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number);
-}
+int
+afr_lock_blocking(call_frame_t *frame, xlator_t *this, int child_index);
 
-static void
-afr_trace_entrylk_out (call_frame_t *frame, xlator_t *this,
-                       afr_lock_call_type_t lock_call_type,
-                       afr_lock_op_type_t lk_op_type, const char *basename,
-                       int op_ret, int op_errno, int32_t child_index)
+void
+afr_set_lk_owner(call_frame_t *frame, xlator_t *this, void *lk_owner)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-        char verdict[16];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        afr_print_verdict (op_ret, op_errno, verdict);
-
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict,
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+    gf_msg_trace(this->name, 0, "Setting lk-owner=%llu",
+                 (unsigned long long)(unsigned long)lk_owner);
 
+    set_lk_owner_from_ptr(&frame->root->lk_owner, lk_owner);
 }
 
-static int
-transaction_lk_op (afr_local_t *local)
+int32_t
+internal_lock_count(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        int ret = -1;
-
-        int_lock = &local->internal_lock;
-
-        if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK) {
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "lk op is for a transaction");
-                ret = 1;
-        }
-        else if (int_lock->transaction_lk_type == AFR_SELFHEAL_LK) {
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "lk op is for a self heal");
-
-                ret = 0;
-        }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int32_t call_count = 0;
+    int i = 0;
 
-        if (ret == -1)
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "lk op is not set");
+    local = frame->local;
+    priv = this->private;
 
-        return ret;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i])
+            ++call_count;
+    }
 
+    return call_count;
 }
 
-static int
-is_afr_lock_transaction (afr_local_t *local)
+int
+afr_add_entry_lockee(afr_local_t *local, loc_t *loc, char *basename,
+                     int child_count)
 {
-        int ret = 0;
+    int ret = -ENOMEM;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count];
 
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                ret = 1;
-                break;
+    GF_ASSERT(int_lock->lockee_count < AFR_LOCKEE_COUNT_MAX);
+    loc_copy(&lockee->loc, loc);
+    lockee->basename = (basename) ? gf_strdup(basename) : NULL;
+    if (basename && !lockee->basename)
+        goto out;
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                ret = 0;
-                break;
+    lockee->locked_count = 0;
+    lockee->locked_nodes = GF_CALLOC(child_count, sizeof(*lockee->locked_nodes),
+                                     gf_afr_mt_afr_node_character);
 
-        }
+    if (!lockee->locked_nodes)
+        goto out;
 
-        return ret;
+    ret = 0;
+    int_lock->lockee_count++;
+out:
+    if (ret) {
+        afr_lockee_cleanup(lockee);
+    }
+    return ret;
 }
 
-static int
-initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
+int
+afr_add_inode_lockee(afr_local_t *local, int child_count)
 {
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
+    int ret = -ENOMEM;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count];
 
-        int i = 0;
+    if (local->fd) {
+        lockee->fd = fd_ref(local->fd);
+    } else {
+        loc_copy(&lockee->loc, &local->loc);
+    }
 
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    lockee->locked_count = 0;
+    lockee->locked_nodes = GF_CALLOC(child_count, sizeof(*lockee->locked_nodes),
+                                     gf_afr_mt_afr_node_character);
 
-        int_lock->entrylk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
-
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->entry_locked_nodes[i] = 0;
-        }
+    if (!lockee->locked_nodes)
+        goto out;
 
-        return 0;
+    ret = 0;
+    int_lock->lockee_count++;
+out:
+    if (ret) {
+        afr_lockee_cleanup(lockee);
+    }
+    return ret;
 }
 
 static int
-initialize_inodelk_variables (call_frame_t *frame, xlator_t *this)
+initialize_internal_lock_variables(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        int i = 0;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->inodelk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_private_t *priv = NULL;
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->inode_locked_nodes[i] = 0;
-        }
+    int i = 0;
 
-        return 0;
-}
-
-loc_t *
-lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2)
-{
-        int ret = 0;
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        ret = uuid_compare (l1->inode->gfid, l2->inode->gfid);
+    int_lock->lock_count = 0;
+    int_lock->lock_op_ret = -1;
+    int_lock->lock_op_errno = 0;
+    int_lock->lk_attempted_count = 0;
 
-        if (ret == 0)
-                ret = strcmp (b1, b2);
+    for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
+        if (!int_lock->lockee[i].locked_nodes)
+            break;
+        int_lock->lockee[i].locked_count = 0;
+        memset(int_lock->lockee[i].locked_nodes, 0,
+               sizeof(*int_lock->lockee[i].locked_nodes) * priv->child_count);
+    }
 
-        if (ret <= 0)
-                return l1;
-        else
-                return l2;
+    return 0;
 }
 
 int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
-
+afr_lockee_locked_nodes_count(afr_internal_lock_t *int_lock)
 {
-        int i = 0;
-        int call_count = 0;
+    int call_count = 0;
+    int i = 0;
 
-        for (i = 0; i < child_count; i++) {
-                if (locked_nodes[i] & LOCKED_YES)
-                        call_count++;
-        }
+    for (i = 0; i < int_lock->lockee_count; i++)
+        call_count += int_lock->lockee[i].locked_count;
 
-        return call_count;
+    return call_count;
 }
 
-/* FIXME: What if UNLOCK fails */
-static int32_t
-afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        int call_count = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "All internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-        }
-
-        return 0;
-}
+int
+afr_locked_nodes_count(unsigned char *locked_nodes, int child_count)
 
-static int32_t
-afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t         *local = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        int32_t             child_index = (long)cookie;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-
-        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
-                               AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, child_index);
-
-        if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_log (this->name, GF_LOG_INFO, "%s: unlock failed on %d "
-                        "unlock by %s", local->loc.path, child_index,
-                        lkowner_utoa (&frame->root->lk_owner));
-        }
+    int i = 0;
+    int call_count = 0;
 
+    for (i = 0; i < child_count; i++) {
+        if (locked_nodes[i] & LOCKED_YES)
+            call_count++;
+    }
 
-        int_lock->inode_locked_nodes[child_index] &= LOCKED_NO;
-        if (local->transaction.eager_lock)
-                local->transaction.eager_lock[child_index] = 0;
-
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-
-        return 0;
-
+    return call_count;
 }
 
-static int
-afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        struct gf_flock flock = {0,};
-        struct gf_flock full_flock = {0,};
-        struct gf_flock *flock_use = NULL;
-        int call_count = 0;
-        int i = 0;
-        int piggyback = 0;
-        afr_fd_ctx_t        *fd_ctx      = NULL;
-
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = F_UNLCK;
-
-        full_flock.l_type = F_UNLCK;
-        call_count = afr_locked_nodes_count (int_lock->inode_locked_nodes,
-                                             priv->child_count);
-
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        if (local->fd)
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if ((int_lock->inode_locked_nodes[i] & LOCKED_YES)
-                    != LOCKED_YES)
-                        continue;
-
-                if (local->fd) {
-                        flock_use = &flock;
-                        if (!local->transaction.eager_lock[i]) {
-                                goto wind;
-                        }
-
-                        piggyback = 0;
-
-                        LOCK (&local->fd->lock);
-                        {
-                                if (fd_ctx->lock_piggyback[i]) {
-                                        fd_ctx->lock_piggyback[i]--;
-                                        piggyback = 1;
-                                } else {
-                                        fd_ctx->lock_acquired[i]--;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
-
-                        if (piggyback) {
-                                afr_unlock_inodelk_cbk (frame, (void *) (long) i,
-                                                        this, 1, 0, NULL);
-                                if (!--call_count)
-                                        break;
-                                continue;
-                        }
-
-                        flock_use = &full_flock;
-                wind:
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_UNLOCK_OP, flock_use, F_SETLK,
-                                              i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                           (void *) (long)i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLK, flock_use, NULL);
-
-                        if (!--call_count)
-                                break;
-
-                } else {
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_UNLOCK_OP, &flock, F_SETLK, i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                           (void *) (long)i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLK, &flock, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-out:
-        return 0;
-}
-
-static int32_t
-afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_local_t         *local = NULL;
-        int32_t             child_index = (long)cookie;
-
-        local = frame->local;
-
-        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, child_index);
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: unlock failed on %d, reason: %s",
-                        local->loc.path, child_index, strerror (op_errno));
-        }
-
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, NULL);
-
-        return 0;
-}
-
-static int
-afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
+static void
+afr_log_locks_failure(call_frame_t *frame, char *where, char *what,
+                      int op_errno)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int call_count = 0;
-        int i = -1;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
-        call_count = afr_locked_nodes_count (int_lock->entry_locked_nodes,
-                                             priv->child_count);
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count){
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->entry_locked_nodes[i] & LOCKED_YES) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    xlator_t *this = frame->this;
+    gf_lkowner_t *lk_owner = &frame->root->lk_owner;
+    afr_local_t *local = frame->local;
+    const char *fop = NULL;
+    char *gfid = NULL;
+    const char *name = NULL;
 
-out:
-        return 0;
+    fop = gf_fop_list[local->op];
 
+    switch (local->transaction.type) {
+        case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
+            switch (local->op) {
+                case GF_FOP_LINK:
+                    gfid = uuid_utoa(local->newloc.pargfid);
+                    name = local->newloc.name;
+                    break;
+                default:
+                    gfid = uuid_utoa(local->loc.pargfid);
+                    name = local->loc.name;
+                    break;
+            }
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   AFR_MSG_INTERNAL_LKS_FAILED,
+                   "Unable to do entry %s with lk-owner:%s on %s "
+                   "while attempting %s on {pgfid:%s, name:%s}.",
+                   what, lkowner_utoa(lk_owner), where, fop, gfid, name);
+            break;
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            gfid = uuid_utoa(local->inode->gfid);
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   AFR_MSG_INTERNAL_LKS_FAILED,
+                   "Unable to do inode %s with lk-owner:%s on %s "
+                   "while attempting %s on gfid:%s.",
+                   what, lkowner_utoa(lk_owner), where, fop, gfid);
+            break;
+    }
 }
 
 static int32_t
-afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_unlock_common_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        int child_index = (long) cookie;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    int lockee_num = 0;
+    int call_count = 0;
+    int child_index = 0;
+    int ret = 0;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+    lockee_num = (int)((long)cookie) / priv->child_count;
+    child_index = (int)((long)cookie) % priv->child_count;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOSYS) {
-                                /* return ENOTSUP */
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "subvolume does not support locking. "
-                                        "please load features/locks xlator on server");
-                                local->op_ret = op_ret;
-                                int_lock->lock_op_ret = op_ret;
-                        }
-
-                        local->op_errno              = op_errno;
-                        int_lock->lock_op_errno      = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if ((op_ret == -1) &&
-            (op_errno == ENOSYS)) {
-                afr_unlock (frame, this);
-        } else {
-                if (op_ret == 0) {
-                        int_lock->locked_nodes[child_index] |= LOCKED_YES;
-                        int_lock->lock_count++;
-                }
-                afr_lock_blocking (frame, this, child_index + 1);
-        }
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        afr_log_locks_failure(frame, priv->children[child_index]->name,
+                              "unlock", op_errno);
+    }
 
-        return 0;
-}
+    int_lock->lockee[lockee_num].locked_nodes[child_index] &= LOCKED_NO;
+    if (local->transaction.type == AFR_DATA_TRANSACTION && op_ret != 1)
+        ret = afr_write_subvol_reset(frame, this);
 
-static int32_t
-afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
+    LOCK(&frame->lock);
+    {
+        call_count = --int_lock->lk_call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-        return 0;
+    if (call_count == 0) {
+        int_lock->lock_cbk(frame, this);
+    }
 
+    return ret;
 }
 
-static int32_t
-afr_lock_lower_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_private_t       *priv        = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *higher_name = NULL;
-        int child_index = (long) cookie;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOSYS) {
-                                /* return ENOTSUP */
-
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "subvolume does not support locking. "
-                                        "please load features/locks xlator on server");
-
-                                local->op_ret   = op_ret;
-                        }
-
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret != 0) {
-                afr_copy_locked_nodes (frame, this);
-                afr_unlock (frame, this);
-                goto out;
-        } else {
-                int_lock->lower_locked_nodes[child_index] |= LOCKED_LOWER;
-                int_lock->lock_count++;
-        }
-
-        /* The lower path has been locked. Now lock the higher path */
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
-                              AFR_LOCK_OP, higher_name, child_index);
-
-
-        STACK_WIND_COOKIE (frame, afr_lock_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->entrylk,
-                           this->name, higher, higher_name,
-                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-
-out:
-        return 0;
-}
-
-static int32_t
-afr_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long)cookie);
-
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-        return 0;
-}
-
-static int
-afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                memcpy (int_lock->inode_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->inodelk_lock_count = int_lock->lock_count;
-                break;
-
-        case AFR_ENTRY_RENAME_TRANSACTION:
+void
+afr_internal_lock_wind(call_frame_t *frame,
+                       int32_t (*cbk)(call_frame_t *, void *, xlator_t *,
+                                      int32_t, int32_t, dict_t *),
+                       void *cookie, int child, int lockee_num,
+                       gf_boolean_t blocking, gf_boolean_t unlock)
+{
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    entrylk_cmd cmd = ENTRYLK_LOCK_NB;
+    int32_t cmd1 = F_SETLK;
+    struct gf_flock flock = {
+        0,
+    };
+
+    switch (local->transaction.type) {
         case AFR_ENTRY_TRANSACTION:
-                memcpy (int_lock->entry_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->entrylk_lock_count = int_lock->lock_count;
-                break;
-        }
-
-        return 0;
-
-}
-
-int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        afr_private_t       *priv        = NULL;
-        loc_t               *lower       = NULL;
-        const char          *lower_name  = NULL;
-        struct gf_flock flock = {0,};
-        uint64_t ctx = 0;
-        int ret = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = int_lock->lk_flock.l_type;
-
-        if (local->fd) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-
-                        afr_copy_locked_nodes (frame, this);
-
-                        afr_unlock (frame, this);
-
-                        return 0;
-                }
-
-                /* skip over children that or down
-                   or don't have the fd open */
-
-                while ((child_index < priv->child_count)
-                       && (!local->child_up[child_index] ||
-                          !local->fd_open_on[child_index]))
-
-                        child_index++;
-        } else {
-                /* skip over children that are down */
-                while ((child_index < priv->child_count)
-                       && !local->child_up[child_index])
-                        child_index++;
-        }
-
-        if ((child_index == priv->child_count) &&
-            int_lock->lock_count == 0) {
-
-                gf_log (this->name, GF_LOG_INFO,
-                        "unable to lock on even one child");
-
-                local->op_ret           = -1;
-                int_lock->lock_op_ret   = -1;
-
-                afr_copy_locked_nodes (frame, this);
-
-                afr_unlock(frame, this);
-
-                return 0;
-
-        }
-
-        if ((child_index == priv->child_count)
-            || (int_lock->lock_count == int_lock->lk_expected_count)) {
-
-                /* we're done locking */
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "we're done locking");
-
-                afr_copy_locked_nodes (frame, this);
-
-                int_lock->lock_op_ret = 0;
-                int_lock->lock_cbk (frame, this);
-                return 0;
-        }
-
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-
-                if (local->fd) {
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_LOCK_OP, &flock, F_SETLKW,
-                                              child_index);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-                                           (void *) (long) child_index,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLKW, &flock, NULL);
-
-                } else {
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_LOCK_OP, &flock, F_SETLKW,
-                                              child_index);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-                                           (void *) (long) child_index,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLKW, &flock, NULL);
-                }
-
-                break;
-
         case AFR_ENTRY_RENAME_TRANSACTION:
-        {
-                lower = lower_path (&local->transaction.parent_loc,
-                                    local->transaction.basename,
-                                    &local->transaction.new_parent_loc,
-                                    local->transaction.new_basename);
+            if (unlock) {
+                cmd = ENTRYLK_UNLOCK;
+            } else if (blocking) { /*Doesn't make sense to have blocking
+                                      unlock*/
+                cmd = ENTRYLK_LOCK;
+            }
+
+            if (local->fd) {
+                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child],
+                                  priv->children[child]->fops->fentrylk,
+                                  int_lock->domain,
+                                  int_lock->lockee[lockee_num].fd,
+                                  int_lock->lockee[lockee_num].basename, cmd,
+                                  ENTRYLK_WRLCK, NULL);
+            } else {
+                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child],
+                                  priv->children[child]->fops->entrylk,
+                                  int_lock->domain,
+                                  &int_lock->lockee[lockee_num].loc,
+                                  int_lock->lockee[lockee_num].basename, cmd,
+                                  ENTRYLK_WRLCK, NULL);
+            }
+            break;
 
-                lower_name = (lower == &local->transaction.parent_loc ?
-                              local->transaction.basename :
-                              local->transaction.new_basename);
-
-                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
-                                      AFR_LOCK_OP, lower_name, child_index);
-
-
-                STACK_WIND_COOKIE (frame, afr_lock_lower_cbk,
-                                   (void *) (long) child_index,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->entrylk,
-                                   this->name, lower, lower_name,
-                                   ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-
-                break;
-        }
-
-        case AFR_ENTRY_TRANSACTION:
-                if (local->fd) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP, local->transaction.basename,
-                                              child_index);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) child_index,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->fentrylk,
-                                           this->name, local->fd,
-                                           local->transaction.basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-                } else {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP, local->transaction.basename,
-                                              child_index);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) child_index,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->entrylk,
-                                           this->name,
-                                           &local->transaction.parent_loc,
-                                           local->transaction.basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-                }
-
-                break;
-        }
-
-        return 0;
-}
-
-int32_t
-afr_blocking_lock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        int                  up_count = 0;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
-                initialize_inodelk_variables (frame, this);
-                break;
+            flock = int_lock->lockee[lockee_num].flock;
+            if (unlock) {
+                flock.l_type = F_UNLCK;
+            } else if (blocking) { /*Doesn't make sense to have blocking
+                                      unlock*/
+                cmd1 = F_SETLKW;
+            }
+
+            if (local->fd) {
+                STACK_WIND_COOKIE(
+                    frame, cbk, cookie, priv->children[child],
+                    priv->children[child]->fops->finodelk, int_lock->domain,
+                    int_lock->lockee[lockee_num].fd, cmd1, &flock, NULL);
+            } else {
+                STACK_WIND_COOKIE(
+                    frame, cbk, cookie, priv->children[child],
+                    priv->children[child]->fops->inodelk, int_lock->domain,
+                    &int_lock->lockee[lockee_num].loc, cmd1, &flock, NULL);
+            }
+            break;
+    }
+}
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                up_count = afr_up_children_count (local->child_up,
-                                                  priv->child_count);
-                int_lock->lk_expected_count = 2 * up_count;
-                //fallthrough
-        case AFR_ENTRY_TRANSACTION:
-                initialize_entrylk_variables (frame, this);
+static int
+afr_unlock_now(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int child_index = 0;
+    int lockee_num = 0;
+    int i = -1;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+
+    call_count = afr_lockee_locked_nodes_count(int_lock);
+
+    int_lock->lk_call_count = call_count;
+
+    if (!call_count) {
+        gf_msg_trace(this->name, 0, "No internal locks unlocked");
+        int_lock->lock_cbk(frame, this);
+        goto out;
+    }
+
+    for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+        lockee_num = i / priv->child_count;
+        child_index = i % priv->child_count;
+        if (int_lock->lockee[lockee_num].locked_nodes[child_index] &
+            LOCKED_YES) {
+            afr_internal_lock_wind(frame, afr_unlock_common_cbk,
+                                   (void *)(long)i, child_index, lockee_num,
+                                   _gf_false, _gf_true);
+            if (!--call_count)
                 break;
         }
+    }
 
-        afr_lock_blocking (frame, this, 0);
-
-        return 0;
+out:
+    return 0;
 }
 
 static int32_t
-afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        int                 call_count  = 0;
-        int                 child_index = (long) cookie;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
-
-        if (op_ret < 0 ) {
-                if (op_errno == ENOSYS) {
-                        /* return ENOTSUP */
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "subvolume does not support locking. "
-                                "please load features/locks xlator on server");
-                        local->op_ret         = op_ret;
-                        int_lock->lock_op_ret = op_ret;
-
-                        int_lock->lock_op_errno      = op_errno;
-                        local->op_errno              = op_errno;
+afr_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int cky = (long)cookie;
+    int child_index = 0;
+    int lockee_num = 0;
+
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
+
+    child_index = ((int)cky) % priv->child_count;
+    lockee_num = ((int)cky) / priv->child_count;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno == ENOSYS) {
+                /* return ENOTSUP */
+                gf_msg(this->name, GF_LOG_ERROR, ENOSYS,
+                       AFR_MSG_LOCK_XLATOR_NOT_LOADED,
+                       "subvolume does not support locking. "
+                       "please load features/locks xlator on server");
+                local->op_ret = op_ret;
+                int_lock->lock_op_ret = op_ret;
+            }
+
+            local->op_errno = op_errno;
+            int_lock->lock_op_errno = op_errno;
+        }
+
+        int_lock->lk_attempted_count++;
+    }
+    UNLOCK(&frame->lock);
+
+    if ((op_ret == -1) && (op_errno == ENOSYS)) {
+        afr_unlock_now(frame, this);
+    } else {
+        if (op_ret == 0) {
+            int_lock->lockee[lockee_num]
+                .locked_nodes[child_index] |= LOCKED_YES;
+            int_lock->lockee[lockee_num].locked_count++;
+            int_lock->lock_count++;
+            if (local->transaction.type == AFR_DATA_TRANSACTION) {
+                LOCK(&local->inode->lock);
+                {
+                    local->inode_ctx->lock_count++;
                 }
-        } else if (op_ret == 0) {
-                int_lock->entry_locked_nodes[child_index] |= LOCKED_YES;
-                int_lock->entrylk_lock_count++;
+                UNLOCK(&local->inode->lock);
+            }
         }
+        afr_lock_blocking(frame, this, cky + 1);
+    }
 
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Last locking reply received");
-                /* all locks successful. Proceed to call FOP */
-                if (int_lock->entrylk_lock_count ==
-                                int_lock->lk_expected_count) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "All servers locked. Calling the cbk");
-                        int_lock->lock_op_ret = 0;
-                        int_lock->lock_cbk (frame, this);
-                }
-                /* Not all locks were successful. Unlock and try locking
-                   again, this time with serially blocking locks */
-                else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%d servers locked. Trying again with blocking calls",
-                                int_lock->lock_count);
-
-                        afr_unlock(frame, this);
-                }
-        }
-
-        return 0;
+    return 0;
 }
 
-void
-afr_mark_fd_open_on (afr_local_t *local, afr_fd_ctx_t *fd_ctx,
-                       size_t child_count)
+static gf_boolean_t
+_is_lock_wind_needed(afr_local_t *local, int child_index)
 {
-        int             i = 0;
-
-        GF_ASSERT (local->fd_open_on);
-
-        memset (local->fd_open_on, 0, sizeof (*local->fd_open_on)*child_count);
-        for (i = 0; i < child_count; i++)
-                if (fd_ctx->opened_on[i] == AFR_FD_OPENED)
-                        local->fd_open_on[i] = 1;
-}
-
-int
-afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int32_t call_count = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        initialize_entrylk_variables (frame, this);
-
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
-        if (local->fd) {
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-                if (!fd_ctx) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
-
-                        return -1;
-                }
-
-                afr_mark_fd_open_on (local, fd_ctx, priv->child_count);
-                call_count = internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
+    if (!local->child_up[child_index])
+        return _gf_false;
 
-                if (!call_count) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "fd not open on any subvolumes. aborting.");
-                        afr_unlock (frame, this);
-                        goto out;
-                }
-
-                /* Send non-blocking entrylk calls only on up children
-                   and where the fd has been opened */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && local->fd_open_on[i]) {
-                                AFR_TRACE_ENTRYLK_IN (frame, this,
-                                                      AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
-
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fentrylk,
-                                                   this->name, local->fd,
-                                                   basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
-                                                   NULL);
-                        }
-                }
-        } else {
-                GF_ASSERT (loc);
-
-                call_count = internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i]) {
-                                AFR_TRACE_ENTRYLK_IN (frame, this,
-                                                      AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
-
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->entrylk,
-                                                   this->name, loc, basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
-                                                   NULL);
-
-                                if (!--call_count)
-                                        break;
-
-                        }
-                }
-        }
-out:
-        return 0;
+    return _gf_true;
 }
 
-int32_t
-afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static gf_boolean_t
+is_blocking_locks_count_sufficient(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        int call_count  = 0;
-        int child_index = (long) cookie;
-        afr_fd_ctx_t        *fd_ctx = NULL;
-
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    int child = 0;
+    int nlockee = 0;
+    int lockee_count = 0;
+    gf_boolean_t ret = _gf_true;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    priv = this->private;
+    int_lock = &local->internal_lock;
+    lockee_count = int_lock->lockee_count;
 
-        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_NB_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
-
-        if (op_ret < 0) {
-                if (op_errno == ENOSYS) {
-                        /* return ENOTSUP */
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "subvolume does not support locking. "
-                                "please load features/locks xlator on server");
-                        local->op_ret                = op_ret;
-                        int_lock->lock_op_ret        = op_ret;
-                        int_lock->lock_op_errno      = op_errno;
-                        local->op_errno              = op_errno;
-                }
-                if (local->transaction.eager_lock)
-                        local->transaction.eager_lock[child_index] = 0;
-        } else {
-                int_lock->inode_locked_nodes[child_index]
-                        |= LOCKED_YES;
-                int_lock->inodelk_lock_count++;
-
-                if (local->transaction.eager_lock &&
-                    local->transaction.eager_lock[child_index] && local->fd) {
-                        fd_ctx = afr_fd_ctx_get (local->fd, this);
-                        /* piggybacked */
-
-                        if (op_ret == 1) {
-                                /* piggybacked */
-                        } else if (op_ret == 0) {
-                                /* lock acquired from server */
-                                LOCK (&local->fd->lock);
-                                {
-                                        fd_ctx->lock_acquired[child_index]++;
-                                }
-                                UNLOCK (&local->fd->lock);
-                        }
-                }
-        }
-
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Last inode locking reply received");
-                /* all locks successful. Proceed to call FOP */
-                if (int_lock->inodelk_lock_count ==
-                                int_lock->lk_expected_count) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "All servers locked. Calling the cbk");
-                        int_lock->lock_op_ret = 0;
-                        int_lock->lock_cbk (frame, this);
-                }
-                /* Not all locks were successful. Unlock and try locking
-                   again, this time with serially blocking locks */
-                else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%d servers locked. Trying again with blocking calls",
-                                int_lock->lock_count);
-
-                        afr_unlock(frame, this);
-                }
+    if (int_lock->lock_count == 0) {
+        afr_log_locks_failure(frame, "any subvolume", "lock",
+                              int_lock->lock_op_errno);
+        return _gf_false;
+    }
+    /* For FOPS that take multiple sets of locks (mkdir, rename),
+     * there must be at least one brick on which the locks from
+     * all lock sets were successful. */
+    for (child = 0; child < priv->child_count; child++) {
+        ret = _gf_true;
+        for (nlockee = 0; nlockee < lockee_count; nlockee++) {
+            if (!(int_lock->lockee[nlockee].locked_nodes[child] & LOCKED_YES))
+                ret = _gf_false;
         }
+        if (ret)
+            return ret;
+    }
+    if (!ret)
+        afr_log_locks_failure(frame, "all", "lock", int_lock->lock_op_errno);
 
-        return 0;
+    return ret;
 }
 
 int
-afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-        int32_t             call_count = 0;
-        int                 i          = 0;
-        int                 ret        = 0;
-        struct              gf_flock flock = {0,};
-        struct              gf_flock full_flock = {0,};
-        struct              gf_flock *flock_use = NULL;
-        int                 piggyback = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = int_lock->lk_flock.l_type;
-
-        full_flock.l_type = int_lock->lk_flock.l_type;
-
-        initialize_inodelk_variables (frame, this);
-
-        if (local->fd) {
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-                if (!fd_ctx) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
-
-                        ret = -1;
-                        goto out;
-                }
-
-                afr_mark_fd_open_on (local, fd_ctx, priv->child_count);
-                call_count = internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                if (!call_count) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "fd not open on any subvolumes. aborting.");
-                        afr_unlock (frame, this);
-                        goto out;
-                }
-
-                /* Send non-blocking inodelk calls only on up children
-                   and where the fd has been opened */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->child_up[i] || !local->fd_open_on[i])
-                                continue;
-
-                        flock_use = &flock;
-                        if (!priv->eager_lock) {
-                                goto wind;
-                        }
-
-                        piggyback = 0;
-                        local->transaction.eager_lock[i] = 1;
-
-			afr_set_delayed_post_op (frame, this);
-
-                        LOCK (&local->fd->lock);
-                        {
-                                if (fd_ctx->lock_acquired[i]) {
-                                        fd_ctx->lock_piggyback[i]++;
-                                        piggyback = 1;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
-
-                        if (piggyback) {
-                                /* (op_ret == 1) => indicate piggybacked lock */
-                                afr_nonblocking_inodelk_cbk (frame, (void *) (long) i,
-                                                             this, 1, 0, NULL);
-                                if (!--call_count)
-                                        break;
-                                continue;
-                        }
-                        flock_use = &full_flock;
-                wind:
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_NB_TRANSACTION,
-                                              AFR_LOCK_OP, flock_use, F_SETLK, i);
-
-                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->finodelk,
-                                           this->name, local->fd,
-                                           F_SETLK, flock_use, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        } else {
-                call_count = internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->child_up[i])
-                                continue;
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_NB_TRANSACTION,
-                                              AFR_LOCK_OP, &flock, F_SETLK, i);
-
-                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->inodelk,
-                                           this->name, &local->loc,
-                                           F_SETLK, &flock, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-out:
-        return ret;
-}
-
-static int
-__is_lower_locked (call_frame_t *frame, xlator_t *this)
+afr_lock_blocking(call_frame_t *frame, xlator_t *this, int cookie)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int count = 0;
-        int i     = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER)
-                        count++;
-        }
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
+    int child_index = 0;
+    int lockee_num = 0;
 
-        return count;
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+    child_index = cookie % priv->child_count;
+    lockee_num = cookie / priv->child_count;
 
-}
+    if (local->fd) {
+        ret = fd_ctx_get(local->fd, this, &ctx);
 
-static int
-__is_higher_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int count = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->locked_nodes[i] & LOCKED_YES)
-                        count++;
-        }
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_FD_CTX_GET_FAILED,
+                   "unable to get fd ctx for fd=%p", local->fd);
 
-        return count;
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
 
-}
+            afr_unlock_now(frame, this);
 
-static int
-afr_unlock_lower_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int call_count = 0;
-        int i = -1;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-        priv  = this->private;
-
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
-        call_count = __is_lower_locked (frame, this);
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count){
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
+            return 0;
         }
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
+    if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+        if (!is_blocking_locks_count_sufficient(frame, this)) {
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
 
-                        if (!--call_count)
-                                break;
+            afr_unlock_now(frame, this);
 
-                }
+            return 0;
         }
+    }
 
-out:
-        return 0;
-
-}
+    if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+        /* we're done locking */
 
+        gf_msg_debug(this->name, 0, "we're done locking");
 
-static int
-afr_post_unlock_higher_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.done (frame, this);
+        int_lock->lock_op_ret = 0;
+        int_lock->lock_cbk(frame, this);
         return 0;
-}
-
-static int
-afr_post_unlock_lower_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *higher_name = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        if (__is_higher_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking higher");
-                int_lock->lk_basename = higher_name;
-                int_lock->lk_loc      = higher;
-                int_lock->lock_cbk    = afr_post_unlock_higher_cbk;
-
-                afr_unlock_entrylk (frame, this);
-        } else
-                local->transaction.done (frame, this);
+    }
 
+    if (!_is_lock_wind_needed(local, child_index)) {
+        afr_lock_blocking(frame, this, cookie + 1);
         return 0;
-}
+    }
 
-static int
-afr_rename_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        const char          *lower_name  = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        lower_name = (lower == &local->transaction.parent_loc ?
-                      local->transaction.basename :
-                      local->transaction.new_basename);
-
-        if (__is_lower_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking lower");
-                int_lock->lk_basename = lower_name;
-                int_lock->lk_loc      = lower;
-                int_lock->lock_cbk    = afr_post_unlock_lower_cbk;
-
-                afr_unlock_lower_entrylk (frame, this);
-        } else
-                afr_post_unlock_lower_cbk (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_transaction (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        return (local->transaction.type ==
-                AFR_ENTRY_RENAME_TRANSACTION);
+    afr_internal_lock_wind(frame, afr_lock_cbk, (void *)(long)cookie,
+                           child_index, lockee_num, _gf_true, _gf_false);
 
+    return 0;
 }
 
 int32_t
-afr_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        if (transaction_lk_op (local)) {
-                if (is_afr_lock_transaction (local))
-                        afr_unlock_inodelk (frame, this);
-                else
-                        if (!afr_rename_transaction (frame, this))
-                                afr_unlock_entrylk (frame, this);
-                        else
-                                afr_rename_unlock (frame, this);
-        } else {
-                if (is_afr_lock_selfheal (local))
-                        afr_unlock_inodelk (frame, this);
-                else
-                        afr_unlock_entrylk (frame, this);
-        }
-
-        return 0;
-}
-
-int
-afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
-                       unsigned char *locked_nodes)
+afr_blocking_lock(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv  = NULL;
-        afr_fd_ctx_t  *fdctx = NULL;
-        uint64_t       tmp   = 0;
-        int            ret   = 0;
-
-        priv = this->private;
-
-        ret = afr_fd_ctx_set (this, fd);
-        if (ret)
-                goto out;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "failed to get the fd ctx");
-                goto out;
-        }
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        GF_ASSERT (fdctx->locked_on);
-
-        memcpy (fdctx->locked_on, locked_nodes,
-                priv->child_count);
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int up_count = 0;
 
-out:
-        return ret;
-}
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-static int
-__is_fd_saved (xlator_t *this, fd_t *fd)
-{
-        afr_locked_fd_t *locked_fd = NULL;
-        afr_private_t   *priv      = NULL;
-        int              found     = 0;
+    up_count = AFR_COUNT(local->child_up, priv->child_count);
+    int_lock->lk_call_count = int_lock->lk_expected_count =
+        (int_lock->lockee_count * up_count);
+    initialize_internal_lock_variables(frame, this);
 
-        priv = this->private;
+    afr_lock_blocking(frame, this, 0);
 
-        list_for_each_entry (locked_fd, &priv->saved_fds, list) {
-                if (locked_fd->fd == fd) {
-                        found = 1;
-                        break;
-                }
-        }
-
-        return found;
+    return 0;
 }
 
-static int
-__afr_save_locked_fd (xlator_t *this, fd_t *fd)
+static int32_t
+afr_nb_internal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_private_t   *priv      = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-        int              ret       = 0;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    int child_index = 0;
+    int lockee_num = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        locked_fd = GF_CALLOC (1, sizeof (*locked_fd),
-                               gf_afr_mt_locked_fd);
-        if (!locked_fd) {
-                ret = -1;
-                goto out;
-        }
-
-        locked_fd->fd = fd;
-        INIT_LIST_HEAD (&locked_fd->list);
-
-        list_add_tail (&locked_fd->list, &priv->saved_fds);
-
-out:
-        return ret;
-}
-
-int
-afr_save_locked_fd (xlator_t *this, fd_t *fd)
-{
-        afr_private_t   *priv      = NULL;
-        int              ret       = 0;
+    child_index = ((long)cookie) % priv->child_count;
+    lockee_num = ((long)cookie) / priv->child_count;
 
-        priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        pthread_mutex_lock (&priv->mutex);
+    if (op_ret == 0 && local->transaction.type == AFR_DATA_TRANSACTION) {
+        LOCK(&local->inode->lock);
         {
-                if (__is_fd_saved (this, fd)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "fd=%p already saved", fd);
-                        goto unlock;
-                }
-
-                ret = __afr_save_locked_fd (this, fd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "fd=%p could not be saved", fd);
-                        goto unlock;
-                }
+            local->inode_ctx->lock_count++;
         }
-unlock:
-        pthread_mutex_unlock (&priv->mutex);
-
-        return ret;
-}
-
-static int
-afr_lock_recovery_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local     = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-
-        local = frame->local;
-
-        locked_fd = local->locked_fd;
-
-        STACK_DESTROY (frame->root);
-        afr_local_cleanup (local, this);
-
-        afr_save_locked_fd (this, locked_fd->fd);
-
-        return 0;
-
-}
-
-static int
-afr_get_source_lock_recovery (xlator_t *this, fd_t *fd)
-{
-        afr_fd_ctx_t  *fdctx        = NULL;
-        afr_private_t *priv         = NULL;
-        uint64_t      tmp           = 0;
-        int           i             = 0;
-        int           source_child  = -1;
-        int           ret           = 0;
+        UNLOCK(&local->inode->lock);
+    }
 
-        priv = this->private;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (fdctx->locked_on[i]) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Found lock recovery source=%d", i);
-                        source_child = i;
-                        break;
-                }
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0) {
+            if (op_errno == ENOSYS) {
+                /* return ENOTSUP */
+                gf_msg(this->name, GF_LOG_ERROR, ENOSYS,
+                       AFR_MSG_LOCK_XLATOR_NOT_LOADED,
+                       "subvolume does not support "
+                       "locking. please load features/locks"
+                       " xlator on server");
+                local->op_ret = op_ret;
+                int_lock->lock_op_ret = op_ret;
+
+                int_lock->lock_op_errno = op_errno;
+                local->op_errno = op_errno;
+            }
+        } else if (op_ret == 0) {
+            int_lock->lockee[lockee_num]
+                .locked_nodes[child_index] |= LOCKED_YES;
+            int_lock->lockee[lockee_num].locked_count++;
+            int_lock->lock_count++;
         }
 
-out:
-        return source_child;
-
-}
+        call_count = --int_lock->lk_call_count;
+    }
+    UNLOCK(&frame->lock);
 
-int32_t
-afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                      dict_t *xdata);
-int32_t
-afr_recover_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                      dict_t *xdata)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int32_t source_child = 0;
-        struct gf_flock flock   = {0,};
-
-        local = frame->local;
-        priv  = this->private;
-
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "lock recovery failed");
-                goto cleanup;
+    if (call_count == 0) {
+        gf_msg_trace(this->name, 0, "Last locking reply received");
+        /* all locks successful. Proceed to call FOP */
+        if (int_lock->lock_count == int_lock->lk_expected_count) {
+            gf_msg_trace(this->name, 0, "All servers locked. Calling the cbk");
+            int_lock->lock_op_ret = 0;
+            int_lock->lock_cbk(frame, this);
         }
+        /* Not all locks were successful. Unlock and try locking
+           again, this time with serially blocking locks */
+        else {
+            gf_msg_trace(this->name, 0,
+                         "%d servers locked. Trying again "
+                         "with blocking calls",
+                         int_lock->lock_count);
 
-        source_child = local->source_child;
-
-        memcpy (&flock, lock, sizeof (*lock));
-
-        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
-                           (void *) (long) source_child,
-                           priv->children[source_child],
-                           priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock, NULL);
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-        return 0;
-}
-
-int
-afr_recover_lock (call_frame_t *frame, xlator_t *this,
-                  struct gf_flock *flock)
-{
-        afr_local_t   *local             = NULL;
-        afr_private_t *priv              = NULL;
-        int32_t      lock_recovery_child = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        lock_recovery_child = local->lock_recovery_child;
-
-        frame->root->lk_owner = flock->l_owner;
-
-        STACK_WIND_COOKIE (frame, afr_recover_lock_cbk,
-                           (void *) (long) lock_recovery_child,
-                           priv->children[lock_recovery_child],
-                           priv->children[lock_recovery_child]->fops->lk,
-                           local->fd, F_SETLK, flock, NULL);
-
-        return 0;
-}
-
-static int
-is_afr_lock_eol (struct gf_flock *lock)
-{
-        int ret = 0;
-
-        if ((lock->l_type == GF_LK_EOL))
-                ret = 1;
-
-        return ret;
-}
-
-int32_t
-afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                      dict_t *xdata)
-{
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Failed to get locks on fd");
-                goto cleanup;
+            afr_unlock_now(frame, this);
         }
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Got a lock on fd");
-
-        if (is_afr_lock_eol (lock)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Reached EOL on locks on fd");
-                goto cleanup;
-        }
-
-        afr_recover_lock (frame, this, lock);
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-
-        return 0;
+    return 0;
 }
 
-static int
-afr_lock_recovery (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local        = NULL;
-        afr_private_t *priv         = NULL;
-        fd_t          *fd           = NULL;
-        int            ret          = 0;
-        int32_t        source_child = 0;
-        struct gf_flock   flock        = {0,};
-
-        priv  = this->private;
-        local = frame->local;
-
-        fd = local->fd;
-
-        source_child = afr_get_source_lock_recovery (this, fd);
-        if (source_child < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not recover locks due to lock "
-                        "split brain");
-                ret = -1;
-                goto out;
+int
+afr_lock_nonblocking(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int child = 0;
+    int lockee_num = 0;
+    int32_t call_count = 0;
+    int i = 0;
+    int ret = 0;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+
+    initialize_internal_lock_variables(frame, this);
+
+    if (local->fd) {
+        fd_ctx = afr_fd_ctx_get(local->fd, this);
+        if (!fd_ctx) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_FD_CTX_GET_FAILED,
+                   "unable to get fd ctx for fd=%p", local->fd);
+
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
+            local->op_errno = EINVAL;
+            int_lock->lock_op_errno = EINVAL;
+
+            afr_unlock_now(frame, this);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    call_count = int_lock->lockee_count * internal_lock_count(frame, this);
+    int_lock->lk_call_count = call_count;
+    int_lock->lk_expected_count = call_count;
+
+    if (!call_count) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INFO_COMMON,
+               "fd not open on any subvolumes. aborting.");
+        afr_unlock_now(frame, this);
+        goto out;
+    }
+
+    /* Send non-blocking lock calls only on up children
+       and where the fd has been opened */
+    for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+        child = i % priv->child_count;
+        lockee_num = i / priv->child_count;
+        if (local->child_up[child]) {
+            afr_internal_lock_wind(frame, afr_nb_internal_lock_cbk,
+                                   (void *)(long)i, child, lockee_num,
+                                   _gf_false, _gf_false);
+            if (!--call_count)
+                break;
         }
-
-        local->source_child = source_child;
-
-        /* the flock can be zero filled as we're querying incrementally
-           the locks held on the fd.
-        */
-        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
-                           (void *) (long) source_child,
-                           priv->children[source_child],
-                           priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock, NULL);
-
+    }
 out:
-        return ret;
-}
-
-
-static int
-afr_mark_fd_opened (xlator_t *this, fd_t *fd, int32_t child_index)
-{
-        afr_fd_ctx_t *fdctx = NULL;
-        uint64_t      tmp   = 0;
-        int           ret   = 0;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        fdctx->opened_on[child_index] = AFR_FD_OPENED;
-
-out:
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_lock_recovery_preopen_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                               int32_t op_ret, int32_t op_errno, fd_t *fd,
-                               dict_t *xdata)
-{
-        int32_t child_index = (long )cookie;
-        int ret = 0;
-
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Reopen during lock-recovery failed");
-                goto cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Open succeeded => proceed to recover locks");
-
-        ret = afr_lock_recovery (frame, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Lock recovery failed");
-                goto cleanup;
-        }
-
-        ret = afr_mark_fd_opened (this, fd, child_index);
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Marking fd open failed");
-                goto cleanup;
-        }
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-        return 0;
-}
-
-static int
-afr_lock_recovery_preopen (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv  = NULL;
-        afr_local_t   *local = NULL;
-        uint64_t       tmp   = 0;
-        afr_fd_ctx_t  *fdctx = NULL;
-        loc_t          loc   = {0,};
-        int32_t        child_index = 0;
-        int            ret   = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        GF_ASSERT (local && local->fd);
-
-        ret = fd_ctx_get (local->fd, this, &tmp);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to get the context of fd",
-                        uuid_utoa (local->fd->inode->gfid));
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-        /* TODO: instead we should return from the function */
-        GF_ASSERT (fdctx);
-
-        child_index = local->lock_recovery_child;
-
-        inode_path (local->fd->inode, NULL, (char **)&loc.path);
-        loc.name   = strrchr (loc.path, '/');
-        loc.inode  = inode_ref (local->fd->inode);
-        loc.parent = inode_parent (local->fd->inode, 0, NULL);
-
-
-        STACK_WIND_COOKIE (frame, afr_lock_recovery_preopen_cbk,
-                           (void *)(long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->open,
-                           &loc, fdctx->flags, local->fd, NULL);
-
-        return 0;
-}
-
-static int
-is_fd_opened (fd_t *fd, int32_t child_index)
-{
-        afr_fd_ctx_t *fdctx = NULL;
-        uint64_t      tmp = 0;
-        int           ret = 0;
-
-        ret = fd_ctx_get (fd, THIS, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        if (fdctx->opened_on[child_index] == AFR_FD_OPENED)
-                ret = 1;
-
-out:
-        return ret;
-}
-
-int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index)
-{
-        call_frame_t    *frame      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_local_t     *local      = NULL;
-        afr_locked_fd_t *locked_fd  = NULL;
-        afr_locked_fd_t  *tmp       = NULL;
-        int              ret        = -1;
-        struct list_head locks_list = {0,};
-        int32_t          op_errno   = 0;
-
-
-        priv = this->private;
-
-        if (list_empty (&priv->saved_fds))
-                goto out;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0) {
-                ret = -1;
-                goto out;
-        }
-
-        frame->local = local;
-
-        INIT_LIST_HEAD (&locks_list);
-
-        pthread_mutex_lock (&priv->mutex);
-        {
-                list_splice_init (&priv->saved_fds, &locks_list);
-        }
-        pthread_mutex_unlock (&priv->mutex);
-
-        list_for_each_entry_safe (locked_fd, tmp,
-                                  &locks_list, list) {
-
-                list_del_init (&locked_fd->list);
-
-                local->fd                  = fd_ref (locked_fd->fd);
-                local->lock_recovery_child = child_index;
-                local->locked_fd           = locked_fd;
-
-                if (!is_fd_opened (locked_fd->fd, child_index)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "attempting open before lock "
-                                "recovery");
-                        afr_lock_recovery_preopen (frame, this);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "attempting lock recovery "
-                                "without a preopen");
-                        afr_lock_recovery (frame, this);
-                }
-        }
+afr_unlock(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+
+    if (!local->transaction.eager_lock_on)
+        goto out;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        list_del_init(&local->transaction.owner_list);
+        if (list_empty(&lock->owners) && list_empty(&lock->post_op)) {
+            local->transaction.do_eager_unlock = _gf_true;
+            /*TODO: Need to get metadata use on_disk and inherit/uninherit
+             *GF_ASSERT (!local->inode_ctx->on_disk[local->transaction.type]);
+             *GF_ASSERT (!local->inode_ctx->inherited[local->transaction.type]);
+             */
+            GF_ASSERT(lock->release);
+        }
+    }
+    UNLOCK(&local->inode->lock);
+    if (!local->transaction.do_eager_unlock) {
+        local->internal_lock.lock_cbk(frame, this);
+        return 0;
+    }
 
 out:
-        if ((ret < 0) && frame)
-                AFR_STACK_DESTROY (frame);
-        return ret;
-}
-
-void
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src,
-                          unsigned int child_count)
-{
-        afr_local_t *dst_local = NULL;
-        afr_local_t *src_local = NULL;
-        afr_internal_lock_t *dst_lock = NULL;
-        afr_internal_lock_t *src_lock = NULL;
-
-        dst_local = dst->local;
-        dst_lock  = &dst_local->internal_lock;
-        src_local = src->local;
-        src_lock  = &src_local->internal_lock;
-        if (src_lock->inode_locked_nodes) {
-                memcpy (dst_lock->inode_locked_nodes,
-                        src_lock->inode_locked_nodes,
-                        sizeof (*dst_lock->inode_locked_nodes) * child_count);
-                memset (src_lock->inode_locked_nodes, 0,
-                        sizeof (*src_lock->inode_locked_nodes) * child_count);
-        }
-
-        dst_lock->transaction_lk_type = src_lock->transaction_lk_type;
-        dst_lock->selfheal_lk_type    = src_lock->selfheal_lk_type;
-        dst_lock->inodelk_lock_count = src_lock->inodelk_lock_count;
-        src_lock->inodelk_lock_count = 0;
+    afr_unlock_now(frame, this);
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index 7e684801fae..816065fb57a 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -8,40 +8,31 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __AFR_MEM_TYPES_H__
 #define __AFR_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_afr_mem_types_ {
-        gf_afr_mt_iovec  = gf_common_mt_end + 1,
-        gf_afr_mt_afr_fd_ctx_t,
-        gf_afr_mt_afr_private_t,
-        gf_afr_mt_int32_t,
-        gf_afr_mt_char,
-        gf_afr_mt_xattr_key,
-        gf_afr_mt_dict_t,
-        gf_afr_mt_xlator_t,
-        gf_afr_mt_iatt,
-        gf_afr_mt_int,
-        gf_afr_mt_afr_node_character,
-        gf_afr_mt_sh_diff_loop_state,
-        gf_afr_mt_uint8_t,
-        gf_afr_mt_loc_t,
-        gf_afr_mt_entry_name,
-        gf_afr_mt_pump_priv,
-        gf_afr_mt_locked_fd,
-        gf_afr_mt_inode_ctx_t,
-        gf_afr_fd_paused_call_t,
-        gf_afr_mt_crawl_data_t,
-        gf_afr_mt_brick_pos_t,
-        gf_afr_mt_shd_bool_t,
-        gf_afr_mt_shd_timer_t,
-        gf_afr_mt_shd_event_t,
-        gf_afr_mt_time_t,
-        gf_afr_mt_pos_data_t,
-        gf_afr_mt_end
+    gf_afr_mt_afr_fd_ctx_t = gf_common_mt_end + 1,
+    gf_afr_mt_afr_private_t,
+    gf_afr_mt_int32_t,
+    gf_afr_mt_char,
+    gf_afr_mt_xattr_key,
+    gf_afr_mt_dict_t,
+    gf_afr_mt_xlator_t,
+    gf_afr_mt_afr_node_character,
+    gf_afr_mt_inode_ctx_t,
+    gf_afr_mt_shd_event_t,
+    gf_afr_mt_reply_t,
+    gf_afr_mt_subvol_healer_t,
+    gf_afr_mt_spbc_timeout_t,
+    gf_afr_mt_spb_status_t,
+    gf_afr_mt_empty_brick_t,
+    gf_afr_mt_child_latency_t,
+    gf_afr_mt_atomic_t,
+    gf_afr_mt_lk_heal_info_t,
+    gf_afr_mt_gf_lock,
+    gf_afr_mt_end
 };
 #endif
-
diff --git a/xlators/cluster/afr/src/afr-messages.h b/xlators/cluster/afr/src/afr-messages.h
new file mode 100644
index 00000000000..e73fd997765
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-messages.h
@@ -0,0 +1,167 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _AFR_MESSAGES_H_
+#define _AFR_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    AFR, AFR_MSG_QUORUM_FAIL, AFR_MSG_QUORUM_MET, AFR_MSG_QUORUM_OVERRIDE,
+    AFR_MSG_INVALID_CHILD_UP, AFR_MSG_SUBVOL_UP, AFR_MSG_SUBVOLS_DOWN,
+    AFR_MSG_ENTRY_UNLOCK_FAIL, AFR_MSG_SPLIT_BRAIN, AFR_MSG_OPEN_FAIL,
+    AFR_MSG_UNLOCK_FAIL, AFR_MSG_REPLACE_BRICK_STATUS, AFR_MSG_GFID_NULL,
+    AFR_MSG_FD_CREATE_FAILED, AFR_MSG_DICT_SET_FAILED,
+    AFR_MSG_EXPUNGING_FILE_OR_DIR, AFR_MSG_MIGRATION_IN_PROGRESS,
+    AFR_MSG_CHILD_MISCONFIGURED, AFR_MSG_VOL_MISCONFIGURED,
+    AFR_MSG_INTERNAL_LKS_FAILED, AFR_MSG_INVALID_FD, AFR_MSG_LOCK_INFO,
+    AFR_MSG_LOCK_XLATOR_NOT_LOADED, AFR_MSG_FD_CTX_GET_FAILED,
+    AFR_MSG_INVALID_SUBVOL, AFR_MSG_PUMP_XLATOR_ERROR, AFR_MSG_SELF_HEAL_INFO,
+    AFR_MSG_READ_SUBVOL_ERROR, AFR_MSG_DICT_GET_FAILED, AFR_MSG_INFO_COMMON,
+    AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR, AFR_MSG_LOCAL_CHILD, AFR_MSG_INVALID_DATA,
+    AFR_MSG_INVALID_ARG, AFR_MSG_INDEX_DIR_GET_FAILED, AFR_MSG_FSYNC_FAILED,
+    AFR_MSG_FAVORITE_CHILD, AFR_MSG_SELF_HEAL_FAILED,
+    AFR_MSG_SPLIT_BRAIN_STATUS, AFR_MSG_ADD_BRICK_STATUS, AFR_MSG_NO_CHANGELOG,
+    AFR_MSG_TIMER_CREATE_FAIL, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+    AFR_MSG_INODE_CTX_GET_FAILED, AFR_MSG_THIN_ARB,
+    AFR_MSG_THIN_ARB_XATTROP_FAILED, AFR_MSG_THIN_ARB_LOC_POP_FAILED,
+    AFR_MSG_GET_PEND_VAL, AFR_MSG_THIN_ARB_SKIP_SHD, AFR_MSG_UNKNOWN_SET,
+    AFR_MSG_NO_XL_ID, AFR_MSG_SELF_HEAL_INFO_START,
+    AFR_MSG_SELF_HEAL_INFO_FINISH, AFR_MSG_INCRE_COUNT,
+    AFR_MSG_ADD_TO_OUTPUT_FAILED, AFR_MSG_SET_TIME_FAILED,
+    AFR_MSG_GFID_MISMATCH_DETECTED, AFR_MSG_GFID_HEAL_MSG,
+    AFR_MSG_THIN_ARB_LOOKUP_FAILED, AFR_MSG_DICT_CREATE_FAILED,
+    AFR_MSG_NO_MAJORITY_TO_RESOLVE, AFR_MSG_TYPE_MISMATCH,
+    AFR_MSG_SIZE_POLICY_NOT_APPLICABLE, AFR_MSG_NO_CHILD_SELECTED,
+    AFR_MSG_INVALID_CHILD, AFR_MSG_RESOLVE_CONFLICTING_DATA,
+    SERROR_GETTING_SRC_BRICK, SNO_DIFF_IN_MTIME, SNO_BIGGER_FILE,
+    SALL_BRICKS_UP_TO_RESOLVE, AFR_MSG_UNLOCK_FAILED, AFR_MSG_POST_OP_FAILED,
+    AFR_MSG_TA_FRAME_CREATE_FAILED, AFR_MSG_SET_KEY_XATTROP_FAILED,
+    AFR_MSG_BLOCKING_ENTRYLKS_FAILED, AFR_MSG_FOP_FAILED,
+    AFR_MSG_CLEAN_UP_FAILED, AFR_MSG_UNABLE_TO_FETCH, AFR_MSG_XATTR_SET_FAILED,
+    AFR_MSG_SPLIT_BRAIN_REPLICA, AFR_MSG_INODE_CTX_FAILED,
+    AFR_MSG_LOOKUP_FAILED, AFR_MSG_ALL_SUBVOLS_DOWN,
+    AFR_MSG_RELEASE_LOCK_FAILED, AFR_MSG_CLEAR_TIME_SPLIT_BRAIN,
+    AFR_MSG_READ_FAILED, AFR_MSG_LAUNCH_FAILED, AFR_MSG_READ_SUBVOL_NOT_UP,
+    AFR_MSG_LK_HEAL_DOM, AFR_MSG_NEW_BRICK, AFR_MSG_SPLIT_BRAIN_SET_FAILED,
+    AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED, AFR_MSG_HEALER_SPAWN_FAILED,
+    AFR_MSG_ADD_CRAWL_EVENT_FAILED, AFR_MSG_NULL_DEREF, AFR_MSG_SET_PEND_XATTR,
+    AFR_MSG_INTERNAL_ATTR);
+
+#define AFR_MSG_DICT_GET_FAILED_STR "Dict get failed"
+#define AFR_MSG_DICT_SET_FAILED_STR "Dict set failed"
+#define AFR_MSG_HEALER_SPAWN_FAILED_STR "Healer spawn failed"
+#define AFR_MSG_ADD_CRAWL_EVENT_FAILED_STR "Adding crawl event failed"
+#define AFR_MSG_INVALID_ARG_STR "Invalid argument"
+#define AFR_MSG_INDEX_DIR_GET_FAILED_STR "unable to get index-dir on "
+#define AFR_MSG_THIN_ARB_LOOKUP_FAILED_STR "Failed lookup on file"
+#define AFR_MSG_DICT_CREATE_FAILED_STR "Failed to create dict."
+#define AFR_MSG_THIN_ARB_XATTROP_FAILED_STR "Xattrop failed."
+#define AFR_MSG_THIN_ARB_LOC_POP_FAILED_STR                                    \
+    "Failed to populate loc for thin-arbiter"
+#define AFR_MSG_GET_PEND_VAL_STR "Error getting value of pending"
+#define AFR_MSG_THIN_ARB_SKIP_SHD_STR "I am not the god shd. skipping."
+#define AFR_MSG_UNKNOWN_SET_STR "Unknown set"
+#define AFR_MSG_NO_XL_ID_STR "xl does not have id"
+#define AFR_MSG_SELF_HEAL_INFO_START_STR "starting full sweep on"
+#define AFR_MSG_SELF_HEAL_INFO_FINISH_STR "finished full sweep on"
+#define AFR_MSG_INCRE_COUNT_STR "Could not increment the counter."
+#define AFR_MSG_ADD_TO_OUTPUT_FAILED_STR "Could not add to output"
+#define AFR_MSG_SET_TIME_FAILED_STR "Could not set time"
+#define AFR_MSG_GFID_HEAL_MSG_STR "Error setting gfid-heal-msg dict"
+#define AFR_MSG_NO_MAJORITY_TO_RESOLVE_STR                                     \
+    "No majority to resolve gfid split brain"
+#define AFR_MSG_GFID_MISMATCH_DETECTED_STR "Gfid mismatch dectected"
+#define AFR_MSG_SELF_HEAL_INFO_STR "performing selfheal"
+#define AFR_MSG_TYPE_MISMATCH_STR "TYPE mismatch"
+#define AFR_MSG_SIZE_POLICY_NOT_APPLICABLE_STR                                 \
+    "Size policy is not applicable to directories."
+#define AFR_MSG_NO_CHILD_SELECTED_STR                                          \
+    "No child selected by favorite-child policy"
+#define AFR_MSG_INVALID_CHILD_STR "Invalid child"
+#define AFR_MSG_RESOLVE_CONFLICTING_DATA_STR                                   \
+    "selected as authentic to resolve conflicting data"
+#define SERROR_GETTING_SRC_BRICK_STR "Error getting the source brick"
+#define SNO_DIFF_IN_MTIME_STR "No difference in mtime"
+#define SNO_BIGGER_FILE_STR "No bigger file"
+#define SALL_BRICKS_UP_TO_RESOLVE_STR                                          \
+    "All the bricks should be up to resolve the gfid split brain"
+#define AFR_MSG_UNLOCK_FAILED_STR "Failed to unlock"
+#define AFR_MSG_POST_OP_FAILED_STR "Post-op on thin-arbiter failed"
+#define AFR_MSG_TA_FRAME_CREATE_FAILED_STR "Failed to create ta_frame"
+#define AFR_MSG_SET_KEY_XATTROP_FAILED_STR "Could not set key during xattrop"
+#define AFR_MSG_BLOCKING_ENTRYLKS_FAILED_STR "Blocking entrylks failed"
+#define AFR_MSG_FSYNC_FAILED_STR "fsync failed"
+#define AFR_MSG_QUORUM_FAIL_STR "quorum is not met"
+#define AFR_MSG_FOP_FAILED_STR "Failing Fop"
+#define AFR_MSG_INVALID_SUBVOL_STR "not a subvolume"
+#define AFR_MSG_VOL_MISCONFIGURED_STR "Volume is dangling"
+#define AFR_MSG_CHILD_MISCONFIGURED_STR                                        \
+    "replicate translator needs more than one subvolume defined"
+#define AFR_MSG_CLEAN_UP_FAILED_STR "Failed to clean up healer threads"
+#define AFR_MSG_QUORUM_OVERRIDE_STR "overriding quorum-count"
+#define AFR_MSG_UNABLE_TO_FETCH_STR                                            \
+    "Unable to fetch afr-pending-xattr option from volfile. Falling back to "  \
+    "using client translator names"
+#define AFR_MSG_NULL_DEREF_STR "possible NULL deref"
+#define AFR_MSG_XATTR_SET_FAILED_STR "Cannot set xattr cookie key"
+#define AFR_MSG_SPLIT_BRAIN_STATUS_STR "Failed to create synctask"
+#define AFR_MSG_SUBVOLS_DOWN_STR "All subvolumes are not up"
+#define AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR_STR                                   \
+    "Failed to cancel split-brain choice"
+#define AFR_MSG_SPLIT_BRAIN_REPLICA_STR                                        \
+    "Cannot set replica. File is not in data/metadata split-brain"
+#define AFR_MSG_INODE_CTX_FAILED_STR "Failed to get inode_ctx"
+#define AFR_MSG_READ_SUBVOL_ERROR_STR "no read subvols"
+#define AFR_MSG_LOCAL_CHILD_STR "selecting local read-child"
+#define AFR_MSG_LOOKUP_FAILED_STR "Failed to lookup/create thin-arbiter id file"
+#define AFR_MSG_TIMER_CREATE_FAIL_STR                                          \
+    "Cannot create timer for delayed initialization"
+#define AFR_MSG_SUBVOL_UP_STR "Subvolume came back up; going online"
+#define AFR_MSG_ALL_SUBVOLS_DOWN_STR                                           \
+    "All subvolumes are down. Going offline until atleast one of them is up"
+#define AFR_MSG_RELEASE_LOCK_FAILED_STR "Failed to release lock"
+#define AFR_MSG_INVALID_CHILD_UP_STR "Received child_up from invalid subvolume"
+#define AFR_MSG_QUORUM_MET_STR "Client-quorum is met"
+#define AFR_MSG_EXPUNGING_FILE_OR_DIR_STR "expunging file or dir"
+#define AFR_MSG_SELF_HEAL_FAILED_STR "Invalid"
+#define AFR_MSG_SPLIT_BRAIN_STR "Skipping conservative mergeon the file"
+#define AFR_MSG_CLEAR_TIME_SPLIT_BRAIN_STR "clear time split brain"
+#define AFR_MSG_READ_FAILED_STR "Failing read since good brick is down"
+#define AFR_MSG_LAUNCH_FAILED_STR "Failed to launch synctask"
+#define AFR_MSG_READ_SUBVOL_NOT_UP_STR                                         \
+    "read subvolume in this generation is not up"
+#define AFR_MSG_INTERNAL_LKS_FAILED_STR                                        \
+    "Unable to work with lk-owner while attempting fop"
+#define AFR_MSG_LOCK_XLATOR_NOT_LOADED_STR                                     \
+    "subvolume does not support locking. please load features/locks xlator "   \
+    "on server."
+#define AFR_MSG_FD_CTX_GET_FAILED_STR "unable to get fd ctx"
+#define AFR_MSG_INFO_COMMON_STR "fd not open on any subvolumes, aborting."
+#define AFR_MSG_REPLACE_BRICK_STATUS_STR "Couldn't acquire lock on any child."
+#define AFR_MSG_NEW_BRICK_STR "New brick"
+#define AFR_MSG_SPLIT_BRAIN_SET_FAILED_STR                                     \
+    "Failed to set split-brain choice to -1"
+#define AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED_STR                               \
+    "Failed to determine split-brain. Aborting split-brain-choice set"
+#define AFR_MSG_OPEN_FAIL_STR "Failed to open subvolume"
+#define AFR_MSG_SET_PEND_XATTR_STR "Set of pending xattr"
+#define AFR_MSG_INTERNAL_ATTR_STR "is an internal extended attribute"
+#endif /* !_AFR_MESSAGES_H_ */
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index 35090d80e4b..64856042b65 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -8,438 +8,346 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "statedump.h"
-
-#include "fd.h"
-
-#include "afr-inode-read.h"
-#include "afr-inode-write.h"
-#include "afr-dir-read.h"
-#include "afr-dir-write.h"
-#include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-int
-afr_stale_child_up (afr_local_t *local, xlator_t *this)
-{
-        int             i = 0;
-        afr_private_t   *priv = NULL;
-        int             up = -1;
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
 
-        priv = this->private;
-
-        if (!local->fresh_children)
-                local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children)
-                goto out;
-
-        afr_inode_get_read_ctx (this, local->fd->inode, local->fresh_children);
-        if (priv->child_count == afr_get_children_count (local->fresh_children,
-                                                         priv->child_count))
-                goto out;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->child_up[i])
-                        continue;
-                if (afr_is_child_present (local->fresh_children,
-                                          priv->child_count, i))
-                        continue;
-                up = i;
-                break;
-        }
-out:
-        return up;
-}
+#include "afr-transaction.h"
 
-void
-afr_perform_data_self_heal (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_fd_fixable(fd_t *fd)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        inode_t         *inode = NULL;
-        int             st_child = -1;
-        char            reason[64] = {0};
-
-        local = frame->local;
-        sh = &local->self_heal;
-        inode = local->fd->inode;
-
-        if (!IA_ISREG (inode->ia_type))
-                goto out;
-
-        st_child = afr_stale_child_up (local, this);
-        if (st_child < 0)
-                goto out;
-
-        sh->do_data_self_heal          = _gf_true;
-        sh->do_metadata_self_heal      = _gf_true;
-        sh->do_gfid_self_heal          = _gf_true;
-        sh->do_missing_entry_self_heal = _gf_true;
-
-        snprintf (reason, sizeof (reason), "stale subvolume %d detected",
-                  st_child);
-        afr_launch_self_heal (frame, this, inode, _gf_true, inode->ia_type,
-                              reason, NULL, NULL);
-out:
-        return;
+    if (!fd || !fd->inode)
+        return _gf_false;
+    else if (fd_is_anonymous(fd))
+        return _gf_false;
+    else if (gf_uuid_is_null(fd->inode->gfid))
+        return _gf_false;
+
+    return _gf_true;
 }
 
 int
-afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+afr_open_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t * local = frame->local;
-        afr_private_t *priv = NULL;
-
-        priv = this->private;
-        if (afr_open_only_data_self_heal (priv->data_self_heal))
-                afr_perform_data_self_heal (frame, this);
-        AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                          local->fd, xdata);
-        return 0;
-}
+    afr_local_t *local = frame->local;
 
+    AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno,
+                     local->cont.open.fd, xdata);
+    return 0;
+}
 
 int
-afr_open_cbk (call_frame_t *frame, void *cookie,
-              xlator_t *this, int32_t op_ret, int32_t op_errno,
-              fd_t *fd, dict_t *xdata)
+afr_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *  local       = NULL;
-        int            ret         = 0;
-        int            call_count  = -1;
-        int            child_index = (long) cookie;
-        afr_private_t *priv        = NULL;
-
-        priv = this->private;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0) {
-                        local->op_ret = op_ret;
-                        local->success_count++;
-
-                        ret = afr_child_fd_ctx_set (this, fd, child_index,
-                                                    local->cont.open.flags);
-                        if (ret) {
-                                local->op_ret = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-                }
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = (long)cookie;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+        } else {
+            local->op_ret = op_ret;
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if ((local->cont.open.flags & O_TRUNC)
-                    && (local->op_ret >= 0)) {
-                        STACK_WIND (frame, afr_open_ftruncate_cbk,
-                                    this, this->fops->ftruncate,
-                                    fd, 0, NULL);
-                } else {
-                        if (afr_open_only_data_self_heal (priv->data_self_heal))
-                                afr_perform_data_self_heal (frame, this);
-                        AFR_STACK_UNWIND (open, frame, local->op_ret,
-                                          local->op_errno, local->fd, xdata);
-                }
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        afr_handle_replies_quorum(frame, this);
+        if (local->op_ret == -1) {
+            AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno, NULL,
+                             NULL);
+        } else if (fd_ctx->flags & O_TRUNC) {
+            STACK_WIND(frame, afr_open_ftruncate_cbk, this,
+                       this->fops->ftruncate, fd, 0, NULL);
+        } else {
+            AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno,
+                             local->cont.open.fd, local->xdata_rsp);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata)
+afr_open_continue(call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_private_t * priv       = NULL;
-        afr_local_t *   local      = NULL;
-        int             i          = 0;
-        int             ret        = -1;
-        int32_t         call_count = 0;
-        int32_t         op_errno   = 0;
-        int32_t         wind_flags = flags & (~O_TRUNC);
-        //We can't let truncation to happen outside transaction.
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
-
-        priv = this->private;
-
-        if (flags & (O_CREAT|O_TRUNC)) {
-                QUORUM_CHECK(open,out);
-        }
-
-        if (afr_is_split_brain (this, loc->inode)) {
-                /* self-heal failed */
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to open as split brain seen, returning EIO");
-                op_errno = EIO;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int i = 0;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+    local = frame->local;
+    priv = this->private;
 
-        call_count   = local->call_count;
-        loc_copy (&local->loc, loc);
-
-        local->cont.open.flags   = flags;
-
-        local->fd = fd_ref (fd);
+    if (err) {
+        AFR_STACK_UNWIND(open, frame, -1, err, NULL, NULL);
+    } else {
+        local->call_count = AFR_COUNT(local->child_up, priv->child_count);
+        call_count = local->call_count;
 
         for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->open,
-                                           loc, wind_flags, fd, xdata);
-
-                        if (!--call_count)
-                                break;
-                }
+            if (local->child_up[i]) {
+                STACK_WIND_COOKIE(frame, afr_open_cbk, (void *)(long)i,
+                                  priv->children[i],
+                                  priv->children[i]->fops->open, &local->loc,
+                                  (local->cont.open.flags & ~O_TRUNC),
+                                  local->cont.open.fd, local->xdata_req);
+                if (!--call_count)
+                    break;
+            }
         }
+    }
+    return 0;
+}
 
-        ret = 0;
+int
+afr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int spb_subvol = 0;
+    int event_generation = 0;
+    int ret = 0;
+    int32_t op_errno = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    // We can't let truncation to happen outside transaction.
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_OPEN;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        op_errno = afr_quorum_errno(priv);
+        goto out;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    local->inode = inode_ref(loc->inode);
+    loc_copy(&local->loc, loc);
+    local->fd_ctx = fd_ctx;
+    fd_ctx->flags = flags;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    local->cont.open.flags = flags;
+    local->cont.open.fd = fd_ref(fd);
+
+    ret = afr_inode_get_readable(frame, local->inode, this, NULL,
+                                 &event_generation, AFR_DATA_TRANSACTION);
+    if ((ret < 0) &&
+        (afr_split_brain_read_subvol_get(local->inode, this, NULL,
+                                         &spb_subvol) == 0) &&
+        spb_subvol < 0) {
+        afr_inode_refresh(frame, this, local->inode, local->inode->gfid,
+                          afr_open_continue);
+    } else {
+        afr_open_continue(frame, this, 0);
+    }
+
+    return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (open, frame, -1, op_errno, fd, xdata);
+    AFR_STACK_UNWIND(open, frame, -1, op_errno, fd, NULL);
 
-        return 0;
+    return 0;
 }
 
-//NOTE: this function should be called with holding the lock on
-//fd to which fd_ctx belongs
-void
-afr_get_resumable_calls (xlator_t *this, afr_fd_ctx_t *fd_ctx,
-                         struct list_head *list)
+int
+afr_openfd_fix_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, fd_t *fd,
+                        dict_t *xdata)
 {
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
-        afr_local_t           *call_local  = NULL;
-        afr_private_t         *priv        = NULL;
-        int                    i = 0;
-        gf_boolean_t           call = _gf_false;
-
-        priv = this->private;
-        list_for_each_entry_safe (paused_call, tmp, &fd_ctx->paused_calls,
-                                  call_list) {
-                call = _gf_true;
-                call_local = paused_call->frame->local;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (call_local->child_up[i] &&
-                            (fd_ctx->opened_on[i] == AFR_FD_OPENING))
-                                call = _gf_false;
-                }
-
-                if (call) {
-                        list_del_init (&paused_call->call_list);
-                        list_add (&paused_call->call_list, list);
-                }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int call_count = 0;
+    int child_index = (long)cookie;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (op_ret >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "fd for %s opened "
+                     "successfully on subvolume %s",
+                     local->loc.path, priv->children[child_index]->name);
+    } else {
+        gf_smsg(this->name, fop_log_level(GF_FOP_OPEN, op_errno), op_errno,
+                AFR_MSG_OPEN_FAIL, "path=%s", local->loc.path, "subvolume=%s",
+                priv->children[child_index]->name, NULL);
+    }
+
+    fd_ctx = local->fd_ctx;
+
+    LOCK(&local->fd->lock);
+    {
+        if (op_ret >= 0) {
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+        } else {
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
         }
+    }
+    UNLOCK(&local->fd->lock);
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        AFR_STACK_DESTROY(frame);
+
+    return 0;
 }
 
-void
-afr_resume_calls (xlator_t *this, struct list_head *list)
+static int
+afr_fd_ctx_need_open(fd_t *fd, xlator_t *this, unsigned char *need_open)
 {
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
-        afr_local_t           *call_local  = NULL;
-
-        list_for_each_entry_safe (paused_call, tmp, list, call_list) {
-                list_del_init (&paused_call->call_list);
-                call_local = paused_call->frame->local;
-                call_local->fop_call_continue (paused_call->frame, this);
-                GF_FREE (paused_call);
+    afr_fd_ctx_t *fd_ctx = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int count = 0;
+
+    priv = this->private;
+
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        return 0;
+
+    LOCK(&fd->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            if (fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED &&
+                priv->child_up[i]) {
+                fd_ctx->opened_on[i] = AFR_FD_OPENING;
+                need_open[i] = 1;
+                count++;
+            } else {
+                need_open[i] = 0;
+            }
         }
+    }
+    UNLOCK(&fd->lock);
+
+    return count;
 }
 
-int
-afr_openfd_fix_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+void
+afr_fix_open(fd_t *fd, xlator_t *this)
 {
-        afr_local_t           *local       = NULL;
-        afr_private_t         *priv        = NULL;
-        afr_fd_ctx_t          *fd_ctx      = NULL;
-        int                    call_count  = 0;
-        int                    child_index = (long) cookie;
-        struct list_head       paused_calls = {0};
-        gf_boolean_t           fop_paused = _gf_false;
-        fd_t                   *local_fd  = NULL;
-
-        priv     = this->private;
-        local    = frame->local;
-        fop_paused = local->fop_paused;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int32_t op_errno = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    unsigned char *need_open = NULL;
+    int call_count = 0;
 
-        if (op_ret >= 0) {
-                gf_log (this->name, GF_LOG_INFO, "fd for %s opened "
-                        "successfully on subvolume %s", local->loc.path,
-                        priv->children[child_index]->name);
-        }
+    priv = this->private;
 
-        local_fd = fd_ref (local->fd);
-        call_count = afr_frame_return (frame);
-        //Note: Do not access any thing using the frame outside call_count 0
+    if (!afr_is_fd_fixable(fd))
+        goto out;
 
-        //Note: No frame locking needed for this block of code
-        fd_ctx = afr_fd_ctx_get (local_fd, this);
-        if (!fd_ctx) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to get fd context, %p", local_fd);
-                goto out;
-        }
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        goto out;
 
-        LOCK (&local_fd->lock);
-        {
-                if (op_ret >= 0) {
-                        fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-                } else {
-                        fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-                }
-                if (call_count == 0) {
-                        INIT_LIST_HEAD (&paused_calls);
-                        afr_get_resumable_calls (this, fd_ctx, &paused_calls);
-                }
-        }
-        UNLOCK (&local_fd->lock);
-out:
-        if (call_count == 0) {
-                afr_resume_calls (this, &paused_calls);
-                //If the fop is paused then resume_calls will continue the fop
-                if (fop_paused)
-                        goto done;
-
-                if (local->fop_call_continue)
-                        local->fop_call_continue (frame, this);
-                else
-                        AFR_STACK_DESTROY (frame);
-        }
+    need_open = alloca0(priv->child_count);
 
-done:
-        fd_unref (local_fd);
-        return 0;
-}
+    call_count = afr_fd_ctx_need_open(fd, this, need_open);
+    if (!call_count)
+        goto out;
 
-int
-afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
-              int need_open_count, int *need_open)
-{
-        afr_local_t       *local = NULL;
-        afr_private_t     *priv  = NULL;
-        int               i      = 0;
-        call_frame_t      *open_frame = NULL;
-        afr_local_t      *open_local = NULL;
-        int               ret    = -1;
-        ia_type_t         ia_type = IA_INVAL;
-        int32_t           op_errno = 0;
-
-        GF_ASSERT (fd_ctx);
-        GF_ASSERT (need_open_count > 0);
-        GF_ASSERT (need_open);
-
-        local = frame->local;
-        priv  = this->private;
-        if (!local->fop_call_continue) {
-                open_frame = copy_frame (frame);
-                if (!open_frame) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-                AFR_LOCAL_ALLOC_OR_GOTO (open_frame->local, out);
-                open_local = open_frame->local;
-                ret = afr_local_init (open_local, priv, &op_errno);
-                if (ret < 0)
-                        goto out;
-                loc_copy (&open_local->loc, &local->loc);
-                open_local->fd = fd_ref (local->fd);
-        } else {
-                ret = 0;
-                open_frame = frame;
-                open_local = local;
-        }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
 
-        open_local->call_count = need_open_count;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        gf_log (this->name, GF_LOG_DEBUG, "need open count: %d",
-                need_open_count);
+    local->loc.inode = inode_ref(fd->inode);
+    ret = loc_path(&local->loc, NULL);
+    if (ret < 0)
+        goto out;
 
-        ia_type = open_local->fd->inode->ia_type;
-        GF_ASSERT (ia_type != IA_INVAL);
-        for (i = 0; i < priv->child_count; i++) {
-                if (!need_open[i])
-                        continue;
-                if (IA_IFDIR == ia_type) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "opening fd for dir %s on subvolume %s",
-                                local->loc.path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (open_frame, afr_openfd_fix_open_cbk,
-                                           (void*) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->opendir,
-                                           &open_local->loc, open_local->fd,
-                                           NULL);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "opening fd for file %s on subvolume %s",
-                                local->loc.path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (open_frame, afr_openfd_fix_open_cbk,
-                                           (void *)(long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->open,
-                                           &open_local->loc, fd_ctx->flags,
-                                           open_local->fd, NULL);
-                }
+    local->fd = fd_ref(fd);
+    local->fd_ctx = fd_ctx;
+
+    local->call_count = call_count;
+
+    gf_msg_debug(this->name, 0, "need open count: %d", call_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!need_open[i])
+            continue;
 
+        if (IA_IFDIR == fd->inode->ia_type) {
+            gf_msg_debug(this->name, 0, "opening fd for dir %s on subvolume %s",
+                         local->loc.path, priv->children[i]->name);
+
+            STACK_WIND_COOKIE(frame, afr_openfd_fix_open_cbk, (void *)(long)i,
+                              priv->children[i],
+                              priv->children[i]->fops->opendir, &local->loc,
+                              local->fd, NULL);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "opening fd for file %s on subvolume %s",
+                         local->loc.path, priv->children[i]->name);
+
+            STACK_WIND_COOKIE(frame, afr_openfd_fix_open_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->open,
+                              &local->loc, fd_ctx->flags & (~O_TRUNC),
+                              local->fd, NULL);
         }
-        op_errno = 0;
-        ret = 0;
+
+        if (!--call_count)
+            break;
+    }
+
+    return;
 out:
-        if (op_errno)
-                ret = -op_errno;
-        if (ret && open_frame)
-                AFR_STACK_DESTROY (open_frame);
-        return ret;
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 }
diff --git a/xlators/cluster/afr/src/afr-read-txn.c b/xlators/cluster/afr/src/afr-read-txn.c
new file mode 100644
index 00000000000..6fc2c75145c
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-read-txn.c
@@ -0,0 +1,494 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "afr.h"
+#include "afr-transaction.h"
+#include "afr-messages.h"
+
+void
+afr_pending_read_increment(afr_private_t *priv, int child_index)
+{
+    if (child_index < 0 || child_index > priv->child_count)
+        return;
+
+    GF_ATOMIC_INC(priv->pending_reads[child_index]);
+}
+
+void
+afr_pending_read_decrement(afr_private_t *priv, int child_index)
+{
+    if (child_index < 0 || child_index > priv->child_count)
+        return;
+
+    GF_ATOMIC_DEC(priv->pending_reads[child_index]);
+}
+
+void
+afr_read_txn_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    afr_pending_read_decrement(priv, local->read_subvol);
+    local->read_subvol = subvol;
+    afr_pending_read_increment(priv, subvol);
+    local->readfn(frame, this, subvol);
+}
+
+int
+afr_read_txn_next_subvol(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int subvol = -1;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->readable[i]) {
+            /* don't even bother trying here.
+               just mark as attempted and move on. */
+            local->read_attempted[i] = 1;
+            continue;
+        }
+
+        if (!local->read_attempted[i]) {
+            subvol = i;
+            break;
+        }
+    }
+
+    /* If no more subvols were available for reading, we leave
+       @subvol as -1, which is an indication we have run out of
+       readable subvols. */
+    if (subvol != -1)
+        local->read_attempted[subvol] = 1;
+    afr_read_txn_wind(frame, this, subvol);
+
+    return 0;
+}
+
+static int
+afr_ta_read_txn_done(int ret, call_frame_t *ta_frame, void *opaque)
+{
+    STACK_DESTROY(ta_frame->root);
+    return 0;
+}
+
+static int
+afr_ta_read_txn(void *opaque)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    int read_subvol = -1;
+    int query_child = AFR_CHILD_UNKNOWN;
+    int possible_bad_child = AFR_CHILD_UNKNOWN;
+    int ret = 0;
+    int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    dict_t *xdata_req = NULL;
+    dict_t *xdata_rsp = NULL;
+    int **pending = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    frame = (call_frame_t *)opaque;
+    this = frame->this;
+    local = frame->local;
+    priv = this->private;
+    query_child = local->read_txn_query_child;
+
+    if (query_child == AFR_CHILD_ZERO) {
+        possible_bad_child = AFR_CHILD_ONE;
+    } else if (query_child == AFR_CHILD_ONE) {
+        possible_bad_child = AFR_CHILD_ZERO;
+    } else {
+        /*read_txn_query_child is AFR_CHILD_UNKNOWN*/
+        goto out;
+    }
+
+    /* Ask the query_child to see if it blames the possibly bad one. */
+    xdata_req = dict_new();
+    if (!xdata_req)
+        goto out;
+
+    pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!pending)
+        goto out;
+
+    ret = afr_set_pending_dict(priv, xdata_req, pending);
+    if (ret < 0)
+        goto out;
+
+    if (local->fd) {
+        ret = syncop_fxattrop(priv->children[query_child], local->fd,
+                              GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                              NULL);
+    } else {
+        ret = syncop_xattrop(priv->children[query_child], &local->loc,
+                             GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                             NULL);
+    }
+    if (ret || !xdata_rsp) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed xattrop for gfid %s on %s",
+               uuid_utoa(local->inode->gfid),
+               priv->children[query_child]->name);
+        op_errno = -ret;
+        goto out;
+    }
+
+    if (afr_ta_dict_contains_pending_xattr(xdata_rsp, priv,
+                                           possible_bad_child)) {
+        read_subvol = query_child;
+        goto out;
+    }
+    dict_unref(xdata_rsp);
+    xdata_rsp = NULL;
+
+    /* It doesn't. So query thin-arbiter to see if it blames any data brick. */
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        goto out;
+    }
+    flock.l_type = F_WRLCK; /*start and length are already zero. */
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, &loc, F_SETLKW, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed to get AFR_TA_DOM_MODIFY lock on %s.",
+               uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+        op_errno = -ret;
+        goto out;
+    }
+
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                         GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                         NULL);
+    if (ret || !xdata_rsp) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed xattrop on %s.", uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+        op_errno = -ret;
+        goto unlock;
+    }
+
+    if (!afr_ta_dict_contains_pending_xattr(xdata_rsp, priv, query_child)) {
+        read_subvol = query_child;
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, EIO, AFR_MSG_THIN_ARB,
+               "Failing read for gfid %s since good brick %s is down",
+               uuid_utoa(local->inode->gfid),
+               priv->children[possible_bad_child]->name);
+        op_errno = EIO;
+    }
+
+unlock:
+    flock.l_type = F_UNLCK;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, &loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed to unlock AFR_TA_DOM_MODIFY lock on "
+               "%s.",
+               uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+    }
+out:
+    if (xdata_req)
+        dict_unref(xdata_req);
+    if (xdata_rsp)
+        dict_unref(xdata_rsp);
+    if (pending)
+        afr_matrix_cleanup(pending, priv->child_count);
+    loc_wipe(&loc);
+
+    if (read_subvol == -1) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
+    afr_read_txn_wind(frame, this, read_subvol);
+    return ret;
+}
+
+void
+afr_ta_read_txn_synctask(call_frame_t *frame, xlator_t *this)
+{
+    call_frame_t *ta_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        goto out;
+    }
+    ret = synctask_new(this->ctx->env, afr_ta_read_txn, afr_ta_read_txn_done,
+                       ta_frame, frame);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to launch "
+               "afr_ta_read_txn synctask for gfid %s.",
+               uuid_utoa(local->inode->gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        STACK_DESTROY(ta_frame->root);
+        goto out;
+    }
+    return;
+out:
+    afr_read_txn_wind(frame, this, -1);
+}
+
+int
+afr_read_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int read_subvol = -1;
+    inode_t *inode = NULL;
+    int ret = -1;
+    int spb_subvol = -1;
+
+    local = frame->local;
+    inode = local->inode;
+    priv = this->private;
+
+    if (err) {
+        if (!priv->thin_arbiter_count)
+            goto readfn;
+        if (err != EINVAL)
+            goto readfn;
+        /* We need to query the good bricks and/or thin-arbiter.*/
+        afr_ta_read_txn_synctask(frame, this);
+        return 0;
+    }
+
+    read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
+                                                   NULL);
+    if (read_subvol == -1) {
+        err = EIO;
+        goto readfn;
+    }
+
+    if (local->read_attempted[read_subvol]) {
+        afr_read_txn_next_subvol(frame, this);
+        return 0;
+    }
+
+    local->read_attempted[read_subvol] = 1;
+readfn:
+    if (read_subvol == -1) {
+        ret = afr_split_brain_read_subvol_get(inode, this, frame, &spb_subvol);
+        if ((ret == 0) && spb_subvol >= 0)
+            read_subvol = spb_subvol;
+    }
+
+    if (read_subvol == -1) {
+        AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, err);
+    }
+    afr_read_txn_wind(frame, this, read_subvol);
+
+    return 0;
+}
+
+int
+afr_read_txn_continue(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (!local->refreshed) {
+        local->refreshed = _gf_true;
+        afr_inode_refresh(frame, this, local->inode, NULL,
+                          afr_read_txn_refresh_done);
+    } else {
+        afr_read_txn_next_subvol(frame, this);
+    }
+
+    return 0;
+}
+
+/* afr_read_txn_wipe:
+
+   clean internal variables in @local in order to make
+   it possible to call afr_read_txn() multiple times from
+   the same frame
+*/
+
+void
+afr_read_txn_wipe(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->readfn = NULL;
+
+    if (local->inode)
+        inode_unref(local->inode);
+
+    for (i = 0; i < priv->child_count; i++) {
+        local->read_attempted[i] = 0;
+        local->readable[i] = 0;
+    }
+}
+
+/*
+  afr_read_txn:
+
+  This is the read transaction function. The way it works:
+
+  - Determine read-subvolume from inode ctx.
+
+  - If read-subvolume's generation was stale, refresh ctx once by
+    calling afr_inode_refresh()
+
+    Else make an attempt to read on read-subvolume.
+
+  - If attempted read on read-subvolume fails, refresh ctx once
+    by calling afr_inode_refresh()
+
+  - After ctx refresh, query read-subvolume freshly and attempt
+    read once.
+
+  - If read fails, try every other readable[] subvolume before
+    finally giving up. readable[] elements are set by afr_inode_refresh()
+    based on dirty and pending flags.
+
+  - If file is in split brain in the backend, generation will be
+    kept 0 by afr_inode_refresh() and readable[] will be set 0 for
+    all elements. Therefore reads always fail.
+*/
+
+int
+afr_read_txn(call_frame_t *frame, xlator_t *this, inode_t *inode,
+             afr_read_txn_wind_t readfn, afr_transaction_type type)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *data = NULL;
+    unsigned char *metadata = NULL;
+    int read_subvol = -1;
+    int event_generation = 0;
+    int ret = -1;
+
+    priv = this->private;
+    local = frame->local;
+    data = alloca0(priv->child_count);
+    metadata = alloca0(priv->child_count);
+
+    afr_read_txn_wipe(frame, this);
+
+    local->readfn = readfn;
+    local->inode = inode_ref(inode);
+    local->is_read_txn = _gf_true;
+    local->transaction.type = type;
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_quorum_errno(priv);
+        goto read;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &local->op_errno)) {
+        local->op_ret = -1;
+        goto read;
+    }
+
+    if (priv->thin_arbiter_count && !afr_ta_has_quorum(priv, local)) {
+        local->op_ret = -1;
+        local->op_errno = -afr_quorum_errno(priv);
+        goto read;
+    }
+
+    if (priv->thin_arbiter_count &&
+        AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) {
+        if (local->child_up[0]) {
+            local->read_txn_query_child = AFR_CHILD_ZERO;
+        } else if (local->child_up[1]) {
+            local->read_txn_query_child = AFR_CHILD_ONE;
+        }
+        afr_ta_read_txn_synctask(frame, this);
+        return 0;
+    }
+
+    ret = afr_inode_read_subvol_get(inode, this, data, metadata,
+                                    &event_generation);
+    if (ret == -1)
+        /* very first transaction on this inode */
+        goto refresh;
+    AFR_INTERSECT(local->readable, data, metadata, priv->child_count);
+
+    gf_msg_debug(this->name, 0,
+                 "%s: generation now vs cached: %d, "
+                 "%d",
+                 uuid_utoa(inode->gfid), local->event_generation,
+                 event_generation);
+    if (afr_is_inode_refresh_reqd(inode, this, local->event_generation,
+                                  event_generation))
+        /* servers have disconnected / reconnected, and possibly
+           rebooted, very likely changing the state of freshness
+           of copies */
+        goto refresh;
+
+    read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
+                                                   NULL);
+
+    if (read_subvol < 0 || read_subvol > priv->child_count) {
+        gf_msg_debug(this->name, 0,
+                     "Unreadable subvolume %d found "
+                     "with event generation %d for gfid %s.",
+                     read_subvol, event_generation, uuid_utoa(inode->gfid));
+        goto refresh;
+    }
+
+    if (!local->child_up[read_subvol]) {
+        /* should never happen, just in case */
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
+               "subvolume %d is the "
+               "read subvolume in this generation, but is not up",
+               read_subvol);
+        goto refresh;
+    }
+
+    local->read_attempted[read_subvol] = 1;
+
+read:
+    afr_read_txn_wind(frame, this, read_subvol);
+
+    return 0;
+
+refresh:
+    afr_inode_refresh(frame, this, inode, NULL, afr_read_txn_refresh_done);
+
+    return 0;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.c b/xlators/cluster/afr/src/afr-self-heal-algorithm.c
deleted file mode 100644
index 201e12c523f..00000000000
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.c
+++ /dev/null
@@ -1,809 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#include <openssl/md5.h>
-#include "glusterfs.h"
-#include "afr.h"
-#include "xlator.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "afr-self-heal-algorithm.h"
-
-/*
-  This file contains the various self-heal algorithms
-*/
-
-static int
-sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
-                gf_boolean_t is_first_call, call_frame_t *old_loop_frame);
-static int
-sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame,
-                int32_t op_ret, int32_t op_errno);
-static int
-sh_destroy_frame (call_frame_t *frame, xlator_t *this)
-{
-        if (!frame)
-                goto out;
-
-        AFR_STACK_DESTROY (frame);
-out:
-        return 0;
-}
-
-static void
-sh_private_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t             *local   = NULL;
-        afr_self_heal_t         *sh      = NULL;
-        afr_sh_algo_private_t   *sh_priv = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh_priv = sh->private;
-        GF_FREE (sh_priv);
-}
-
-static int
-sh_number_of_writes_needed (unsigned char *write_needed, int child_count)
-{
-        int writes = 0;
-        int i      = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (write_needed[i])
-                        writes++;
-        }
-
-        return writes;
-}
-
-
-static int
-sh_loop_driver_done (call_frame_t *sh_frame, xlator_t *this,
-                     call_frame_t *last_loop_frame)
-{
-        afr_local_t             *local        = NULL;
-        afr_self_heal_t         *sh           = NULL;
-        afr_sh_algo_private_t   *sh_priv      = NULL;
-        int32_t                 total_blocks = 0;
-        int32_t                 diff_blocks  = 0;
-
-        local        = sh_frame->local;
-        sh           = &local->self_heal;
-        sh_priv      = sh->private;
-        if (sh_priv) {
-                total_blocks = sh_priv->total_blocks;
-                diff_blocks  = sh_priv->diff_blocks;
-        }
-
-        sh_private_cleanup (sh_frame, this);
-        if (sh->op_failed) {
-                GF_ASSERT (!last_loop_frame);
-                //loop_finish should have happened and the old_loop should be NULL
-                gf_log (this->name, GF_LOG_INFO,
-                        "self-heal aborting on %s",
-                        local->loc.path);
-
-                local->self_heal.algo_abort_cbk (sh_frame, this);
-        } else {
-                GF_ASSERT (last_loop_frame);
-                if (diff_blocks == total_blocks) {
-                        gf_log (this->name, GF_LOG_INFO, "full self-heal "
-                                "completed on %s",local->loc.path);
-                } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "diff self-heal on %s: completed. "
-                                "(%d blocks of %d were different (%.2f%%))",
-                                local->loc.path, diff_blocks, total_blocks,
-                                ((diff_blocks * 1.0)/total_blocks) * 100);
-                }
-
-                sh->old_loop_frame = last_loop_frame;
-                local->self_heal.algo_completion_cbk (sh_frame, this);
-        }
-
-        return 0;
-}
-
-int
-sh_loop_finish (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_local_t             *loop_local = NULL;
-        afr_self_heal_t         *loop_sh = NULL;
-
-        if (!loop_frame)
-                goto out;
-
-        loop_local = loop_frame->local;
-        if (loop_local) {
-                loop_sh = &loop_local->self_heal;
-        }
-
-        if (loop_sh && loop_sh->data_lock_held) {
-                afr_sh_data_unlock (loop_frame, this,
-                                    sh_destroy_frame);
-        } else {
-                sh_destroy_frame (loop_frame, this);
-        }
-out:
-        return 0;
-}
-
-static int
-sh_loop_lock_success (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_local_t                 *loop_local = NULL;
-        afr_self_heal_t             *loop_sh    = NULL;
-
-        loop_local = loop_frame->local;
-        loop_sh = &loop_local->self_heal;
-
-        sh_loop_finish (loop_sh->old_loop_frame, this);
-        loop_sh->old_loop_frame = NULL;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Acquired lock for range %"PRIu64
-                " %"PRIu64, loop_sh->offset, loop_sh->block_size);
-        loop_sh->data_lock_held = _gf_true;
-        loop_sh->sh_data_algo_start (loop_frame, this);
-        return 0;
-}
-
-static int
-sh_loop_lock_failure (call_frame_t *loop_frame, xlator_t *this)
-{
-        call_frame_t                *sh_frame = NULL;
-        afr_local_t                 *loop_local = NULL;
-        afr_self_heal_t             *loop_sh    = NULL;
-
-        loop_local = loop_frame->local;
-        loop_sh = &loop_local->self_heal;
-        sh_frame = loop_sh->sh_frame;
-
-        gf_log (this->name, GF_LOG_ERROR, "failed lock for range %"PRIu64
-                " %"PRIu64, loop_sh->offset, loop_sh->block_size);
-        sh_loop_finish (loop_sh->old_loop_frame, this);
-        loop_sh->old_loop_frame = NULL;
-        sh_loop_return (sh_frame, this, loop_frame, -1, ENOTCONN);
-        return 0;
-}
-
-static int
-sh_loop_frame_create (call_frame_t *sh_frame, xlator_t *this,
-                      call_frame_t *old_loop_frame, call_frame_t **loop_frame)
-{
-        call_frame_t                *new_loop_frame = NULL;
-        afr_local_t                 *local          = NULL;
-        afr_self_heal_t             *sh             = NULL;
-        afr_local_t                 *new_loop_local = NULL;
-        afr_self_heal_t             *new_loop_sh    = NULL;
-        afr_private_t               *priv           = NULL;
-
-        GF_ASSERT (sh_frame);
-        GF_ASSERT (loop_frame);
-
-        *loop_frame = NULL;
-        local   = sh_frame->local;
-        sh      = &local->self_heal;
-        priv    = this->private;
-
-        new_loop_frame = copy_frame (sh_frame);
-        if (!new_loop_frame)
-                goto out;
-        //We want the frame to have same lk_owner as sh_frame
-        //so that locks translator allows conflicting locks
-        new_loop_local = afr_local_copy (local, this);
-        if (!new_loop_local)
-                goto out;
-        new_loop_frame->local = new_loop_local;
-
-        new_loop_sh = &new_loop_local->self_heal;
-        new_loop_sh->sources = memdup (sh->sources,
-                                       priv->child_count * sizeof (*sh->sources));
-        if (!new_loop_sh->sources)
-                goto out;
-        new_loop_sh->write_needed = GF_CALLOC (priv->child_count,
-                                               sizeof (*new_loop_sh->write_needed),
-                                               gf_afr_mt_char);
-        if (!new_loop_sh->write_needed)
-                goto out;
-        new_loop_sh->checksum = GF_CALLOC (priv->child_count, MD5_DIGEST_LENGTH,
-                                           gf_afr_mt_uint8_t);
-        if (!new_loop_sh->checksum)
-                goto out;
-        new_loop_sh->inode      = inode_ref (sh->inode);
-        new_loop_sh->sh_data_algo_start = sh->sh_data_algo_start;
-        new_loop_sh->source = sh->source;
-        new_loop_sh->active_sinks = sh->active_sinks;
-        new_loop_sh->healing_fd = fd_ref (sh->healing_fd);
-        new_loop_sh->file_has_holes = sh->file_has_holes;
-        new_loop_sh->old_loop_frame = old_loop_frame;
-        new_loop_sh->sh_frame = sh_frame;
-        *loop_frame = new_loop_frame;
-        return 0;
-out:
-        sh_destroy_frame (new_loop_frame, this);
-        return -ENOMEM;
-}
-
-static int
-sh_loop_start (call_frame_t *sh_frame, xlator_t *this, off_t offset,
-               call_frame_t *old_loop_frame)
-{
-        call_frame_t                *new_loop_frame = NULL;
-        afr_local_t                 *local          = NULL;
-        afr_self_heal_t             *sh             = NULL;
-        afr_local_t                 *new_loop_local = NULL;
-        afr_self_heal_t             *new_loop_sh    = NULL;
-        int                         ret             = 0;
-
-        GF_ASSERT (sh_frame);
-
-        local   = sh_frame->local;
-        sh      = &local->self_heal;
-
-        ret = sh_loop_frame_create (sh_frame, this, old_loop_frame,
-                                    &new_loop_frame);
-        if (ret)
-                goto out;
-        new_loop_local = new_loop_frame->local;
-        new_loop_sh = &new_loop_local->self_heal;
-        new_loop_sh->offset = offset;
-        new_loop_sh->block_size = sh->block_size;
-        afr_sh_data_lock (new_loop_frame, this, offset, new_loop_sh->block_size,
-                          sh_loop_lock_success, sh_loop_lock_failure);
-        return 0;
-out:
-        sh->op_failed = 1;
-        if (old_loop_frame)
-                sh_loop_finish (old_loop_frame, this);
-        sh_loop_return (sh_frame, this, new_loop_frame, -1, ENOMEM);
-        return 0;
-}
-
-static int
-sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
-                gf_boolean_t is_first_call, call_frame_t *old_loop_frame)
-{
-        afr_local_t *               local          = NULL;
-        afr_self_heal_t *           sh             = NULL;
-        afr_sh_algo_private_t       *sh_priv        = NULL;
-        gf_boolean_t                is_driver_done = _gf_false;
-        blksize_t                   block_size     = 0;
-        int                         loop           = 0;
-        off_t                       offset         = 0;
-        afr_private_t               *priv          = NULL;
-
-        priv    = this->private;
-        local   = sh_frame->local;
-        sh      = &local->self_heal;
-        sh_priv = sh->private;
-
-        LOCK (&sh_priv->lock);
-        {
-                if (!is_first_call)
-                        sh_priv->loops_running--;
-                offset = sh_priv->offset;
-                block_size = sh->block_size;
-                while ((!sh->eof_reached) && (0 == sh->op_failed) &&
-                       (sh_priv->loops_running < priv->data_self_heal_window_size)
-                       && (sh_priv->offset < sh->file_size)) {
-
-                        loop++;
-                        sh_priv->offset += block_size;
-                        sh_priv->loops_running++;
-
-                        if (!is_first_call)
-                                break;
-                }
-                if (0 == sh_priv->loops_running) {
-                        is_driver_done = _gf_true;
-                }
-        }
-        UNLOCK (&sh_priv->lock);
-
-        if (0 == loop) {
-                //loop finish does unlock, but the erasing of the pending
-                //xattrs needs to happen before that so do not finish the loop
-                if (is_driver_done && !sh->op_failed)
-                        goto driver_done;
-                if (old_loop_frame) {
-                        sh_loop_finish (old_loop_frame, this);
-                        old_loop_frame = NULL;
-                }
-        }
-
-        //If we have more loops to form we should finish previous loop after
-        //the next loop lock
-        while (loop--) {
-                if (sh->op_failed) {
-                        // op failed in other loop, stop spawning more loops
-                        if (old_loop_frame) {
-                                sh_loop_finish (old_loop_frame, this);
-                                old_loop_frame = NULL;
-                        }
-                        sh_loop_driver (sh_frame, this, _gf_false, NULL);
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE, "spawning a loop "
-                                "for offset %"PRId64, offset);
-
-                        sh_loop_start (sh_frame, this, offset, old_loop_frame);
-                        old_loop_frame = NULL;
-                        offset += block_size;
-                }
-        }
-
-driver_done:
-        if (is_driver_done) {
-                sh_loop_driver_done (sh_frame, this, old_loop_frame);
-        }
-        return 0;
-}
-
-static int
-sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame,
-                int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t *               loop_local = NULL;
-        afr_self_heal_t *           loop_sh    = NULL;
-        afr_local_t *               sh_local = NULL;
-        afr_self_heal_t            *sh       = NULL;
-
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        if (loop_frame) {
-                loop_local = loop_frame->local;
-                if (loop_local)
-                        loop_sh    = &loop_local->self_heal;
-                if (loop_sh)
-                        gf_log (this->name, GF_LOG_TRACE, "loop for offset "
-                                "%"PRId64" returned", loop_sh->offset);
-        }
-
-        if (op_ret == -1) {
-                sh->op_failed = 1;
-                afr_sh_set_error (sh, op_errno);
-                if (loop_frame) {
-                        sh_loop_finish (loop_frame, this);
-                        loop_frame = NULL;
-                }
-        }
-
-        sh_loop_driver (sh_frame, this, _gf_false, loop_frame);
-
-        return 0;
-}
-
-static int
-sh_loop_write_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        afr_private_t *             priv        = NULL;
-        afr_local_t *               loop_local    = NULL;
-        afr_self_heal_t *           loop_sh       = NULL;
-        call_frame_t               *sh_frame    = NULL;
-        afr_local_t *               sh_local    = NULL;
-        afr_self_heal_t            *sh          = NULL;
-        int                         call_count  = 0;
-        int                         child_index = 0;
-
-        priv     = this->private;
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        sh_frame = loop_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        child_index =  (long) cookie;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "wrote %d bytes of data from %s to child %d, offset %"PRId64"",
-                op_ret, sh_local->loc.path, child_index, loop_sh->offset);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "write to %s failed on subvolume %s (%s)",
-                        sh_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-
-                sh->op_failed = 1;
-                afr_sh_set_error (loop_sh, op_errno);
-        }
-
-        call_count = afr_frame_return (loop_frame);
-
-        if (call_count == 0) {
-                sh_loop_return (sh_frame, this, loop_frame,
-                                loop_sh->op_ret, loop_sh->op_errno);
-        }
-
-        return 0;
-}
-
-static void
-sh_prune_writes_needed (call_frame_t *sh_frame, call_frame_t *loop_frame,
-                        afr_private_t *priv)
-{
-        afr_local_t     *sh_local     = NULL;
-        afr_self_heal_t *sh           = NULL;
-        afr_local_t     *loop_local   = NULL;
-        afr_self_heal_t *loop_sh      = NULL;
-        int             i             = 0;
-
-        sh_local   = sh_frame->local;
-        sh         = &sh_local->self_heal;
-
-        if (!strcmp (sh->algo->name, "diff"))
-                return;
-
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        /* full self-heal guarantees there exists atleast 1 file with size 0
-         * That means for other files we can preserve holes that come after
-         * its size before 'trim'
-         */
-        for (i = 0; i < priv->child_count; i++) {
-                if (loop_sh->write_needed[i] &&
-                    ((loop_sh->offset + 1) > sh->buf[i].ia_size))
-                        loop_sh->write_needed[i] = 0;
-        }
-}
-
-static int
-sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  struct iovec *vector, int32_t count, struct iatt *buf,
-                  struct iobref *iobref, dict_t *xdata)
-{
-        afr_private_t *               priv       = NULL;
-        afr_local_t *                 loop_local   = NULL;
-        afr_self_heal_t *             loop_sh      = NULL;
-        call_frame_t                 *sh_frame   = NULL;
-        int                           i          = 0;
-        int                           call_count = 0;
-        afr_local_t *                 sh_local   = NULL;
-        afr_self_heal_t *             sh      = NULL;
-
-        priv       = this->private;
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        sh_frame = loop_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "read %d bytes of data from %s, offset %"PRId64"",
-                op_ret, loop_local->loc.path, loop_sh->offset);
-
-        if (op_ret <= 0) {
-                if (op_ret < 0) {
-                        sh->op_failed = 1;
-                        gf_log (this->name, GF_LOG_ERROR, "read failed on %d "
-                                "for %s reason :%s", sh->source,
-                                sh_local->loc.path, strerror (errno));
-                } else {
-                        sh->eof_reached = _gf_true;
-                        gf_log (this->name, GF_LOG_DEBUG, "Eof reached for %s",
-                                sh_local->loc.path);
-                }
-                sh_loop_return (sh_frame, this, loop_frame, op_ret, op_errno);
-                goto out;
-        }
-
-        if (loop_sh->file_has_holes && iov_0filled (vector, count) == 0)
-                sh_prune_writes_needed (sh_frame, loop_frame, priv);
-
-        call_count = sh_number_of_writes_needed (loop_sh->write_needed,
-                                                 priv->child_count);
-        if (call_count == 0) {
-                sh_loop_return (sh_frame, this, loop_frame, 0, 0);
-                goto out;
-        }
-        loop_local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!loop_sh->write_needed[i])
-                        continue;
-                STACK_WIND_COOKIE (loop_frame, sh_loop_write_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->writev,
-                                   loop_sh->healing_fd, vector, count,
-                                   loop_sh->offset, 0, iobref, NULL);
-
-                if (!--call_count)
-                        break;
-        }
-
-out:
-        return 0;
-}
-
-
-static int
-sh_loop_read (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_private_t           *priv       = NULL;
-        afr_local_t             *loop_local   = NULL;
-        afr_self_heal_t         *loop_sh      = NULL;
-
-        priv     = this->private;
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        STACK_WIND_COOKIE (loop_frame, sh_loop_read_cbk,
-                           (void *) (long) loop_sh->source,
-                           priv->children[loop_sh->source],
-                           priv->children[loop_sh->source]->fops->readv,
-                           loop_sh->healing_fd, loop_sh->block_size,
-                           loop_sh->offset, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      uint32_t weak_checksum, uint8_t *strong_checksum,
-                      dict_t *xdata)
-{
-        afr_private_t                 *priv         = NULL;
-        afr_local_t                   *loop_local   = NULL;
-        afr_self_heal_t               *loop_sh      = NULL;
-        call_frame_t                  *sh_frame     = NULL;
-        afr_local_t                   *sh_local     = NULL;
-        afr_self_heal_t               *sh           = NULL;
-        afr_sh_algo_private_t         *sh_priv      = NULL;
-        int                           child_index  = 0;
-        int                           call_count   = 0;
-        int                           i            = 0;
-        int                           write_needed = 0;
-
-        priv  = this->private;
-
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        sh_frame = loop_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        sh_priv = sh->private;
-
-        child_index = (long) cookie;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "checksum on %s failed on subvolume %s (%s)",
-                        sh_local->loc.path, priv->children[child_index]->name,
-                        strerror (op_errno));
-                sh->op_failed = 1;
-        } else {
-                memcpy (loop_sh->checksum + child_index * MD5_DIGEST_LENGTH,
-                        strong_checksum, MD5_DIGEST_LENGTH);
-        }
-
-        call_count = afr_frame_return (loop_frame);
-
-        if (call_count == 0) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (sh->sources[i] || !sh_local->child_up[i])
-                                continue;
-
-                        if (memcmp (loop_sh->checksum + (i * MD5_DIGEST_LENGTH),
-                                    loop_sh->checksum + (sh->source * MD5_DIGEST_LENGTH),
-                                    MD5_DIGEST_LENGTH)) {
-                                /*
-                                  Checksums differ, so this block
-                                  must be written to this sink
-                                */
-
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "checksum on subvolume %s at offset %"
-                                        PRId64" differs from that on source",
-                                        priv->children[i]->name, loop_sh->offset);
-
-                                write_needed = loop_sh->write_needed[i] = 1;
-                        }
-                }
-
-                LOCK (&sh_priv->lock);
-                {
-                        sh_priv->total_blocks++;
-                        if (write_needed)
-                                sh_priv->diff_blocks++;
-                }
-                UNLOCK (&sh_priv->lock);
-
-                if (write_needed && !sh->op_failed) {
-                        sh_loop_read (loop_frame, this);
-                } else {
-                        sh_loop_return (sh_frame, this, loop_frame,
-                                        op_ret, op_errno);
-                }
-        }
-
-        return 0;
-}
-
-static int
-sh_diff_checksum (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_private_t           *priv         = NULL;
-        afr_local_t             *loop_local   = NULL;
-        afr_self_heal_t         *loop_sh      = NULL;
-        int                     call_count    = 0;
-        int                     i             = 0;
-
-        priv         = this->private;
-        loop_local   = loop_frame->local;
-        loop_sh      = &loop_local->self_heal;
-
-        call_count = loop_sh->active_sinks + 1;  /* sinks and source */
-
-        loop_local->call_count = call_count;
-
-        STACK_WIND_COOKIE (loop_frame, sh_diff_checksum_cbk,
-                           (void *) (long) loop_sh->source,
-                           priv->children[loop_sh->source],
-                           priv->children[loop_sh->source]->fops->rchecksum,
-                           loop_sh->healing_fd,
-                           loop_sh->offset, loop_sh->block_size, NULL);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (loop_sh->sources[i] || !loop_local->child_up[i])
-                        continue;
-
-                STACK_WIND_COOKIE (loop_frame, sh_diff_checksum_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->rchecksum,
-                                   loop_sh->healing_fd,
-                                   loop_sh->offset, loop_sh->block_size, NULL);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
-}
-
-static int
-sh_full_read_write_to_sinks (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_private_t           *priv         = NULL;
-        afr_local_t             *loop_local   = NULL;
-        afr_self_heal_t         *loop_sh      = NULL;
-        int                     i             = 0;
-
-        priv         = this->private;
-        loop_local   = loop_frame->local;
-        loop_sh      = &loop_local->self_heal;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (loop_sh->sources[i] || !loop_local->child_up[i])
-                        continue;
-                loop_sh->write_needed[i] = 1;
-        }
-        sh_loop_read (loop_frame, this);
-        return 0;
-}
-
-afr_sh_algo_private_t*
-afr_sh_priv_init ()
-{
-        afr_sh_algo_private_t   *sh_priv = NULL;
-
-        sh_priv = GF_CALLOC (1, sizeof (*sh_priv),
-                             gf_afr_mt_afr_private_t);
-        if (!sh_priv)
-                goto out;
-
-        LOCK_INIT (&sh_priv->lock);
-out:
-        return sh_priv;
-}
-
-void
-afr_sh_transfer_lock (call_frame_t *dst, call_frame_t *src,
-                      unsigned int child_count)
-{
-        afr_local_t             *dst_local   = NULL;
-        afr_self_heal_t         *dst_sh      = NULL;
-        afr_local_t             *src_local   = NULL;
-        afr_self_heal_t         *src_sh      = NULL;
-
-        dst_local = dst->local;
-        dst_sh = &dst_local->self_heal;
-        src_local = src->local;
-        src_sh = &src_local->self_heal;
-        GF_ASSERT (src_sh->data_lock_held);
-        GF_ASSERT (!dst_sh->data_lock_held);
-        afr_lk_transfer_datalock (dst, src, child_count);
-        src_sh->data_lock_held = _gf_false;
-        dst_sh->data_lock_held = _gf_true;
-}
-
-int
-afr_sh_start_loops (call_frame_t *sh_frame, xlator_t *this,
-                    afr_sh_algo_fn sh_data_algo_start)
-{
-        call_frame_t            *first_loop_frame = NULL;
-        afr_local_t             *local   = NULL;
-        afr_self_heal_t         *sh      = NULL;
-        int                     ret      = 0;
-        afr_private_t           *priv    = NULL;
-
-        local = sh_frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        sh->sh_data_algo_start = sh_data_algo_start;
-        local->call_count = 0;
-        ret = sh_loop_frame_create (sh_frame, this, NULL, &first_loop_frame);
-        if (ret)
-                goto out;
-        afr_sh_transfer_lock (first_loop_frame, sh_frame, priv->child_count);
-        sh->private = afr_sh_priv_init ();
-        if (!sh->private) {
-                ret = -1;
-                goto out;
-        }
-        sh_loop_driver (sh_frame, this, _gf_true, first_loop_frame);
-        ret = 0;
-out:
-        if (ret) {
-                sh->op_failed = 1;
-                sh_loop_driver_done (sh_frame, this, NULL);
-        }
-        return 0;
-}
-
-int
-afr_sh_algo_diff (call_frame_t *sh_frame, xlator_t *this)
-{
-        afr_sh_start_loops (sh_frame, this, sh_diff_checksum);
-        return 0;
-}
-
-int
-afr_sh_algo_full (call_frame_t *sh_frame, xlator_t *this)
-{
-        afr_sh_start_loops (sh_frame, this, sh_full_read_write_to_sinks);
-        return 0;
-}
-
-struct afr_sh_algorithm afr_self_heal_algorithms[] = {
-        {.name = "full",  .fn = afr_sh_algo_full},
-        {.name = "diff",  .fn = afr_sh_algo_diff},
-        {0, 0},
-};
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.h b/xlators/cluster/afr/src/afr-self-heal-algorithm.h
deleted file mode 100644
index 6b20789b1bb..00000000000
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __AFR_SELF_HEAL_ALGORITHM_H__
-#define __AFR_SELF_HEAL_ALGORITHM_H__
-
-typedef int (*afr_sh_algo_fn) (call_frame_t *frame,
-                               xlator_t *this);
-
-struct afr_sh_algorithm {
-        const char *name;
-        afr_sh_algo_fn fn;
-};
-
-extern struct afr_sh_algorithm afr_self_heal_algorithms[3];
-typedef struct {
-        gf_lock_t lock;
-        unsigned int loops_running;
-        off_t offset;
-
-        int32_t total_blocks;
-        int32_t diff_blocks;
-} afr_sh_algo_private_t;
-
-#endif /* __AFR_SELF_HEAL_ALGORITHM_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index f082685e207..a580a1584cc 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,2508 +8,2927 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "byte-order.h"
-
 #include "afr.h"
-#include "afr-transaction.h"
-#include "afr-self-heal-common.h"
 #include "afr-self-heal.h"
-#include "pump.h"
-
-void
-afr_sh_reset (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
+#include <glusterfs/byte-order.h>
+#include "protocol-common.h"
+#include "afr-messages.h"
+#include <glusterfs/events.h>
 
-        memset (sh->child_errno, 0,
-                sizeof (*sh->child_errno) * priv->child_count);
-        memset (sh->buf, 0, sizeof (*sh->buf) * priv->child_count);
-        memset (sh->parentbufs, 0,
-                sizeof (*sh->parentbufs) * priv->child_count);
-        memset (sh->success, 0, sizeof (*sh->success) * priv->child_count);
-        memset (sh->locked_nodes, 0,
-                sizeof (*sh->locked_nodes) * priv->child_count);
-        sh->active_sinks = 0;
-
-        afr_reset_xattr (sh->xattr, priv->child_count);
-}
-
-//Intersection[child]=1 if child is part of intersection
 void
-afr_children_intersection_get (int32_t *set1, int32_t *set2,
-                               int *intersection, unsigned int child_count)
-{
-        int                      i = 0;
+afr_heal_synctask(xlator_t *this, afr_local_t *local);
 
-        memset (intersection, 0, sizeof (*intersection) * child_count);
-        for (i = 0; i < child_count; i++) {
-                intersection[i] = afr_is_child_present (set1, child_count, i)
-                                     && afr_is_child_present (set2, child_count,
-                                                              i);
+int
+afr_lookup_and_heal_gfid(xlator_t *this, inode_t *parent, const char *name,
+                         inode_t *inode, struct afr_reply *replies, int source,
+                         unsigned char *sources, void *gfid, int *gfid_idx)
+{
+    afr_private_t *priv = NULL;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    unsigned char *wind_on = NULL;
+    ia_type_t ia_type = IA_INVAL;
+    dict_t *xdata = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+    int i = 0;
+
+    priv = this->private;
+    wind_on = alloca0(priv->child_count);
+    if (source >= 0 && replies[source].valid && replies[source].op_ret == 0)
+        ia_type = replies[source].poststat.ia_type;
+
+    if (ia_type != IA_INVAL)
+        goto heal;
+
+    /* If ia_type is still invalid, it means either
+     * (a)'source' was -1, i.e. parent dir pending xattrs are in split-brain
+     * (or) (b) The parent dir pending xattrs are all zeroes (i.e. all bricks
+     * are sources) and the 'source' we selected earlier might be the one where
+     * the file is not actually present.
+     *
+     * In both cases, let us pick a brick with a successful reply and use its
+     * ia_type.
+     * */
+    for (i = 0; i < priv->child_count; i++) {
+        if (source == -1) {
+            /* case (a) above. */
+            if (replies[i].valid && replies[i].op_ret == 0 &&
+                replies[i].poststat.ia_type != IA_INVAL) {
+                ia_type = replies[i].poststat.ia_type;
+                break;
+            }
+        } else {
+            /* case (b) above. */
+            if (i == source)
+                continue;
+            if (sources[i] && replies[i].valid && replies[i].op_ret == 0 &&
+                replies[i].poststat.ia_type != IA_INVAL) {
+                ia_type = replies[i].poststat.ia_type;
+                break;
+            }
         }
-}
+    }
 
-/**
- * select_source - select a source and return it
- */
+heal:
+    /* gfid heal on those subvolumes that do not have gfid associated
+     * with the inode and update those replies.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-int
-afr_sh_select_source (int sources[], int child_count)
-{
-        int i = 0;
-        for (i = 0; i < child_count; i++)
-                if (sources[i])
-                        return i;
+        if (gf_uuid_is_null(gfid) &&
+            !gf_uuid_is_null(replies[i].poststat.ia_gfid) &&
+            replies[i].poststat.ia_type == ia_type)
+            gfid = replies[i].poststat.ia_gfid;
 
-        return -1;
-}
+        if (!gf_uuid_is_null(replies[i].poststat.ia_gfid) ||
+            replies[i].poststat.ia_type != ia_type)
+            continue;
 
-void
-afr_sh_mark_source_sinks (call_frame_t *frame, xlator_t *this)
-{
-        int              i = 0;
-        afr_local_t     *local      = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              active_sinks = 0;
+        wind_on[i] = 1;
+    }
 
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
+    if (AFR_COUNT(wind_on, priv->child_count) == 0)
+        return 0;
 
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_gfuuid(xdata, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    local = frame->local;
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
+
+    AFR_ONLIST(wind_on, frame, afr_selfheal_discover_cbk, lookup, &loc, xdata);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!wind_on[i])
+            continue;
+        afr_reply_wipe(&replies[i]);
+        afr_reply_copy(&replies[i], &local->replies[i]);
+    }
+    if (gfid_idx && (*gfid_idx == -1)) {
+        /*Pick a brick where the gifd heal was successful.*/
         for (i = 0; i < priv->child_count; i++) {
-                if (sh->sources[i] == 0 && local->child_up[i] == 1) {
-                        active_sinks++;
-                        sh->success[i] = 1;
-                } else if (sh->sources[i] == 1 && local->child_up[i] == 1) {
-                        sh->success[i] = 1;
-                }
+            if (!wind_on[i])
+                continue;
+            if (replies[i].valid && replies[i].op_ret == 0 &&
+                !gf_uuid_is_null(replies[i].poststat.ia_gfid)) {
+                *gfid_idx = i;
+                break;
+            }
         }
-        sh->active_sinks = active_sinks;
+    }
+out:
+    if (gfid_idx && (*gfid_idx == -1) && (ret == 0) && local) {
+        ret = -afr_final_errno(local, priv);
+    }
+    loc_wipe(&loc);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    if (xdata)
+        dict_unref(xdata);
+
+    return ret;
 }
 
 int
-afr_sh_source_count (int sources[], int child_count)
+afr_gfid_sbrain_source_from_src_brick(xlator_t *this, struct afr_reply *replies,
+                                      char *src_brick)
 {
-        int i = 0;
-        int nsource = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        for (i = 0; i < child_count; i++)
-                if (sources[i])
-                        nsource++;
-        return nsource;
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (strcmp(priv->children[i]->name, src_brick) == 0)
+            return i;
+    }
+    return -1;
 }
 
-void
-afr_sh_set_error (afr_self_heal_t *sh, int32_t op_errno)
-{
-        sh->op_ret = -1;
-        if (afr_error_more_important (sh->op_errno, op_errno))
-                sh->op_errno = op_errno;
-}
-
-void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this)
+int
+afr_selfheal_gfid_mismatch_by_majority(struct afr_reply *replies,
+                                       int child_count)
 {
-        afr_private_t *  priv = this->private;
-        char            *buf  = NULL;
-        char            *ptr  = NULL;
-        int              i    = 0;
-        int              j    = 0;
+    int j = 0;
+    int i = 0;
+    int votes;
 
-        /* 10 digits per entry + 1 space + '[' and ']' */
-        buf = GF_MALLOC (priv->child_count * 11 + 8, gf_afr_mt_char);
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ptr = buf;
-                ptr += sprintf (ptr, "[ ");
-                for (j = 0; j < priv->child_count; j++) {
-                        ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
-                }
-                sprintf (ptr, "]");
-                gf_log (this->name, GF_LOG_DEBUG, "pending_matrix: %s", buf);
+        votes = 1;
+        for (j = i + 1; j < child_count; j++) {
+            if ((!gf_uuid_compare(replies[i].poststat.ia_gfid,
+                                  replies[j].poststat.ia_gfid)))
+                votes++;
+            if (votes > child_count / 2)
+                return i;
         }
+    }
 
-        GF_FREE (buf);
+    return -1;
 }
 
-void
-afr_init_pending_matrix (int32_t **pending_matrix, size_t child_count)
+int
+afr_gfid_sbrain_source_from_bigger_file(struct afr_reply *replies,
+                                        int child_count)
 {
-        int             i   = 0;
-        int             j   = 0;
+    int i = 0;
+    int src = -1;
+    uint64_t size = 0;
 
-        GF_ASSERT (pending_matrix);
-
-        for (i = 0; i < child_count; i++) {
-                for (j = 0; j < child_count; j++) {
-                        pending_matrix[i][j] = 0;
-                }
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (size < replies[i].poststat.ia_size) {
+            src = i;
+            size = replies[i].poststat.ia_size;
+        } else if (replies[i].poststat.ia_size == size) {
+            src = -1;
         }
+    }
+    return src;
 }
 
-void
-afr_mark_ignorant_subvols_as_pending (int32_t **pending_matrix,
-                                      unsigned char *ignorant_subvols,
-                                      size_t  child_count)
-{
-        int            i                = 0;
-        int            j                = 0;
-
-        GF_ASSERT (pending_matrix);
-        GF_ASSERT (ignorant_subvols);
-
-        for (i = 0; i < child_count; i++) {
-                if (ignorant_subvols[i]) {
-                        for (j = 0; j < child_count; j++) {
-                                if (!ignorant_subvols[j])
-                                        pending_matrix[j][i] += 1;
-                        }
-                }
-        }
+int
+afr_gfid_sbrain_source_from_latest_mtime(struct afr_reply *replies,
+                                         int child_count)
+{
+    int i = 0;
+    int src = -1;
+    uint32_t mtime = 0;
+    uint32_t mtime_nsec = 0;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+        if ((mtime < replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
+            src = i;
+            mtime = replies[i].poststat.ia_mtime;
+            mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+        } else if ((mtime == replies[i].poststat.ia_mtime) &&
+                   (mtime_nsec == replies[i].poststat.ia_mtime_nsec)) {
+            src = -1;
+        }
+    }
+    return src;
 }
 
 int
-afr_build_pending_matrix (char **pending_key, int32_t **pending_matrix,
-                          unsigned char *ignorant_subvols,
-                          dict_t *xattr[], afr_transaction_type type,
-                          size_t child_count)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]       = {0,};
-        void          *pending_raw      = NULL;
-        int            ret              = -1;
-        int            i                = 0;
-        int            j                = 0;
-        int            k                = 0;
-
-        afr_init_pending_matrix (pending_matrix, child_count);
-
-        for (i = 0; i < child_count; i++) {
-                pending_raw = NULL;
-
-                for (j = 0; j < child_count; j++) {
-                        ret = dict_get_ptr (xattr[i], pending_key[j],
-                                            &pending_raw);
-
-                        if (ret != 0) {
-                                /*
-                                 * There is no xattr present. This means this
-                                 * subvolume should be considered an 'ignorant'
-                                 * subvolume.
-                                 */
-
-                                if (ignorant_subvols)
-                                        ignorant_subvols[i] = 1;
-                                continue;
-                        }
-
-                        memcpy (pending, pending_raw, sizeof(pending));
-                        k = afr_index_for_transaction_type (type);
-
-                        pending_matrix[i][j] = ntoh32 (pending[k]);
+afr_gfid_split_brain_source(xlator_t *this, struct afr_reply *replies,
+                            inode_t *inode, uuid_t pargfid, const char *bname,
+                            int src_idx, int child_idx,
+                            unsigned char *locked_on, int *src, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    char g1[64] = {
+        0,
+    };
+    char g2[64] = {
+        0,
+    };
+    int up_count = 0;
+    int heal_op = -1;
+    int ret = -1;
+    char *src_brick = NULL;
+
+    *src = -1;
+    priv = this->private;
+    up_count = AFR_COUNT(locked_on, priv->child_count);
+    if (up_count != priv->child_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "All the bricks should be up to resolve the gfid split "
+               "barin");
+        if (xdata) {
+            ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                           SALL_BRICKS_UP_TO_RESOLVE);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                       "Error setting"
+                       " gfid-heal-msg dict");
+        }
+        goto out;
+    }
+
+    if (xdata) {
+        ret = dict_get_int32_sizen(xdata, "heal-op", &heal_op);
+        if (ret)
+            goto fav_child;
+    } else {
+        goto fav_child;
+    }
+
+    switch (heal_op) {
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+            *src = afr_gfid_sbrain_source_from_bigger_file(replies,
+                                                           priv->child_count);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SNO_BIGGER_FILE);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SNO_BIGGER_FILE);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               " setting gfid-heal-msg dict");
                 }
-        }
-
-        return ret;
-}
-
-typedef enum {
-        AFR_NODE_INVALID,
-        AFR_NODE_INNOCENT,
-        AFR_NODE_FOOL,
-        AFR_NODE_WISE,
-} afr_node_type;
+            }
+            break;
+
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            *src = afr_gfid_sbrain_source_from_latest_mtime(replies,
+                                                            priv->child_count);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SNO_DIFF_IN_MTIME);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SNO_DIFF_IN_MTIME);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               "setting gfid-heal-msg dict");
+                }
+            }
+            break;
+
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_get_str_sizen(xdata, "child-name", &src_brick);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Error getting the source "
+                       "brick");
+                break;
+            }
+            *src = afr_gfid_sbrain_source_from_src_brick(this, replies,
+                                                         src_brick);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SERROR_GETTING_SRC_BRICK);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SERROR_GETTING_SRC_BRICK);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               " setting gfid-heal-msg dict");
+                }
+            }
+            break;
 
-typedef struct {
-        afr_node_type type;
-        int           wisdom;
-} afr_node_character;
+        default:
+            break;
+    }
+    goto out;
+
+fav_child:
+    switch (priv->fav_child_policy) {
+        case AFR_FAV_CHILD_BY_SIZE:
+            *src = afr_sh_fav_by_size(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_MTIME:
+            *src = afr_sh_fav_by_mtime(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_CTIME:
+            *src = afr_sh_fav_by_ctime(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_MAJORITY:
+            if (priv->child_count != 2)
+                *src = afr_selfheal_gfid_mismatch_by_majority(
+                    replies, priv->child_count);
+            else
+                *src = -1;
+
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "No majority to resolve "
+                       "gfid split brain");
+            }
+            break;
+        default:
+            break;
+    }
 
+out:
+    if (*src == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "Gfid mismatch detected for <gfid:%s>/%s>, %s on %s and"
+               " %s on %s.",
+               uuid_utoa(pargfid), bname,
+               uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1),
+               priv->children[child_idx]->name,
+               uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2),
+               priv->children[src_idx]->name);
+        gf_event(EVENT_AFR_SPLIT_BRAIN,
+                 "client-pid=%d;"
+                 "subvol=%s;type=gfid;file="
+                 "<gfid:%s>/%s>;count=2;child-%d=%s;gfid-%d=%s;"
+                 "child-%d=%s;gfid-%d=%s",
+                 this->ctx->cmd_args.client_pid, this->name, uuid_utoa(pargfid),
+                 bname, child_idx, priv->children[child_idx]->name, child_idx,
+                 uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1), src_idx,
+                 priv->children[src_idx]->name, src_idx,
+                 uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2));
+        return -1;
+    }
+    return 0;
+}
 
-static int
-afr_sh_is_innocent (int32_t *array, int child_count)
+int
+afr_selfheal_post_op_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int i   = 0;
-        int ret = 1;   /* innocent until proven guilty */
+    afr_local_t *local = NULL;
 
-        for (i = 0; i < child_count; i++) {
-                if (array[i]) {
-                        ret = 0;
-                        break;
-                }
-        }
-
-        return ret;
-}
+    local = frame->local;
 
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    syncbarrier_wake(&local->barrier);
 
-static int
-afr_sh_is_fool (int32_t *array, int i, int child_count)
-{
-        return array[i];   /* fool if accuses itself */
+    return 0;
 }
 
-
-static int
-afr_sh_is_wise (int32_t *array, int i, int child_count)
+int
+afr_selfheal_post_op(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     int subvol, dict_t *xattr, dict_t *xdata)
 {
-        return !array[i];  /* wise if does not accuse itself */
-}
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
 
+    priv = this->private;
+    local = frame->local;
 
-static int
-afr_sh_all_nodes_innocent (afr_node_character *characters,
-                           int child_count)
-{
-        int i   = 0;
-        int ret = 1;
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_INNOCENT) {
-                        ret = 0;
-                        break;
-                }
-        }
-
-        return ret;
-}
+    local->op_ret = 0;
 
+    STACK_WIND(frame, afr_selfheal_post_op_cbk, priv->children[subvol],
+               priv->children[subvol]->fops->xattrop, &loc,
+               GF_XATTROP_ADD_ARRAY, xattr, xdata);
 
-static int
-afr_sh_wise_nodes_exist (afr_node_character *characters, int child_count)
-{
-        int i   = 0;
-        int ret = 0;
+    syncbarrier_wait(&local->barrier, 1);
+    if (local->op_ret < 0)
+        ret = -local->op_errno;
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_WISE) {
-                        ret = 1;
-                        break;
-                }
-        }
+    loc_wipe(&loc);
+    local->op_ret = 0;
 
-        return ret;
+    return ret;
 }
 
+int
+afr_check_stale_error(struct afr_reply *replies, afr_private_t *priv)
+{
+    int i = 0;
+    int op_errno = 0;
+    int tmp_errno = 0;
+    int stale_count = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        tmp_errno = replies[i].op_errno;
+        if (tmp_errno == ENOENT || tmp_errno == ESTALE) {
+            op_errno = afr_higher_errno(op_errno, tmp_errno);
+            stale_count++;
+        }
+    }
+    if (stale_count != priv->child_count)
+        return -ENOTCONN;
+    else
+        return -op_errno;
+}
 
-/*
- * The 'wisdom' of a wise node is 0 if any other wise node accuses it.
- * It is 1 if no other wise node accuses it.
- * Only wise nodes with wisdom 1 are sources.
- *
- * If no nodes with wisdom 1 exist, a split-brain has occurred.
- */
-
-static void
-afr_sh_compute_wisdom (int32_t *pending_matrix[],
-                       afr_node_character characters[], int child_count)
+int
+afr_sh_generic_fop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata)
 {
-        int i = 0;
-        int j = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_WISE) {
-                        characters[i].wisdom = 1;
+    int i = (long)cookie;
+    afr_local_t *local = NULL;
 
-                        for (j = 0; j < child_count; j++) {
-                                if ((characters[j].type == AFR_NODE_WISE)
-                                    && pending_matrix[j][i]) {
+    local = frame->local;
 
-                                        characters[i].wisdom = 0;
-                                }
-                        }
-                }
-        }
-}
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (pre)
+        local->replies[i].prestat = *pre;
+    if (post)
+        local->replies[i].poststat = *post;
+    if (xdata)
+        local->replies[i].xdata = dict_ref(xdata);
 
+    syncbarrier_wake(&local->barrier);
 
-static int
-afr_sh_wise_nodes_conflict (afr_node_character *characters,
-                            int child_count)
-{
-        int i   = 0;
-        int ret = 1;
-
-        for (i = 0; i < child_count; i++) {
-                if ((characters[i].type == AFR_NODE_WISE)
-                    && characters[i].wisdom == 1) {
+    return 0;
+}
 
-                        /* There is atleast one bona-fide wise node */
-                        ret = 0;
-                        break;
-                }
+int
+afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          int source, unsigned char *healed_sinks,
+                          struct afr_reply *replies)
+{
+    loc_t loc = {
+        0,
+    };
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, setattr, &loc,
+               &replies[source].poststat,
+               (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME),
+               NULL);
+
+    loc_wipe(&loc);
+
+    return 0;
+}
+
+dict_t *
+afr_selfheal_output_xattr(xlator_t *this, gf_boolean_t is_full_crawl,
+                          afr_transaction_type type, int *output_dirty,
+                          int **output_matrix, int subvol,
+                          int **full_heal_mtx_out)
+{
+    int j = 0;
+    int idx = 0;
+    int d_idx = 0;
+    int ret = 0;
+    int *raw = 0;
+    dict_t *xattr = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    idx = afr_index_for_transaction_type(type);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    xattr = dict_new();
+    if (!xattr)
+        return NULL;
+
+    /* clear dirty */
+    raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+    if (!raw)
+        goto err;
+
+    raw[idx] = hton32(output_dirty[subvol]);
+    ret = dict_set_bin(xattr, AFR_DIRTY, raw,
+                       sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret) {
+        GF_FREE(raw);
+        goto err;
+    }
+
+    /* clear/set pending */
+    for (j = 0; j < priv->child_count; j++) {
+        raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+        if (!raw)
+            goto err;
+
+        raw[idx] = hton32(output_matrix[subvol][j]);
+        if (is_full_crawl)
+            raw[d_idx] = hton32(full_heal_mtx_out[subvol][j]);
+
+        ret = dict_set_bin(xattr, priv->pending_key[j], raw,
+                           sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            GF_FREE(raw);
+            goto err;
         }
+    }
 
-        return ret;
+    return xattr;
+err:
+    if (xattr)
+        dict_unref(xattr);
+    return NULL;
 }
 
+int
+afr_selfheal_undo_pending(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          unsigned char *sources, unsigned char *sinks,
+                          unsigned char *healed_sinks,
+                          unsigned char *undid_pending,
+                          afr_transaction_type type, struct afr_reply *replies,
+                          unsigned char *locked_on)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int j = 0;
+    unsigned char *pending = NULL;
+    int *input_dirty = NULL;
+    int **input_matrix = NULL;
+    int **full_heal_mtx_in = NULL;
+    int **full_heal_mtx_out = NULL;
+    int *output_dirty = NULL;
+    int **output_matrix = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    pending = alloca0(priv->child_count);
+
+    input_dirty = alloca0(priv->child_count * sizeof(int));
+    input_matrix = ALLOC_MATRIX(priv->child_count, int);
+    full_heal_mtx_in = ALLOC_MATRIX(priv->child_count, int);
+    full_heal_mtx_out = ALLOC_MATRIX(priv->child_count, int);
+    output_dirty = alloca0(priv->child_count * sizeof(int));
+    output_matrix = ALLOC_MATRIX(priv->child_count, int);
+
+    xdata = dict_new();
+    if (!xdata)
+        return -1;
 
-static int
-afr_sh_mark_wisest_as_sources (int sources[],
-                               afr_node_character *characters,
-                               int child_count)
-{
-        int nsources = 0;
-        int i        = 0;
+    afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].wisdom == 1) {
-                        sources[i] = 1;
-                        nsources++;
-                }
-        }
+    if (local->need_full_crawl)
+        afr_selfheal_extract_xattr(this, replies, AFR_DATA_TRANSACTION, NULL,
+                                   full_heal_mtx_in);
 
-        return nsources;
-}
+    for (i = 0; i < priv->child_count; i++)
+        if (sinks[i] && !healed_sinks[i])
+            pending[i] = 1;
 
-static void
-afr_compute_witness_of_fools (int32_t *witnesses, int32_t **pending_matrix,
-                              afr_node_character *characters,
-                              int32_t child_count)
-{
-        int i       = 0;
-        int j       = 0;
-        int witness = 0;
-
-        GF_ASSERT (witnesses);
-        GF_ASSERT (pending_matrix);
-        GF_ASSERT (characters);
-        GF_ASSERT (child_count > 0);
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_FOOL)
-                        continue;
-
-                witness = 0;
-                for (j = 0; j < child_count; j++) {
-                        if (i == j)
-                                continue;
-                        witness += pending_matrix[i][j];
-                }
-                witnesses[i] = witness;
+    for (i = 0; i < priv->child_count; i++) {
+        for (j = 0; j < priv->child_count; j++) {
+            if (pending[j]) {
+                output_matrix[i][j] = 1;
+                if (type == AFR_ENTRY_TRANSACTION)
+                    full_heal_mtx_out[i][j] = 1;
+            } else if (locked_on[j]) {
+                output_matrix[i][j] = -input_matrix[i][j];
+                if (type == AFR_ENTRY_TRANSACTION)
+                    full_heal_mtx_out[i][j] = -full_heal_mtx_in[i][j];
+            }
         }
-}
+    }
 
-static int32_t
-afr_find_biggest_witness_among_fools (int32_t *witnesses,
-                                      afr_node_character *characters,
-                                      int32_t child_count)
-{
-        int i               = 0;
-        int biggest_witness = -1;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!pending[i])
+            output_dirty[i] = -input_dirty[i];
+    }
 
-        GF_ASSERT (witnesses);
-        GF_ASSERT (characters);
-        GF_ASSERT (child_count > 0);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!locked_on[i])
+            /* perform post-op only on subvols we had locked
+               and inspected on.
+            */
+            continue;
+        if (undid_pending[i])
+            /* We already unset the pending xattrs in
+             * _afr_fav_child_reset_sink_xattrs(). */
+            continue;
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_FOOL)
-                        continue;
+        xattr = afr_selfheal_output_xattr(this, local->need_full_crawl, type,
+                                          output_dirty, output_matrix, i,
+                                          full_heal_mtx_out);
+        if (!xattr) {
+            continue;
+        }
 
-                if (biggest_witness < witnesses[i])
-                        biggest_witness = witnesses[i];
+        if ((type == AFR_ENTRY_TRANSACTION) && (priv->esh_granular)) {
+            if (xdata && dict_set_int8(xdata, GF_XATTROP_PURGE_INDEX, 1))
+                gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_DICT_SET_FAILED,
+                       "Failed to set"
+                       " dict value for %s",
+                       GF_XATTROP_PURGE_INDEX);
         }
-        return biggest_witness;
-}
 
-int
-afr_mark_fool_as_source_by_witness (int32_t *sources, int32_t *witnesses,
-                                    afr_node_character *characters,
-                                    int32_t child_count, int32_t witness)
-{
-        int i        = 0;
-        int nsources = 0;
+        afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
+        dict_unref(xattr);
+    }
 
-        GF_ASSERT (sources);
-        GF_ASSERT (witnesses);
-        GF_ASSERT (characters);
-        GF_ASSERT (child_count > 0);
+    if (xdata)
+        dict_unref(xdata);
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_FOOL)
-                        continue;
+    return 0;
+}
 
-                if (witness == witnesses[i]) {
-                        sources[i] = 1;
-                        nsources++;
-                }
-        }
-        return nsources;
+void
+afr_reply_copy(struct afr_reply *dst, struct afr_reply *src)
+{
+    dict_t *xdata = NULL;
+
+    dst->valid = src->valid;
+    dst->op_ret = src->op_ret;
+    dst->op_errno = src->op_errno;
+    dst->prestat = src->prestat;
+    dst->poststat = src->poststat;
+    dst->preparent = src->preparent;
+    dst->postparent = src->postparent;
+    dst->preparent2 = src->preparent2;
+    dst->postparent2 = src->postparent2;
+    if (src->xdata)
+        xdata = dict_ref(src->xdata);
+    else
+        xdata = NULL;
+    if (dst->xdata)
+        dict_unref(dst->xdata);
+    dst->xdata = xdata;
+    if (xdata && dict_get_str_boolean(xdata, "fips-mode-rchecksum",
+                                      _gf_false) == _gf_true) {
+        memcpy(dst->checksum, src->checksum, SHA256_DIGEST_LENGTH);
+    } else {
+        memcpy(dst->checksum, src->checksum, MD5_DIGEST_LENGTH);
+    }
+    dst->fips_mode_rchecksum = src->fips_mode_rchecksum;
 }
 
-static int
-afr_mark_biggest_of_fools_as_source (int32_t *sources, int32_t **pending_matrix,
-                                     afr_node_character *characters,
-                                     int child_count)
+void
+afr_replies_copy(struct afr_reply *dst, struct afr_reply *src, int count)
 {
-        int32_t       biggest_witness = 0;
-        int           nsources        = 0;
-        int32_t       *witnesses      = NULL;
+    int i = 0;
 
-        GF_ASSERT (child_count > 0);
-
-        witnesses = GF_CALLOC (child_count, sizeof (*witnesses),
-                               gf_afr_mt_int32_t);
-        if (NULL == witnesses) {
-                nsources = -1;
-                goto out;
-        }
+    if (dst == src)
+        return;
 
-        afr_compute_witness_of_fools (witnesses, pending_matrix, characters,
-                                      child_count);
-        biggest_witness = afr_find_biggest_witness_among_fools (witnesses,
-                                                                characters,
-                                                                child_count);
-        nsources = afr_mark_fool_as_source_by_witness (sources, witnesses,
-                                                       characters, child_count,
-                                                       biggest_witness);
-out:
-        GF_FREE (witnesses);
-        return nsources;
+    for (i = 0; i < count; i++) {
+        afr_reply_copy(&dst[i], &src[i]);
+    }
 }
 
 int
-afr_mark_child_as_source_by_uid (int32_t *sources, struct iatt *bufs,
-                                 int32_t *success_children,
-                                 unsigned int child_count, uint32_t uid)
-{
-        int     i        = 0;
-        int     nsources = 0;
-        int     child    = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (-1 == success_children[i])
-                        break;
-
-                child = success_children[i];
-                if (uid == bufs[child].ia_uid) {
-                        sources[child] = 1;
-                        nsources++;
-                }
-        }
-        return nsources;
-}
+afr_selfheal_fill_dirty(xlator_t *this, int *dirty, int subvol, int idx,
+                        dict_t *xdata)
+{
+    void *pending_raw = NULL;
+    int pending[3] = {
+        0,
+    };
 
-int
-afr_get_child_with_lowest_uid (struct iatt *bufs, int32_t *success_children,
-                               unsigned int child_count)
-{
-        int     i        = 0;
-        int     smallest = -1;
-        int     child    = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (-1 == success_children[i])
-                        break;
-                child = success_children[i];
-                if ((smallest == -1) ||
-                    (bufs[child].ia_uid < bufs[smallest].ia_uid)) {
-                        smallest = child;
-                }
-        }
-        return smallest;
-}
+    if (!dirty)
+        return 0;
 
-static int
-afr_sh_mark_lowest_uid_as_source (struct iatt *bufs, int32_t *success_children,
-                                  int child_count, int32_t *sources)
-{
-        int   nsources              = 0;
-        int   smallest              = 0;
+    if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw))
+        return -1;
 
-        smallest = afr_get_child_with_lowest_uid (bufs, success_children,
-                                                  child_count);
-        if (smallest < 0) {
-                nsources = -1;
-                goto out;
-        }
-        nsources = afr_mark_child_as_source_by_uid (sources, bufs,
-                                                    success_children, child_count,
-                                                    bufs[smallest].ia_uid);
-out:
-        return nsources;
-}
+    if (!pending_raw)
+        return -1;
 
-int
-afr_get_no_xattr_dir_read_child (xlator_t *this, int32_t *success_children,
-                                 struct iatt *bufs)
-{
-        afr_private_t *priv = NULL;
-        int            i = 0;
-        int            child = -1;
-        int            read_child = -1;
+    memcpy(pending, pending_raw, sizeof(pending));
 
-        priv = this->private;
-        for (i = 0; i < priv->child_count; i++) {
-                child = success_children[i];
-                if (child < 0)
-                        break;
-                if (read_child < 0)
-                        read_child = child;
-                else if (bufs[read_child].ia_size < bufs[child].ia_size)
-                        read_child = child;
-        }
-        return read_child;
+    dirty[subvol] = ntoh32(pending[idx]);
+
+    return 0;
 }
 
 int
-afr_sh_mark_zero_size_file_as_sink (struct iatt *bufs, int32_t *success_children,
-                                    int child_count, int32_t *sources)
-{
-        int             nsources = 0;
-        int             i = 0;
-        int             child = 0;
-        gf_boolean_t    sink_exists = _gf_false;
-        gf_boolean_t    source_exists = _gf_false;
-        int             source = -1;
-
-        for (i = 0; i < child_count; i++) {
-                child = success_children[i];
-                if (child < 0)
-                        break;
-                if (!bufs[child].ia_size) {
-                        sink_exists = _gf_true;
-                        continue;
-                }
-                if (!source_exists) {
-                        source_exists = _gf_true;
-                        source = child;
-                        continue;
-                }
-                if (bufs[source].ia_size != bufs[child].ia_size) {
-                        nsources = -1;
-                        goto out;
-                }
-        }
-        if (!source_exists && !sink_exists) {
-                nsources = -1;
-                goto out;
-        }
+afr_selfheal_fill_matrix(xlator_t *this, int **matrix, int subvol, int idx,
+                         dict_t *xdata)
+{
+    int i = 0;
+    void *pending_raw = NULL;
+    int pending[3] = {
+        0,
+    };
+    afr_private_t *priv = NULL;
 
-        if (!source_exists || !sink_exists)
-                goto out;
+    priv = this->private;
 
-        for (i = 0; i < child_count; i++) {
-                child = success_children[i];
-                if (child < 0)
-                        break;
-                if (bufs[child].ia_size) {
-                        sources[child] = 1;
-                        nsources++;
-                }
-        }
-out:
-        return nsources;
-}
+    if (!matrix)
+        return 0;
 
-char *
-afr_get_character_str (afr_node_type type)
-{
-        char *character = NULL;
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
 
-        switch (type) {
-        case AFR_NODE_INNOCENT:
-                character = "innocent";
-                break;
-        case AFR_NODE_FOOL:
-                character = "fool";
-                break;
-        case AFR_NODE_WISE:
-                character = "wise";
-                break;
-        default:
-                character = "invalid";
-                break;
-        }
-        return character;
-}
+        if (!pending_raw)
+            continue;
 
-afr_node_type
-afr_find_child_character_type (int32_t *pending_row, int32_t child,
-                               unsigned int child_count)
-{
-        afr_node_type type = AFR_NODE_INVALID;
+        memcpy(pending, pending_raw, sizeof(pending));
 
-        GF_ASSERT ((child >= 0) && (child < child_count));
+        matrix[subvol][i] = ntoh32(pending[idx]);
+    }
 
-        if (afr_sh_is_innocent (pending_row, child_count))
-                type = AFR_NODE_INNOCENT;
-        else if (afr_sh_is_fool (pending_row, child, child_count))
-                type = AFR_NODE_FOOL;
-        else if (afr_sh_is_wise (pending_row, child, child_count))
-                type = AFR_NODE_WISE;
-        return type;
+    return 0;
 }
 
 int
-afr_build_sources (xlator_t *this, dict_t **xattr, struct iatt *bufs,
-                   int32_t **pending_matrix, int32_t *sources,
-                   int32_t *success_children, afr_transaction_type type,
-                   int32_t *subvol_status, gf_boolean_t ignore_ignorant)
+afr_selfheal_extract_xattr(xlator_t *this, struct afr_reply *replies,
+                           afr_transaction_type type, int *dirty, int **matrix)
 {
-        afr_private_t           *priv = NULL;
-        afr_self_heal_type      sh_type    = AFR_SELF_HEAL_INVALID;
-        int                     nsources   = -1;
-        unsigned char           *ignorant_subvols = NULL;
-        unsigned int            child_count = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    dict_t *xdata = NULL;
+    int idx = -1;
 
-        priv = this->private;
-        child_count = priv->child_count;
+    idx = afr_index_for_transaction_type(type);
 
-        if (afr_get_children_count (success_children, priv->child_count) == 0)
-                goto out;
+    priv = this->private;
 
-        if (!ignore_ignorant) {
-                ignorant_subvols = GF_CALLOC (sizeof (*ignorant_subvols),
-                                              child_count, gf_afr_mt_char);
-                if (NULL == ignorant_subvols)
-                        goto out;
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-        afr_build_pending_matrix (priv->pending_key, pending_matrix,
-                                  ignorant_subvols, xattr, type,
-                                  priv->child_count);
+        if (!replies[i].xdata)
+            continue;
 
-        if (!ignore_ignorant)
-                afr_mark_ignorant_subvols_as_pending (pending_matrix,
-                                                      ignorant_subvols,
-                                                      priv->child_count);
-        sh_type = afr_self_heal_type_for_transaction (type);
-        if (AFR_SELF_HEAL_INVALID == sh_type)
-                goto out;
+        xdata = replies[i].xdata;
 
-        afr_sh_print_pending_matrix (pending_matrix, this);
+        afr_selfheal_fill_dirty(this, dirty, i, idx, xdata);
+        afr_selfheal_fill_matrix(this, matrix, i, idx, xdata);
+    }
 
-        nsources = afr_mark_sources (this, sources, pending_matrix, bufs,
-                                     sh_type, success_children, subvol_status);
-out:
-        GF_FREE (ignorant_subvols);
-        return nsources;
+    return 0;
 }
 
+/*
+ * If by chance there are multiple sources with differing sizes, select
+ * the largest file as the source.
+ *
+ * This can happen if data was directly modified in the backend or for snapshots
+ */
 void
-afr_find_character_types (afr_node_character *characters,
-                          int32_t **pending_matrix, int32_t *success_children,
-                          unsigned int child_count)
-{
-        afr_node_type type  = AFR_NODE_INVALID;
-        int           child = 0;
-        int           i     = 0;
-
-        for (i = 0; i < child_count; i++) {
-                child = success_children[i];
-                if (child == -1)
-                        break;
-                type = afr_find_child_character_type (pending_matrix[child],
-                                                      child, child_count);
-                characters[child].type = type;
+afr_mark_largest_file_as_source(xlator_t *this, unsigned char *sources,
+                                struct afr_reply *replies)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t size = 0;
+
+    /* Find source with biggest file size */
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (!replies[i].valid || replies[i].op_ret != 0) {
+            sources[i] = 0;
+            continue;
         }
+        if (size <= replies[i].poststat.ia_size) {
+            size = replies[i].poststat.ia_size;
+        }
+    }
+
+    /* Mark sources with less size as not source */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (size > replies[i].poststat.ia_size)
+            sources[i] = 0;
+    }
 }
 
 void
-afr_mark_success_children_sources (int32_t *sources, int32_t *success_children,
-                                   unsigned int child_count)
-{
-        int i = 0;
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                sources[success_children[i]] = 1;
-        }
+afr_mark_latest_mtime_file_as_source(xlator_t *this, unsigned char *sources,
+                                     struct afr_reply *replies)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint32_t mtime = 0;
+    uint32_t mtime_nsec = 0;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (!replies[i].valid || replies[i].op_ret != 0) {
+            sources[i] = 0;
+            continue;
+        }
+        if ((mtime < replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
+            mtime = replies[i].poststat.ia_mtime;
+            mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+        }
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if ((mtime > replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec > replies[i].poststat.ia_mtime_nsec))) {
+            sources[i] = 0;
+        }
+    }
 }
-/**
- * mark_sources: Mark all 'source' nodes and return number of source
- * nodes found
- *
- * A node (a row in the pending matrix) belongs to one of
- * three categories:
- *
- * M is the pending matrix.
- *
- * 'innocent' - M[i] is all zeroes
- * 'fool'     - M[i] has i'th element = 1 (self-reference)
- * 'wise'     - M[i] has i'th element = 0, others are 1 or 0.
- *
- * All 'innocent' nodes are sinks. If all nodes are innocent, no self-heal is
- * needed.
- *
- * A 'wise' node can be a source. If two 'wise' nodes conflict, it is
- * a split-brain. If one wise node refers to the other but the other doesn't
- * refer back, the referrer is a source.
- *
- * All fools are sinks, unless there are no 'wise' nodes. In that case,
- * one of the fools is made a source.
- */
 
-int
-afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
-                  struct iatt *bufs, afr_self_heal_type type,
-                  int32_t *success_children, int32_t *subvol_status)
-{
-        /* stores the 'characters' (innocent, fool, wise) of the nodes */
-        afr_node_character *characters =  NULL;
-        int                nsources    = -1;
-        unsigned int       child_count = 0;
-        afr_private_t      *priv       = NULL;
-
-        priv = this->private;
-        child_count = priv->child_count;
-        characters = GF_CALLOC (sizeof (afr_node_character),
-                                child_count, gf_afr_mt_afr_node_character);
-        if (!characters)
-                goto out;
+void
+afr_mark_active_sinks(xlator_t *this, unsigned char *sources,
+                      unsigned char *locked_on, unsigned char *sinks)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        this = THIS;
-
-        /* start clean */
-        memset (sources, 0, sizeof (*sources) * child_count);
-        nsources = 0;
-        afr_find_character_types (characters, pending_matrix, success_children,
-                                  child_count);
-        if (afr_sh_all_nodes_innocent (characters, child_count)) {
-                switch (type) {
-                case AFR_SELF_HEAL_METADATA:
-                        nsources = afr_sh_mark_lowest_uid_as_source (bufs,
-                                                             success_children,
-                                                             child_count,
-                                                             sources);
-                        break;
-                case AFR_SELF_HEAL_DATA:
-                        nsources = afr_sh_mark_zero_size_file_as_sink (bufs,
-                                                             success_children,
-                                                             child_count,
-                                                             sources);
-                        if ((nsources < 0) && subvol_status)
-                                *subvol_status |= SPLIT_BRAIN;
-                        break;
-                default:
-                        break;
-                }
-                goto out;
-        }
+    priv = this->private;
 
-        if (afr_sh_wise_nodes_exist (characters, child_count)) {
-                afr_sh_compute_wisdom (pending_matrix, characters, child_count);
-
-                if (afr_sh_wise_nodes_conflict (characters, child_count)) {
-                        if (subvol_status)
-                                *subvol_status |= SPLIT_BRAIN;
-                        nsources = -1;
-                } else {
-                        nsources = afr_sh_mark_wisest_as_sources (sources,
-                                                                  characters,
-                                                                  child_count);
-                }
-        } else {
-                if (subvol_status)
-                        *subvol_status |= ALL_FOOLS;
-                nsources = afr_mark_biggest_of_fools_as_source (sources,
-                                                                pending_matrix,
-                                                                characters,
-                                                                child_count);
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] && locked_on[i])
+            sinks[i] = 1;
+        else
+            sinks[i] = 0;
+    }
+}
 
-out:
-        if (nsources == 0)
-                afr_mark_success_children_sources (sources, success_children,
-                                                   child_count);
-        GF_FREE (characters);
+gf_boolean_t
+afr_dict_contains_heal_op(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    dict_t *xdata_req = NULL;
+    int ret = 0;
+    int heal_op = -1;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Number of sources: %d", nsources);
-        return nsources;
+    local = frame->local;
+    xdata_req = local->xdata_req;
+    ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
+    if (ret)
+        return _gf_false;
+    if (local->xdata_rsp == NULL) {
+        local->xdata_rsp = dict_new();
+        if (!local->xdata_rsp)
+            return _gf_true;
+    }
+    ret = dict_set_sizen_str_sizen(local->xdata_rsp, "sh-fail-msg",
+                                   SFILE_NOT_IN_SPLIT_BRAIN);
+
+    return _gf_true;
 }
 
-void
-afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
-                         int32_t *delta_matrix[], unsigned char success[],
-                         int child_count, afr_transaction_type type)
+gf_boolean_t
+afr_can_decide_split_brain_source_sinks(struct afr_reply *replies,
+                                        int child_count)
 {
-        int i = 0;
-        int j = 0;
+    int i = 0;
+
+    for (i = 0; i < child_count; i++)
+        if (replies[i].valid != 1 || replies[i].op_ret != 0)
+            return _gf_false;
 
-        afr_build_pending_matrix (priv->pending_key, delta_matrix, NULL,
-                                  xattr, type, priv->child_count);
-        for (i = 0; i < priv->child_count; i++)
-                for (j = 0; j < priv->child_count; j++)
-                        delta_matrix[i][j] = -delta_matrix[i][j];
+    return _gf_true;
 }
 
+int
+afr_mark_split_brain_source_sinks_by_heal_op(
+    call_frame_t *frame, xlator_t *this, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type, int heal_op)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+    dict_t *xdata_rsp = NULL;
+    int ret = 0;
+    int i = 0;
+    char *name = NULL;
+    int source = -1;
+
+    local = frame->local;
+    priv = this->private;
+    xdata_req = local->xdata_req;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (locked_on[i])
+            if (sources[i] || !sinks[i] || !healed_sinks[i]) {
+                ret = -1;
+                goto out;
+            }
+    }
+    if (local->xdata_rsp == NULL) {
+        local->xdata_rsp = dict_new();
+        if (!local->xdata_rsp) {
+            ret = -1;
+            goto out;
+        }
+    }
+    xdata_rsp = local->xdata_rsp;
+
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                       SBRAIN_HEAL_NO_GO_MSG);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++)
+        if (locked_on[i])
+            sources[i] = 1;
+    switch (heal_op) {
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+            if (type == AFR_METADATA_TRANSACTION) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SUSE_SOURCE_BRICK_TO_HEAL);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            afr_mark_largest_file_as_source(this, sources, replies);
+            if (AFR_COUNT(sources, priv->child_count) != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SNO_BIGGER_FILE);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            if (type == AFR_METADATA_TRANSACTION) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SUSE_SOURCE_BRICK_TO_HEAL);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            afr_mark_latest_mtime_file_as_source(this, sources, replies);
+            if (AFR_COUNT(sources, priv->child_count) != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SNO_DIFF_IN_MTIME);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_get_str_sizen(xdata_req, "child-name", &name);
+            if (ret)
+                goto out;
+            source = afr_get_child_index_from_name(this, name);
+            if (source < 0) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SINVALID_BRICK_NAME);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            if (locked_on[source] != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SBRICK_IS_NOT_UP);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            memset(sources, 0, sizeof(*sources) * priv->child_count);
+            sources[source] = 1;
+            break;
+        default:
+            ret = -1;
+            goto out;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
+        }
+    }
+    sinks[source] = 0;
+    healed_sinks[source] = 0;
+    ret = source;
+out:
+    if (ret < 0)
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+    return ret;
+}
 
 int
-afr_sh_delta_to_xattr (xlator_t *this,
-                       int32_t *delta_matrix[], dict_t *xattr[],
-                       int child_count, afr_transaction_type type)
-{
-        int              i       = 0;
-        int              j       = 0;
-        int              k       = 0;
-        int              ret     = 0;
-        int32_t         *pending = NULL;
-        int32_t         *local_pending = NULL;
-        afr_private_t   *priv = NULL;
-
-        priv = this->private;
-        for (i = 0; i < child_count; i++) {
-                if (!xattr[i])
-                        continue;
-
-                local_pending = NULL;
-                for (j = 0; j < child_count; j++) {
-                        pending = GF_CALLOC (sizeof (int32_t), 3,
-                                             gf_afr_mt_int32_t);
-
-                        if (!pending)
-                                continue;
-                        /* 3 = data+metadata+entry */
-
-                        k = afr_index_for_transaction_type (type);
-
-                        pending[k] = hton32 (delta_matrix[i][j]);
-
-                        if (j == i) {
-                                local_pending = pending;
-                                continue;
-                        }
-                        ret = dict_set_bin (xattr[i], priv->pending_key[j],
-                                            pending,
-                                        AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Unable to set dict value.");
-                                GF_FREE (pending);
-                        }
-                }
-                if (local_pending) {
-                        ret = dict_set_bin (xattr[i], priv->pending_key[i],
-                                            local_pending,
-                                        AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Unable to set dict value.");
-                                GF_FREE (local_pending);
-                        }
+afr_sh_fav_by_majority(xlator_t *this, struct afr_reply *replies,
+                       inode_t *inode)
+{
+    afr_private_t *priv;
+    int vote_count = -1;
+    int fav_child = -1;
+    int i = 0;
+    int k = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s mtime_sec = %" PRId64 ", size = %" PRIu64
+                         " for gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_mtime,
+                         replies[i].poststat.ia_size, uuid_utoa(inode->gfid));
+            vote_count = 0;
+            for (k = 0; k < priv->child_count; k++) {
+                if ((replies[k].poststat.ia_mtime ==
+                     replies[i].poststat.ia_mtime) &&
+                    (replies[k].poststat.ia_size ==
+                     replies[i].poststat.ia_size)) {
+                    vote_count++;
                 }
+            }
+            if (vote_count > priv->child_count / 2) {
+                fav_child = i;
+                break;
+            }
         }
-        return 0;
+    }
+    return fav_child;
 }
 
-
+/*
+ * afr_sh_fav_by_mtime: Choose favorite child by mtime.
+ */
 int
-afr_sh_has_metadata_pending (dict_t *xattr, xlator_t *this)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
-
-                if (ret != 0)
-                        return 0;
-
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-
-                if (pending[j])
-                        return 1;
-        }
+afr_sh_fav_by_mtime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint32_t cmp_mtime = 0;
+    uint32_t cmp_mtime_nsec = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s mtime = %" PRId64
+                         ", mtime_nsec = %d for "
+                         "gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_mtime,
+                         replies[i].poststat.ia_mtime_nsec,
+                         uuid_utoa(inode->gfid));
+            if (replies[i].poststat.ia_mtime > cmp_mtime) {
+                cmp_mtime = replies[i].poststat.ia_mtime;
+                cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+                fav_child = i;
+            } else if ((replies[i].poststat.ia_mtime == cmp_mtime) &&
+                       (replies[i].poststat.ia_mtime_nsec > cmp_mtime_nsec)) {
+                cmp_mtime = replies[i].poststat.ia_mtime;
+                cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+                fav_child = i;
+            }
+        }
+    }
+    return fav_child;
+}
 
-        return 0;
+/*
+ * afr_sh_fav_by_ctime: Choose favorite child by ctime.
+ */
+int
+afr_sh_fav_by_ctime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint32_t cmp_ctime = 0;
+    uint32_t cmp_ctime_nsec = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s ctime = %" PRId64
+                         ", ctime_nsec = %d for "
+                         "gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_ctime,
+                         replies[i].poststat.ia_ctime_nsec,
+                         uuid_utoa(inode->gfid));
+            if (replies[i].poststat.ia_ctime > cmp_ctime) {
+                cmp_ctime = replies[i].poststat.ia_ctime;
+                cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
+                fav_child = i;
+            } else if ((replies[i].poststat.ia_ctime == cmp_ctime) &&
+                       (replies[i].poststat.ia_ctime_nsec > cmp_ctime_nsec)) {
+                cmp_ctime = replies[i].poststat.ia_ctime;
+                cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
+                fav_child = i;
+            }
+        }
+    }
+    return fav_child;
 }
 
+/*
+ * afr_sh_fav_by_size: Choose favorite child by size
+ * when not all files are of zero size.
+ */
+int
+afr_sh_fav_by_size(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint64_t cmp_sz = 0;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid) {
+            continue;
+        }
+        gf_msg_debug(this->name, 0,
+                     "Child:%s file size = %" PRIu64 " for gfid %s",
+                     priv->children[i]->name, replies[i].poststat.ia_size,
+                     uuid_utoa(inode->gfid));
+        if (replies[i].poststat.ia_type == IA_IFDIR) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+                   "Cannot perform selfheal on %s. "
+                   "Size policy is not applicable to directories.",
+                   uuid_utoa(inode->gfid));
+            break;
+        }
+        if (replies[i].poststat.ia_size > cmp_sz) {
+            cmp_sz = replies[i].poststat.ia_size;
+            fav_child = i;
+        } else if (replies[i].poststat.ia_size == cmp_sz) {
+            fav_child = -1;
+        }
+    }
+    if (fav_child == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "No bigger file");
+    }
+    return fav_child;
+}
 
 int
-afr_sh_has_data_pending (dict_t *xattr, xlator_t *this)
+afr_sh_get_fav_by_policy(xlator_t *this, struct afr_reply *replies,
+                         inode_t *inode, char **policy_str)
 {
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
+    afr_private_t *priv = NULL;
+    int fav_child = -1;
 
-        priv = this->private;
+    priv = this->private;
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        return -1;
+    }
+
+    switch (priv->fav_child_policy) {
+        case AFR_FAV_CHILD_BY_SIZE:
+            fav_child = afr_sh_fav_by_size(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "SIZE";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_CTIME:
+            fav_child = afr_sh_fav_by_ctime(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "CTIME";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_MTIME:
+            fav_child = afr_sh_fav_by_mtime(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "MTIME";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_MAJORITY:
+            fav_child = afr_sh_fav_by_majority(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "MAJORITY";
+            }
+            break;
+        case AFR_FAV_CHILD_NONE:
+        default:
+            break;
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
+    return fav_child;
+}
 
-                if (ret != 0)
-                        return 0;
+int
+afr_mark_split_brain_source_sinks_by_policy(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type)
+{
+    afr_private_t *priv = NULL;
+    int fav_child = -1;
+    char mtime_str[256];
+    char ctime_str[256];
+    char *policy_str = NULL;
+    struct tm *tm_ptr;
+    time_t time;
+
+    priv = this->private;
+
+    fav_child = afr_sh_get_fav_by_policy(this, replies, inode, &policy_str);
+    if (fav_child == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "No child selected by favorite-child policy.");
+    } else if (fav_child > priv->child_count - 1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "Invalid child (%d) "
+               "selected by policy %s.",
+               fav_child, policy_str);
+    } else if (fav_child >= 0) {
+        time = replies[fav_child].poststat.ia_mtime;
+        tm_ptr = localtime(&time);
+        strftime(mtime_str, sizeof(mtime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
+        time = replies[fav_child].poststat.ia_ctime;
+        tm_ptr = localtime(&time);
+        strftime(ctime_str, sizeof(ctime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "Source %s selected as authentic to resolve conflicting data "
+               "in file (gfid:%s) by %s (%" PRIu64
+               " bytes @ %s mtime, %s "
+               "ctime).",
+               priv->children[fav_child]->name, uuid_utoa(inode->gfid),
+               policy_str, replies[fav_child].poststat.ia_size, mtime_str,
+               ctime_str);
+
+        sources[fav_child] = 1;
+        sinks[fav_child] = 0;
+        healed_sinks[fav_child] = 0;
+    }
+    return fav_child;
+}
 
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
+gf_boolean_t
+afr_is_file_empty_on_all_children(afr_private_t *priv,
+                                  struct afr_reply *replies)
+{
+    int i = 0;
 
-                if (pending[j])
-                        return 1;
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if ((!replies[i].valid) || (replies[i].op_ret != 0) ||
+            (replies[i].poststat.ia_size != 0))
+            return _gf_false;
+    }
 
-        return 0;
+    return _gf_true;
 }
 
+int
+afr_mark_source_sinks_if_file_empty(xlator_t *this, unsigned char *sources,
+                                    unsigned char *sinks,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on,
+                                    struct afr_reply *replies,
+                                    afr_transaction_type type)
+{
+    int source = -1;
+    int i = 0;
+    afr_private_t *priv = this->private;
+    struct iatt stbuf = {
+        0,
+    };
+
+    if ((AFR_COUNT(locked_on, priv->child_count) < priv->child_count) ||
+        (afr_success_count(replies, priv->child_count) < priv->child_count))
+        return -1;
 
+    if (type == AFR_DATA_TRANSACTION) {
+        if (!afr_is_file_empty_on_all_children(priv, replies))
+            return -1;
+        goto mark;
+    }
+
+    /*For AFR_METADATA_TRANSACTION, metadata must be same on all bricks.*/
+    stbuf = replies[0].poststat;
+    for (i = 1; i < priv->child_count; i++) {
+        if ((!IA_EQUAL(stbuf, replies[i].poststat, type)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, prot)))
+            return -1;
+    }
+    for (i = 1; i < priv->child_count; i++) {
+        if (!afr_xattrs_are_equal(replies[0].xdata, replies[i].xdata))
+            return -1;
+    }
+
+mark:
+    /* data/metadata is same on all bricks. Pick one of them as source. Rest
+     * are sinks.*/
+    for (i = 0; i < priv->child_count; i++) {
+        if (source == -1) {
+            source = i;
+            sources[i] = 1;
+            sinks[i] = 0;
+            healed_sinks[i] = 0;
+            continue;
+        }
+        sources[i] = 0;
+        sinks[i] = 1;
+        healed_sinks[i] = 1;
+    }
+
+    return source;
+}
+
+/* Return a source depending on the type of heal_op, and set sources[source],
+ * sinks[source] and healed_sinks[source] to 1, 0 and 0 respectively. Do so
+ * only if the following condition is met:
+ * ∀i((i ∈ locked_on[] ∧ i=1)==>(sources[i]=0 ∧ sinks[i]=1 ∧ healed_sinks[i]=1))
+ * i.e. for each locked node, sources[node] is 0; healed_sinks[node] and
+ * sinks[node] are 1. This should be the case if the file is in split-brain.
+ */
 int
-afr_sh_has_entry_pending (dict_t *xattr, xlator_t *this)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
+afr_mark_split_brain_source_sinks(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+    int heal_op = -1;
+    int ret = -1;
+    int source = -1;
+
+    local = frame->local;
+    priv = this->private;
+    xdata_req = local->xdata_req;
+
+    source = afr_mark_source_sinks_if_file_empty(
+        this, sources, sinks, healed_sinks, locked_on, replies, type);
+    if (source >= 0)
+        return source;
+
+    ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
+    if (ret)
+        goto autoheal;
+
+    source = afr_mark_split_brain_source_sinks_by_heal_op(
+        frame, this, sources, sinks, healed_sinks, locked_on, replies, type,
+        heal_op);
+    return source;
+
+autoheal:
+    /* Automatically heal if fav_child_policy is set. */
+    if (priv->fav_child_policy != AFR_FAV_CHILD_NONE) {
+        source = afr_mark_split_brain_source_sinks_by_policy(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, type);
+        if (source != -1) {
+            ret = dict_set_int32_sizen(xdata_req, "fav-child-policy", 1);
+            if (ret)
+                return -1;
+        }
+    }
+
+    return source;
+}
 
-        priv = this->private;
+int
+_afr_fav_child_reset_sink_xattrs(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, int source,
+                                 unsigned char *healed_sinks,
+                                 unsigned char *undid_pending,
+                                 afr_transaction_type type,
+                                 unsigned char *locked_on,
+                                 struct afr_reply *replies)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int *input_dirty = NULL;
+    int **input_matrix = NULL;
+    int *output_dirty = NULL;
+    int **output_matrix = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (!dict_get_sizen(local->xdata_req, "fav-child-policy"))
+        return 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
+    xdata = dict_new();
+    if (!xdata)
+        return -1;
 
-                if (ret != 0)
-                        return 0;
+    input_dirty = alloca0(priv->child_count * sizeof(int));
+    input_matrix = ALLOC_MATRIX(priv->child_count, int);
+    output_dirty = alloca0(priv->child_count * sizeof(int));
+    output_matrix = ALLOC_MATRIX(priv->child_count, int);
 
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
+    afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
 
-                if (pending[j])
-                        return 1;
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || !healed_sinks[i])
+            continue;
+        output_dirty[i] = -input_dirty[i];
+        output_matrix[i][source] = -input_matrix[i][source];
+    }
 
-        return 0;
-}
-
-int
-afr_sh_missing_entries_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i] || !locked_on[i])
+            continue;
+        xattr = afr_selfheal_output_xattr(this, _gf_false, type, output_dirty,
+                                          output_matrix, i, NULL);
 
-        afr_sh_reset (frame, this);
+        afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
 
-        if (local->govinda_gOvinda) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "split brain found, aborting selfheal of %s",
-                        local->loc.path);
-                sh->op_failed = 1;
-        }
+        undid_pending[i] = 1;
+        dict_unref(xattr);
+    }
 
-        if (sh->op_failed) {
-                sh->completion_cbk (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "proceeding to metadata check on %s",
-                        local->loc.path);
-                afr_self_heal_metadata (frame, this);
-        }
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
-static int
-afr_sh_missing_entries_finish (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_does_witness_exist(xlator_t *this, uint64_t *witness)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        int_lock->lock_cbk = afr_sh_missing_entries_done;
-        afr_unlock (frame, this);
+    priv = this->private;
 
-        return 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (witness[i])
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
-int
-afr_sh_common_create (afr_self_heal_t *sh, unsigned int child_count)
+unsigned int
+afr_get_quorum_count(afr_private_t *priv)
 {
-        int     ret = -ENOMEM;
-        sh->buf = GF_CALLOC (child_count, sizeof (*sh->buf),
-                             gf_afr_mt_iatt);
-        if (!sh->buf)
-                goto out;
-        sh->parentbufs = GF_CALLOC (child_count, sizeof (*sh->parentbufs),
-                                    gf_afr_mt_iatt);
-        if (!sh->parentbufs)
-                goto out;
-        sh->child_errno = GF_CALLOC (child_count, sizeof (*sh->child_errno),
-                                     gf_afr_mt_int);
-        if (!sh->child_errno)
-                goto out;
-        sh->success_children = afr_children_create (child_count);
-        if (!sh->success_children)
-                goto out;
-        sh->fresh_children = afr_children_create (child_count);
-        if (!sh->fresh_children)
-                goto out;
-        sh->xattr = GF_CALLOC (child_count, sizeof (*sh->xattr),
-                               gf_afr_mt_dict_t);
-        if (!sh->xattr)
-                goto out;
-        ret = 0;
-out:
-        return ret;
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        return priv->child_count / 2 + 1;
+    } else {
+        return priv->quorum_count;
+    }
 }
 
 void
-afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
-                                   xlator_t *this,
-                                   int32_t op_ret, int32_t op_errno,
-                                   inode_t *inode, struct iatt *buf,
-                                   dict_t *xattr, struct iatt *postparent,
-                                   loc_t *loc)
-{
-        int              child_index = 0;
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        priv = this->private;
-        sh   = &local->self_heal;
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        sh->buf[child_index] = *buf;
-                        sh->parentbufs[child_index] = *postparent;
-                        sh->success_children[sh->success_count] = child_index;
-                        sh->success_count++;
-                        sh->xattr[child_index] = dict_ref (xattr);
-                } else {
-                        gf_log (this->name, GF_LOG_ERROR, "path %s on subvolume"
-                                " %s => -1 (%s)", loc->path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        local->self_heal.child_errno[child_index] = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
+afr_selfheal_post_op_failure_accounting(afr_private_t *priv, char *accused,
+                                        unsigned char *sources,
+                                        unsigned char *locked_on)
+{
+    int i = 0;
+    unsigned int quorum_count = 0;
+
+    if (AFR_COUNT(sources, priv->child_count) != 0)
         return;
-}
 
-gf_boolean_t
-afr_valid_ia_type (ia_type_t ia_type)
-{
-        switch (ia_type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-        case IA_IFLNK:
-        case IA_IFDIR:
-                return _gf_true;
-        default:
-                return _gf_false;
+    quorum_count = afr_get_quorum_count(priv);
+    for (i = 0; i < priv->child_count; i++) {
+        if ((accused[i] < quorum_count) && locked_on[i]) {
+            sources[i] = 1;
         }
-        return _gf_false;
+    }
+    return;
 }
 
-int
-afr_impunge_frame_create (call_frame_t *frame, xlator_t *this,
-                          int active_source, call_frame_t **impunge_frame)
-{
-        afr_local_t     *local         = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh    = NULL;
-        int32_t         op_errno       = 0;
-        afr_private_t   *priv          = NULL;
-        int             ret            = 0;
-        call_frame_t    *new_frame     = NULL;
+/*
+ * This function determines if a self-heal is required for a given inode,
+ * and if needed, in what direction.
+ *
+ * locked_on[] is the array representing servers which have been locked and
+ * from which xattrs have been fetched for analysis.
+ *
+ * The output of the function is by filling the arrays sources[] and sinks[].
+ *
+ * sources[i] is set if i'th server is an eligible source for a selfheal.
+ *
+ * sinks[i] is set if i'th server needs to be healed.
+ *
+ * if sources[0..N] are all set, there is no need for a selfheal.
+ *
+ * if sinks[0..N] are all set, the inode is in split brain.
+ *
+ */
 
-        op_errno = ENOMEM;
-        priv = this->private;
-        new_frame = copy_frame (frame);
-        if (!new_frame) {
-                goto out;
+int
+afr_selfheal_find_direction(call_frame_t *frame, xlator_t *this,
+                            struct afr_reply *replies,
+                            afr_transaction_type type, unsigned char *locked_on,
+                            unsigned char *sources, unsigned char *sinks,
+                            uint64_t *witness, unsigned char *pflag)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int j = 0;
+    int *dirty = NULL;         /* Denotes if dirty xattr is set */
+    int **matrix = NULL;       /* Changelog matrix */
+    char *accused = NULL;      /* Accused others without any self-accusal */
+    char *pending = NULL;      /* Have pending operations on others */
+    char *self_accused = NULL; /* Accused itself */
+
+    priv = this->private;
+
+    dirty = alloca0(priv->child_count * sizeof(int));
+    accused = alloca0(priv->child_count);
+    pending = alloca0(priv->child_count);
+    self_accused = alloca0(priv->child_count);
+    matrix = ALLOC_MATRIX(priv->child_count, int);
+    memset(witness, 0, sizeof(*witness) * priv->child_count);
+
+    /* First construct the pending matrix for further analysis */
+    afr_selfheal_extract_xattr(this, replies, type, dirty, matrix);
+
+    if (pflag) {
+        for (i = 0; i < priv->child_count; i++) {
+            for (j = 0; j < priv->child_count; j++)
+                if (matrix[i][j])
+                    *pflag |= PFLAG_PENDING;
+            if (*pflag)
+                break;
         }
+    }
+
+    if (afr_success_count(replies, priv->child_count) < priv->child_count) {
+        /* Treat this just like locks not being acquired */
+        return -ENOTCONN;
+    }
+
+    /* short list all self-accused */
+    for (i = 0; i < priv->child_count; i++) {
+        if (matrix[i][i])
+            self_accused[i] = 1;
+    }
+
+    /* Next short list all accused to exclude them from being sources */
+    /* Self-accused can't accuse others as they are FOOLs */
+    for (i = 0; i < priv->child_count; i++) {
+        for (j = 0; j < priv->child_count; j++) {
+            if (matrix[i][j]) {
+                if (!self_accused[i])
+                    accused[j] += 1;
+                if (i != j)
+                    pending[i] += 1;
+            }
+        }
+    }
+
+    /* Short list all non-accused as sources */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!accused[i] && locked_on[i])
+            sources[i] = 1;
+        else
+            sources[i] = 0;
+    }
+
+    /* Everyone accused by non-self-accused sources are sinks */
+    memset(sinks, 0, priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (self_accused[i])
+            continue;
+        for (j = 0; j < priv->child_count; j++) {
+            if (matrix[i][j])
+                sinks[j] = 1;
+        }
+    }
+
+    /* For breaking ties provide with number of fops they witnessed */
+
+    /*
+     * count the pending fops witnessed from itself to others when it is
+     * self-accused
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!self_accused[i])
+            continue;
+        for (j = 0; j < priv->child_count; j++) {
+            if (i == j)
+                continue;
+            witness[i] += matrix[i][j];
+        }
+    }
+
+    if (type == AFR_DATA_TRANSACTION || type == AFR_METADATA_TRANSACTION)
+        afr_selfheal_post_op_failure_accounting(priv, accused, sources,
+                                                locked_on);
+
+    /* If no sources, all locked nodes are sinks - split brain */
+    if (AFR_COUNT(sources, priv->child_count) == 0) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (locked_on[i])
+                sinks[i] = 1;
+        }
+        if (pflag)
+            *pflag |= PFLAG_SBRAIN;
+    }
+
+    /* One more class of witness similar to dirty in v2 is where no pending
+     * exists but we have self-accusing markers. This can happen in afr-v1
+     * if the brick crashes just after doing xattrop on self but
+     * before xattrop on the other xattrs on the brick in pre-op. */
+    if (AFR_COUNT(pending, priv->child_count) == 0) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (self_accused[i])
+                witness[i] += matrix[i][i];
+        }
+    } else {
+        /* In afr-v1 if a file is self-accused and has pending
+         * operations on others then it is similar to 'dirty' in afr-v2.
+         * Consider such cases as witness.
+         */
+        for (i = 0; i < priv->child_count; i++) {
+            if (self_accused[i] && pending[i])
+                witness[i] += matrix[i][i];
+        }
+    }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (impunge_local, out);
-
-        local = frame->local;
-        new_frame->local = impunge_local;
-        impunge_sh = &impunge_local->self_heal;
-        impunge_sh->sh_frame = frame;
-        impunge_sh->active_source = active_source;
-        impunge_local->child_up  = memdup (local->child_up,
-                                           sizeof (*local->child_up) *
-                                           priv->child_count);
-        if (!impunge_local->child_up)
-                goto out;
-
-        impunge_local->pending = afr_matrix_create (priv->child_count,
-                                                    AFR_NUM_CHANGE_LOGS);
-        if (!impunge_local->pending)
-                goto out;
+    /* count the number of dirty fops witnessed */
+    for (i = 0; i < priv->child_count; i++)
+        witness[i] += dirty[i];
 
-        ret = afr_sh_common_create (impunge_sh, priv->child_count);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-        op_errno = 0;
-        *impunge_frame = new_frame;
-out:
-        if (op_errno && new_frame)
-                AFR_STACK_DESTROY (new_frame);
-        return -op_errno;
+    return 0;
 }
 
 void
-afr_sh_missing_entry_call_impunge_recreate (call_frame_t *frame, xlator_t *this,
-                                            struct iatt *buf,
-                                            struct iatt *postparent,
-                                            afr_impunge_done_cbk_t impunge_done)
-{
-        call_frame_t    *impunge_frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int             ret = 0;
-        unsigned int    enoent_count = 0;
-        afr_private_t   *priv = NULL;
-        int             i = 0;
-        int32_t         op_errno = 0;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        enoent_count = afr_errno_count (NULL, sh->child_errno,
-                                        priv->child_count, ENOENT);
-        if (!enoent_count) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no missing files - %s. proceeding to metadata check",
-                        local->loc.path);
-                goto out;
-        }
-        sh->impunge_done = impunge_done;
-        ret = afr_impunge_frame_create (frame, this, sh->source, &impunge_frame);
-        if (ret)
-                goto out;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-        loc_copy (&impunge_local->loc, &local->loc);
-        ret = afr_build_parent_loc (&impunge_sh->parent_loc,
-                                    &impunge_local->loc, &op_errno);
-        if (ret) {
-                ret = -op_errno;
-                goto out;
-        }
-        impunge_local->call_count = enoent_count;
-        impunge_sh->entrybuf = sh->buf[sh->source];
-        impunge_sh->parentbuf = sh->parentbufs[sh->source];
-        for (i = 0; i < priv->child_count; i++) {
-                if (!impunge_local->child_up[i]) {
-                        impunge_sh->child_errno[i] = ENOTCONN;
-                        continue;
-                }
-                if (sh->child_errno[i] != ENOENT) {
-                        impunge_sh->child_errno[i] = EEXIST;
-                        continue;
-                }
-        }
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->child_errno[i] != ENOENT)
-                        continue;
-                afr_sh_entry_impunge_create (impunge_frame, this, i);
-                enoent_count--;
-        }
-        GF_ASSERT (!enoent_count);
-        return;
-out:
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "impunge of %s failed, "
-                        "reason: %s", local->loc.path, strerror (-ret));
-                sh->op_failed = 1;
-        }
-        afr_sh_missing_entries_finish (frame, this);
+afr_log_selfheal(uuid_t gfid, xlator_t *this, int ret, char *type, int source,
+                 unsigned char *sources, unsigned char *healed_sinks)
+{
+    char *status = NULL;
+    char *sinks_str = NULL;
+    char *p = NULL;
+    char *sources_str = NULL;
+    char *q = NULL;
+    afr_private_t *priv = NULL;
+    gf_loglevel_t loglevel = GF_LOG_NONE;
+    int i = 0;
+
+    priv = this->private;
+    sinks_str = alloca0(priv->child_count * 8);
+    p = sinks_str;
+    sources_str = alloca0(priv->child_count * 8);
+    q = sources_str;
+    for (i = 0; i < priv->child_count; i++) {
+        if (healed_sinks[i])
+            p += sprintf(p, "%d ", i);
+        if (sources[i]) {
+            if (source == i) {
+                q += sprintf(q, "[%d] ", i);
+            } else {
+                q += sprintf(q, "%d ", i);
+            }
+        }
+    }
+
+    if (ret < 0) {
+        status = "Failed";
+        loglevel = GF_LOG_DEBUG;
+    } else {
+        status = "Completed";
+        loglevel = GF_LOG_INFO;
+    }
+
+    gf_msg(this->name, loglevel, 0, AFR_MSG_SELF_HEAL_INFO,
+           "%s %s selfheal on %s. "
+           "sources=%s sinks=%s",
+           status, type, uuid_utoa(gfid), sources_str, sinks_str);
 }
 
 int
-afr_sh_create_entry_cbk (call_frame_t *frame, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno)
+afr_selfheal_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
+    afr_local_t *local = NULL;
+    int i = -1;
+    GF_UNUSED int ret = -1;
+    int8_t need_heal = 1;
 
-        local = frame->local;
-        sh = &local->self_heal;
-        if (op_ret < 0)
-                sh->op_failed = 1;
-        afr_sh_missing_entries_finish (frame, this);
-        return 0;
-}
+    local = frame->local;
+    i = (long)cookie;
 
-static int
-sh_missing_entries_create (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int              type = 0;
-        struct iatt     *buf = NULL;
-        struct iatt     *postparent = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        buf = &sh->buf[sh->source];
-        postparent = &sh->parentbufs[sh->source];
-
-        type = buf->ia_type;
-        if (!afr_valid_ia_type (type)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: unknown file type: 0%o", local->loc.path, type);
-                local->govinda_gOvinda = 1;
-                afr_sh_missing_entries_finish (frame, this);
-                goto out;
-        }
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (buf)
+        local->replies[i].poststat = *buf;
+    if (parbuf)
+        local->replies[i].postparent = *parbuf;
+    if (xdata) {
+        local->replies[i].xdata = dict_ref(xdata);
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+    }
 
-        afr_sh_missing_entry_call_impunge_recreate (frame, this,
-                                                    buf, postparent,
-                                                    afr_sh_create_entry_cbk);
-out:
-        return 0;
+    local->replies[i].need_heal = need_heal;
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
 }
 
-void
-afr_sh_missing_entries_lookup_done (call_frame_t *frame, xlator_t *this,
-                                    int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        ia_type_t       ia_type = IA_INVAL;
-        int32_t         nsources = 0;
-        loc_t           *loc = NULL;
-        int32_t         subvol_status = 0;
-        afr_transaction_type txn_type = AFR_DATA_TRANSACTION;
-        gf_boolean_t    split_brain = _gf_false;
-        int             read_child = -1;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-        loc = &local->loc;
-
-        if (op_ret < 0) {
-                if (op_errno == EIO)
-                        local->govinda_gOvinda = 1;
-                // EIO can happen if finding the fresh parent dir failed
-                goto out;
-        }
+inode_t *
+afr_selfheal_unlocked_lookup_on(call_frame_t *frame, inode_t *parent,
+                                const char *name, struct afr_reply *replies,
+                                unsigned char *lookup_on, dict_t *xattr)
+{
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
 
-        //now No chance for the ia_type to conflict
-        ia_type = sh->buf[sh->success_children[0]].ia_type;
-        txn_type = afr_transaction_type_get (ia_type);
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children, txn_type,
-                                      &subvol_status, _gf_false);
-        if (nsources < 0) {
-                gf_log (this->name, GF_LOG_INFO, "No sources for dir of %s,"
-                        " in missing entry self-heal, continuing with the rest"
-                        " of the self-heals", local->loc.path);
-                if (subvol_status & SPLIT_BRAIN) {
-                        split_brain = _gf_true;
-                        switch (txn_type) {
-                        case AFR_DATA_TRANSACTION:
-                                nsources = 1;
-                                sh->sources[sh->success_children[0]] = 1;
-                                break;
-                        case AFR_ENTRY_TRANSACTION:
-                                read_child = afr_get_no_xattr_dir_read_child
-                                                          (this,
-                                                           sh->success_children,
-                                                           sh->buf);
-                                sh->sources[read_child] = 1;
-                                nsources = 1;
-                                break;
-                        default:
-                                op_errno = EIO;
-                                goto out;
-                        }
-                } else {
-                        op_errno = EIO;
-                        goto out;
-                }
-        }
+    local = frame->local;
+    priv = frame->this->private;
 
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, priv->child_count);
-        sh->source = sh->fresh_children[0];
-        if (sh->source == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "No active sources found.");
-                op_errno = EIO;
-                goto out;
-        }
+    xattr_req = dict_new();
+    if (!xattr_req)
+        return NULL;
 
-        if (sh->gfid_sh_success_cbk)
-                sh->gfid_sh_success_cbk (frame, this);
-        sh->type = sh->buf[sh->source].ia_type;
-        if (uuid_is_null (loc->inode->gfid))
-                uuid_copy (loc->gfid, sh->buf[sh->source].ia_gfid);
-        if (split_brain) {
-                afr_sh_missing_entries_finish (frame, this);
-        } else {
-                sh_missing_entries_create (frame, this);
-        }
-        return;
-out:
-        sh->op_failed = 1;
-        afr_sh_set_error (sh, op_errno);
-        afr_sh_missing_entries_finish (frame, this);
-        return;
+    if (xattr)
+        dict_copy(xattr, xattr_req);
+
+    if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
+        dict_unref(xattr_req);
+        return NULL;
+    }
+
+    inode = inode_new(parent->table);
+    if (!inode) {
+        dict_unref(xattr_req);
+        return NULL;
+    }
+
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
+
+    AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xattr_req);
+
+    afr_replies_copy(replies, local->replies, priv->child_count);
+
+    loc_wipe(&loc);
+    dict_unref(xattr_req);
+
+    return inode;
 }
 
 static int
-afr_sh_common_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, inode_t *inode,
-                          struct iatt *buf, dict_t *xattr,
-                          struct iatt *postparent)
-{
-        int                     call_count = 0;
-        afr_local_t             *local = NULL;
-        afr_self_heal_t         *sh    = NULL;
-        afr_private_t           *priv  = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        afr_sh_common_lookup_resp_handler (frame, cookie, this, op_ret,
-                                           op_errno, inode, buf, xattr,
-                                           postparent, &sh->lookup_loc);
-        call_count = afr_frame_return (frame);
-
-        if (call_count)
-                goto out;
-        op_ret = -1;
-        if (!sh->success_count) {
-                op_errno = afr_resultant_errno_get (NULL, sh->child_errno,
-                                                    priv->child_count);
-                gf_log (this->name, GF_LOG_ERROR, "Failed to lookup %s, "
-                        "reason %s", sh->lookup_loc.path,
-                        strerror (op_errno));
-                goto done;
-        }
+afr_set_multi_dom_lock_count_request(xlator_t *this, dict_t *dict)
+{
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    char *key1 = NULL;
+    char *key2 = NULL;
 
-        if ((sh->lookup_flags & AFR_LOOKUP_FAIL_CONFLICTS) &&
-            (afr_conflicting_iattrs (sh->buf, sh->success_children,
-                                     priv->child_count,
-                                     sh->lookup_loc.path, this->name))) {
-                op_errno = EIO;
-                gf_log (this->name, GF_LOG_ERROR, "Conflicting entries "
-                        "for %s", sh->lookup_loc.path);
-                goto done;
-        }
+    priv = this->private;
+    key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(this->name));
+    key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(priv->sh_domain));
 
-        if ((sh->lookup_flags & AFR_LOOKUP_FAIL_MISSING_GFIDS) &&
-            (afr_gfid_missing_count (this->name, sh->success_children,
-                                     sh->buf, priv->child_count,
-                                     sh->lookup_loc.path))) {
-                op_errno = ENODATA;
-                gf_log (this->name, GF_LOG_ERROR, "Missing Gfids "
-                        "for %s", sh->lookup_loc.path);
-                goto done;
-        }
-        op_ret = 0;
+    ret = dict_set_uint32(dict, GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS, 1);
+    if (ret)
+        return ret;
 
-done:
-        sh->lookup_done (frame, this, op_ret, op_errno);
-out:
-        return 0;
+    sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
+    ret = dict_set_uint32(dict, key1, 1);
+    if (ret)
+        return ret;
+
+    sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
+    ret = dict_set_uint32(dict, key2, 1);
+    if (ret)
+        return ret;
+
+    return 0;
 }
 
 int
-afr_sh_remove_entry_cbk (call_frame_t *frame, xlator_t *this, int child,
-                         int32_t op_ret, int32_t op_errno)
-{
-        int             call_count = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        GF_ASSERT (sh->post_remove_call);
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "purge entry %s failed, on child %d reason, %s",
-                        local->loc.path, child, strerror (op_errno));
-                LOCK (&frame->lock);
-                {
-                        afr_sh_set_error (sh, EIO);
-                        sh->op_failed = 1;
-                }
-                UNLOCK (&frame->lock);
-        }
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                sh->post_remove_call (frame, this);
-        return 0;
-}
+afr_selfheal_unlocked_discover_on(call_frame_t *frame, inode_t *inode,
+                                  uuid_t gfid, struct afr_reply *replies,
+                                  unsigned char *discover_on, dict_t *dict)
+{
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = frame->this->private;
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        return -ENOMEM;
+    if (dict)
+        dict_copy(dict, xattr_req);
+
+    if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
+        dict_unref(xattr_req);
+        return -ENOMEM;
+    }
+
+    if (afr_set_multi_dom_lock_count_request(frame->this, xattr_req)) {
+        dict_unref(xattr_req);
+        return -1;
+    }
 
-void
-afr_sh_call_entry_expunge_remove (call_frame_t *frame, xlator_t *this,
-                                  int child_index, struct iatt *buf,
-                                  struct iatt *parentbuf,
-                                  afr_expunge_done_cbk_t expunge_done)
-{
-        call_frame_t    *expunge_frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int32_t         op_errno = 0;
-        int             ret = 0;
-
-        expunge_frame = copy_frame (frame);
-        if (!expunge_frame) {
-                goto out;
-        }
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, gfid);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
+    AFR_ONLIST(discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xattr_req);
 
-        local = frame->local;
-        sh = &local->self_heal;
-        expunge_frame->local = expunge_local;
-        expunge_sh = &expunge_local->self_heal;
-        expunge_sh->sh_frame = frame;
-        loc_copy (&expunge_local->loc, &local->loc);
-        ret = afr_build_parent_loc (&expunge_sh->parent_loc,
-                                    &expunge_local->loc, &op_errno);
-        if (ret) {
-                ret = -op_errno;
-                goto out;
-        }
-        sh->expunge_done = expunge_done;
-        afr_sh_entry_expunge_remove (expunge_frame, this, child_index, buf,
-                                     parentbuf);
-        return;
-out:
-        gf_log (this->name, GF_LOG_ERROR, "Expunge of %s failed, reason: %s",
-                local->loc.path, strerror (op_errno));
-        expunge_done (frame, this, child_index, -1, op_errno);
-}
+    afr_replies_copy(replies, local->replies, priv->child_count);
 
-void
-afr_sh_remove_stale_lookup_info (afr_self_heal_t *sh, int32_t *success_children,
-                                 int32_t *fresh_children,
-                                 unsigned int child_count)
-{
-        int     i = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (afr_is_child_present (success_children, child_count, i) &&
-                    !afr_is_child_present (fresh_children, child_count, i)) {
-                        sh->child_errno[i] = ENOENT;
-                        GF_ASSERT (sh->xattr[i]);
-                        dict_unref (sh->xattr[i]);
-                        sh->xattr[i] = NULL;
-                }
-        }
+    loc_wipe(&loc);
+    dict_unref(xattr_req);
+
+    return 0;
 }
 
 int
-afr_sh_purge_stale_entries_done (call_frame_t *frame, xlator_t *this)
+afr_selfheal_unlocked_discover(call_frame_t *frame, inode_t *inode, uuid_t gfid,
+                               struct afr_reply *replies)
 {
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-        afr_private_t       *priv     = NULL;
+    afr_local_t *local = NULL;
+    dict_t *dict = NULL;
 
-        local    = frame->local;
-        sh       = &local->self_heal;
-        priv     = this->private;
+    local = frame->local;
 
-        if (sh->op_failed) {
-                afr_sh_missing_entries_finish (frame, this);
-        } else {
-                if (afr_gfid_missing_count (this->name, sh->fresh_children,
-                                            sh->buf, priv->child_count,
-                                            local->loc.path)) {
-                        afr_sh_common_lookup (frame, this, &local->loc,
-                                              afr_sh_missing_entries_lookup_done,
-                                              sh->sh_gfid_req,
-                                              AFR_LOOKUP_FAIL_CONFLICTS|
-                                              AFR_LOOKUP_FAIL_MISSING_GFIDS,
-                                              NULL);
-                } else {
-                        //No need to set gfid so goto missing entries lookup done
-                        //Behave as if you have done the lookup
-                        afr_sh_remove_stale_lookup_info (sh,
-                                                         sh->success_children,
-                                                         sh->fresh_children,
-                                                         priv->child_count);
-                        afr_children_copy (sh->success_children,
-                                           sh->fresh_children,
-                                           priv->child_count);
-                        afr_sh_missing_entries_lookup_done (frame, this, 0, 0);
-                }
-        }
-        return 0;
+    if (local->xattr_req)
+        dict = local->xattr_req;
+
+    return afr_selfheal_unlocked_discover_on(frame, inode, gfid, replies,
+                                             local->child_up, dict);
 }
 
-gf_boolean_t
-afr_sh_purge_entry_condition (afr_local_t *local, afr_private_t *priv,
-                              int child)
+unsigned int
+afr_success_count(struct afr_reply *replies, unsigned int count)
 {
-        afr_self_heal_t *sh = NULL;
-
-        sh = &local->self_heal;
+    int i = 0;
+    unsigned int success = 0;
 
-        if (local->child_up[child] &&
-            (!afr_is_child_present (sh->fresh_parent_dirs, priv->child_count,
-                                    child))
-            && (sh->child_errno[child] != ENOENT))
-                return _gf_true;
-
-        return _gf_false;
+    for (i = 0; i < count; i++)
+        if (replies[i].valid && replies[i].op_ret == 0)
+            success++;
+    return success;
 }
 
-gf_boolean_t
-afr_sh_purge_stale_entry_condition (afr_local_t *local, afr_private_t *priv,
-                                    int child)
+int
+afr_selfheal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata)
 {
-        afr_self_heal_t *sh = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        sh = &local->self_heal;
+    local = frame->local;
+    i = (long)cookie;
 
-        if (local->child_up[child] &&
-            (!afr_is_child_present (sh->fresh_children, priv->child_count,
-                                    child))
-             && (sh->child_errno[child] != ENOENT))
-                return _gf_true;
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-        return _gf_false;
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
 }
 
-void
-afr_sh_purge_entry_common (call_frame_t *frame, xlator_t *this,
-                           gf_boolean_t purge_condition (afr_local_t *local,
-                                                         afr_private_t *priv,
-                                                         int child))
+int
+afr_locked_fill(call_frame_t *frame, xlator_t *this, unsigned char *locked_on)
 {
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv = NULL;
-        afr_self_heal_t *sh = NULL;
-        int             i = 0;
-        int             call_count = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int count = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (purge_condition (local, priv, i))
-                        call_count++;
-        }
+    local = frame->local;
+    priv = this->private;
 
-        if (call_count == 0) {
-                sh->post_remove_call (frame, this);
-                goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+            locked_on[i] = 1;
+            count++;
+        } else {
+            locked_on[i] = 0;
         }
+    }
 
-        local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!purge_condition (local, priv, i))
-                        continue;
-                gf_log (this->name, GF_LOG_INFO, "purging the stale entry %s "
-                        "on %d", local->loc.path, i);
-                afr_sh_call_entry_expunge_remove (frame, this,
-                                                  (long) i, &sh->buf[i],
-                                                  &sh->parentbufs[i],
-                                                  afr_sh_remove_entry_cbk);
-        }
-out:
-        return;
+    return count;
 }
 
-void
-afr_sh_purge_entry (call_frame_t *frame, xlator_t *this)
+int
+afr_selfheal_tryinodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, off_t off, size_t size,
+                        unsigned char *locked_on)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-        local = frame->local;
-        sh = &local->self_heal;
-        sh->post_remove_call = afr_sh_missing_entries_finish;
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-        afr_sh_purge_entry_common (frame, this, afr_sh_purge_entry_condition);
+    loc_wipe(&loc);
+
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-void
-afr_sh_purge_stale_entry (call_frame_t *frame, xlator_t *this)
+int
+afr_selfheal_inodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, off_t off, size_t size,
+                     unsigned char *locked_on)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int             i = 0;
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
+    priv = this->private;
+    local = frame->local;
 
-        sh->post_remove_call = afr_sh_purge_stale_entries_done;
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (afr_is_child_present (sh->fresh_children,
-                                          priv->child_count, i))
-                        continue;
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-                if ((!local->child_up[i]) || sh->child_errno[i] != 0)
-                        continue;
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-                GF_ASSERT (!uuid_is_null (sh->entrybuf.ia_gfid) ||
-                           uuid_is_null (sh->buf[i].ia_gfid));
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == -1 &&
+            local->replies[i].op_errno == EAGAIN) {
+            afr_locked_fill(frame, this, locked_on);
+            afr_selfheal_uninodelk(frame, this, inode, dom, off, size,
+                                   locked_on);
 
-                if ((sh->entrybuf.ia_type != sh->buf[i].ia_type) ||
-                    (uuid_compare (sh->buf[i].ia_gfid,
-                                   sh->entrybuf.ia_gfid)))
-                        continue;
+            AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
+                    &flock, NULL);
+            break;
+        }
+    }
 
-                afr_children_add_child (sh->fresh_children, i,
-                                        priv->child_count);
+    loc_wipe(&loc);
 
-        }
-        afr_sh_purge_entry_common (frame, this,
-                                   afr_sh_purge_stale_entry_condition);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-void
-afr_sh_save_child_iatts_from_policy (int32_t *children, struct iatt *bufs,
-                                     struct iatt *save,
-                                     unsigned int child_count)
-{
-        int             i = 0;
-        int             child = 0;
-        gf_boolean_t    saved = _gf_false;
-
-        GF_ASSERT (save);
-        //if iatt buf with gfid exists sets it
-        for (i = 0; i < child_count; i++) {
-                child = children[i];
-                if (child == -1)
-                        break;
-                *save = bufs[child];
-                saved = _gf_true;
-                if (!uuid_is_null (save->ia_gfid))
-                        break;
+static void
+afr_get_lock_and_eagain_counts(afr_private_t *priv, struct afr_reply *replies,
+                               int *lock_count, int *eagain_count)
+{
+    int i = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == 0) {
+            (*lock_count)++;
+        } else if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            (*eagain_count)++;
         }
-        GF_ASSERT (saved);
+    }
 }
 
-void
-afr_get_children_of_fresh_parent_dirs (afr_self_heal_t *sh,
-                                       unsigned int child_count)
+/*Do blocking locks if number of locks acquired is majority and there were some
+ * EAGAINs. Useful for odd-way replication*/
+int
+afr_selfheal_tie_breaker_inodelk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, off_t off,
+                                 size_t size, unsigned char *locked_on)
 {
-        afr_children_intersection_get (sh->success_children,
-                                       sh->fresh_parent_dirs,
-                                       sh->sources, child_count);
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, child_count);
-        memset (sh->sources, 0, sizeof (*sh->sources) * child_count);
-}
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int lock_count = 0;
+    int eagain_count = 0;
 
-void
-afr_sh_children_lookup_done (call_frame_t *frame, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t      *local = NULL;
-        afr_self_heal_t  *sh = NULL;
-        afr_private_t    *priv = NULL;
-        int32_t          fresh_child_enoents = 0;
-        int32_t          fresh_parent_count = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0)
-                goto fail;
-        afr_get_children_of_fresh_parent_dirs (sh, priv->child_count);
-        fresh_parent_count = afr_get_children_count (sh->fresh_parent_dirs,
-                                                     priv->child_count);
-        //we need the enoent count of the subvols present in fresh_parent_dirs
-        fresh_child_enoents = afr_errno_count (sh->fresh_parent_dirs,
-                                               sh->child_errno,
-                                               priv->child_count, ENOENT);
-        if (fresh_child_enoents == fresh_parent_count) {
-                gf_log (this->name, GF_LOG_INFO, "Deleting stale file %s",
-                        local->loc.path);
-                afr_sh_set_error (sh, ENOENT);
-                sh->op_failed = 1;
-                afr_sh_purge_entry (frame, this);
-        } else if (!afr_conflicting_iattrs (sh->buf, sh->fresh_children,
-                                            priv->child_count, local->loc.path,
-                                            this->name)) {
-                afr_sh_save_child_iatts_from_policy (sh->fresh_children,
-                                                     sh->buf, &sh->entrybuf,
-                                                     priv->child_count);
-                afr_update_gfid_from_iatts (sh->sh_gfid_req, sh->buf,
-                                            sh->fresh_children,
-                                            priv->child_count);
-                afr_sh_purge_stale_entry (frame, this);
-        } else {
-                op_errno = EIO;
-                local->govinda_gOvinda = 1;
-                goto fail;
-        }
+    priv = this->private;
+    local = frame->local;
 
-        return;
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-fail:
-        sh->op_failed = 1;
-        afr_sh_set_error (sh, op_errno);
-        afr_sh_missing_entries_finish (frame, this);
-        return;
-}
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-static void
-afr_sh_find_fresh_parents (call_frame_t *frame, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno)
-{
-        afr_self_heal_t *sh  = NULL;
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local = NULL;
-        int             enoent_count = 0;
-        int             nsources = 0;
-        int             source  = -1;
-        int32_t         subvol_status = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0)
-                goto out;
-        enoent_count = afr_errno_count (NULL, sh->child_errno,
-                                        priv->child_count, ENOENT);
-        if (enoent_count > 0) {
-                gf_log (this->name, GF_LOG_INFO, "Parent dir missing for %s,"
-                        " in missing entry self-heal, aborting missing-entry "
-                        "self-heal",
-                        local->loc.path);
-                afr_sh_missing_entries_finish (frame, this);
-                return;
-        }
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      AFR_ENTRY_TRANSACTION, &subvol_status,
-                                      _gf_true);
-        if ((subvol_status & ALL_FOOLS) ||
-            (subvol_status & SPLIT_BRAIN)) {
-                gf_log (this->name, GF_LOG_INFO, "%s: Performing conservative "
-                        "merge", sh->parent_loc.path);
-                afr_mark_success_children_sources (sh->sources,
-                                                   sh->success_children,
-                                                   priv->child_count);
-        } else if (nsources < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "No sources for dir "
-                        "of %s, in missing entry self-heal, aborting "
-                        "self-heal", local->loc.path);
-                op_errno = EIO;
-                goto out;
-        }
+    afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
+                                   &eagain_count);
 
-        source = afr_sh_select_source (sh->sources, priv->child_count);
-        if (source == -1) {
-                GF_ASSERT (0);
-                gf_log (this->name, GF_LOG_DEBUG, "No active sources found.");
-                op_errno = EIO;
-                goto out;
-        }
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_parent_dirs, priv->child_count);
-        afr_sh_common_lookup (frame, this, &local->loc,
-                              afr_sh_children_lookup_done, NULL, 0,
-                              NULL);
-        return;
+    if (lock_count > priv->child_count / 2 && eagain_count) {
+        afr_locked_fill(frame, this, locked_on);
+        afr_selfheal_uninodelk(frame, this, inode, dom, off, size, locked_on);
 
-out:
-        afr_sh_set_error (sh, op_errno);
-        sh->op_failed = 1;
-        afr_sh_missing_entries_finish (frame, this);
-        return;
-}
+        AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
+                &flock, NULL);
+    }
 
-void
-afr_sh_common_reset (afr_self_heal_t *sh, unsigned int child_count)
-{
-        int             i = 0;
+    loc_wipe(&loc);
 
-        for (i = 0; i < child_count; i++) {
-                memset (&sh->buf[i], 0, sizeof (sh->buf[i]));
-                memset (&sh->parentbufs[i], 0, sizeof (sh->parentbufs[i]));
-                sh->child_errno[i] = 0;
-        }
-        memset (&sh->parentbuf, 0, sizeof (sh->parentbuf));
-        sh->success_count = 0;
-        afr_reset_children (sh->success_children, child_count);
-        afr_reset_children (sh->fresh_children, child_count);
-        afr_reset_xattr (sh->xattr, child_count);
-        loc_wipe (&sh->lookup_loc);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-/* afr self-heal state will be lost if this call is made
- * please check the afr_sh_common_reset that is called in this function
- */
 int
-afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      afr_lookup_done_cbk_t lookup_done , uuid_t gfid,
-                      int32_t flags, dict_t *xdata)
-{
-        afr_local_t    *local = NULL;
-        int             i = 0;
-        int             call_count = 0;
-        afr_private_t  *priv = NULL;
-        dict_t         *xattr_req = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        priv  = this->private;
-        sh    = &local->self_heal;
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-        local->call_count = call_count;
-
-        xattr_req = dict_new();
-
-        if (xattr_req) {
-                afr_xattr_req_prepare (this, xattr_req, loc->path);
-                if (gfid) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "looking up %s with gfid: %s",
-                                loc->path, uuid_utoa (gfid));
-                        GF_ASSERT (!uuid_is_null (gfid));
-                        afr_set_dict_gfid (xattr_req, gfid);
-                }
-        }
+afr_selfheal_uninodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, off_t off, size_t size,
+                       const unsigned char *locked_on)
+{
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
 
-        afr_sh_common_reset (sh, priv->child_count);
-        sh->lookup_done = lookup_done;
-        loc_copy (&sh->lookup_loc, loc);
-        sh->lookup_flags = flags;
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "looking up %s on subvolume %s",
-                                loc->path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (frame,
-                                           afr_sh_common_lookup_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           loc, xattr_req);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    flock.l_type = F_UNLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-        return 0;
-}
+    AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, inodelk, dom, &loc,
+               F_SETLK, &flock, NULL);
 
+    loc_wipe(&loc);
 
+    return 0;
+}
 
 int
-afr_sh_post_nb_entrylk_conflicting_sh_cbk (call_frame_t *frame, xlator_t *this)
+afr_selfheal_tryentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, const char *name, unsigned char *locked_on)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
+    loc_t loc = {
+        0,
+    };
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Non blocking entrylks failed.");
-                sh->op_failed = -1;
-                afr_sh_missing_entries_done (frame, this);
-        } else {
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                afr_sh_common_lookup (frame, this, &sh->parent_loc,
-                                      afr_sh_find_fresh_parents,
-                                      NULL, AFR_LOOKUP_FAIL_CONFLICTS,
-                                      NULL);
-        }
+    loc_wipe(&loc);
 
-        return 0;
+    return afr_locked_fill(frame, this, locked_on);
 }
 
 int
-afr_sh_post_nb_entrylk_gfid_sh_cbk (call_frame_t *frame, xlator_t *this)
+afr_selfheal_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, const char *name, unsigned char *locked_on)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
+    loc_t loc = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        local    = frame->local;
-        sh       = &local->self_heal;
-        int_lock = &local->internal_lock;
+    priv = this->private;
+    local = frame->local;
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Non blocking entrylks failed.");
-                afr_sh_missing_entries_done (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                afr_sh_common_lookup (frame, this, &local->loc,
-                                      afr_sh_missing_entries_lookup_done,
-                                      sh->sh_gfid_req, AFR_LOOKUP_FAIL_CONFLICTS|
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
-                                      NULL);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == -1 &&
+            local->replies[i].op_errno == EAGAIN) {
+            afr_locked_fill(frame, this, locked_on);
+            afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on,
+                                   NULL);
+
+            AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+                    ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+            break;
         }
+    }
 
-        return 0;
+    loc_wipe(&loc);
+
+    return afr_locked_fill(frame, this, locked_on);
 }
 
 int
-afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                char *base_name, afr_lock_cbk_t lock_cbk)
+afr_selfheal_tie_breaker_entrylk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, const char *name,
+                                 unsigned char *locked_on)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    loc_t loc = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int lock_count = 0;
+    int eagain_count = 0;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    priv = this->private;
+    local = frame->local;
 
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_ENTRY_SELF_HEAL_LK;
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        afr_set_lock_number (frame, this);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-        int_lock->lk_basename = base_name;
-        int_lock->lk_loc      = loc;
-        int_lock->lock_cbk    = lock_cbk;
+    afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
+                                   &eagain_count);
 
-        afr_nonblocking_entrylk (frame, this);
+    if (lock_count > priv->child_count / 2 && eagain_count) {
+        afr_locked_fill(frame, this, locked_on);
+        afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on, NULL);
 
-        return 0;
+        AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+                ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+    }
+
+    loc_wipe(&loc);
+
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-static int
-afr_self_heal_parent_entrylk (call_frame_t *frame, xlator_t *this,
-                              afr_lock_cbk_t lock_cbk)
+int
+afr_selfheal_unentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, const char *name, unsigned char *locked_on,
+                       dict_t *xdata)
 {
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        int                 ret       = -1;
-        int32_t             op_errno  = 0;
+    loc_t loc = {
+        0,
+    };
 
-        local    = frame->local;
-        sh       = &local->self_heal;
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "attempting to recreate missing entries for path=%s",
-                local->loc.path);
+    AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
+               name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
 
-        ret = afr_build_parent_loc (&sh->parent_loc, &local->loc, &op_errno);
-        if (ret)
-                goto out;
+    loc_wipe(&loc);
 
-        afr_sh_entrylk (frame, this, &sh->parent_loc, NULL,
-                        lock_cbk);
-        return 0;
-out:
-        int_lock = &local->internal_lock;
-        int_lock->lock_op_ret = -1;
-        lock_cbk (frame, this);
-        return 0;
+    return 0;
 }
 
-static int
-afr_self_heal_conflicting_entries (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_data_set(xlator_t *this, dict_t *xdata)
 {
-        afr_self_heal_parent_entrylk (frame, this,
-                                      afr_sh_post_nb_entrylk_conflicting_sh_cbk);
-        return 0;
+    return afr_is_pending_set(this, xdata, AFR_DATA_TRANSACTION);
 }
 
-static int
-afr_self_heal_gfids (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_metadata_set(xlator_t *this, dict_t *xdata)
 {
-        afr_self_heal_parent_entrylk (frame, this,
-                                      afr_sh_post_nb_entrylk_gfid_sh_cbk);
-        return 0;
+    return afr_is_pending_set(this, xdata, AFR_METADATA_TRANSACTION);
 }
 
-afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
+gf_boolean_t
+afr_is_entry_set(xlator_t *this, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t   *lc     = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *shc = NULL;
-
-        priv = this->private;
-
-        sh = &l->self_heal;
-
-        lc = mem_get0 (this->local_pool);
-        if (!lc)
-                goto out;
-
-        shc = &lc->self_heal;
-
-        shc->unwind = sh->unwind;
-        shc->gfid_sh_success_cbk = sh->gfid_sh_success_cbk;
-        shc->do_missing_entry_self_heal = sh->do_missing_entry_self_heal;
-        shc->do_gfid_self_heal = sh->do_gfid_self_heal;
-        shc->do_data_self_heal = sh->do_data_self_heal;
-        shc->do_metadata_self_heal = sh->do_metadata_self_heal;
-        shc->do_entry_self_heal = sh->do_entry_self_heal;
-        shc->forced_merge = sh->forced_merge;
-        shc->background = sh->background;
-        shc->type = sh->type;
-
-        uuid_copy (shc->sh_gfid_req, sh->sh_gfid_req);
-        if (l->loc.path)
-                loc_copy (&lc->loc, &l->loc);
-
-        lc->child_up  = memdup (l->child_up,
-                                sizeof (*lc->child_up) * priv->child_count);
-        if (l->xattr_req)
-                lc->xattr_req = dict_ref (l->xattr_req);
-
-        if (l->cont.lookup.inode)
-                lc->cont.lookup.inode = inode_ref (l->cont.lookup.inode);
-        if (l->cont.lookup.xattr)
-                lc->cont.lookup.xattr = dict_ref (l->cont.lookup.xattr);
-        if (l->internal_lock.inode_locked_nodes)
-                lc->internal_lock.inode_locked_nodes =
-                        memdup (l->internal_lock.inode_locked_nodes,
-                                sizeof (*lc->internal_lock.inode_locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.inode_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.inode_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.entry_locked_nodes)
-                lc->internal_lock.entry_locked_nodes =
-                        memdup (l->internal_lock.entry_locked_nodes,
-                                sizeof (*lc->internal_lock.entry_locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.entry_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.entry_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.locked_nodes)
-                lc->internal_lock.locked_nodes =
-                        memdup (l->internal_lock.locked_nodes,
-                                sizeof (*lc->internal_lock.locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
+    return afr_is_pending_set(this, xdata, AFR_ENTRY_TRANSACTION);
+}
 
-        lc->internal_lock.inodelk_lock_count =
-                l->internal_lock.inodelk_lock_count;
-        lc->internal_lock.entrylk_lock_count =
-                l->internal_lock.entrylk_lock_count;
+/*
+ * This function inspects the looked up replies (in an unlocked manner)
+ * and decides whether a locked verification and possible healing is
+ * required or not. It updates the three booleans for each type
+ * of healing. If the boolean flag gets set to FALSE, then we are sure
+ * no healing is required. If the boolean flag gets set to TRUE then
+ * we have to proceed with locked reinspection.
+ */
 
+int
+afr_selfheal_unlocked_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                              inode_t **link_inode, gf_boolean_t *data_selfheal,
+                              gf_boolean_t *metadata_selfheal,
+                              gf_boolean_t *entry_selfheal,
+                              struct afr_reply *replies_dst)
+{
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    int i = 0;
+    int valid_cnt = 0;
+    struct iatt first = {
+        0,
+    };
+    int first_idx = 0;
+    struct afr_reply *replies = NULL;
+    int ret = -1;
+
+    priv = this->private;
+
+    inode = afr_inode_find(this, gfid);
+    if (!inode)
+        goto out;
+
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+
+    ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == -1)
+            continue;
+
+        /* The data segment of the changelog can be non-zero to indicate
+         * the directory needs a full heal. So the check below ensures
+         * it's not a directory before setting the data_selfheal boolean.
+         */
+        if (data_selfheal && !IA_ISDIR(replies[i].poststat.ia_type) &&
+            afr_is_data_set(this, replies[i].xdata))
+            *data_selfheal = _gf_true;
+
+        if (metadata_selfheal && afr_is_metadata_set(this, replies[i].xdata))
+            *metadata_selfheal = _gf_true;
+
+        if (entry_selfheal && afr_is_entry_set(this, replies[i].xdata))
+            *entry_selfheal = _gf_true;
+
+        valid_cnt++;
+        if (valid_cnt == 1) {
+            first = replies[i].poststat;
+            first_idx = i;
+            continue;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, type)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                   "TYPE mismatch %d vs %d on %s for gfid:%s",
+                   (int)first.ia_type, (int)replies[i].poststat.ia_type,
+                   priv->children[i]->name,
+                   uuid_utoa(replies[i].poststat.ia_gfid));
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;"
+                     "type=file;gfid=%s;"
+                     "ia_type-%d=%s;ia_type-%d=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(replies[i].poststat.ia_gfid), first_idx,
+                     gf_inode_type_to_str(first.ia_type), i,
+                     gf_inode_type_to_str(replies[i].poststat.ia_type));
+            ret = -EIO;
+            goto out;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, uid)) {
+            gf_msg_debug(this->name, 0,
+                         "UID mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, gid)) {
+            gf_msg_debug(this->name, 0,
+                         "GID mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, prot)) {
+            gf_msg_debug(this->name, 0,
+                         "MODE mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)st_mode_from_ia(first.ia_prot, 0),
+                         (int)st_mode_from_ia(replies[i].poststat.ia_prot, 0),
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (IA_ISREG(first.ia_type) &&
+            !IA_EQUAL(first, replies[i].poststat, size)) {
+            gf_msg_debug(this->name, 0,
+                         "SIZE mismatch "
+                         "%lld vs %lld on %s for gfid:%s",
+                         (long long)first.ia_size,
+                         (long long)replies[i].poststat.ia_size,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (data_selfheal)
+                *data_selfheal = _gf_true;
+        }
+    }
+
+    if (valid_cnt > 0 && link_inode) {
+        *link_inode = inode_link(inode, NULL, NULL, &first);
+        if (!*link_inode) {
+            ret = -EINVAL;
+            goto out;
+        }
+    } else if (valid_cnt < 2) {
+        ret = afr_check_stale_error(replies, priv);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return lc;
+    if (replies && replies_dst)
+        afr_replies_copy(replies_dst, replies, priv->child_count);
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+
+    return ret;
 }
 
-int
-afr_self_heal_completion_cbk (call_frame_t *bgsh_frame, xlator_t *this)
+inode_t *
+afr_inode_find(xlator_t *this, uuid_t gfid)
 {
-        afr_private_t *   priv  = NULL;
-        afr_local_t *     local = NULL;
-        afr_self_heal_t * sh    = NULL;
-        char              sh_type_str[256] = {0,};
-        gf_boolean_t      split_brain = _gf_false;
-
-        priv  = this->private;
-        local = bgsh_frame->local;
-        sh    = &local->self_heal;
-
-        if (local->govinda_gOvinda)
-                split_brain = _gf_true;
+    inode_table_t *table = NULL;
+    inode_t *inode = NULL;
 
-        afr_set_split_brain (this, sh->inode, split_brain);
+    table = this->itable;
+    if (!table)
+        return NULL;
 
-        afr_self_heal_type_str_get (sh, sh_type_str,
-                                    sizeof(sh_type_str));
-        if (sh->op_failed) {
-                gf_log (this->name, GF_LOG_ERROR, "background %s self-heal "
-                        "failed on %s", sh_type_str, local->loc.path);
-        } else {
-                gf_log (this->name, GF_LOG_INFO, "background %s self-heal "
-                        "completed on %s", sh_type_str, local->loc.path);
-        }
-
-        FRAME_SU_UNDO (bgsh_frame, afr_local_t);
-
-        if (!sh->unwound && sh->unwind) {
-                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno,
-                            sh->op_failed);
-        }
+    inode = inode_find(table, gfid);
+    if (inode)
+        return inode;
 
-        if (sh->background) {
-                LOCK (&priv->lock);
-                {
-                        priv->background_self_heals_started--;
-                }
-                UNLOCK (&priv->lock);
-        }
+    inode = inode_new(table);
+    if (!inode)
+        return NULL;
 
-        AFR_STACK_DESTROY (bgsh_frame);
+    gf_uuid_copy(inode->gfid, gfid);
 
-        return 0;
+    return inode;
 }
 
-int
-afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int32_t          op_errno = 0;
-        int              ret = 0;
-        afr_self_heal_t *orig_sh = NULL;
-        call_frame_t    *sh_frame = NULL;
-        afr_local_t     *sh_local = NULL;
-        loc_t           *loc   = NULL;
-
-        local = frame->local;
-        orig_sh = &local->self_heal;
-        priv  = this->private;
-
-        GF_ASSERT (local->loc.path);
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "performing self heal on %s (metadata=%d data=%d entry=%d)",
-                local->loc.path,
-                local->self_heal.do_metadata_self_heal,
-                local->self_heal.do_data_self_heal,
-                local->self_heal.do_entry_self_heal);
-
-        op_errno        = ENOMEM;
-        sh_frame        = copy_frame (frame);
-        if (!sh_frame)
-                goto out;
-        afr_set_lk_owner (sh_frame, this, sh_frame->root);
-        afr_set_low_priority (sh_frame);
+call_frame_t *
+afr_frame_create(xlator_t *this, int32_t *op_errno)
+{
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    pid_t pid = GF_CLIENT_PID_SELF_HEALD;
 
-        sh_local        = afr_local_copy (local, this);
-        if (!sh_local)
-                goto out;
-        sh_frame->local = sh_local;
-        sh              = &sh_local->self_heal;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        return NULL;
+    }
 
-        sh->inode       = inode_ref (inode);
-        sh->orig_frame  = frame;
+    local = AFR_FRAME_INIT(frame, (*op_errno));
+    if (!local) {
+        STACK_DESTROY(frame->root);
+        return NULL;
+    }
 
-        sh->completion_cbk = afr_self_heal_completion_cbk;
+    syncopctx_setfspid(&pid);
 
-        sh->success = GF_CALLOC (priv->child_count, sizeof (*sh->success),
-                                 gf_afr_mt_char);
-        if (!sh->success)
-                goto out;
-        sh->sources = GF_CALLOC (sizeof (*sh->sources), priv->child_count,
-                                 gf_afr_mt_int);
-        if (!sh->sources)
-                goto out;
-        sh->locked_nodes = GF_CALLOC (sizeof (*sh->locked_nodes),
-                                      priv->child_count,
-                                      gf_afr_mt_int);
-        if (!sh->locked_nodes)
-                goto out;
+    frame->root->pid = pid;
 
-        sh->pending_matrix = afr_matrix_create (priv->child_count,
-                                                priv->child_count);
-        if (!sh->pending_matrix)
-                goto out;
+    afr_set_lk_owner(frame, this, frame->root);
 
-        sh->delta_matrix = afr_matrix_create (priv->child_count,
-                                              priv->child_count);
-        if (!sh->delta_matrix)
-                goto out;
+    return frame;
+}
 
-        sh->fresh_parent_dirs = afr_children_create (priv->child_count);
-        if (!sh->fresh_parent_dirs)
-                goto out;
-        ret = afr_sh_common_create (sh, priv->child_count);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+int
+afr_selfheal_newentry_mark(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, struct afr_reply *replies,
+                           unsigned char *sources, unsigned char *newentry)
+{
+    int ret = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int **changelog = NULL;
 
-        if (local->self_heal.background) {
-                LOCK (&priv->lock);
-                {
-                        if (priv->background_self_heals_started
-                            < priv->background_self_heal_count) {
-                                priv->background_self_heals_started++;
+    priv = this->private;
 
+    gf_uuid_copy(inode->gfid, replies[source].poststat.ia_gfid);
 
-                        } else {
-                                local->self_heal.background = _gf_false;
-                                sh->background = _gf_false;
-                        }
-                }
-                UNLOCK (&priv->lock);
-        }
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
 
-        if (!local->loc.parent) {
-                sh->do_missing_entry_self_heal = _gf_false;
-                sh->do_gfid_self_heal = _gf_false;
-        }
+    changelog = afr_mark_pending_changelog(priv, newentry, xattr,
+                                           replies[source].poststat.ia_type);
 
-        FRAME_SU_DO (sh_frame, afr_local_t);
-        if (sh->do_missing_entry_self_heal) {
-                afr_self_heal_conflicting_entries (sh_frame, this);
-        } else if (sh->do_gfid_self_heal) {
-                GF_ASSERT (!uuid_is_null (sh->sh_gfid_req));
-                afr_self_heal_gfids (sh_frame, this);
-        } else {
-                loc = &sh_local->loc;
-                if (uuid_is_null (loc->inode->gfid) && uuid_is_null (loc->gfid)) {
-                        if (!uuid_is_null (inode->gfid))
-                                GF_ASSERT (!uuid_compare (inode->gfid,
-                                           sh->sh_gfid_req));
-                        uuid_copy (loc->gfid, sh->sh_gfid_req);
-                }
-                gf_log (this->name, GF_LOG_TRACE,
-                        "proceeding to metadata check on %s",
-                        local->loc.path);
+    if (!changelog) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-                afr_sh_missing_entries_done (sh_frame, this);
-        }
-        op_errno = 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        ret |= afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
+    }
+out:
+    if (changelog)
+        afr_matrix_cleanup(changelog, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
+    return ret;
+}
+
+int
+afr_selfheal_do(call_frame_t *frame, xlator_t *this, uuid_t gfid)
+{
+    int ret = -1;
+    int entry_ret = 1;
+    int metadata_ret = 1;
+    int data_ret = 1;
+    int or_ret = 0;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    gf_boolean_t data_selfheal = _gf_false;
+    gf_boolean_t metadata_selfheal = _gf_false;
+    gf_boolean_t entry_selfheal = _gf_false;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = afr_selfheal_unlocked_inspect(frame, this, gfid, &inode,
+                                        &data_selfheal, &metadata_selfheal,
+                                        &entry_selfheal, NULL);
+    if (ret)
+        goto out;
+
+    if (!(data_selfheal || metadata_selfheal || entry_selfheal)) {
+        ret = 2;
+        goto out;
+    }
+
+    if (inode->ia_type == IA_IFREG) {
+        ret = afr_selfheal_data_open(this, inode, &fd);
+        if (!fd) {
+            ret = -EIO;
+            goto out;
+        }
+    }
+
+    if (data_selfheal && priv->data_self_heal)
+        data_ret = afr_selfheal_data(frame, this, fd);
+
+    if (metadata_selfheal && priv->metadata_self_heal)
+        metadata_ret = afr_selfheal_metadata(frame, this, inode);
+
+    if (entry_selfheal && priv->entry_self_heal)
+        entry_ret = afr_selfheal_entry(frame, this, inode);
+
+    or_ret = (data_ret | metadata_ret | entry_ret);
+
+    if (data_ret == -EIO || metadata_ret == -EIO || entry_ret == -EIO)
+        ret = -EIO;
+    else if (data_ret == 1 && metadata_ret == 1 && entry_ret == 1)
+        ret = 1;
+    else if (or_ret < 0)
+        ret = or_ret;
+    else
+        ret = 0;
 
 out:
-        if (op_errno) {
-                orig_sh->unwind (frame, this, -1, op_errno, 1);
-                if (sh_frame)
-                        AFR_STACK_DESTROY (sh_frame);
-        }
-        return 0;
+    if (inode)
+        inode_unref(inode);
+    if (fd)
+        fd_unref(fd);
+    return ret;
 }
+/*
+ * This is the entry point for healing a given GFID. The return values for this
+ * function are as follows:
+ * '0' if the self-heal is successful
+ * '1' if the afr-xattrs are non-zero (due to on-going IO) and no heal is needed
+ * '2' if the afr-xattrs are all-zero and no heal is needed
+ * $errno if the heal on the gfid failed.
+ */
 
-void
-afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
-                            size_t size)
+int
+afr_selfheal(xlator_t *this, uuid_t gfid)
 {
-        GF_ASSERT (str && (size > strlen (" missing-entry gfid "
-                                          "meta-data data entry")));
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
 
-        if (self_heal_p->do_metadata_self_heal) {
-                snprintf (str, size, " meta-data");
-        }
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        return ret;
 
-        if (self_heal_p->do_data_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str), " data");
-        }
+    local = frame->local;
+    local->xdata_req = dict_new();
 
-        if (self_heal_p->do_entry_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str), " entry");
-        }
+    ret = afr_selfheal_do(frame, this, gfid);
 
-        if (self_heal_p->do_missing_entry_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str),
-                         " missing-entry");
-        }
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 
-        if (self_heal_p->do_gfid_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str), " gfid");
-        }
+    return ret;
 }
 
-afr_self_heal_type
-afr_self_heal_type_for_transaction (afr_transaction_type type)
+afr_local_t *
+__afr_dequeue_heals(afr_private_t *priv)
 {
-        afr_self_heal_type sh_type = AFR_SELF_HEAL_INVALID;
+    afr_local_t *local = NULL;
 
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-                sh_type = AFR_SELF_HEAL_DATA;
-                break;
-        case AFR_METADATA_TRANSACTION:
-                sh_type = AFR_SELF_HEAL_METADATA;
-                break;
-        case AFR_ENTRY_TRANSACTION:
-                sh_type = AFR_SELF_HEAL_ENTRY;
-                break;
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                GF_ASSERT (0);
-                break;
-        }
-        return sh_type;
+    if (list_empty(&priv->heal_waiting))
+        goto none;
+    if ((priv->background_self_heal_count > 0) &&
+        (priv->healers >= priv->background_self_heal_count))
+        goto none;
+
+    local = list_entry(priv->heal_waiting.next, afr_local_t, healer);
+    priv->heal_waiters--;
+    GF_ASSERT(priv->heal_waiters >= 0);
+    list_del_init(&local->healer);
+    list_add(&local->healer, &priv->healing);
+    priv->healers++;
+    return local;
+none:
+    gf_msg_debug(THIS->name, 0,
+                 "Nothing dequeued. "
+                 "Num healers: %d, Num Waiters: %d",
+                 priv->healers, priv->heal_waiters);
+    return NULL;
 }
 
 int
-afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
+afr_refresh_selfheal_wrap(void *opaque)
 {
-        int   ret = -1;
-        uuid_t pargfid = {0};
+    call_frame_t *heal_frame = opaque;
+    afr_local_t *local = heal_frame->local;
+    int ret = 0;
 
-        if (!child)
-                goto out;
+    ret = afr_selfheal(heal_frame->this, local->refreshinode->gfid);
+    return ret;
+}
 
-        if (!uuid_is_null (parent->inode->gfid))
-                uuid_copy (pargfid, parent->inode->gfid);
-        else if (!uuid_is_null (parent->gfid))
-                uuid_copy (pargfid, parent->gfid);
+int
+afr_refresh_heal_done(int ret, call_frame_t *frame, void *opaque)
+{
+    call_frame_t *heal_frame = opaque;
+    xlator_t *this = heal_frame->this;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = heal_frame->local;
 
-        if (uuid_is_null (pargfid))
-                goto out;
+    LOCK(&priv->lock);
+    {
+        list_del_init(&local->healer);
+        priv->healers--;
+        GF_ASSERT(priv->healers >= 0);
+        local = __afr_dequeue_heals(priv);
+    }
+    UNLOCK(&priv->lock);
 
-        if (strcmp (parent->path, "/") == 0)
-                ret = gf_asprintf ((char **)&child->path, "/%s", name);
-        else
-                ret = gf_asprintf ((char **)&child->path, "%s/%s", parent->path,
-                                   name);
+    AFR_STACK_DESTROY(heal_frame);
 
-        if (-1 == ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "asprintf failed while setting child path");
-        }
+    if (local)
+        afr_heal_synctask(this, local);
+    return 0;
+}
 
-        child->name = strrchr (child->path, '/');
-        if (child->name)
-                child->name++;
+void
+afr_heal_synctask(xlator_t *this, afr_local_t *local)
+{
+    int ret = 0;
+    call_frame_t *heal_frame = NULL;
 
-        child->parent = inode_ref (parent->inode);
-        child->inode = inode_new (parent->inode->table);
-        uuid_copy (child->pargfid, pargfid);
+    heal_frame = local->heal_frame;
+    ret = synctask_new(this->ctx->env, afr_refresh_selfheal_wrap,
+                       afr_refresh_heal_done, heal_frame, heal_frame);
+    if (ret < 0)
+        /* Heal not launched. Will be queued when the next inode
+         * refresh happens and shd hasn't healed it yet. */
+        afr_refresh_heal_done(ret, heal_frame, heal_frame);
+}
 
-        if (!child->inode) {
-                ret = -1;
-                goto out;
+gf_boolean_t
+afr_throttled_selfheal(call_frame_t *frame, xlator_t *this)
+{
+    gf_boolean_t can_heal = _gf_true;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+
+    LOCK(&priv->lock);
+    {
+        if ((priv->background_self_heal_count > 0) &&
+            (priv->heal_wait_qlen + priv->background_self_heal_count) >
+                (priv->heal_waiters + priv->healers)) {
+            list_add_tail(&local->healer, &priv->heal_waiting);
+            priv->heal_waiters++;
+            local = __afr_dequeue_heals(priv);
+        } else {
+            can_heal = _gf_false;
         }
+    }
+    UNLOCK(&priv->lock);
 
-        ret = 0;
-out:
-        if ((ret == -1) && child)
-                loc_wipe (child);
+    if (can_heal) {
+        if (local)
+            afr_heal_synctask(this, local);
+        else
+            gf_msg_debug(this->name, 0,
+                         "Max number of heals are "
+                         "pending, background self-heal rejected.");
+    }
 
-        return ret;
+    return can_heal;
 }
 
 int
-afr_sh_erase_pending (call_frame_t *frame, xlator_t *this,
-                      afr_transaction_type type, afr_fxattrop_cbk_t cbk,
-                      int (*finish)(call_frame_t *frame, xlator_t *this))
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
-        int             ret = -1;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix,
-                                 sh->success, priv->child_count, type);
-
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-        if (!erase_xattr)
-                goto out;
+afr_choose_source_by_policy(afr_private_t *priv, unsigned char *sources,
+                            afr_transaction_type type)
+{
+    int source = -1;
+    int i = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
-                        erase_xattr[i] = dict_new ();
-                        if (!erase_xattr[i])
-                                goto out;
-                }
-        }
+    /* Give preference to local child to save on bandwidth */
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->local[i] && sources[i]) {
+            if ((type == AFR_DATA_TRANSACTION) && AFR_IS_ARBITER_BRICK(priv, i))
+                continue;
 
-        afr_sh_delta_to_xattr (this, sh->delta_matrix, erase_xattr,
-                               priv->child_count, type);
+            source = i;
+            goto out;
+        }
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Delta matrix for: %s",
-                lkowner_utoa (&frame->root->lk_owner));
-        afr_sh_print_pending_matrix (sh->delta_matrix, this);
-        local->call_count = call_count;
-        if (call_count == 0) {
-                ret = 0;
-                finish (frame, this);
-                goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            goto out;
         }
+    }
+out:
+    return source;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
-                        continue;
-
-                if (sh->healing_fd) {//true for ENTRY, reg file DATA transaction
-                        STACK_WIND_COOKIE (frame, cbk, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fxattrop,
-                                           sh->healing_fd,
-                                           GF_XATTROP_ADD_ARRAY, erase_xattr[i],
-                                           NULL);
-                } else {
-                        STACK_WIND_COOKIE (frame, cbk, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->xattrop,
-                                           &local->loc,
-                                           GF_XATTROP_ADD_ARRAY, erase_xattr[i],
-                                           NULL);
-                }
+static int
+afr_anon_inode_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->replies[i].poststat = *buf;
+        local->replies[i].preparent = *preparent;
+        local->replies[i].postparent = *postparent;
+    }
+    if (xdata) {
+        local->replies[i].xdata = dict_ref(xdata);
+    }
+
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+int
+afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode)
+{
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
+    unsigned char *mkdir_on = alloca0(priv->child_count);
+    unsigned char *lookup_on = alloca0(priv->child_count);
+    loc_t loc = {0};
+    int32_t op_errno = 0;
+    int32_t child_op_errno = 0;
+    struct iatt iatt = {0};
+    dict_t *xdata = NULL;
+    uuid_t anon_inode_gfid = {0};
+    int mkdir_count = 0;
+    int i = 0;
+
+    /*Try to mkdir everywhere and return success if the dir exists on 'child'
+     */
+
+    if (!priv->use_anon_inode) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &op_errno);
+    if (op_errno) {
+        goto out;
+    }
+    local = frame->local;
+    if (!local->child_up[child]) {
+        /*Other bricks may need mkdir so don't error out yet*/
+        child_op_errno = ENOTCONN;
+    }
+    gf_uuid_parse(priv->anon_gfid_str, anon_inode_gfid);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+
+        if (priv->anon_inode[i]) {
+            mkdir_on[i] = 0;
+        } else {
+            mkdir_on[i] = 1;
+            mkdir_count++;
         }
+    }
 
-        ret = 0;
-out:
-        if (erase_xattr) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (erase_xattr[i]) {
-                                dict_unref (erase_xattr[i]);
-                        }
-                }
+    if (mkdir_count == 0) {
+        *linked_inode = inode_find(this->itable, anon_inode_gfid);
+        if (*linked_inode) {
+            op_errno = 0;
+            goto out;
         }
+    }
 
-        GF_FREE (erase_xattr);
+    loc.parent = inode_ref(this->itable->root);
+    loc.name = priv->anon_inode_name;
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        if (ret < 0) {
-                sh->op_failed = _gf_true;
-                finish (frame, this);
+    xdata = dict_new();
+    if (!xdata) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_errno = -dict_set_gfuuid(xdata, "gfid-req", anon_inode_gfid, _gf_true);
+    if (op_errno) {
+        goto out;
+    }
+
+    if (mkdir_count == 0) {
+        memcpy(lookup_on, local->child_up, priv->child_count);
+        goto lookup;
+    }
+
+    AFR_ONLIST(mkdir_on, frame, afr_anon_inode_mkdir_cbk, mkdir, &loc, 0755, 0,
+               xdata);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!mkdir_on[i]) {
+            continue;
+        }
+
+        if (local->replies[i].op_ret == 0) {
+            priv->anon_inode[i] = 1;
+            iatt = local->replies[i].poststat;
+        } else if (local->replies[i].op_ret < 0 &&
+                   local->replies[i].op_errno == EEXIST) {
+            lookup_on[i] = 1;
+        } else if (i == child) {
+            child_op_errno = local->replies[i].op_errno;
+        }
+    }
+
+    if (AFR_COUNT(lookup_on, priv->child_count) == 0) {
+        goto link;
+    }
+
+lookup:
+    AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xdata);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!lookup_on[i]) {
+            continue;
+        }
+
+        if (local->replies[i].op_ret == 0) {
+            if (gf_uuid_compare(anon_inode_gfid,
+                                local->replies[i].poststat.ia_gfid) == 0) {
+                priv->anon_inode[i] = 1;
+                iatt = local->replies[i].poststat;
+            } else {
+                if (i == child)
+                    child_op_errno = EINVAL;
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_DATA,
+                       "%s has gfid: %s", priv->anon_inode_name,
+                       uuid_utoa(local->replies[i].poststat.ia_gfid));
+            }
+        } else if (i == child) {
+            child_op_errno = local->replies[i].op_errno;
+        }
+    }
+link:
+    if (!gf_uuid_is_null(iatt.ia_gfid)) {
+        *linked_inode = inode_link(loc.inode, loc.parent, loc.name, &iatt);
+        if (*linked_inode) {
+            op_errno = 0;
+            inode_lookup(*linked_inode);
+        } else {
+            op_errno = ENOMEM;
         }
+        goto out;
+    }
 
-        return 0;
+out:
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
+    /*child_op_errno takes precedence*/
+    if (child_op_errno == 0) {
+        child_op_errno = op_errno;
+    }
+
+    if (child_op_errno && *linked_inode) {
+        inode_unref(*linked_inode);
+        *linked_inode = NULL;
+    }
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return -child_op_errno;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.h b/xlators/cluster/afr/src/afr-self-heal-common.h
deleted file mode 100644
index 6eabd1766c0..00000000000
--- a/xlators/cluster/afr/src/afr-self-heal-common.h
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __AFR_SELF_HEAL_COMMON_H__
-#define __AFR_SELF_HEAL_COMMON_H__
-
-#define FILE_HAS_HOLES(buf) (((buf)->ia_size) > ((buf)->ia_blocks * 512))
-#define AFR_SH_MIN_PARTICIPANTS 2
-
-typedef enum {
-        AFR_SELF_HEAL_ENTRY,
-        AFR_SELF_HEAL_METADATA,
-        AFR_SELF_HEAL_DATA,
-        AFR_SELF_HEAL_INVALID = -1,
-} afr_self_heal_type;
-
-typedef enum {
-        AFR_LOOKUP_FAIL_CONFLICTS = 1,
-        AFR_LOOKUP_FAIL_MISSING_GFIDS = 2,
-} afr_lookup_flags_t;
-
-int
-afr_sh_select_source (int sources[], int child_count);
-
-int
-afr_sh_source_count (int sources[], int child_count);
-
-void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this);
-
-int
-afr_build_pending_matrix (char **pending_key, int32_t **pending_matrix,
-                          unsigned char *ignorant_subvols,
-                          dict_t *xattr[], afr_transaction_type type,
-                          size_t child_count);
-
-void
-afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
-                         int32_t *delta_matrix[], unsigned char success[],
-                         int child_count, afr_transaction_type type);
-
-int
-afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
-                  struct iatt *bufs, afr_self_heal_type type,
-                  int32_t *success_children, int32_t *subvol_status);
-
-int
-afr_sh_delta_to_xattr (xlator_t *this,
-                       int32_t *delta_matrix[], dict_t *xattr[],
-		       int child_count, afr_transaction_type type);
-
-void
-afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
-                            size_t size);
-
-afr_self_heal_type
-afr_self_heal_type_for_transaction (afr_transaction_type type);
-
-int
-afr_build_sources (xlator_t *this, dict_t **xattr, struct iatt *bufs,
-                   int32_t **pending_matrix, int32_t *sources,
-                   int32_t *success_children, afr_transaction_type type,
-                   int32_t *subvol_status, gf_boolean_t ignore_ignorant);
-void
-afr_sh_common_reset (afr_self_heal_t *sh, unsigned int child_count);
-
-void
-afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
-                                   xlator_t *this,
-                                   int32_t op_ret, int32_t op_errno,
-                                   inode_t *inode, struct iatt *buf,
-                                   dict_t *xattr, struct iatt *postparent,
-                                   loc_t *loc);
-
-int
-afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      afr_lookup_done_cbk_t lookup_cbk, uuid_t uuid,
-                      int32_t flags, dict_t *xdata);
-int
-afr_sh_entry_expunge_remove (call_frame_t *expunge_frame, xlator_t *this,
-                             int active_src, struct iatt *buf,
-                             struct iatt *parentbuf);
-int
-afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                char *base_name, afr_lock_cbk_t lock_cbk);
-int
-afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
-                             int child_index);
-int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this,
-                    afr_lock_cbk_t lock_cbk);
-afr_local_t *
-afr_local_copy (afr_local_t *l, xlator_t *this);
-int
-afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
-                  off_t start, off_t len,
-                  afr_lock_cbk_t success_handler,
-                  afr_lock_cbk_t failure_handler);
-void
-afr_sh_set_error (afr_self_heal_t *sh, int32_t op_errno);
-void
-afr_sh_mark_source_sinks (call_frame_t *frame, xlator_t *this);
-typedef int
-(*afr_fxattrop_cbk_t) (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xattr, dict_t *xdata);
-int
-afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
-int
-afr_impunge_frame_create (call_frame_t *frame, xlator_t *this,
-                          int active_source, call_frame_t **impunge_frame);
-void
-afr_sh_reset (call_frame_t *frame, xlator_t *this);
-
-void
-afr_children_intersection_get (int32_t *set1, int32_t *set2,
-                               int *intersection, unsigned int child_count);
-int
-afr_get_no_xattr_dir_read_child (xlator_t *this, int32_t *success_children,
-                                 struct iatt *bufs);
-int
-afr_sh_erase_pending (call_frame_t *frame, xlator_t *this,
-                      afr_transaction_type type, afr_fxattrop_cbk_t cbk,
-                      int (*finish)(call_frame_t *frame, xlator_t *this));
-#endif /* __AFR_SELF_HEAL_COMMON_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 1672b4f282e..37bcc2b3f9e 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,1376 +8,884 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "afr-self-heal-algorithm.h"
+#include <glusterfs/byte-order.h>
+#include "protocol-common.h"
+#include "afr-messages.h"
+#include <glusterfs/events.h>
 
-int
-afr_sh_data_fail (call_frame_t *frame, xlator_t *this);
-
-static inline gf_boolean_t
-afr_sh_data_proceed (unsigned int success_count)
-{
-        return (success_count >= AFR_SH_MIN_PARTICIPANTS);
+#define HAS_HOLES(i) ((i->ia_blocks * 512) < (i->ia_size))
+static int
+__checksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, uint32_t weak, uint8_t *strong, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    struct afr_reply *replies = NULL;
+    int i = (long)cookie;
+
+    local = frame->local;
+    replies = local->replies;
+
+    replies[i].valid = 1;
+    replies[i].op_ret = op_ret;
+    replies[i].op_errno = op_errno;
+    if (xdata) {
+        replies[i].buf_has_zeroes = dict_get_str_boolean(
+            xdata, "buf-has-zeroes", _gf_false);
+        replies[i].fips_mode_rchecksum = dict_get_str_boolean(
+            xdata, "fips-mode-rchecksum", _gf_false);
+    }
+    if (strong) {
+        if (replies[i].fips_mode_rchecksum) {
+            memcpy(local->replies[i].checksum, strong, SHA256_DIGEST_LENGTH);
+        } else {
+            memcpy(local->replies[i].checksum, strong, MD5_DIGEST_LENGTH);
+        }
+    }
+
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+static gf_boolean_t
+__afr_can_skip_data_block_heal(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               int source, unsigned char *healed_sinks,
+                               off_t offset, size_t size, struct iatt *poststat)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    unsigned char *wind_subvols = NULL;
+    gf_boolean_t checksum_match = _gf_true;
+    struct afr_reply *replies = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+
+    xdata = dict_new();
+    if (!xdata)
+        goto out;
+    if (dict_set_int32_sizen(xdata, "check-zero-filled", 1)) {
+        dict_unref(xdata);
+        goto out;
+    }
+
+    wind_subvols = alloca0(priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || healed_sinks[i])
+            wind_subvols[i] = 1;
+    }
+
+    AFR_ONLIST(wind_subvols, frame, __checksum_cbk, rchecksum, fd, offset, size,
+               xdata);
+    if (xdata)
+        dict_unref(xdata);
+
+    if (!replies[source].valid || replies[source].op_ret != 0)
+        return _gf_false;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source)
+            continue;
+        if (replies[i].valid) {
+            if (memcmp(replies[source].checksum, replies[i].checksum,
+                       replies[source].fips_mode_rchecksum
+                           ? SHA256_DIGEST_LENGTH
+                           : MD5_DIGEST_LENGTH)) {
+                checksum_match = _gf_false;
+                break;
+            }
+        }
+    }
+
+    if (checksum_match) {
+        if (HAS_HOLES(poststat))
+            return _gf_true;
+
+        /* For non-sparse files, we might be better off writing the
+         * zeroes to sinks to avoid mismatch of disk-usage in bricks. */
+        if (local->replies[source].buf_has_zeroes)
+            return _gf_false;
+        else
+            return _gf_true;
+    }
+out:
+    return _gf_false;
+}
+
+static gf_boolean_t
+__afr_is_sink_zero_filled(xlator_t *this, fd_t *fd, size_t size, off_t offset,
+                          int sink)
+{
+    afr_private_t *priv = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec *iovec = NULL;
+    int count = 0;
+    int ret = 0;
+    gf_boolean_t zero_filled = _gf_false;
+
+    priv = this->private;
+    ret = syncop_readv(priv->children[sink], fd, size, offset, 0, &iovec,
+                       &count, &iobref, NULL, NULL, NULL);
+    if (ret < 0)
+        goto out;
+    ret = iov_0filled(iovec, count);
+    if (!ret)
+        zero_filled = _gf_true;
+out:
+    if (iovec)
+        GF_FREE(iovec);
+    if (iobref)
+        iobref_unref(iobref);
+    return zero_filled;
 }
 
-extern int
-sh_loop_finish (call_frame_t *loop_frame, xlator_t *this);
-
-int
-afr_post_sh_big_lock_success (call_frame_t *frame, xlator_t *this);
-
-int
-afr_post_sh_big_lock_failure (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_data_finish (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_data_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->completion_cbk (frame, this);
+static int
+__afr_selfheal_data_read_write(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               int source, unsigned char *healed_sinks,
+                               off_t offset, size_t size,
+                               struct afr_reply *replies, int type)
+{
+    struct iovec *iovec = NULL;
+    int count = 0;
+    struct iobref *iobref = NULL;
+    int ret = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = syncop_readv(priv->children[source], fd, size, offset, 0, &iovec,
+                       &count, &iobref, NULL, NULL, NULL);
+    if (ret <= 0)
+        return ret;
 
-        return 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
+
+            /*
+             * TODO: Use fiemap() and discard() to heal holes
+             * in the future.
+             *
+             * For now,
+             *
+             * - if the source had any holes at all,
+             * AND
+             * - if we are writing past the original file size
+             *   of the sink
+             * AND
+             * - is NOT the last block of the source file. if
+             *   the block contains EOF, it has to be written
+             *   in order to set the file size even if the
+             *   last block is 0-filled.
+             * AND
+             * - if the read buffer is filled with only 0's
+             *
+             * then, skip writing to this source. We don't depend
+             * on the write to happen to update the size as we
+             * have performed an ftruncate() upfront anyways.
+             */
+#define is_last_block(o, b, s) ((s >= o) && (s <= (o + b)))
+        if (HAS_HOLES((&replies[source].poststat)) &&
+            offset >= replies[i].poststat.ia_size &&
+            !is_last_block(offset, size, replies[source].poststat.ia_size) &&
+            (iov_0filled(iovec, count) == 0))
+            continue;
+
+        /* Avoid filling up sparse regions of the sink with 0-filled
+         * writes.*/
+        if (type == AFR_SELFHEAL_DATA_FULL &&
+            HAS_HOLES((&replies[source].poststat)) &&
+            ((offset + size) <= replies[i].poststat.ia_size) &&
+            (iov_0filled(iovec, count) == 0) &&
+            __afr_is_sink_zero_filled(this, fd, size, offset, i)) {
+            continue;
+        }
+
+        ret = syncop_writev(priv->children[i], fd, iovec, count, offset, iobref,
+                            0, NULL, NULL, NULL, NULL);
+        if (ret != iov_length(iovec, count)) {
+            /* write() failed on this sink. unset the corresponding
+               member in sinks[] (which is healed_sinks[] in the
+               caller) so that this server does NOT get considered
+               as successfully healed.
+            */
+            healed_sinks[i] = 0;
+        }
+    }
+    if (iovec)
+        GF_FREE(iovec);
+    if (iobref)
+        iobref_unref(iobref);
+
+    return ret;
+}
+
+static gf_boolean_t
+afr_source_sinks_locked(xlator_t *this, unsigned char *locked_on, int source,
+                        unsigned char *healed_sinks)
+{
+    afr_private_t *priv = this->private;
+    int i = 0;
+
+    if (!locked_on[source])
+        return _gf_false;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (healed_sinks[i] && locked_on[i])
+            return _gf_true;
+    }
+
+    return _gf_false;
 }
 
-
-int
-afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+afr_selfheal_data_block(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        int source, unsigned char *healed_sinks, off_t offset,
+                        size_t size, int type, struct afr_reply *replies)
 {
-        afr_local_t   *local       = NULL;
-        afr_private_t *priv        = NULL;
-        int            call_count  = 0;
-        int            child_index = (long) cookie;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "flush failed on %s on subvolume %s: %s",
-                                local->loc.path, priv->children[child_index]->name,
-                                strerror (op_errno));
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_sh_data_done (frame, this);
-        }
-
-        return 0;
-}
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    unsigned char *data_lock = NULL;
 
-int
-afr_sh_data_close (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_self_heal_t *sh         = NULL;
-        int              i          = 0;
-        int              call_count = 0;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        if (!sh->healing_fd) {
-                //This happens when file is non-reg
-                afr_sh_data_done (frame, this);
-                return 0;
-        }
-        call_count        = afr_set_elem_count_get (sh->success,
-                                                    priv->child_count);
-        local->call_count = call_count;
+    priv = this->private;
+    data_lock = alloca0(priv->child_count);
 
-        if (call_count == 0) {
-                afr_sh_data_done (frame, this);
-                return 0;
+    ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, offset, size,
+                               data_lock);
+    {
+        if (!afr_source_sinks_locked(this, data_lock, source, healed_sinks)) {
+            ret = -ENOTCONN;
+            goto unlock;
         }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sh->success[i])
-                        continue;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "closing fd of %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_flush_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->flush,
-                                   sh->healing_fd, NULL);
-
-                if (!--call_count)
-                        break;
+        if (type == AFR_SELFHEAL_DATA_DIFF &&
+            __afr_can_skip_data_block_heal(frame, this, fd, source,
+                                           healed_sinks, offset, size,
+                                           &replies[source].poststat)) {
+            ret = 0;
+            goto unlock;
         }
 
-        return 0;
+        ret = __afr_selfheal_data_read_write(
+            frame, this, fd, source, healed_sinks, offset, size, replies, type);
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, offset, size,
+                           data_lock);
+    return ret;
 }
 
-int
-afr_sh_data_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                         struct iatt *statpost, dict_t *xdata)
+static int
+afr_selfheal_data_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        unsigned char *healed_sinks)
 {
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-        afr_local_t   *local       = NULL;
-        afr_private_t *priv        = NULL;
-        int            call_count  = 0;
-        int            child_index = (long) cookie;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "setattr failed on %s on subvolume %s: %s",
-                                local->loc.path, priv->children[child_index]->name,
-                                strerror (op_errno));
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_sh_data_finish (frame, this);
-        }
+    local = frame->local;
+    priv = this->private;
 
+    if (!priv->ensure_durability)
         return 0;
-}
-
-int
-afr_sh_data_setattr (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_self_heal_t *sh         = NULL;
-        int              i          = 0;
-        int              call_count = 0;
-        int              source     = 0;
-        int32_t          valid      = 0;
-        struct iatt      stbuf      = {0,};
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        source = sh->source;
-
-        valid |= (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
-
-        stbuf.ia_atime = sh->buf[source].ia_atime;
-        stbuf.ia_atime_nsec = sh->buf[source].ia_atime_nsec;
-        stbuf.ia_mtime = sh->buf[source].ia_mtime;
-        stbuf.ia_mtime_nsec = sh->buf[source].ia_mtime_nsec;
-
-        call_count        = afr_set_elem_count_get (sh->success,
-                                                    priv->child_count);
-        local->call_count = call_count;
-
-        if (call_count == 0) {
-                GF_ASSERT (0);
-                afr_sh_data_finish (frame, this);
-                return 0;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sh->success[i])
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_setattr_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &local->loc, &stbuf, valid, NULL);
 
-                if (!--call_count)
-                        break;
-        }
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, fsync, fd, 0, NULL);
 
-        return 0;
+    for (i = 0; i < priv->child_count; i++)
+        if (healed_sinks[i] && local->replies[i].op_ret != 0)
+            /* fsync() failed. Do NOT consider this server
+               as successfully healed. Mark it so.
+            */
+            healed_sinks[i] = 0;
+    return 0;
 }
 
-int
-afr_sh_data_setattr_fstat_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               struct iatt *buf, dict_t *xdata)
+static int
+afr_data_self_heal_type_get(afr_private_t *priv, unsigned char *healed_sinks,
+                            int source, struct afr_reply *replies)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int child_index = (long) cookie;
-
-        local = frame->local;
-        sh = &local->self_heal;
+    int type = AFR_SELFHEAL_DATA_FULL;
+    int i = 0;
 
-        GF_ASSERT (sh->source == child_index);
-        if (op_ret != -1) {
-                sh->buf[child_index] = *buf;
-                afr_sh_data_setattr (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
-                        "time-stamps after self-heal", local->loc.path);
-                afr_sh_data_fail (frame, this);
+    if (priv->data_self_heal_algorithm == AFR_SELFHEAL_DATA_DYNAMIC) {
+        type = AFR_SELFHEAL_DATA_FULL;
+        for (i = 0; i < priv->child_count; i++) {
+            if (!healed_sinks[i] && i != source)
+                continue;
+            if (replies[i].poststat.ia_size) {
+                type = AFR_SELFHEAL_DATA_DIFF;
+                break;
+            }
         }
-
-        return 0;
-}
-
-/*
- * If there are any writes after the self-heal is triggered then the
- * stbuf stored in local->self_heal.buf[] will be invalid so we do one more
- * stat on the source and then set the [am]times
- */
-int
-afr_sh_set_timestamps (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_self_heal_t *sh         = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        STACK_WIND_COOKIE (frame, afr_sh_data_setattr_fstat_cbk,
-                           (void *) (long) sh->source,
-                           priv->children[sh->source],
-                           priv->children[sh->source]->fops->fstat,
-                           sh->healing_fd, NULL);
-        return 0;
+    } else {
+        type = priv->data_self_heal_algorithm;
+    }
+    return type;
 }
 
-//Fun fact, lock_cbk is being used for both lock & unlock
-int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this,
-                    afr_lock_cbk_t lock_cbk)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
-
-        GF_ASSERT (sh->data_lock_held);
-
-        sh->data_lock_held = _gf_false;
-        int_lock->lock_cbk = lock_cbk;
-        afr_unlock (frame, this);
-
-        return 0;
-}
-
-int
-afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
+static int
+afr_selfheal_data_do(call_frame_t *frame, xlator_t *this, fd_t *fd, int source,
+                     unsigned char *healed_sinks, struct afr_reply *replies)
 {
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "finishing data selfheal of %s", local->loc.path);
-
-        if (sh->data_lock_held)
-                afr_sh_data_unlock (frame, this, afr_sh_data_close);
-        else
-                afr_sh_data_close (frame, this);
-
-        return 0;
-}
+    afr_private_t *priv = NULL;
+    off_t off = 0;
+    size_t block = 0;
+    int type = AFR_SELFHEAL_DATA_FULL;
+    int ret = -1;
+    call_frame_t *iter_frame = NULL;
+    unsigned char arbiter_sink_status = 0;
 
-int
-afr_sh_data_fail (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing data selfheal on %s", uuid_utoa(fd->inode->gfid));
 
-        local = frame->local;
-        sh = &local->self_heal;
+    priv = this->private;
+    if (priv->arbiter_count) {
+        arbiter_sink_status = healed_sinks[ARBITER_BRICK_INDEX];
+        healed_sinks[ARBITER_BRICK_INDEX] = 0;
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "finishing failed data selfheal of %s", local->loc.path);
+    block = 128 * 1024 * priv->data_self_heal_window_size;
 
-        sh->op_failed = 1;
-        if (sh->data_lock_held)
-                afr_sh_data_unlock (frame, this, afr_sh_data_close);
-        else
-                afr_sh_data_close (frame, this);
-        return 0;
-}
+    type = afr_data_self_heal_type_get(priv, healed_sinks, source, replies);
 
-int
-afr_sh_data_erase_pending_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret,
-                               int32_t op_errno, dict_t *xattr, dict_t *xdata)
-{
-        int             call_count = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int32_t         child_index = (long) cookie;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Erasing of pending change "
-                        "log failed on %s for subvol %s, reason: %s",
-                        local->loc.path, priv->children[child_index]->name,
-                        strerror (op_errno));
-                sh->op_failed = 1;
-        }
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (sh->op_failed) {
-                        if (sh->old_loop_frame)
-                                sh_loop_finish (sh->old_loop_frame, this);
-                        sh->old_loop_frame = NULL;
-                        afr_sh_data_fail (frame, this);
-                        goto out;
-                }
-                if (!IA_ISREG (sh->type)) {
-                        afr_sh_data_finish (frame, this);
-                        goto out;
-                }
-                GF_ASSERT (sh->old_loop_frame);
-                afr_sh_data_lock (frame, this, 0, 0,
-                                  afr_post_sh_big_lock_success,
-                                  afr_post_sh_big_lock_failure);
+    for (off = 0; off < replies[source].poststat.ia_size; off += block) {
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            ret = -ENOTCONN;
+            goto out;
         }
-out:
-        return 0;
-}
-
-int
-afr_sh_data_erase_pending (call_frame_t *frame, xlator_t *this)
-{
-        afr_sh_erase_pending (frame, this, AFR_DATA_TRANSACTION,
-                              afr_sh_data_erase_pending_cbk,
-                              afr_sh_data_finish);
-        return 0;
-}
 
+        ret = afr_selfheal_data_block(iter_frame, this, fd, source,
+                                      healed_sinks, off, block, type, replies);
+        if (ret < 0)
+            goto out;
 
-static struct afr_sh_algorithm *
-sh_algo_from_name (xlator_t *this, char *name)
-{
-        int i = 0;
+        AFR_STACK_RESET(iter_frame);
+        if (iter_frame->local == NULL) {
+            ret = -ENOTCONN;
+            goto out;
+        }
+    }
 
-        while (afr_self_heal_algorithms[i].name) {
-                if (!strcmp (name, afr_self_heal_algorithms[i].name)) {
-                        return &afr_self_heal_algorithms[i];
-                }
+    ret = afr_selfheal_data_fsync(frame, this, fd, healed_sinks);
 
-                i++;
-        }
+out:
+    if (arbiter_sink_status)
+        healed_sinks[ARBITER_BRICK_INDEX] = arbiter_sink_status;
 
-        return NULL;
+    if (iter_frame)
+        AFR_STACK_DESTROY(iter_frame);
+    return ret;
 }
 
-
 static int
-sh_zero_byte_files_exist (afr_local_t *local, int child_count)
+__afr_selfheal_truncate_sinks(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              unsigned char *healed_sinks, uint64_t size)
 {
-        int             i = 0;
-        int             ret = 0;
-        afr_self_heal_t *sh = NULL;
-
-        sh = &local->self_heal;
-        for (i = 0; i < child_count; i++) {
-                if (!local->child_up[i] || sh->child_errno[i])
-                        continue;
-                if (sh->buf[i].ia_size == 0) {
-                        ret = 1;
-                        break;
-                }
-        }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-        return ret;
-}
+    local = frame->local;
+    priv = this->private;
 
+    /* This will send truncate on the arbiter brick as well if it is marked as
+     * sink. If changelog is enabled on the volume it captures truncate as a
+     * data transactions on the arbiter brick. This will help geo-rep to
+     * properly sync the data from master to slave if arbiter is the ACTIVE
+     * brick during syncing and which had got some entries healed for data as
+     * part of self heal.
+     */
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, ftruncate, fd, size,
+               NULL);
 
-struct afr_sh_algorithm *
-afr_sh_data_pick_algo (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *           priv  = NULL;
-        struct afr_sh_algorithm * algo  = NULL;
-        afr_local_t *             local = NULL;
-        afr_self_heal_t *         sh    = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-        algo  = sh_algo_from_name (this, priv->data_self_heal_algorithm);
-
-        if (algo == NULL) {
-                /* option not set, so fall back on heuristics */
-
-                if (sh_zero_byte_files_exist (local, priv->child_count)
-                    || (sh->file_size <= (priv->data_self_heal_window_size *
-                                          this->ctx->page_size))) {
-
-                        /*
-                         * If the file does not exist on one of the subvolumes,
-                         * or a zero-byte file exists (created by entry self-heal)
-                         * the entire content has to be copied anyway, so there
-                         * is no benefit from using the "diff" algorithm.
-                         *
-                         * If the file size is about the same as page size,
-                         * the entire file can be read and written with a few
-                         * (pipelined) STACK_WINDs, which will be faster
-                         * than "diff" which has to read checksums and then
-                         * read and write.
-                         */
-
-                        algo = sh_algo_from_name (this, "full");
-
-                } else {
-                        algo = sh_algo_from_name (this, "diff");
-                }
-        }
+    for (i = 0; i < priv->child_count; i++)
+        if (healed_sinks[i] && local->replies[i].op_ret == -1)
+            /* truncate() failed. Do NOT consider this server
+               as successfully healed. Mark it so.
+            */
+            healed_sinks[i] = 0;
 
-        return algo;
+    return 0;
 }
 
-
-int
-afr_sh_data_sync_prepare (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_has_source_witnesses(xlator_t *this, unsigned char *sources,
+                         uint64_t *witness)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        struct afr_sh_algorithm *sh_algo = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        sh = &local->self_heal;
+    priv = this->private;
 
-        sh->algo_completion_cbk = afr_sh_data_erase_pending;
-        sh->algo_abort_cbk      = afr_sh_data_fail;
-
-        sh_algo = afr_sh_data_pick_algo (frame, this);
-
-        sh->algo = sh_algo;
-        sh_algo->fn (frame, this);
-
-        return 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i] && witness[i])
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
-int
-afr_sh_data_trim_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+static gf_boolean_t
+afr_does_size_mismatch(xlator_t *this, unsigned char *sources,
+                       struct afr_reply *replies)
 {
-        int              call_count = 0;
-        int              child_index = 0;
-        afr_private_t    *priv = NULL;
-        afr_local_t      *local  = NULL;
-        afr_self_heal_t  *sh = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "ftruncate of %s on subvolume %s failed (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->op_failed = 1;
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "ftruncate of %s on subvolume %s completed",
-                                local->loc.path,
-                                priv->children[child_index]->name);
-                }
-        }
-        UNLOCK (&frame->lock);
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct iatt *min = NULL;
+    struct iatt *max = NULL;
 
-        call_count = afr_frame_return (frame);
+    priv = this->private;
 
-        if (call_count == 0) {
-                if (sh->op_failed)
-                        afr_sh_data_fail (frame, this);
-                else
-                        afr_sh_data_sync_prepare (frame, this);
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-        return 0;
-}
+        if (replies[i].op_ret < 0)
+            continue;
 
+        if (!sources[i])
+            continue;
 
-int
-afr_sh_data_trim_sinks (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv = NULL;
-        afr_local_t * local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int             *sources = NULL;
-        int              call_count = 0;
-        int              i = 0;
+        if (AFR_IS_ARBITER_BRICK(priv, i) && (replies[i].poststat.ia_size == 0))
+            continue;
 
+        if (!min)
+            min = &replies[i].poststat;
 
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
+        if (!max)
+            max = &replies[i].poststat;
 
-        sources = sh->sources;
-        call_count = sh->active_sinks;
+        if (min->ia_size > replies[i].poststat.ia_size)
+            min = &replies[i].poststat;
 
-        local->call_count = call_count;
+        if (max->ia_size < replies[i].poststat.ia_size)
+            max = &replies[i].poststat;
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] || !local->child_up[i])
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_trim_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->ftruncate,
-                                   sh->healing_fd, sh->file_size,
-                                   NULL);
-
-                if (!--call_count)
-                        break;
-        }
+    if (min && max) {
+        if (min->ia_size != max->ia_size)
+            return _gf_true;
+    }
 
-        return 0;
+    return _gf_false;
 }
 
-int
-afr_sh_inode_set_read_ctx (afr_self_heal_t *sh, xlator_t *this)
+static void
+afr_mark_biggest_witness_as_source(xlator_t *this, unsigned char *sources,
+                                   uint64_t *witness)
 {
-        afr_private_t   *priv = NULL;
-        int             ret = 0;
-        int             i = 0;
-
-        priv = this->private;
-        sh->source = afr_sh_select_source (sh->sources, priv->child_count);
-        if (sh->source < 0) {
-                ret = -1;
-                goto out;
-        }
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t biggest_witness = 0;
 
-        /* detect changes not visible through pending flags -- JIC */
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == sh->source || sh->child_errno[i])
-                        continue;
+    priv = this->private;
+    /* Find source with biggest witness count */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (biggest_witness < witness[i])
+            biggest_witness = witness[i];
+    }
 
-                if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[sh->source]))
-                        sh->sources[i] = 0;
-        }
+    /* Mark files with less witness count as not source */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (witness[i] < biggest_witness)
+            sources[i] = 0;
+    }
 
-        afr_reset_children (sh->fresh_children, priv->child_count);
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, priv->child_count);
-        afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                sh->fresh_children);
-out:
-        return ret;
+    return;
 }
 
-void
-afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
+/* This is a tie breaker function. Only one source be assigned here */
+static void
+afr_mark_newest_file_as_source(xlator_t *this, unsigned char *sources,
+                               struct afr_reply *replies)
 {
-        int              source = 0;
-        afr_local_t     *local      = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source     = sh->source;
-        sh->block_size = this->ctx->page_size;
-        sh->file_size  = sh->buf[source].ia_size;
-
-        if (FILE_HAS_HOLES (&sh->buf[source]))
-                sh->file_has_holes = 1;
-
-        if (sh->background && sh->unwind && !sh->unwound) {
-                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno,
-                            sh->op_failed);
-                sh->unwound = _gf_true;
-        }
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    uint32_t max_ctime = 0;
 
-        afr_sh_mark_source_sinks (frame, this);
-        if (sh->active_sinks == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no active sinks for performing self-heal on file %s",
-                        local->loc.path);
-                afr_sh_data_finish (frame, this);
-                return;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "self-healing file %s from subvolume %s to %d other",
-                local->loc.path, priv->children[sh->source]->name,
-                sh->active_sinks);
-        afr_sh_data_trim_sinks (frame, this);
-}
-
-int
-afr_sh_data_fxattrop_fstat_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local      = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              nsources = 0;
-        int              ret = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Pending matrix for: %s",
-                lkowner_utoa (&frame->root->lk_owner));
-
-        nsources = afr_build_sources (this, sh->xattr, sh->buf, sh->pending_matrix,
-                                      sh->sources, sh->success_children,
-                                      AFR_DATA_TRANSACTION, NULL, _gf_true);
-        if ((nsources == 0) && !sh->sync_done) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                local->govinda_gOvinda = 0;
-                afr_sh_data_finish (frame, this);
-                return 0;
-        }
-
-        if ((nsources == -1)
-            && (priv->favorite_child != -1)
-            && (sh->child_errno[priv->favorite_child] == 0)) {
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Picking favorite child %s as authentic source to "
-                        "resolve conflicting data of %s",
-                        priv->children[priv->favorite_child]->name,
-                        local->loc.path);
-
-                sh->sources[priv->favorite_child] = 1;
-
-                nsources = afr_sh_source_count (sh->sources,
-                                                priv->child_count);
-        }
-
-        if (nsources == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to self-heal contents of '%s' (possible "
-                        "split-brain). Please delete the file from all but "
-                        "the preferred subvolume.", local->loc.path);
-
-                local->govinda_gOvinda = 1;
-                afr_sh_data_fail (frame, this);
-                return 0;
-        }
-
-        local->govinda_gOvinda = 0;
-        ret = afr_sh_inode_set_read_ctx (sh, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No active sources found.");
-
-                afr_sh_data_fail (frame, this);
-                return 0;
-        }
+    priv = this->private;
+    /* Find source with latest ctime */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
 
-        if (sh->sync_done) {
-                afr_sh_data_setattr (frame, this);
-        } else {
-                afr_sh_data_fix (frame, this);
+        if (max_ctime <= replies[i].poststat.ia_ctime) {
+            source = i;
+            max_ctime = replies[i].poststat.ia_ctime;
         }
-        return 0;
-}
+    }
 
-int
-afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
-                                          dict_t **xattr,
-                                          afr_transaction_type txn_type,
-                                          uuid_t gfid)
-{
-        afr_private_t            *priv      = NULL;
-        int                      read_child = -1;
-        int32_t                  **pending_matrix = NULL;
-        int32_t                  *sources         = NULL;
-        int32_t                  *success_children   = NULL;
-        struct iatt              *bufs            = NULL;
-        int32_t                  nsources         = 0;
-        int32_t                  prev_read_child  = -1;
-        int32_t                  config_read_child = -1;
-        int32_t                  subvol_status = 0;
-
-        priv = this->private;
-        bufs = local->cont.lookup.bufs;
-        success_children = local->cont.lookup.success_children;
-
-        pending_matrix = afr_matrix_create (priv->child_count,
-                                            priv->child_count);
-        if (NULL == pending_matrix)
-                goto out;
-
-        sources = local->cont.lookup.sources;
-        memset (sources, 0, sizeof (*sources) * priv->child_count);
-
-        nsources = afr_build_sources (this, xattr, bufs, pending_matrix,
-                                      sources, success_children, txn_type,
-                                      &subvol_status, _gf_false);
-        if (subvol_status & SPLIT_BRAIN) {
-                gf_log (this->name, GF_LOG_WARNING, "%s: Possible split-brain",
-                        local->loc.path);
-                switch (txn_type) {
-                case AFR_DATA_TRANSACTION:
-                        afr_set_split_brain (this,
-                                             local->cont.lookup.inode,
-                                             _gf_true);
-                        nsources = 1;
-                        sources[success_children[0]] = 1;
-                        break;
-                case AFR_ENTRY_TRANSACTION:
-                        read_child = afr_get_no_xattr_dir_read_child (this,
-                                                             success_children,
-                                                             bufs);
-                        sources[read_child] = 1;
-                        nsources = 1;
-                        break;
-                default:
-                        break;
-                }
-        }
-        if (nsources < 0)
-                goto out;
-
-        prev_read_child = local->read_child_index;
-        config_read_child = priv->read_child;
-        read_child = afr_select_read_child_from_policy (success_children,
-                                                        priv->child_count,
-                                                        prev_read_child,
-                                                        config_read_child,
-                                                        sources,
-                                                        priv->hash_mode, gfid);
-out:
-        afr_matrix_cleanup (pending_matrix, priv->child_count);
-        gf_log (this->name, GF_LOG_DEBUG, "returning read_child: %d",
-                read_child);
-        return read_child;
+    /* Only mark one of the files as source to break ties */
+    memset(sources, 0, sizeof(*sources) * priv->child_count);
+    sources[source] = 1;
 }
 
-int
-afr_sh_data_fstat_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       struct iatt *buf, dict_t *xdata)
-{
-        afr_private_t   *priv  = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fstat of %s on %s succeeded",
-                                local->loc.path,
-                                priv->children[child_index]->name);
-
-                        sh->buf[child_index] = *buf;
-                        sh->success_children[sh->success_count] = child_index;
-                        sh->success_count++;
-                } else {
-                        gf_log (this->name, GF_LOG_ERROR, "%s: fstat failed "
-                                "on %s, reason %s", local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->child_errno[child_index] = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                /* Previous versions of glusterfs might have set
-                 * the pending data xattrs which need to be erased
-                 */
-                if (!afr_sh_data_proceed (sh->success_count)) {
-                        gf_log (this->name, GF_LOG_ERROR, "inspecting metadata "
-                                "succeeded on < %d children, aborting "
-                                "self-heal for %s", AFR_SH_MIN_PARTICIPANTS,
-                                local->loc.path);
-                        afr_sh_data_fail (frame, this);
-                        goto out;
-                }
-                afr_sh_data_fxattrop_fstat_done (frame, this);
-        }
-out:
-        return 0;
-}
-
-
-int
-afr_sh_data_fstat (call_frame_t *frame, xlator_t *this)
-{
-        afr_self_heal_t *sh    = NULL;
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv  = NULL;
-        int             call_count = 0;
-        int             i = 0;
-        int             child = 0;
-        int32_t         *fstat_children = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        fstat_children = memdup (sh->success_children,
-                                 sizeof (*fstat_children) * priv->child_count);
-        if (!fstat_children) {
-                afr_sh_data_fail (frame, this);
-                goto out;
-        }
-        call_count = sh->success_count;
-        local->call_count = call_count;
+static int
+__afr_selfheal_data_finalize_source(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    unsigned char *undid_pending, struct afr_reply *replies, uint64_t *witness)
+{
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count) {
+        /* split brain */
+        source = afr_mark_split_brain_source_sinks(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, AFR_DATA_TRANSACTION);
+        if (source < 0) {
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;type=data;"
+                     "file=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(inode->gfid));
+            return -EIO;
+        }
+
+        _afr_fav_child_reset_sink_xattrs(
+            frame, this, inode, source, healed_sinks, undid_pending,
+            AFR_DATA_TRANSACTION, locked_on, replies);
+        goto out;
+    }
+
+    /* No split brain at this point. If we were called from
+     * afr_heal_splitbrain_file(), abort.*/
+    if (afr_dict_contains_heal_op(frame))
+        return -EIO;
+
+    /* If there are no witnesses/size-mismatches on sources we are done*/
+    if (!afr_does_size_mismatch(this, sources, replies) &&
+        !afr_has_source_witnesses(this, sources, witness))
+        goto out;
+
+    afr_mark_largest_file_as_source(this, sources, replies);
+    afr_mark_biggest_witness_as_source(this, sources, witness);
+    afr_mark_newest_file_as_source(this, sources, replies);
+    if (priv->arbiter_count)
+        /* Choose non-arbiter brick as source for empty files. */
+        afr_mark_source_sinks_if_file_empty(this, sources, sinks, healed_sinks,
+                                            locked_on, replies,
+                                            AFR_DATA_TRANSACTION);
 
-        memset (sh->buf, 0, sizeof (*sh->buf) * priv->child_count);
-        afr_reset_children (sh->success_children, priv->child_count);
-        sh->success_count = 0;
-        for (i = 0; i < priv->child_count; i++) {
-                child = fstat_children[i];
-                if (child == -1)
-                        break;
-                STACK_WIND_COOKIE (frame, afr_sh_data_fstat_cbk,
-                                   (void *) (long) child,
-                                   priv->children[child],
-                                   priv->children[child]->fops->fstat,
-                                   sh->healing_fd, NULL);
-                --call_count;
-        }
-        GF_ASSERT (!call_count);
 out:
-        GF_FREE (fstat_children);
-        return 0;
-}
+    afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+    source = afr_choose_source_by_policy(priv, sources, AFR_DATA_TRANSACTION);
 
-void
-afr_sh_common_fxattrop_resp_handler (call_frame_t *frame, void *cookie,
-                                     xlator_t *this, int32_t op_ret,
-                                     int32_t op_errno, dict_t *xattr)
-{
-        afr_private_t   *priv  = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int child_index = (long) cookie;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fxattrop of %s on %s succeeded",
-                                local->loc.path,
-                                priv->children[child_index]->name);
-
-                        sh->xattr[child_index] = dict_ref (xattr);
-                        sh->success_children[sh->success_count] = child_index;
-                        sh->success_count++;
-                } else {
-                        gf_log (this->name, GF_LOG_ERROR, "fxattrop of %s "
-                                "failed on %s, reason %s", local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->child_errno[child_index] = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-}
-
-int
-afr_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                          xlator_t *this, int32_t op_ret, int32_t op_errno,
-                          dict_t *xattr, dict_t *xdata)
-{
-        int             call_count  = -1;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh     = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        afr_sh_common_fxattrop_resp_handler (frame, cookie, this, op_ret,
-                                             op_errno, xattr);
-
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                if (!afr_sh_data_proceed (sh->success_count)) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s, inspecting "
-                                "change log succeeded on < %d children",
-                                local->loc.path, AFR_SH_MIN_PARTICIPANTS);
-                        afr_sh_data_fail (frame, this);
-                        goto out;
-                }
-                afr_sh_data_fstat (frame, this);
-        }
-out:
-        return 0;
-}
-
-
-int
-afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this)
-{
-        afr_self_heal_t *sh    = NULL;
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv  = NULL;
-        dict_t          *xattr_req = NULL;
-        int32_t         *zero_pending = NULL;
-        int call_count = 0;
-        int i = 0;
-        int ret = 0;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        call_count = afr_up_children_count (local->child_up,
-                                            priv->child_count);
-
-        local->call_count = call_count;
-
-        xattr_req = dict_new();
-        if (!xattr_req) {
-                ret = -1;
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                zero_pending = GF_CALLOC (3, sizeof (*zero_pending),
-                                          gf_afr_mt_int32_t);
-                if (!zero_pending) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynptr (xattr_req, priv->pending_key[i],
-                                       zero_pending,
-                                       3 * sizeof (*zero_pending));
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unable to set dict value");
-                        goto out;
-                } else {
-                        zero_pending = NULL;
-                }
-        }
-
-        afr_reset_xattr (sh->xattr, priv->child_count);
-        afr_reset_children (sh->success_children, priv->child_count);
-        memset (sh->child_errno, 0,
-                sizeof (*sh->child_errno) * priv->child_count);
-        sh->success_count = 0;
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_sh_data_fxattrop_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fxattrop,
-                                           sh->healing_fd, GF_XATTROP_ADD_ARRAY,
-                                           xattr_req, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        if (ret) {
-                GF_FREE (zero_pending);
-                afr_sh_data_fail (frame, this);
-        }
-
-        return 0;
+    return source;
 }
 
+/*
+ * __afr_selfheal_data_prepare:
+ *
+ * This function inspects the on-disk xattrs and determines which subvols
+ * are sources and sinks.
+ *
+ * The return value is the index of the subvolume to be used as the source
+ * for self-healing, or -1 if no healing is necessary/split brain.
+ */
 int
-afr_sh_data_big_lock_success (call_frame_t *frame, xlator_t *this)
+__afr_selfheal_data_prepare(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            unsigned char *undid_pending,
+                            struct afr_reply *replies, unsigned char *pflag)
 {
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
 
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->data_lock_held = _gf_true;
-        afr_sh_data_fxattrop (frame, this);
-        return 0;
-}
-
-int
-afr_sh_data_post_blocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
+    priv = this->private;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Blocking data inodelks "
-                        "failed for %s. by %s",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
+    if (ret)
+        return ret;
 
-                sh->data_lock_failure_handler (frame, this);
-        } else {
+    witness = alloca0(priv->child_count * sizeof(*witness));
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_DATA_TRANSACTION, locked_on, sources,
+                                      sinks, witness, pflag);
+    if (ret)
+        return ret;
 
-                gf_log (this->name, GF_LOG_DEBUG, "Blocking data inodelks "
-                        "done for %s by %s. Proceding to self-heal",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
 
-                sh->data_lock_success_handler (frame, this);
-        }
+    source = __afr_selfheal_data_finalize_source(
+        frame, this, inode, sources, sinks, healed_sinks, locked_on,
+        undid_pending, replies, witness);
+    if (source < 0)
+        return -EIO;
 
-        return 0;
+    return source;
 }
 
-int
-afr_sh_data_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
+static int
+__afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    unsigned char *locked_on)
+{
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    int source = -1;
+    gf_boolean_t did_sh = _gf_true;
+    gf_boolean_t is_arbiter_the_only_sink = _gf_false;
+    gf_boolean_t empty_file = _gf_false;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, 0, 0,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "self-heal as only %d number "
+                         "of subvolumes "
+                         "could be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_data_prepare(frame, this, fd->inode, data_lock,
+                                          sources, sinks, healed_sinks,
+                                          undid_pending, locked_replies, NULL);
+        if (ret < 0)
+            goto unlock;
+
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
+        }
+
+        source = ret;
+
+        if (AFR_IS_ARBITER_BRICK(priv, source)) {
+            empty_file = afr_is_file_empty_on_all_children(priv,
+                                                           locked_replies);
+            if (empty_file)
+                goto restore_time;
+
+            did_sh = _gf_false;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_truncate_sinks(
+            frame, this, fd, healed_sinks,
+            locked_replies[source].poststat.ia_size);
+        if (ret < 0)
+            goto unlock;
+
+        if (priv->arbiter_count &&
+            AFR_COUNT(healed_sinks, priv->child_count) == 1 &&
+            healed_sinks[ARBITER_BRICK_INDEX]) {
+            is_arbiter_the_only_sink = _gf_true;
+            goto restore_time;
+        }
+        ret = 0;
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, 0, 0, data_lock);
+    if (ret < 0)
+        goto out;
+
+    if (!did_sh)
+        goto out;
+
+    ret = afr_selfheal_data_do(frame, this, fd, source, healed_sinks,
+                               locked_replies);
+    if (ret)
+        goto out;
+restore_time:
+    afr_selfheal_restore_time(frame, this, fd->inode, source, healed_sinks,
+                              locked_replies);
+
+    if (!is_arbiter_the_only_sink && !empty_file) {
+        ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, 0, 0,
+                                   data_lock);
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            did_sh = _gf_false;
+            goto skip_undo_pending;
+        }
+    }
+    ret = afr_selfheal_undo_pending(
+        frame, this, fd->inode, sources, sinks, healed_sinks, undid_pending,
+        AFR_DATA_TRANSACTION, locked_replies, data_lock);
+skip_undo_pending:
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, 0, 0, data_lock);
+out:
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking data inodelks "
-                        "failed for %s. by %s",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
+    if (did_sh)
+        afr_log_selfheal(fd->inode->gfid, this, ret, "data", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
 
-                int_lock->lock_cbk = afr_sh_data_post_blocking_inodelk_cbk;
-                afr_blocking_lock (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking data inodelks "
-                        "done for %s by %s. Proceeding to self-heal",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
-                sh->data_lock_success_handler (frame, this);
-        }
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
 
-        return 0;
+    return ret;
 }
 
 int
-afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, off_t start, off_t len)
+afr_selfheal_data_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, fd_t *fd,
+                           dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    afr_local_t *local = NULL;
+    int i = (long)cookie;
 
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_DATA_SELF_HEAL_LK;
+    local = frame->local;
 
-        afr_set_lock_number (frame, this);
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-        int_lock->lk_flock.l_start = start;
-        int_lock->lk_flock.l_len   = len;
-        int_lock->lk_flock.l_type  = F_WRLCK;
-        int_lock->lock_cbk         = afr_sh_data_post_nonblocking_inodelk_cbk;
+    syncbarrier_wake(&local->barrier);
 
-        afr_nonblocking_inodelk (frame, this);
-
-        return 0;
+    return 0;
 }
 
 int
-afr_post_sh_big_lock_success (call_frame_t *frame, xlator_t *this)
+afr_selfheal_data_open(xlator_t *this, inode_t *inode, fd_t **fd)
 {
-        afr_local_t     *local     = NULL;
-        afr_self_heal_t *sh        = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        GF_ASSERT (sh->old_loop_frame);
-        sh_loop_finish (sh->old_loop_frame, this);
-        sh->old_loop_frame = NULL;
-        sh->data_lock_held = _gf_true;
-        sh->sync_done = _gf_true;
-        afr_sh_data_fxattrop (frame, this);
-        return 0;
-}
+    int ret = 0;
+    fd_t *fd_tmp = NULL;
+    loc_t loc = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-int
-afr_post_sh_big_lock_failure (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local     = NULL;
-        afr_self_heal_t *sh        = NULL;
+    priv = this->private;
 
-        local = frame->local;
-        sh = &local->self_heal;
+    fd_tmp = fd_create(inode, 0);
+    if (!fd_tmp)
+        return -ENOMEM;
 
-        GF_ASSERT (sh->old_loop_frame);
-        sh_loop_finish (sh->old_loop_frame, this);
-        sh->old_loop_frame = NULL;
-        afr_sh_set_timestamps (frame, this);
-        return 0;
-}
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        fd_unref(fd_tmp);
+        goto out;
+    }
+    local = frame->local;
 
-int
-afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
-                  off_t start, off_t len,
-                  afr_lock_cbk_t success_handler,
-                  afr_lock_cbk_t failure_handler)
-{
-        afr_local_t *   local = NULL;
-        afr_self_heal_t * sh  = NULL;
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_data_open_cbk, open, &loc,
+               O_RDWR | O_LARGEFILE, fd_tmp, NULL);
 
-        local = frame->local;
-        sh    = &local->self_heal;
+    ret = -ENOTCONN;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-        sh->data_lock_success_handler = success_handler;
-        sh->data_lock_failure_handler = failure_handler;
-        return afr_sh_data_lock_rec (frame, this, start, len);
-}
-
-int
-afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              child_index = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        /* TODO: some of the open's might fail.
-           In that case, modify cleanup fn to send flush on those
-           fd's which are already open */
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "open of %s failed on child %s (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->op_failed = 1;
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "open of %s succeeded on child %s",
-                                local->loc.path,
-                                priv->children[child_index]->name);
-                }
+        if (local->replies[i].op_ret < 0) {
+            ret = -local->replies[i].op_errno;
+            continue;
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                if (sh->op_failed) {
-                        afr_sh_data_fail (frame, this);
-                        return 0;
-                }
+        ret = 0;
+        break;
+    }
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "fd for %s opened, commencing sync",
-                        local->loc.path);
+    if (ret < 0) {
+        fd_unref(fd_tmp);
+        goto out;
+    } else {
+        fd_bind(fd_tmp);
+    }
 
-                afr_sh_data_lock (frame, this, 0, 0,
-                                  afr_sh_data_big_lock_success,
-                                  afr_sh_data_fail);
-        }
-
-        return 0;
+    *fd = fd_tmp;
+out:
+    loc_wipe(&loc);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
 }
 
-
 int
-afr_sh_data_open (call_frame_t *frame, xlator_t *this)
+afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd)
 {
-        int i = 0;
-        int call_count = 0;
-        fd_t *fd = NULL;
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    int ret = 0;
+    inode_t *inode = fd->inode;
 
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-        local->call_count = call_count;
+    priv = this->private;
 
-        fd = fd_create (local->loc.inode, frame->root->pid);
-        sh->healing_fd = fd;
+    locked_on = alloca0(priv->child_count);
 
-        /* open sinks */
-        for (i = 0; i < priv->child_count; i++) {
-                if(!local->child_up[i])
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_open_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->open,
-                                   &local->loc,
-                                   O_RDWR|O_LARGEFILE, fd, NULL);
-
-                if (!--call_count)
-                        break;
+    ret = afr_selfheal_tie_breaker_inodelk(frame, this, inode, priv->sh_domain,
+                                           0, 0, locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "self-heal as only %d number of "
+                         "subvolumes could be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            /* Either less than two subvols available, or another
+               selfheal (from another server) is in progress. Skip
+               for now in any case there isn't anything to do.
+            */
+            ret = -ENOTCONN;
+            goto unlock;
         }
 
-        return 0;
-}
-
-void
-afr_sh_non_reg_fix (call_frame_t *frame, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        afr_private_t   *priv = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_local_t     *local = NULL;
-        int             i = 0;
-
-        if (op_ret < 0) {
-                afr_sh_data_fail (frame, this);
-                return;
-        }
+        ret = __afr_selfheal_data(frame, this, fd, locked_on);
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, inode, priv->sh_domain, 0, 0,
+                           locked_on);
 
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count ; i++) {
-                if (1 == local->child_up[i])
-                        sh->success[i] = 1;
-        }
-
-        afr_sh_erase_pending (frame, this, AFR_DATA_TRANSACTION,
-                              afr_sh_data_erase_pending_cbk,
-                              afr_sh_data_finish);
-}
-
-int
-afr_sh_non_reg_lock_success (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        sh->data_lock_held = _gf_true;
-        afr_sh_common_lookup (frame, this, &local->loc,
-                              afr_sh_non_reg_fix, NULL,
-                              AFR_LOOKUP_FAIL_CONFLICTS |
-                              AFR_LOOKUP_FAIL_MISSING_GFIDS,
-                              NULL);
-        return 0;
-}
-
-int
-afr_self_heal_data (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t *priv = this->private;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        local->govinda_gOvinda = afr_is_split_brain (this, sh->inode);
-
-        if (sh->do_data_self_heal &&
-            afr_data_self_heal_enabled (priv->data_self_heal)) {
-                if (IA_ISREG (sh->type)) {
-                        afr_sh_data_open (frame, this);
-                } else {
-                        afr_sh_data_lock (frame, this, 0, 0,
-                                          afr_sh_non_reg_lock_success,
-                                          afr_sh_data_fail);
-                }
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "not doing data self heal on %s",
-                        local->loc.path);
-                afr_sh_data_done (frame, this);
-        }
-
-        return 0;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 14ce3aa9fdd..64893f441e3 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,2351 +8,1269 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "inode.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-#define AFR_INIT_SH_FRAME_VALS(_frame, _local, _sh, _sh_frame, _sh_local, _sh_sh)\
-        do {\
-                _local = _frame->local;\
-                _sh = &_local->self_heal;\
-                _sh_frame = _sh->sh_frame;\
-                _sh_local = _sh_frame->local;\
-                _sh_sh    = &_sh_local->self_heal;\
-        } while (0);
-
-int
-afr_sh_entry_impunge_create_file (call_frame_t *impunge_frame, xlator_t *this,
-                                  int child_index);
-int
-afr_sh_entry_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->completion_cbk (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_cbk = afr_sh_entry_done;
-        afr_unlock (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_finish (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-
-        local = frame->local;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "finishing entry selfheal of %s", local->loc.path);
-
-        afr_sh_entry_unlock (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_erase_pending_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, dict_t *xattr, dict_t *xdata)
-{
-        long                 i          = 0;
-        int                  call_count = 0;
-        afr_local_t         *local      = NULL;
-        afr_self_heal_t     *sh         = NULL;
-        afr_local_t         *orig_local = NULL;
-        call_frame_t        *orig_frame = NULL;
-        afr_private_t       *priv       = NULL;
-        int32_t             read_child  = -1;
-
-        local = frame->local;
-        priv  = this->private;
-        sh = &local->self_heal;
-        i = (long)cookie;
-
-
-        afr_children_add_child (sh->fresh_children, i, priv->child_count);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: failed to erase pending xattrs on %s (%s)",
-                        local->loc.path, priv->children[i]->name,
-                        strerror (op_errno));
-        }
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (sh->source == -1) {
-                        //this happens if the forced merge option is set
-                        read_child = sh->fresh_children[0];
-                } else {
-                        read_child = sh->source;
-                }
-                afr_inode_set_read_ctx (this, sh->inode, read_child,
-                                        sh->fresh_children);
-                orig_frame = sh->orig_frame;
-                orig_local = orig_frame->local;
-
-                if (sh->source != -1) {
-                        orig_local->cont.lookup.buf.ia_nlink = sh->buf[sh->source].ia_nlink;
-                }
-
-                afr_sh_entry_finish (frame, this);
-        }
-
-        return 0;
-}
-
+#include <glusterfs/byte-order.h>
+#include "afr-transaction.h"
+#include "afr-messages.h"
+#include <glusterfs/syncop-utils.h>
+#include <glusterfs/events.h>
 
 int
-afr_sh_entry_erase_pending (call_frame_t *frame, xlator_t *this)
+afr_selfheal_entry_anon_inode(xlator_t *this, inode_t *dir, const char *name,
+                              inode_t *inode, int child,
+                              struct afr_reply *replies,
+                              gf_boolean_t *anon_inode)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        if (sh->entries_skipped) {
-                sh->op_failed = _gf_true;
-                goto out;
-        }
-        afr_sh_erase_pending (frame, this, AFR_ENTRY_TRANSACTION,
-                              afr_sh_entry_erase_pending_cbk,
-                              afr_sh_entry_finish);
-        return 0;
-out:
-        afr_sh_entry_finish (frame, this);
-        return 0;
-}
-
-
-
-static int
-next_active_source (call_frame_t *frame, xlator_t *this,
-                    int current_active_source)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              source = -1;
-        int              next_active_source = -1;
-        int              i = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        source = sh->source;
-
-        if (source != -1) {
-                if (current_active_source != source)
-                        next_active_source = source;
-                goto out;
-        }
-
-        /*
-          the next active sink becomes the source for the
-          'conservative decision' of merging all entries
-        */
-
-        for (i = 0; i < priv->child_count; i++) {
-                if ((sh->sources[i] == 0)
-                    && (local->child_up[i] == 1)
-                    && (i > current_active_source)) {
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    int i = 0;
+    char g[64] = {0};
+    unsigned char *lookup_success = NULL;
+    call_frame_t *frame = NULL;
+    loc_t loc2 = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[child];
+    lookup_success = alloca0(priv->child_count);
+    uuid_utoa_r(replies[child].poststat.ia_gfid, g);
+    loc.inode = inode_new(inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    if (replies[child].poststat.ia_type == IA_IFDIR) {
+        /* This directory may have sub-directory hierarchy which may need to
+         * be preserved for subsequent heals. So unconditionally move the
+         * directory to anonymous-inode directory*/
+        *anon_inode = _gf_true;
+        goto anon_inode;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+    local = frame->local;
+    gf_uuid_copy(loc.gfid, replies[child].poststat.ia_gfid);
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == 0) {
+            lookup_success[i] = 1;
+        } else if (local->replies[i].op_errno != ENOENT &&
+                   local->replies[i].op_errno != ESTALE) {
+            ret = -local->replies[i].op_errno;
+        }
+    }
+
+    if (priv->quorum_count) {
+        if (afr_has_quorum(lookup_success, this, NULL)) {
+            *anon_inode = _gf_true;
+        }
+    } else if (AFR_COUNT(lookup_success, priv->child_count) > 1) {
+        *anon_inode = _gf_true;
+    } else if (ret) {
+        goto out;
+    }
+
+anon_inode:
+    if (!*anon_inode) {
+        ret = 0;
+        goto out;
+    }
+
+    loc.parent = inode_ref(dir);
+    gf_uuid_copy(loc.pargfid, dir->gfid);
+    loc.name = name;
+
+    ret = afr_anon_inode_create(this, child, &loc2.parent);
+    if (ret < 0)
+        goto out;
+
+    loc2.name = g;
+    ret = syncop_rename(subvol, &loc, &loc2, NULL, NULL);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "Rename to %s dir %s/%s (%s) on %s failed",
+               priv->anon_inode_name, uuid_utoa(dir->gfid), name, g,
+               subvol->name);
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "Rename to %s dir %s/%s (%s) on %s successful",
+               priv->anon_inode_name, uuid_utoa(dir->gfid), name, g,
+               subvol->name);
+    }
 
-                        next_active_source = i;
-                        break;
-                }
-        }
 out:
-        return next_active_source;
-}
-
-
-
-static int
-next_active_sink (call_frame_t *frame, xlator_t *this,
-                  int current_active_sink)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              next_active_sink = -1;
-        int              i = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        /*
-          the next active sink becomes the source for the
-          'conservative decision' of merging all entries
-        */
-
-        for (i = 0; i < priv->child_count; i++) {
-                if ((sh->sources[i] == 0)
-                    && (local->child_up[i] == 1)
-                    && (i > current_active_sink)) {
+    loc_wipe(&loc);
+    loc_wipe(&loc2);
+    if (frame) {
+        AFR_STACK_DESTROY(frame);
+    }
 
-                        next_active_sink = i;
-                        break;
-                }
-        }
-
-        return next_active_sink;
-}
-
-int
-afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_expunge_subvol (call_frame_t *frame, xlator_t *this,
-                             int active_src);
-
-int
-afr_sh_entry_expunge_entry_done (call_frame_t *frame, xlator_t *this,
-                                 int active_src, int32_t op_ret,
-                                 int32_t op_errno)
-{
-        int              call_count = 0;
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                afr_sh_entry_expunge_subvol (frame, this, active_src);
-
-        return 0;
-}
-
-int
-afr_sh_entry_expunge_parent_setattr_cbk (call_frame_t *expunge_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop,
-                                         dict_t *xdata)
-{
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh    = NULL;
-        call_frame_t    *frame         = NULL;
-        int              active_src    = (long) cookie;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-
-        priv          = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh    = &expunge_local->self_heal;
-        frame         = expunge_sh->sh_frame;
-        local         = frame->local;
-        sh            = &local->self_heal;
-
-        if (op_ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "setattr on parent directory of %s on subvolume %s failed: %s",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name, strerror (op_errno));
-        }
-
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
+    return ret;
 }
 
-
-int
-afr_sh_entry_expunge_remove_cbk (call_frame_t *expunge_frame, void *cookie,
-                                 xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent, dict_t *xdata)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int              active_src = 0;
-        int32_t          valid = 0;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-
-        active_src = (long) cookie;
-
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "removed %s on %s",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "removing %s on %s failed (%s)",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
-        }
-
-        valid = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_parent_setattr_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->setattr,
-                           &expunge_sh->parent_loc,
-                           &expunge_sh->parentbuf,
-                           valid, NULL);
-
-        return 0;
-}
-
-
 int
-afr_sh_entry_expunge_unlink (call_frame_t *expunge_frame, xlator_t *this,
-                             int active_src)
+afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name,
+                          inode_t *inode, int child, struct afr_reply *replies)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-
-        priv          = this->private;
-        expunge_local = expunge_frame->local;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "expunging file %s on %s",
-                expunge_local->loc.path, priv->children[active_src]->name);
-
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->unlink,
-                           &expunge_local->loc, 0, NULL);
-
-        return 0;
-}
-
-
-
-int
-afr_sh_entry_expunge_rmdir (call_frame_t *expunge_frame, xlator_t *this,
-                            int active_src)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-
-        priv          = this->private;
-        expunge_local = expunge_frame->local;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "expunging directory %s on %s",
-                expunge_local->loc.path, priv->children[active_src]->name);
-
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->rmdir,
-                           &expunge_local->loc, 1, NULL);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_remove (call_frame_t *expunge_frame, xlator_t *this,
-                             int active_src, struct iatt *buf,
-                             struct iatt *parentbuf)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        int              type = 0;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-        loc_t           *loc           = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-        frame = expunge_sh->sh_frame;
-        local         = frame->local;
-        sh            = &local->self_heal;
-        loc           = &expunge_local->loc;
-
-        type = buf->ia_type;
-        if (loc->parent && uuid_is_null (loc->parent->gfid))
-                uuid_copy (loc->pargfid, parentbuf->ia_gfid);
-
-        switch (type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-        case IA_IFLNK:
-                afr_sh_entry_expunge_unlink (expunge_frame, this, active_src);
-                break;
+    char g[64] = {0};
+    afr_private_t *priv = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    gf_boolean_t anon_inode = _gf_false;
+
+    priv = this->private;
+    subvol = priv->children[child];
+
+    if ((!replies[child].valid) || (replies[child].op_ret < 0)) {
+        /*Nothing to do*/
+        ret = 0;
+        goto out;
+    }
+
+    if (priv->use_anon_inode) {
+        ret = afr_selfheal_entry_anon_inode(this, dir, name, inode, child,
+                                            replies, &anon_inode);
+        if (ret < 0 || anon_inode)
+            goto out;
+    }
+
+    loc.parent = inode_ref(dir);
+    loc.inode = inode_new(inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    loc.name = name;
+    switch (replies[child].poststat.ia_type) {
         case IA_IFDIR:
-                afr_sh_entry_expunge_rmdir (expunge_frame, this, active_src);
-                break;
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+                   "expunging dir %s/%s (%s) on %s", uuid_utoa(dir->gfid), name,
+                   uuid_utoa_r(replies[child].poststat.ia_gfid, g),
+                   subvol->name);
+            ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL);
+            break;
         default:
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s has unknown file type on %s: 0%o",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name, type);
-                goto out;
-                break;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+                   "expunging file %s/%s (%s) on %s", uuid_utoa(dir->gfid),
+                   name, uuid_utoa_r(replies[child].poststat.ia_gfid, g),
+                   subvol->name);
+            ret = syncop_unlink(subvol, &loc, NULL, NULL);
+            break;
+    }
 
-        return 0;
 out:
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, -1, EINVAL);
-
-        return 0;
+    loc_wipe(&loc);
+    return ret;
 }
 
-
 int
-afr_sh_entry_expunge_lookup_cbk (call_frame_t *expunge_frame, void *cookie,
-                                 xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno,
-                                 inode_t *inode, struct iatt *buf, dict_t *x,
-                                 struct iatt *postparent)
+afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source,
+                            unsigned char *sources, inode_t *dir,
+                            const char *name, inode_t *inode,
+                            struct afr_reply *replies)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        int              active_src = 0;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-        frame = expunge_sh->sh_frame;
-        active_src = (long) cookie;
-        local         = frame->local;
-        sh            = &local->self_heal;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "lookup of %s on %s failed (%s)",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    loc_t srcloc = {
+        0,
+    };
+    loc_t anonloc = {
+        0,
+    };
+    xlator_t *this = frame->this;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    struct iatt *iatt = NULL;
+    char *linkname = NULL;
+    mode_t mode = 0;
+    struct iatt newent = {
+        0,
+    };
+    unsigned char *newentry = NULL;
+    char iatt_uuid_str[64] = {0};
+    char dir_uuid_str[64] = {0};
+
+    priv = this->private;
+    iatt = &replies[source].poststat;
+    uuid_utoa_r(iatt->ia_gfid, iatt_uuid_str);
+    if (iatt->ia_type == IA_INVAL || gf_uuid_is_null(iatt->ia_gfid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SELF_HEAL_FAILED,
+               "Invalid ia_type (%d) or gfid(%s). source brick=%d, "
+               "pargfid=%s, name=%s",
+               iatt->ia_type, iatt_uuid_str, source,
+               uuid_utoa_r(dir->gfid, dir_uuid_str), name);
+        ret = -EINVAL;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata)
+        return -ENOMEM;
+    newentry = alloca0(priv->child_count);
+    loc.parent = inode_ref(dir);
+    gf_uuid_copy(loc.pargfid, dir->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
+
+    ret = afr_selfheal_entry_delete(this, dir, name, inode, dst, replies);
+    if (ret)
+        goto out;
+
+    ret = dict_set_gfuuid(xdata, "gfid-req", replies[source].poststat.ia_gfid,
+                          true);
+    if (ret)
+        goto out;
+
+    srcloc.inode = inode_ref(inode);
+    gf_uuid_copy(srcloc.gfid, iatt->ia_gfid);
+    ret = syncop_lookup(priv->children[dst], &srcloc, 0, 0, 0, 0);
+    if (ret == -ENOENT || ret == -ESTALE) {
+        newentry[dst] = 1;
+        ret = afr_selfheal_newentry_mark(frame, this, inode, source, replies,
+                                         sources, newentry);
+        if (ret)
+            goto out;
+    } else if (ret == 0 && iatt->ia_type == IA_IFDIR && priv->use_anon_inode) {
+        // Try rename from hidden directory
+        ret = afr_anon_inode_create(this, dst, &anonloc.parent);
+        if (ret < 0)
+            goto out;
+        anonloc.inode = inode_ref(inode);
+        anonloc.name = iatt_uuid_str;
+        ret = syncop_rename(priv->children[dst], &anonloc, &loc, NULL, NULL);
+        if (ret == -ENOENT || ret == -ESTALE)
+            ret = -1; /*This sets 'mismatch' to true*/
+        goto out;
+    }
+
+    mode = st_mode_from_ia(iatt->ia_prot, iatt->ia_type);
+
+    switch (iatt->ia_type) {
+        case IA_IFDIR:
+            ret = syncop_mkdir(priv->children[dst], &loc, mode, 0, xdata, NULL);
+            break;
+        case IA_IFLNK:
+            if (!newentry[dst]) {
+                ret = syncop_link(priv->children[dst], &srcloc, &loc, &newent,
+                                  NULL, NULL);
+            } else {
+                ret = syncop_readlink(priv->children[source], &srcloc,
+                                      &linkname, 4096, NULL, NULL);
+                if (ret <= 0)
+                    goto out;
+                ret = syncop_symlink(priv->children[dst], &loc, linkname, NULL,
+                                     xdata, NULL);
+            }
+            break;
+        default:
+            ret = dict_set_int32_sizen(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+            if (ret)
                 goto out;
-        }
+            ret = syncop_mknod(
+                priv->children[dst], &loc, mode,
+                makedev(ia_major(iatt->ia_rdev), ia_minor(iatt->ia_rdev)),
+                &newent, xdata, NULL);
+            break;
+    }
 
-        afr_sh_entry_expunge_remove (expunge_frame, this, active_src, buf,
-                                     postparent);
-
-        return 0;
 out:
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
+    if (xdata)
+        dict_unref(xdata);
+    GF_FREE(linkname);
+    loc_wipe(&loc);
+    loc_wipe(&srcloc);
+    loc_wipe(&anonloc);
+    return ret;
 }
 
-
-int
-afr_sh_entry_expunge_purge (call_frame_t *expunge_frame, xlator_t *this,
-                            int active_src)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "looking up %s on %s",
-                expunge_local->loc.path, priv->children[active_src]->name);
-
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_lookup_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->lookup,
-                           &expunge_local->loc, NULL);
-
-        return 0;
-}
-
-int
-afr_sh_entry_expunge_entry_cbk (call_frame_t *expunge_frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *buf, dict_t *x,
-                                struct iatt *postparent)
+static int
+__afr_selfheal_heal_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                           char *name, inode_t *inode, int source,
+                           unsigned char *sources, unsigned char *healed_sinks,
+                           unsigned char *locked_on, struct afr_reply *replies)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int              source = 0;
-        call_frame_t    *frame = NULL;
-        int              active_src = 0;
-        int              need_expunge = 0;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-        frame = expunge_sh->sh_frame;
-        active_src = expunge_sh->active_source;
-        source = (long) cookie;
-        local         = frame->local;
-        sh            = &local->self_heal;
-
-        if (op_ret == -1 && op_errno == ENOENT)
-                need_expunge = 1;
-        else if (op_ret == -1)
-                goto out;
-
-        if (!uuid_is_null (expunge_sh->entrybuf.ia_gfid) &&
-            !uuid_is_null (buf->ia_gfid) &&
-            (uuid_compare (expunge_sh->entrybuf.ia_gfid, buf->ia_gfid) != 0)) {
-                char uuidbuf1[64];
-                char uuidbuf2[64];
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "entry %s found on %s with mismatching gfid (%s/%s)",
-                        expunge_local->loc.path,
-                        priv->children[source]->name,
-                        uuid_utoa_r (expunge_sh->entrybuf.ia_gfid, uuidbuf1),
-                        uuid_utoa_r (buf->ia_gfid, uuidbuf2));
-                need_expunge = 1;
-        }
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-        if (need_expunge) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "missing entry %s on %s",
-                        expunge_local->loc.path,
-                        priv->children[source]->name);
+    priv = this->private;
 
-                if (postparent)
-                        expunge_sh->parentbuf = *postparent;
+    if (!replies[source].valid)
+        return -EIO;
 
-                afr_sh_entry_expunge_purge (expunge_frame, this, active_src);
+    /* Skip healing this entry if the last lookup on it failed for reasons
+     * other than ENOENT.
+     */
+    if ((replies[source].op_ret < 0) && (replies[source].op_errno != ENOENT))
+        return -replies[source].op_errno;
 
-                return 0;
-        }
-
-out:
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s exists under %s",
-                        expunge_local->loc.path,
-                        priv->children[source]->name);
+    if (replies[source].op_ret == 0) {
+        ret = afr_lookup_and_heal_gfid(this, fd->inode, name, inode, replies,
+                                       source, sources,
+                                       &replies[source].poststat.ia_gfid, NULL);
+        if (ret)
+            return ret;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
+        if (replies[source].op_ret == -1 &&
+            replies[source].op_errno == ENOENT) {
+            ret = afr_selfheal_entry_delete(this, fd->inode, name, inode, i,
+                                            replies);
         } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "looking up %s under %s failed (%s)",
-                        expunge_local->loc.path,
-                        priv->children[source]->name,
-                        strerror (op_errno));
-        }
-
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
-                            gf_dirent_t *entry)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              ret = -1;
-        call_frame_t    *expunge_frame = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int              active_src = 0;
-        int              source = 0;
-        int              op_errno = 0;
-        char            *name = NULL;
-        int             op_ret = -1;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-        source = sh->source;
-        sh->expunge_done = afr_sh_entry_expunge_entry_done;
-
-        name = entry->d_name;
-
-        if ((strcmp (name, ".") == 0)
-            || (strcmp (name, "..") == 0)) {
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "skipping inspection of %s under %s",
-                        name, local->loc.path);
-                op_ret = 0;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "inspecting existence of %s under %s",
-                name, local->loc.path);
-
-        expunge_frame = copy_frame (frame);
-        if (!expunge_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
-
-        expunge_frame->local = expunge_local;
-        expunge_sh = &expunge_local->self_heal;
-        expunge_sh->sh_frame = frame;
-        expunge_sh->active_source = active_src;
-        expunge_sh->entrybuf = entry->d_stat;
-        loc_copy (&expunge_sh->parent_loc, &local->loc);
-
-        ret = afr_build_child_loc (this, &expunge_local->loc, &local->loc,
-                                   name);
-        if (ret != 0) {
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "looking up %s on %s", expunge_local->loc.path,
-                priv->children[source]->name);
-
-        STACK_WIND_COOKIE (expunge_frame,
-                           afr_sh_entry_expunge_entry_cbk,
-                           (void *) (long) source,
-                           priv->children[source],
-                           priv->children[source]->fops->lookup,
-                           &expunge_local->loc, NULL);
-
-        ret = 0;
-out:
-        if (ret == -1)
-                sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_readdir_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  gf_dirent_t *entries, dict_t *xdata)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        gf_dirent_t     *entry = NULL;
-        off_t            last_offset = 0;
-        int              active_src = 0;
-        int              entry_count = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-
-        if (op_ret <= 0) {
-                if (op_ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "readdir of %s on subvolume %s failed (%s)",
-                                local->loc.path,
-                                priv->children[active_src]->name,
-                                strerror (op_errno));
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "readdir of %s on subvolume %s complete",
-                                local->loc.path,
-                                priv->children[active_src]->name);
-                }
-
-                afr_sh_entry_expunge_all (frame, this);
-                return 0;
-        }
-
-        list_for_each_entry (entry, &entries->list, list) {
-                last_offset = entry->d_off;
-                entry_count++;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "readdir'ed %d entries from %s",
-                entry_count, priv->children[active_src]->name);
-
-        sh->offset = last_offset;
-        local->call_count = entry_count;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                afr_sh_entry_expunge_entry (frame, this, entry);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_entry_expunge_subvol (call_frame_t *frame, xlator_t *this,
-                             int active_src)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        STACK_WIND (frame, afr_sh_entry_expunge_readdir_cbk,
-                    priv->children[active_src],
-                    priv->children[active_src]->fops->readdirp,
-                    sh->healing_fd, sh->block_size, sh->offset, NULL);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              active_src = -1;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->offset = 0;
-
-        if (sh->source == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no active sources for %s to expunge entries",
-                        local->loc.path);
-                goto out;
-        }
-
-        active_src = next_active_sink (frame, this, sh->active_source);
-        sh->active_source = active_src;
+            if (!gf_uuid_compare(replies[i].poststat.ia_gfid,
+                                 replies[source].poststat.ia_gfid))
+                continue;
 
-        if (sh->op_failed) {
-                goto out;
+            ret = afr_selfheal_recreate_entry(frame, i, source, sources,
+                                              fd->inode, name, inode, replies);
         }
+        if (ret < 0)
+            break;
+    }
 
-        if (active_src == -1) {
-                /* completed creating missing files on all subvolumes */
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "expunging entries of %s on %s to other sinks",
-                local->loc.path, priv->children[active_src]->name);
-
-        afr_sh_entry_expunge_subvol (frame, this, active_src);
-
-        return 0;
-out:
-        afr_sh_entry_impunge_all (frame, this);
-        return 0;
-
-}
-
-
-int
-afr_sh_entry_impunge_entry_done (call_frame_t *frame, xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno)
-{
-        int              call_count = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        if (op_ret < 0)
-                sh->entries_skipped = _gf_true;
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                afr_sh_entry_impunge_subvol (frame, this);
-
-        return 0;
+    return ret;
 }
 
-void
-afr_sh_entry_call_impunge_done (call_frame_t *impunge_frame, xlator_t *this,
-                                int32_t op_ret, int32_t op_errno)
+static int
+afr_selfheal_detect_gfid_and_type_mismatch(xlator_t *this,
+                                           struct afr_reply *replies,
+                                           inode_t *inode, uuid_t pargfid,
+                                           char *bname, int src_idx,
+                                           unsigned char *locked_on, int *src)
 {
-        afr_local_t     *impunge_local = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-
-        AFR_STACK_DESTROY (impunge_frame);
-        sh->impunge_done (frame, this, op_ret, op_errno);
+    int i = 0;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    void *gfid = NULL;
+    ia_type_t ia_type = IA_INVAL;
+
+    priv = this->private;
+    gfid = &replies[src_idx].poststat.ia_gfid;
+    ia_type = replies[src_idx].poststat.ia_type;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == src_idx)
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        if (gf_uuid_is_null(replies[i].poststat.ia_gfid))
+            continue;
+
+        if (replies[i].poststat.ia_type == IA_INVAL)
+            continue;
+
+        if (ia_type == IA_INVAL || gf_uuid_is_null(gfid)) {
+            src_idx = i;
+            ia_type = replies[src_idx].poststat.ia_type;
+            gfid = &replies[src_idx].poststat.ia_gfid;
+            continue;
+        }
+
+        if (gf_uuid_compare(gfid, replies[i].poststat.ia_gfid) &&
+            (ia_type == replies[i].poststat.ia_type)) {
+            ret = afr_gfid_split_brain_source(this, replies, inode, pargfid,
+                                              bname, src_idx, i, locked_on, src,
+                                              NULL);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Skipping conservative merge on the "
+                       "file.");
+            return ret;
+        }
+
+        if (ia_type != replies[i].poststat.ia_type) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                   "Type mismatch detected "
+                   "for <gfid:%s>/%s>, %s on %s and %s on %s. "
+                   "Skipping conservative merge on the file.",
+                   uuid_utoa(pargfid), bname,
+                   gf_inode_type_to_str(replies[i].poststat.ia_type),
+                   priv->children[i]->name,
+                   gf_inode_type_to_str(replies[src_idx].poststat.ia_type),
+                   priv->children[src_idx]->name);
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;type=file;"
+                     "file=<gfid:%s>/%s>;count=2;child-%d=%s;type-"
+                     "%d=%s;child-%d=%s;type-%d=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(pargfid), bname, i, priv->children[i]->name, i,
+                     gf_inode_type_to_str(replies[i].poststat.ia_type), src_idx,
+                     priv->children[src_idx]->name, src_idx,
+                     gf_inode_type_to_str(replies[src_idx].poststat.ia_type));
+            return -1;
+        }
+    }
+
+    return 0;
 }
 
-int
-afr_sh_entry_impunge_setattr_cbk (call_frame_t *impunge_frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  struct iatt *preop, struct iatt *postop,
-                                  dict_t *xdata)
+static int
+__afr_selfheal_merge_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            char *name, inode_t *inode, unsigned char *sources,
+                            unsigned char *healed_sinks,
+                            unsigned char *locked_on, struct afr_reply *replies)
 {
-        int              call_count = 0;
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        int              child_index = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        child_index = (long) cookie;
-
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setattr done for %s on %s",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "setattr (%s) on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-        }
+    int ret = 0;
+    int i = 0;
+    int source = -1;
+    int src = -1;
+    afr_private_t *priv = NULL;
 
-        call_count = afr_frame_return (impunge_frame);
-        if (call_count == 0) {
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                0, op_errno);
-        }
+    priv = this->private;
 
-        return 0;
-}
-
-int
-afr_sh_entry_impunge_parent_setattr_cbk (call_frame_t *setattr_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop,
-                                         dict_t *xdata)
-{
-        int             call_count = 0;
-        afr_local_t     *setattr_local = NULL;
-
-        setattr_local = setattr_frame->local;
-        if (op_ret != 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "setattr on parent directory (%s) failed: %s",
-                        setattr_local->loc.path, strerror (op_errno));
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            source = i;
+            break;
         }
+    }
 
-        call_count = afr_frame_return (setattr_frame);
-        if (call_count == 0)
-                AFR_STACK_DESTROY (setattr_frame);
+    if (source == -1) {
+        /* entry got deleted in the mean time? */
         return 0;
-}
-
-int
-afr_sh_entry_impunge_setattr (call_frame_t *impunge_frame, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_local_t     *setattr_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *setattr_frame = NULL;
-        int32_t          valid = 0;
-        int32_t          op_errno = 0;
-        int              child_index = 0;
-        int              call_count = 0;
-        int              i = 0;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setting ownership of %s on %s to %d/%d",
-                impunge_local->loc.path,
-                priv->children[child_index]->name,
-                impunge_sh->entrybuf.ia_uid,
-                impunge_sh->entrybuf.ia_gid);
-
-        setattr_frame = copy_frame (impunge_frame);
-        if (!setattr_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-        AFR_LOCAL_ALLOC_OR_GOTO (setattr_frame->local, out);
-        setattr_local = setattr_frame->local;
-        call_count = afr_errno_count (NULL, impunge_sh->child_errno,
-                                      priv->child_count, 0);
-        loc_copy (&setattr_local->loc, &impunge_sh->parent_loc);
-        impunge_local->call_count = call_count;
-        setattr_local->call_count = call_count;
+    }
+
+    /* Set all the sources as 1, otheriwse newentry_mark won't be set */
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            sources[i] = 1;
+        }
+    }
+
+    ret = afr_lookup_and_heal_gfid(this, fd->inode, name, inode, replies,
+                                   source, sources,
+                                   &replies[source].poststat.ia_gfid, NULL);
+    if (ret)
+        return ret;
+
+    /* In case of type mismatch / unable to resolve gfid mismatch on the
+     * entry, return -1.*/
+    ret = afr_selfheal_detect_gfid_and_type_mismatch(
+        this, replies, inode, fd->inode->gfid, name, source, locked_on, &src);
+
+    if (ret < 0)
+        return ret;
+    if (src != -1) {
+        source = src;
         for (i = 0; i < priv->child_count; i++) {
-                if (impunge_sh->child_errno[i])
-                        continue;
-                valid         = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-                STACK_WIND_COOKIE (setattr_frame,
-                                   afr_sh_entry_impunge_parent_setattr_cbk,
-                                   (void *) (long) i, priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &setattr_local->loc,
-                                   &impunge_sh->parentbuf, valid, NULL);
-
-                valid = GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
-                        GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-                STACK_WIND_COOKIE (impunge_frame,
-                                   afr_sh_entry_impunge_setattr_cbk,
-                                   (void *) (long) i, priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &impunge_local->loc,
-                                   &impunge_sh->entrybuf, valid, NULL);
-                call_count--;
+            if (i != src && replies[i].valid &&
+                gf_uuid_compare(replies[src].poststat.ia_gfid,
+                                replies[i].poststat.ia_gfid)) {
+                sources[i] = 0;
+            }
         }
-        GF_ASSERT (!call_count);
-        return 0;
-out:
-        if (setattr_frame)
-                AFR_STACK_DESTROY (setattr_frame);
-        afr_sh_entry_call_impunge_done (impunge_frame, this, 0, op_errno);
-        return 0;
-}
+    }
 
-int
-afr_sh_entry_impunge_xattrop_cbk (call_frame_t *impunge_frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  dict_t *xattr, dict_t *xdata)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        int              child_index = 0;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || !healed_sinks[i])
+            continue;
 
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: failed to perform xattrop on %s (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-                goto out;
+        if (src != -1) {
+            if (!gf_uuid_compare(replies[src].poststat.ia_gfid,
+                                 replies[i].poststat.ia_gfid))
+                continue;
+        } else if (replies[i].op_errno != ENOENT) {
+            continue;
         }
 
-        afr_sh_entry_impunge_setattr (impunge_frame, this);
-        return 0;
-out:
-        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                        -1, op_errno);
-        return 0;
-}
+        ret |= afr_selfheal_recreate_entry(frame, i, source, sources, fd->inode,
+                                           name, inode, replies);
+    }
 
-void
-afr_sh_prepare_new_entry_pending_matrix (int32_t **pending,
-                                         int *child_errno,
-                                         struct iatt *buf,
-                                         unsigned int child_count)
-{
-        int              midx             = 0;
-        int              idx              = 0;
-        int              i                = 0;
-
-        midx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-        if (IA_ISDIR (buf->ia_type))
-                idx = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
-        else if (IA_ISREG (buf->ia_type))
-                idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-        else
-                idx = -1;
-        for (i = 0; i < child_count; i++) {
-                if (child_errno[i])
-                        continue;
-                pending[i][midx] = hton32 (1);
-                if (idx == -1)
-                        continue;
-                pending[i][idx] = hton32 (1);
-        }
-}
-
-int
-afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
-                                      xlator_t *this)
-{
-        int              active_src       = 0;
-        dict_t          *xattr            = NULL;
-        afr_private_t   *priv             = NULL;
-        afr_local_t     *impunge_local    = NULL;
-        afr_self_heal_t *impunge_sh       = NULL;
-        int32_t         op_errno          = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        active_src = impunge_sh->active_source;
-
-        afr_sh_prepare_new_entry_pending_matrix (impunge_local->pending,
-                                                 impunge_sh->child_errno,
-                                                 &impunge_sh->entrybuf,
-                                                 priv->child_count);
-        xattr = dict_new ();
-        if (!xattr) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        afr_set_pending_dict (priv, xattr, impunge_local->pending, active_src,
-                              LOCAL_LAST);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_xattrop_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->xattrop,
-                           &impunge_local->loc, GF_XATTROP_ADD_ARRAY, xattr, NULL);
-
-        if (xattr)
-                dict_unref (xattr);
-        return 0;
-out:
-        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                        -1, op_errno);
-        return 0;
+    return ret;
 }
 
-int
-afr_sh_entry_impunge_newfile_cbk (call_frame_t *impunge_frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  inode_t *inode, struct iatt *stbuf,
-                                  struct iatt *preparent,
-                                  struct iatt *postparent, dict_t *xdata)
+static int
+__afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            char *name, inode_t *inode, int source,
+                            unsigned char *sources, unsigned char *healed_sinks,
+                            unsigned char *locked_on, struct afr_reply *replies)
 {
-        int              call_count       = 0;
-        afr_private_t   *priv             = NULL;
-        afr_local_t     *impunge_local    = NULL;
-        afr_self_heal_t *impunge_sh       = NULL;
-        int              child_index      = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                impunge_sh->child_errno[child_index] = op_errno;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "creation of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-        } else {
-                impunge_sh->child_errno[child_index] = 0;
-        }
-
-        call_count = afr_frame_return (impunge_frame);
-        if (call_count == 0) {
-                if (!afr_errno_count (NULL, impunge_sh->child_errno,
-                                      priv->child_count, 0)) {
-                        // new_file creation failed every where
-                        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                        -1, op_errno);
-                        goto out;
-                }
-                afr_sh_entry_impunge_perform_xattrop (impunge_frame, this);
-        }
-out:
-        return 0;
+    int ret = -1;
+
+    if (source < 0)
+        ret = __afr_selfheal_merge_dirent(frame, this, fd, name, inode, sources,
+                                          healed_sinks, locked_on, replies);
+    else
+        ret = __afr_selfheal_heal_dirent(frame, this, fd, name, inode, source,
+                                         sources, healed_sinks, locked_on,
+                                         replies);
+    return ret;
 }
 
-int
-afr_sh_entry_impunge_hardlink_cbk (call_frame_t *impunge_frame, void *cookie,
-                                   xlator_t *this, int32_t op_ret,
-                                   int32_t op_errno, inode_t *inode,
-                                   struct iatt *buf, struct iatt *preparent,
-                                   struct iatt *postparent, dict_t *xdata)
+static gf_boolean_t
+is_full_heal_marker_present(xlator_t *this, dict_t *xdata, int idx)
 {
-        int              call_count        = 0;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh  = NULL;
-
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-
-        if (IA_IFLNK == impunge_sh->entrybuf.ia_type) {
-                //For symlinks impunge is attempted un-conditionally
-                //So the file can already exist.
-                if ((op_ret < 0) && (op_errno == EEXIST))
-                        op_ret = 0;
-        }
-
-        call_count = afr_frame_return (impunge_frame);
-        if (call_count == 0)
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                op_ret, op_errno);
-
-        return 0;
+    int i = 0;
+    int pending[3] = {
+        0,
+    };
+    void *pending_raw = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (!xdata)
+        return _gf_false;
+
+    /* Iterate over each of the priv->pending_keys[] elements and then
+     * see if any of them have data segment non-zero. If they do, return
+     * true. Else return false.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
+
+        if (!pending_raw)
+            continue;
+
+        memcpy(pending, pending_raw, sizeof(pending));
+        if (ntoh32(pending[idx]))
+            return _gf_true;
+    }
+
+    return _gf_false;
 }
 
-int
-afr_sh_entry_impunge_hardlink (call_frame_t *impunge_frame, xlator_t *this,
-                               int child_index)
+static gf_boolean_t
+afr_need_full_heal(xlator_t *this, struct afr_reply *replies, int source,
+                   unsigned char *healed_sinks, afr_transaction_type type)
 {
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh  = NULL;
-        loc_t           *loc           = NULL;
-        struct iatt     *buf           = NULL;
-        loc_t            oldloc        = {0};
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        loc = &impunge_local->loc;
-        buf = &impunge_sh->entrybuf;
-
-        oldloc.inode = inode_ref (loc->inode);
-        uuid_copy (oldloc.gfid, buf->ia_gfid);
-        gf_log (this->name, GF_LOG_DEBUG, "linking missing file %s on %s",
-                loc->path, priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_hardlink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->link,
-                           &oldloc, loc, NULL);
-        loc_wipe (&oldloc);
+    int i = 0;
+    int idx = 0;
+    afr_private_t *priv = NULL;
 
-        return 0;
-}
-
-int
-afr_sh_nameless_lookup_cbk (call_frame_t *impunge_frame, void *cookie,
-                            xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, inode_t *inode,
-                            struct iatt *buf, dict_t *xattr,
-                            struct iatt *postparent)
-{
-        if (op_ret < 0) {
-                 afr_sh_entry_impunge_create_file (impunge_frame, this,
-                                                   (long)cookie);
-        } else {
-                afr_sh_entry_impunge_hardlink (impunge_frame, this,
-                                               (long)cookie);
-        }
-        return 0;
-}
+    priv = this->private;
 
-int
-afr_sh_entry_impunge_check_hardlink (call_frame_t *impunge_frame,
-                                     xlator_t *this,
-                                     int child_index, struct iatt *stbuf)
-{
-        afr_private_t   *priv          = NULL;
-        call_frame_t    *frame             = NULL;
-        afr_local_t     *impunge_local     = NULL;
-        afr_local_t     *local             = NULL;
-        afr_self_heal_t *impunge_sh        = NULL;
-        afr_self_heal_t *sh                = NULL;
-        loc_t           *loc           = NULL;
-        dict_t          *xattr_req     = NULL;
-        loc_t            oldloc        = {0};
-        int              ret           = -1;
-
-        priv = this->private;
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        loc = &impunge_local->loc;
-
-        xattr_req = dict_new ();
-        if (!xattr_req)
-                goto out;
-        oldloc.inode = inode_ref (loc->inode);
-        uuid_copy (oldloc.gfid, stbuf->ia_gfid);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_nameless_lookup_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->lookup,
-                           &oldloc, xattr_req);
-        ret = 0;
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        loc_wipe (&oldloc);
-        if (ret)
-                sh->impunge_done (frame, this, -1, ENOMEM);
-        return 0;
-}
+    if (!priv->esh_granular)
+        return _gf_true;
 
-int
-afr_sh_entry_impunge_mknod (call_frame_t *impunge_frame, xlator_t *this,
-                            int child_index, struct iatt *stbuf)
-{
-        afr_private_t *priv          = NULL;
-        afr_local_t   *impunge_local = NULL;
-        dict_t        *dict          = NULL;
-        int            ret           = 0;
+    if (type != AFR_ENTRY_TRANSACTION)
+        return _gf_true;
 
-        priv = this->private;
-        impunge_local = impunge_frame->local;
+    priv = this->private;
+    idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "creating missing file %s on %s",
-                impunge_local->loc.path,
-                priv->children[child_index]->name);
+    /* If there is a clear source, check whether the full-heal-indicator
+     * is present in its xdata. Otherwise, we need to examine all the
+     * participating bricks and then figure if *even* one of them has a
+     * full-heal-indicator.
+     */
 
-        dict = dict_new ();
-        if (!dict)
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
+    if (source != -1) {
+        if (is_full_heal_marker_present(this, replies[source].xdata, idx))
+            return _gf_true;
+    }
 
-        GF_ASSERT (!uuid_is_null (stbuf->ia_gfid));
-        ret = afr_set_dict_gfid (dict, stbuf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_INFO, "%s: gfid set failed",
-                        impunge_local->loc.path);
+    /* else ..*/
 
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->mknod,
-                           &impunge_local->loc,
-                           st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
-                           makedev (ia_major (stbuf->ia_rdev),
-                                    ia_minor (stbuf->ia_rdev)), 0, dict);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
 
-        if (dict)
-                dict_unref (dict);
+        if (is_full_heal_marker_present(this, replies[i].xdata, idx))
+            return _gf_true;
+    }
 
-        return 0;
+    return _gf_false;
 }
 
-
-
-int
-afr_sh_entry_impunge_mkdir (call_frame_t *impunge_frame, xlator_t *this,
-                            int child_index, struct iatt *stbuf)
+static int
+__afr_selfheal_entry_finalize_source(xlator_t *this, unsigned char *sources,
+                                     unsigned char *healed_sinks,
+                                     unsigned char *locked_on,
+                                     struct afr_reply *replies,
+                                     uint64_t *witness)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        dict_t          *dict = NULL;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+    int i = 0;
 
-        int ret = 0;
+    priv = this->private;
 
-        priv = this->private;
-        impunge_local = impunge_frame->local;
+    sources_count = AFR_COUNT(sources, priv->child_count);
 
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                return 0;
-        }
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count || afr_does_witness_exist(this, witness)) {
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+        afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+        return -1;
+    }
 
-        GF_ASSERT (!uuid_is_null (stbuf->ia_gfid));
-        ret = afr_set_dict_gfid (dict, stbuf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_INFO, "%s: gfid set failed",
-                        impunge_local->loc.path);
+    source = afr_choose_source_by_policy(priv, sources, AFR_ENTRY_TRANSACTION);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "creating missing directory %s on %s",
-                impunge_local->loc.path,
-                priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->mkdir,
-                           &impunge_local->loc,
-                           st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
-                           0, dict);
-
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_symlink (call_frame_t *impunge_frame, xlator_t *this,
-                              int child_index, const char *linkname)
-{
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *impunge_local = NULL;
-        dict_t          *dict          = NULL;
-        struct iatt     *buf           = NULL;
-        int              ret           = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-
-        buf = &impunge_local->cont.symlink.buf;
-
-        dict = dict_new ();
-        if (!dict) {
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                -1, ENOMEM);
-                goto out;
+    /*If the selected source does not blame any other brick, then mark
+     * everything as sink to trigger conservative merge.
+     */
+    if (source != -1 && !AFR_COUNT(healed_sinks, priv->child_count)) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (locked_on[i]) {
+                sources[i] = 0;
+                healed_sinks[i] = 1;
+            }
         }
+        return -1;
+    }
 
-        GF_ASSERT (!uuid_is_null (buf->ia_gfid));
-        ret = afr_set_dict_gfid (dict, buf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: dict set gfid failed",
-                        impunge_local->loc.path);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "creating missing symlink %s -> %s on %s",
-                impunge_local->loc.path, linkname,
-                priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->symlink,
-                           linkname, &impunge_local->loc, 0, dict);
-
-        if (dict)
-                dict_unref (dict);
-out:
-        return 0;
+    return source;
 }
 
-
 int
-afr_sh_entry_impunge_symlink_unlink_cbk (call_frame_t *impunge_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preparent,
-                                         struct iatt *postparent, dict_t *xdata)
+__afr_selfheal_entry_prepare(call_frame_t *frame, xlator_t *this,
+                             inode_t *inode, unsigned char *locked_on,
+                             unsigned char *sources, unsigned char *sinks,
+                             unsigned char *healed_sinks,
+                             struct afr_reply *replies, int *source_p,
+                             unsigned char *pflag)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              child_index = -1;
-        int              call_count = -1;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "unlink of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        afr_sh_entry_impunge_symlink (impunge_frame, this, child_index,
-                                      impunge_sh->linkname);
-
-        return 0;
-out:
-        LOCK (&impunge_frame->lock);
-        {
-                call_count = --impunge_local->call_count;
-        }
-        UNLOCK (&impunge_frame->lock);
-
-        if (call_count == 0)
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                op_ret, op_errno);
-
-        return 0;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
+    if (ret)
+        return ret;
+
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_ENTRY_TRANSACTION, locked_on, sources,
+                                      sinks, witness, pflag);
+    if (ret)
+        return ret;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    source = __afr_selfheal_entry_finalize_source(this, sources, healed_sinks,
+                                                  locked_on, replies, witness);
+
+    if (source < 0) {
+        /* If source is < 0 (typically split-brain), we perform a
+           conservative merge of entries rather than erroring out */
+    }
+    *source_p = source;
+
+    return ret;
 }
 
-
-int
-afr_sh_entry_impunge_symlink_unlink (call_frame_t *impunge_frame, xlator_t *this,
-                                     int child_index)
+static int
+afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                          char *name, inode_t *parent_idx_inode,
+                          xlator_t *subvol, gf_boolean_t full_crawl)
 {
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *impunge_local = NULL;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "unlinking symlink %s with wrong target on %s",
-                impunge_local->loc.path,
-                priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_symlink_unlink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->unlink,
-                           &impunge_local->loc, 0, NULL);
-
+    int ret = 0;
+    int source = -1;
+    unsigned char *locked_on = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    inode_t *inode = NULL;
+    struct afr_reply *replies = NULL;
+    struct afr_reply *par_replies = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+
+    priv = this->private;
+
+    if (afr_is_private_directory(priv, fd->inode->gfid, name,
+                                 GF_CLIENT_PID_SELF_HEALD)) {
         return 0;
+    }
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+    ret = dict_set_int32_sizen(xattr, GF_GFIDLESS_LOOKUP, 1);
+    if (ret) {
+        dict_unref(xattr);
+        return -1;
+    }
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    locked_on = alloca0(priv->child_count);
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+    par_replies = alloca0(priv->child_count * sizeof(*par_replies));
+
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes "
+                         " could be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, this->name);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_entry_prepare(frame, this, fd->inode, locked_on,
+                                           sources, sinks, healed_sinks,
+                                           par_replies, &source, NULL);
+        if (ret < 0)
+            goto unlock;
+
+        inode = afr_selfheal_unlocked_lookup_on(frame, fd->inode, name, replies,
+                                                locked_on, xattr);
+        if (!inode) {
+            ret = -ENOMEM;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_entry_dirent(frame, this, fd, name, inode, source,
+                                          sources, healed_sinks, locked_on,
+                                          replies);
+
+        if ((ret == 0) && (priv->esh_granular) && parent_idx_inode) {
+            ret = afr_shd_entry_purge(subvol, parent_idx_inode, name,
+                                      inode->ia_type);
+            /* Why is ret force-set to 0? We do not care about
+             * index purge failing for full heal as it is quite
+             * possible during replace-brick that not all files
+             * and directories have their name indices present in
+             * entry-changes/.
+             */
+            ret = 0;
+        }
+    }
+
+unlock:
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL, locked_on,
+                           NULL);
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    if (par_replies)
+        afr_replies_wipe(par_replies, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
+
+    return ret;
 }
 
-
-int
-afr_sh_entry_impunge_readlink_sink_cbk (call_frame_t *impunge_frame, void *cookie,
-                                        xlator_t *this,
-                                        int32_t op_ret, int32_t op_errno,
-                                        const char *linkname, struct iatt *sbuf, dict_t *xdata)
+static inode_t *
+afr_shd_entry_changes_index_inode(xlator_t *this, xlator_t *subvol,
+                                  uuid_t pargfid)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              child_index = -1;
-        int              call_count = -1;
-        int              active_src = -1;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-        active_src    = impunge_sh->active_source;
-
-        child_index = (long) cookie;
-
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "readlink of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        /* symlink doesn't exist on the sink */
-
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
-                afr_sh_entry_impunge_symlink (impunge_frame, this,
-                                              child_index, impunge_sh->linkname);
-                return 0;
-        }
-
-
-        /* symlink exists on the sink, so check if targets match */
-
-        if (strcmp (linkname, impunge_sh->linkname) == 0) {
-                /* targets match, nothing to do */
-
-                goto out;
-        } else {
-                /*
-                 * Hah! Sneaky wolf in sheep's clothing!
-                 */
-                afr_sh_entry_impunge_symlink_unlink (impunge_frame, this,
-                                                     child_index);
-                return 0;
-        }
+    int ret = -1;
+    void *index_gfid = NULL;
+    loc_t rootloc = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
+
+    ret = syncop_getxattr(subvol, &rootloc, &xattr,
+                          GF_XATTROP_ENTRY_CHANGES_GFID, NULL, NULL);
+    if (ret || !xattr) {
+        errno = -ret;
+        goto out;
+    }
+
+    ret = dict_get_ptr(xattr, GF_XATTROP_ENTRY_CHANGES_GFID, &index_gfid);
+    if (ret) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_copy(loc.pargfid, index_gfid);
+    loc.name = gf_strdup(uuid_utoa(pargfid));
+
+    ret = syncop_lookup(subvol, &loc, &iatt, NULL, NULL, NULL);
+    if (ret < 0) {
+        errno = -ret;
+        goto out;
+    }
+
+    inode = inode_link(loc.inode, NULL, NULL, &iatt);
 
 out:
-        LOCK (&impunge_frame->lock);
-        {
-                call_count = --impunge_local->call_count;
-        }
-        UNLOCK (&impunge_frame->lock);
-
-        if (call_count == 0)
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                op_ret, op_errno);
-
-        return 0;
-}
+    if (xattr)
+        dict_unref(xattr);
+    loc_wipe(&rootloc);
+    GF_FREE((char *)loc.name);
+    loc_wipe(&loc);
 
-
-int
-afr_sh_entry_impunge_readlink_sink (call_frame_t *impunge_frame, xlator_t *this,
-                                    int child_index)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "checking symlink target of %s on %s",
-                impunge_local->loc.path, priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_sink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->readlink,
-                           &impunge_local->loc, 4096, NULL);
-
-        return 0;
+    return inode;
 }
 
-
-int
-afr_sh_entry_impunge_readlink_cbk (call_frame_t *impunge_frame, void *cookie,
-                                   xlator_t *this,
-                                   int32_t op_ret, int32_t op_errno,
-                                   const char *linkname, struct iatt *sbuf, dict_t *xdata)
+static int
+afr_selfheal_entry_do_subvol(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                             int child)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              child_index = -1;
-        int              call_count = -1;
-        int              active_src = -1;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        active_src = impunge_sh->active_source;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "readlink of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        impunge_sh->linkname = gf_strdup (linkname);
-        afr_sh_entry_impunge_readlink_sink (impunge_frame, this, child_index);
-
-        return 0;
-
-out:
-        LOCK (&impunge_frame->lock);
+    int ret = 0;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    off_t offset = 0;
+    call_frame_t *iter_frame = NULL;
+    xlator_t *subvol = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t mismatch = _gf_false;
+    afr_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[child];
+
+    INIT_LIST_HEAD(&entries.list);
+
+    local = frame->local;
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame)
+        return -ENOMEM;
+
+    loc.inode = afr_shd_entry_changes_index_inode(this, subvol,
+                                                  fd->inode->gfid);
+
+    while ((ret = syncop_readdir(subvol, fd, 131072, offset, &entries, NULL,
+                                 NULL))) {
+        if (ret > 0)
+            ret = 0;
+        list_for_each_entry(entry, &entries.list, list)
         {
-                call_count = --impunge_local->call_count;
-        }
-        UNLOCK (&impunge_frame->lock);
+            offset = entry->d_off;
 
-        if (call_count == 0)
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                op_ret, op_errno);
+            if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                continue;
 
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_readlink (call_frame_t *impunge_frame, xlator_t *this,
-                               int child_index, struct iatt *stbuf)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              active_src = -1;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        active_src = impunge_sh->active_source;
-        impunge_local->cont.symlink.buf = *stbuf;
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->readlink,
-                           &impunge_local->loc, 4096, NULL);
-
-        return 0;
-}
-
-int
-afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
-                             int child_index)
-{
-        call_frame_t    *frame             = NULL;
-        afr_local_t     *impunge_local     = NULL;
-        afr_local_t     *local             = NULL;
-        afr_self_heal_t *impunge_sh        = NULL;
-        afr_self_heal_t *sh                = NULL;
-        afr_private_t   *priv = NULL;
-        ia_type_t       type = IA_INVAL;
-        int             active_src = 0;
-        struct iatt     *buf = NULL;
-
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src = impunge_sh->active_source;
-        afr_update_loc_gfids (&impunge_local->loc, &impunge_sh->entrybuf,
-                              &impunge_sh->parentbuf);
-
-        buf = &impunge_sh->entrybuf;
-        type = buf->ia_type;
-
-        switch (type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-        case IA_IFLNK:
-                afr_sh_entry_impunge_check_hardlink (impunge_frame, this,
-                                                     child_index, buf);
+            ret = afr_selfheal_entry_dirent(iter_frame, this, fd, entry->d_name,
+                                            loc.inode, subvol,
+                                            local->need_full_crawl);
+            AFR_STACK_RESET(iter_frame);
+            if (iter_frame->local == NULL) {
+                ret = -ENOTCONN;
                 break;
-        case IA_IFDIR:
-                afr_sh_entry_impunge_mkdir (impunge_frame, this,
-                                            child_index, buf);
-                break;
-        default:
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s has unknown file type on %s: 0%o",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name, type);
-                sh->impunge_done (frame, this, -1, EINVAL);
+            }
+
+            if (ret == -1) {
+                /* gfid or type mismatch. */
+                mismatch = _gf_true;
+                ret = 0;
+            }
+            if (ret)
                 break;
         }
 
-        return 0;
-}
-
-int
-afr_sh_entry_impunge_create_file (call_frame_t *impunge_frame, xlator_t *this,
-                                  int child_index)
-{
-        call_frame_t    *frame             = NULL;
-        afr_local_t     *impunge_local     = NULL;
-        afr_local_t     *local             = NULL;
-        afr_self_heal_t *impunge_sh        = NULL;
-        afr_self_heal_t *sh                = NULL;
-        afr_private_t   *priv = NULL;
-        ia_type_t       type = IA_INVAL;
-        int             active_src = 0;
-        struct iatt     *buf = NULL;
-
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src = impunge_sh->active_source;
-        buf = &impunge_sh->entrybuf;
-        type = buf->ia_type;
-
-        switch (type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-                afr_sh_entry_impunge_mknod (impunge_frame, this,
-                                            child_index, buf);
-                break;
-        case IA_IFLNK:
-                afr_sh_entry_impunge_readlink (impunge_frame, this,
-                                               child_index, buf);
-                break;
-        default:
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s has unknown file type on %s: 0%o",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name, type);
-                sh->impunge_done (frame, this, -1, EINVAL);
-                break;
-        }
-
-        return 0;
-}
-
-gf_boolean_t
-afr_sh_need_recreate (afr_self_heal_t *impunge_sh, unsigned int child,
-                      unsigned int child_count)
-{
-        gf_boolean_t    recreate = _gf_false;
-
-        GF_ASSERT (impunge_sh->child_errno);
-
-        if (child == impunge_sh->active_source)
-                goto out;
+        gf_dirent_free(&entries);
+        if (ret)
+            break;
+    }
 
-        if (IA_IFLNK == impunge_sh->entrybuf.ia_type) {
-                recreate = _gf_true;
-                goto out;
-        }
+    loc_wipe(&loc);
 
-        if (impunge_sh->child_errno[child] == ENOENT)
-                recreate = _gf_true;
-out:
-        return recreate;
+    AFR_STACK_DESTROY(iter_frame);
+    if (mismatch == _gf_true)
+        /* undo pending will be skipped */
+        ret = -1;
+    return ret;
 }
 
-unsigned int
-afr_sh_recreate_count (afr_self_heal_t *impunge_sh, int *sources,
-                       unsigned int child_count)
+static int
+afr_selfheal_entry_granular_dirent(xlator_t *subvol, gf_dirent_t *entry,
+                                   loc_t *parent, void *data)
 {
-        int             count = 0;
-        int             i = 0;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    afr_granular_esh_args_t *args = data;
+
+    /* Look up the actual inode associated with entry. If the lookup returns
+     * ESTALE or ENOENT, then it means we have a stale index. Remove it.
+     * This is analogous to the check in afr_shd_index_heal() except that
+     * here it is achieved through LOOKUP and in afr_shd_index_heal() through
+     * a GETXATTR.
+     */
+
+    loc.inode = inode_new(args->xl->itable);
+    loc.parent = inode_ref(args->heal_fd->inode);
+    gf_uuid_copy(loc.pargfid, loc.parent->gfid);
+    loc.name = entry->d_name;
+
+    ret = syncop_lookup(args->xl, &loc, &iatt, NULL, NULL, NULL);
+    if ((ret == -ENOENT) || (ret == -ESTALE)) {
+        /* The name indices under the pgfid index dir are guaranteed
+         * to be regular files. Hence the hardcoding.
+         */
+        afr_shd_entry_purge(subvol, parent->inode, entry->d_name, IA_IFREG);
+        ret = 0;
+        goto out;
+    }
+    /* TBD: afr_shd_zero_xattrop? */
 
-        for (i = 0; i < child_count; i++) {
-                if (afr_sh_need_recreate (impunge_sh, i, child_count))
-                        count++;
-        }
+    ret = afr_selfheal_entry_dirent(args->frame, args->xl, args->heal_fd,
+                                    entry->d_name, parent->inode, subvol,
+                                    _gf_false);
+    AFR_STACK_RESET(args->frame);
+    if (args->frame->local == NULL)
+        ret = -ENOTCONN;
 
-        return count;
-}
+    if (ret == -1)
+        args->mismatch = _gf_true;
 
-int
-afr_sh_entry_call_impunge_recreate (call_frame_t *impunge_frame,
-                                    xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        unsigned int     recreate_count = 0;
-        int              i = 0;
-        int              active_src = 0;
-
-        priv          = this->private;
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src = impunge_sh->active_source;
-        impunge_sh->entrybuf = impunge_sh->buf[active_src];
-        impunge_sh->parentbuf = impunge_sh->parentbufs[active_src];
-        recreate_count = afr_sh_recreate_count (impunge_sh, sh->sources,
-                                                priv->child_count);
-        if (!recreate_count) {
-                afr_sh_entry_call_impunge_done (impunge_frame, this, 0, 0);
-                goto out;
-        }
-        impunge_local->call_count = recreate_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!impunge_local->child_up[i]) {
-                        impunge_sh->child_errno[i] = ENOTCONN;
-                        continue;
-                }
-                if (!afr_sh_need_recreate (impunge_sh, i, priv->child_count)) {
-                        impunge_sh->child_errno[i] = EEXIST;
-                        continue;
-                }
-        }
-        for (i = 0; i < priv->child_count; i++) {
-                if (!afr_sh_need_recreate (impunge_sh, i, priv->child_count))
-                        continue;
-                (void)afr_sh_entry_impunge_create (impunge_frame, this, i);
-                recreate_count--;
-        }
-        GF_ASSERT (!recreate_count);
 out:
-        return 0;
-}
-
-void
-afr_sh_entry_common_lookup_done (call_frame_t *impunge_frame, xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        unsigned int     gfid_miss_count = 0;
-        unsigned int     children_up_count = 0;
-        uuid_t           gfid = {0};
-        int              active_src = 0;
-
-        priv          = this->private;
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src    = impunge_sh->active_source;
-
-        if (op_ret < 0)
-                goto done;
-        if (impunge_sh->child_errno[active_src]) {
-                op_ret = -1;
-                op_errno = impunge_sh->child_errno[active_src];
-                goto done;
-        }
-
-        gfid_miss_count = afr_gfid_missing_count (this->name,
-                                                  impunge_sh->success_children,
-                                                  impunge_sh->buf, priv->child_count,
-                                                  impunge_local->loc.path);
-        children_up_count = afr_up_children_count (impunge_local->child_up,
-                                                   priv->child_count);
-        if ((gfid_miss_count == children_up_count) &&
-            (children_up_count < priv->child_count)) {
-                op_ret = -1;
-                op_errno = ENODATA;
-                gf_log (this->name, GF_LOG_ERROR, "Not all children are up, "
-                        "gfid should not be assigned in this state for %s",
-                        impunge_local->loc.path);
-                goto done;
-        }
-
-        if (gfid_miss_count) {
-                afr_update_gfid_from_iatts (gfid, impunge_sh->buf,
-                                            impunge_sh->success_children,
-                                            priv->child_count);
-                if (uuid_is_null (gfid)) {
-                        sh->entries_skipped = _gf_true;
-                        gf_log (this->name, GF_LOG_INFO, "%s: Skipping entry "
-                                "self-heal because of gfid absence",
-                                impunge_local->loc.path);
-                        goto done;
-                }
-                afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
-                                      afr_sh_entry_common_lookup_done, gfid,
-                                      AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
-                                      NULL);
-        } else {
-                afr_sh_entry_call_impunge_recreate (impunge_frame, this);
-        }
-        return;
-done:
-        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                        op_ret, op_errno);
-        return;
+    loc_wipe(&loc);
+    return 0;
 }
 
-int
-afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
-                            gf_dirent_t *entry)
+static int
+afr_selfheal_entry_granular(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            int subvol_idx, gf_boolean_t is_src)
 {
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        afr_self_heal_t *impunge_sh  = NULL;
-        int              ret = -1;
-        call_frame_t    *impunge_frame = NULL;
-        afr_local_t     *impunge_local = NULL;
-        int              active_src = 0;
-        int              op_errno = 0;
-        int              op_ret = -1;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-        sh->impunge_done = afr_sh_entry_impunge_entry_done;
-
-        if ((strcmp (entry->d_name, ".") == 0)
-            || (strcmp (entry->d_name, "..") == 0)) {
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "skipping inspection of %s under %s",
-                        entry->d_name, local->loc.path);
-                op_ret = 0;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "inspecting existence of %s under %s",
-                entry->d_name, local->loc.path);
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    afr_private_t *priv = NULL;
+    afr_granular_esh_args_t args = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[subvol_idx];
+
+    args.frame = afr_copy_frame(frame);
+    if (!args.frame)
+        goto out;
+    args.xl = this;
+    /* args.heal_fd represents the fd associated with the original directory
+     * on which entry heal is being attempted.
+     */
+    args.heal_fd = fd;
+
+    /* @subvol here represents the subvolume of AFR where
+     * indices/entry-changes/<pargfid> will be processed
+     */
+    loc.inode = afr_shd_entry_changes_index_inode(this, subvol,
+                                                  fd->inode->gfid);
+    if (!loc.inode) {
+        /* If granular heal failed on the sink (as it might sometimes
+         * because it is the src that would mostly contain the granular
+         * changelogs and the sink's entry-changes would be empty),
+         * do not treat heal as failure.
+         */
+        if (is_src)
+            ret = -errno;
+        else
+            ret = 0;
+        goto out;
+    }
 
-        ret = afr_impunge_frame_create (frame, this, active_src,
-                                        &impunge_frame);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+    ret = syncop_dir_scan(subvol, &loc, GF_CLIENT_PID_SELF_HEALD, &args,
+                          afr_selfheal_entry_granular_dirent);
 
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        ret = afr_build_child_loc (this, &impunge_local->loc, &local->loc,
-                                   entry->d_name);
-        loc_copy (&impunge_sh->parent_loc, &local->loc);
-        if (ret != 0) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    loc_wipe(&loc);
 
-        afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
-                              afr_sh_entry_common_lookup_done, NULL,
-                              AFR_LOOKUP_FAIL_CONFLICTS, NULL);
-
-        op_ret = 0;
+    if (args.mismatch == _gf_true)
+        ret = -1;
 out:
-        if (ret) {
-                if (impunge_frame)
-                        AFR_STACK_DESTROY (impunge_frame);
-                sh->impunge_done (frame, this, op_ret, op_errno);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_readdir_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  gf_dirent_t *entries, dict_t *xdata)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        gf_dirent_t     *entry = NULL;
-        off_t            last_offset = 0;
-        int              active_src = 0;
-        int              entry_count = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-
-        if (op_ret <= 0) {
-                if (op_ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "readdir of %s on subvolume %s failed (%s)",
-                                local->loc.path,
-                                priv->children[active_src]->name,
-                                strerror (op_errno));
-                        sh->op_failed = 1;
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "readdir of %s on subvolume %s complete",
-                                local->loc.path,
-                                priv->children[active_src]->name);
-                }
-
-                afr_sh_entry_impunge_all (frame, this);
-                return 0;
-        }
-
-        list_for_each_entry (entry, &entries->list, list) {
-                last_offset = entry->d_off;
-                entry_count++;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "readdir'ed %d entries from %s",
-                entry_count, priv->children[active_src]->name);
-
-        sh->offset = last_offset;
-        local->call_count = entry_count;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                afr_sh_entry_impunge_entry (frame, this, entry);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int32_t         active_src = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-        active_src = sh->active_source;
-        gf_log (this->name, GF_LOG_DEBUG, "%s: readdir from offset %zd",
-                local->loc.path, sh->offset);
-
-        STACK_WIND (frame, afr_sh_entry_impunge_readdir_cbk,
-                    priv->children[active_src],
-                    priv->children[active_src]->fops->readdirp,
-                    sh->healing_fd, sh->block_size, sh->offset, NULL);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              active_src = -1;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->offset = 0;
-
-        active_src = next_active_source (frame, this, sh->active_source);
-        sh->active_source = active_src;
-
-        if (sh->op_failed) {
-                afr_sh_entry_finish (frame, this);
-                return 0;
-        }
-
-        if (active_src == -1) {
-                /* completed creating missing files on all subvolumes */
-                afr_sh_entry_erase_pending (frame, this);
-                return 0;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "impunging entries of %s on %s to other sinks",
-                local->loc.path, priv->children[active_src]->name);
-
-        afr_sh_entry_impunge_subvol (frame, this);
-
-        return 0;
+    if (args.frame)
+        AFR_STACK_DESTROY(args.frame);
+    return ret;
 }
 
-
-int
-afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              child_index = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        /* TODO: some of the open's might fail.
-           In that case, modify cleanup fn to send flush on those
-           fd's which are already open */
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "opendir of %s failed on child %s (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->op_failed = 1;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (sh->op_failed) {
-                        afr_sh_entry_finish (frame, this);
-                        return 0;
-                }
-                gf_log (this->name, GF_LOG_TRACE,
-                        "fd for %s opened, commencing sync",
-                        local->loc.path);
-
-                sh->active_source = -1;
-                afr_sh_entry_expunge_all (frame, this);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
+static int
+afr_selfheal_entry_do(call_frame_t *frame, xlator_t *this, fd_t *fd, int source,
+                      unsigned char *sources, unsigned char *healed_sinks)
 {
-        int i = 0;
-        int call_count = 0;
-
-        int source = -1;
-        int *sources = NULL;
-
-        fd_t *fd = NULL;
-
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source  = local->self_heal.source;
-        sources = local->self_heal.sources;
-
-        sh->block_size = 65536; //131072
-        sh->offset = 0;
-
-        call_count = sh->active_sinks;
-        if (source != -1)
-                call_count++;
-
-        local->call_count = call_count;
-
-        fd = fd_create (local->loc.inode, frame->root->pid);
-        sh->healing_fd = fd;
-
-        if (source != -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "opening directory %s on subvolume %s (source)",
-                        local->loc.path, priv->children[source]->name);
-
-                /* open source */
-                STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
-                                   (void *) (long) source,
-                                   priv->children[source],
-                                   priv->children[source]->fops->opendir,
-                                   &local->loc, fd, NULL);
-                call_count--;
-        }
-
-        /* open sinks */
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] || !local->child_up[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "opening directory %s on subvolume %s (sink)",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->opendir,
-                                   &local->loc, fd, NULL);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
-}
-
+    int i = 0;
+    int ret = 0;
+    gf_boolean_t mismatch = _gf_false;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing entry selfheal on %s", uuid_utoa(fd->inode->gfid));
+
+    for (i = 0; i < priv->child_count; i++) {
+        /* Expunge */
+        if (!healed_sinks[i])
+            continue;
+
+        if (!local->need_full_crawl)
+            /* Why call afr_selfheal_entry_granular() on a "healed sink",
+             * given that it is the source that contains the granular
+             * indices?
+             * If the index for this directory is non-existent or empty on
+             * this subvol (=> clear sink), then it will return early
+             * without failure status.
+             * If the index is non-empty and it is yet a 'healed sink', then
+             * it is due to a split-brain in which case we anyway need to
+             * crawl the indices/entry-changes/pargfid directory.
+             */
+            ret = afr_selfheal_entry_granular(frame, this, fd, i, _gf_false);
+        else
+            ret = afr_selfheal_entry_do_subvol(frame, this, fd, i);
 
-int
-afr_sh_entry_sync_prepare (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-
-        afr_sh_mark_source_sinks (frame, this);
-        if (source != -1)
-                sh->success[source] = 1;
-
-        if (sh->active_sinks == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "no active sinks for self-heal on dir %s",
-                        local->loc.path);
-                afr_sh_entry_finish (frame, this);
-                return 0;
-        }
-        if (source == -1 && sh->active_sinks < 2) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "cannot sync with 0 sources and 1 sink on dir %s",
-                        local->loc.path);
-                afr_sh_entry_finish (frame, this);
-                return 0;
+        if (ret == -1) {
+            /* gfid or type mismatch. */
+            mismatch = _gf_true;
+            ret = 0;
         }
+        if (ret)
+            break;
+    }
 
-        if (source != -1)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "self-healing directory %s from subvolume %s to "
-                        "%d other",
-                        local->loc.path, priv->children[source]->name,
-                        sh->active_sinks);
+    if (!ret && source != -1) {
+        /* Impunge */
+        if (local->need_full_crawl)
+            ret = afr_selfheal_entry_do_subvol(frame, this, fd, source);
         else
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no active sources for %s found. "
-                        "merging all entries as a conservative decision",
-                        local->loc.path);
-
-        afr_sh_entry_open (frame, this);
-
-        return 0;
+            ret = afr_selfheal_entry_granular(frame, this, fd, source,
+                                              _gf_true);
+    }
+
+    if (mismatch == _gf_true)
+        /* undo pending will be skipped */
+        ret = -1;
+    return ret;
 }
 
-
-void
-afr_sh_entry_fix (call_frame_t *frame, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+static int
+__afr_selfheal_entry(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     unsigned char *locked_on)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-        int              nsources = 0;
-        int32_t          subvol_status = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0) {
-                sh->op_failed = 1;
-                afr_sh_set_error (sh, op_errno);
-                afr_sh_entry_finish (frame, this);
-                goto out;
-        }
-
-        if (sh->forced_merge) {
-                sh->source = -1;
-                goto heal;
-        }
-
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      AFR_ENTRY_TRANSACTION, &subvol_status,
-                                      _gf_true);
-        if ((subvol_status & ALL_FOOLS) ||
-            (subvol_status & SPLIT_BRAIN)) {
-                gf_log (this->name, GF_LOG_INFO, "%s: Performing conservative "
-                        "merge", local->loc.path);
-                source = -1;
-                memset (sh->sources, 0,
-                        sizeof (*sh->sources) * priv->child_count);
-        } else if (nsources == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_entry_finish (frame, this);
-                return;
-        } else {
-                source = afr_sh_select_source (sh->sources, priv->child_count);
-        }
-
-        sh->source = source;
-
-        afr_reset_children (sh->fresh_children, priv->child_count);
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, priv->child_count);
-        if (sh->source >= 0)
-                afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                        sh->fresh_children);
-
-heal:
-        afr_sh_entry_sync_prepare (frame, this);
+    int ret = -1;
+    int source = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *postop_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t did_sh = _gf_true;
+
+    priv = this->private;
+    local = frame->local;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+    postop_lock = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes could "
+                         "be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, this->name);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_entry_prepare(frame, this, fd->inode, data_lock,
+                                           sources, sinks, healed_sinks,
+                                           locked_replies, &source, NULL);
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
+        }
+
+        local->need_full_crawl = afr_need_full_heal(
+            this, locked_replies, source, healed_sinks, AFR_ENTRY_TRANSACTION);
+    }
+unlock:
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL, data_lock,
+                           NULL);
+    if (ret < 0)
+        goto out;
+
+    if (!did_sh)
+        goto out;
+
+    ret = afr_selfheal_entry_do(frame, this, fd, source, sources, healed_sinks);
+    if (ret)
+        goto out;
+
+    /* Take entrylks in xlator domain before doing post-op (undo-pending) in
+     * entry self-heal. This is to prevent a parallel name self-heal on
+     * an entry under @fd->inode from reading pending xattrs while it is
+     * being modified by SHD after entry sh below, given that
+     * name self-heal takes locks ONLY in xlator domain and is free to read
+     * pending changelog in the absence of the following locking.
+     */
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               postop_lock);
+    {
+        if (AFR_CMP(data_lock, postop_lock, priv->child_count) != 0) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "post-op after entry self-heal as %d "
+                         "sub-volumes, as opposed to %d, "
+                         "could be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret,
+                         AFR_COUNT(data_lock, priv->child_count), this->name);
+            ret = -ENOTCONN;
+            goto postop_unlock;
+        }
+
+        afr_selfheal_restore_time(frame, this, fd->inode, source, healed_sinks,
+                                  locked_replies);
+        ret = afr_selfheal_undo_pending(
+            frame, this, fd->inode, sources, sinks, healed_sinks, undid_pending,
+            AFR_ENTRY_TRANSACTION, locked_replies, postop_lock);
+    }
+postop_unlock:
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL,
+                           postop_lock, NULL);
 out:
-        return;
+    if (did_sh)
+        afr_log_selfheal(fd->inode->gfid, this, ret, "entry", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
+
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
+    return ret;
 }
 
-int
-afr_sh_post_nonblocking_entry_cbk (call_frame_t *frame, xlator_t *this)
+static fd_t *
+afr_selfheal_data_opendir(xlator_t *this, inode_t *inode)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Non Blocking entrylks "
-                        "failed for %s.", local->loc.path);
-                sh->op_failed = 1;
-                afr_sh_entry_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking entrylks done "
-                        "for %s. Proceeding to FOP", local->loc.path);
-                afr_sh_common_lookup (frame, this, &local->loc,
-                                      afr_sh_entry_fix, NULL,
-                                      AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
-                                      NULL);
-        }
-
-        return 0;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+    fd_t *fd = NULL;
+
+    fd = fd_create(inode, 0);
+    if (!fd)
+        return NULL;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = syncop_opendir(this, &loc, fd, NULL, NULL);
+    if (ret) {
+        fd_unref(fd);
+        fd = NULL;
+    } else {
+        fd_bind(fd);
+    }
+
+    loc_wipe(&loc);
+    return fd;
 }
 
 int
-afr_self_heal_entry (call_frame_t *frame, xlator_t *this)
+afr_selfheal_entry(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t   *priv = NULL;
-
-
-        priv = this->private;
-        local = frame->local;
-
-        if (local->self_heal.do_entry_self_heal && priv->entry_self_heal) {
-                afr_sh_entrylk (frame, this, &local->loc, NULL,
-                                afr_sh_post_nonblocking_entry_cbk);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "proceeding to completion on %s",
-                        local->loc.path);
-                afr_sh_entry_done (frame, this);
-        }
-
-        return 0;
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    fd_t *fd = NULL;
+    int ret = 0;
+
+    priv = this->private;
+
+    fd = afr_selfheal_data_opendir(this, inode);
+    if (!fd)
+        return -EIO;
+
+    locked_on = alloca0(priv->child_count);
+
+    ret = afr_selfheal_tie_breaker_entrylk(frame, this, inode, priv->sh_domain,
+                                           NULL, locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes could "
+                         "be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, priv->sh_domain);
+            /* Either less than two subvols available, or another
+               selfheal (from another server) is in progress. Skip
+               for now in any case there isn't anything to do.
+            */
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_entry(frame, this, fd, locked_on);
+    }
+unlock:
+    afr_selfheal_unentrylk(frame, this, inode, priv->sh_domain, NULL, locked_on,
+                           NULL);
+
+    if (fd)
+        fd_unref(fd);
+
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index 147ab98d409..03f43bad16e 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,525 +8,539 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
+#include <glusterfs/byte-order.h>
+#include "protocol-common.h"
+#include <glusterfs/events.h>
 
+#define AFR_HEAL_ATTR (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE)
 
-int
-afr_sh_metadata_done (call_frame_t *frame, xlator_t *this)
+static gf_boolean_t
+_afr_ignorable_key_match(dict_t *d, char *k, data_t *val, void *mdata)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        afr_sh_reset (frame, this);
-        if (local->govinda_gOvinda) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "split-brain detected, aborting selfheal of %s",
-                        local->loc.path);
-                sh->op_failed = 1;
-                sh->completion_cbk (frame, this);
-        } else {
-                if (IA_ISDIR (sh->type)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "proceeding to entry check on %s",
-                                local->loc.path);
-                        afr_self_heal_entry (frame, this);
-                        return 0;
-                }
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "proceeding to data check on %s",
-                        local->loc.path);
-                afr_self_heal_data (frame, this);
-        }
-
-        return 0;
+    return afr_is_xattr_ignorable(k);
 }
 
-int
-afr_sh_inode_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_cbk = afr_sh_metadata_done;
-        afr_unlock (frame, this);
-
-        return 0;
-}
-
-int
-afr_sh_metadata_finish (call_frame_t *frame, xlator_t *this)
+void
+afr_delete_ignorable_xattrs(dict_t *xattr)
 {
-        afr_sh_inode_unlock (frame, this);
-
-        return 0;
+    dict_foreach_match(xattr, _afr_ignorable_key_match, NULL,
+                       dict_remove_foreach_fn, NULL);
 }
 
-
 int
-afr_sh_metadata_erase_pending_cbk (call_frame_t *frame, void *cookie,
-                                   xlator_t *this, int32_t op_ret,
-                                   int32_t op_errno, dict_t *xattr, dict_t *xdata)
+__afr_selfheal_metadata_do(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, unsigned char *healed_sinks,
+                           struct afr_reply *locked_replies)
 {
-        afr_local_t     *local     = NULL;
-        int             call_count = 0;
-        long            i          = 0;
-        afr_self_heal_t *sh        = NULL;
-        afr_private_t   *priv      = NULL;
-
-        local = frame->local;
-        priv  = this->private;
-        sh = &local->self_heal;
-        i = (long)cookie;
-
-        if ((!IA_ISREG (sh->buf[sh->source].ia_type)) &&
-            (!IA_ISDIR (sh->buf[sh->source].ia_type))) {
-                afr_children_add_child (sh->fresh_children, i,
-                                        priv->child_count);
-        }
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if ((!IA_ISREG (sh->buf[sh->source].ia_type)) &&
-                    (!IA_ISDIR (sh->buf[sh->source].ia_type))) {
-                        afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                                sh->fresh_children);
-                }
-                afr_sh_metadata_finish (frame, this);
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    dict_t *old_xattr = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing metadata selfheal on %s", uuid_utoa(inode->gfid));
+
+    ret = syncop_getxattr(priv->children[source], &loc, &xattr, NULL, NULL,
+                          NULL);
+    if (ret < 0) {
+        ret = -EIO;
+        goto out;
+    }
+
+    afr_delete_ignorable_xattrs(xattr);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (old_xattr) {
+            dict_unref(old_xattr);
+            old_xattr = NULL;
         }
 
-        return 0;
-}
-
-int
-afr_sh_metadata_erase_pending (call_frame_t *frame, xlator_t *this)
-{
-         afr_sh_erase_pending (frame, this, AFR_METADATA_TRANSACTION,
-                               afr_sh_metadata_erase_pending_cbk,
-                               afr_sh_metadata_finish);
-         return 0;
-}
-
-
-int
-afr_sh_metadata_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              child_index = 0;
-
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "setting attributes failed for %s on %s (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-
-                        sh->success[child_index] = 0;
-                }
+        if (!healed_sinks[i])
+            continue;
+
+        ret = syncop_setattr(priv->children[i], &loc,
+                             &locked_replies[source].poststat, AFR_HEAL_ATTR,
+                             NULL, NULL, NULL, NULL);
+        if (ret)
+            healed_sinks[i] = 0;
+
+        ret = syncop_getxattr(priv->children[i], &loc, &old_xattr, 0, NULL,
+                              NULL);
+        if (old_xattr) {
+            afr_delete_ignorable_xattrs(old_xattr);
+            ret = syncop_removexattr(priv->children[i], &loc, "", old_xattr,
+                                     NULL);
+            if (ret)
+                healed_sinks[i] = 0;
         }
-        UNLOCK (&frame->lock);
 
-        call_count = afr_frame_return (frame);
+        ret = syncop_setxattr(priv->children[i], &loc, xattr, 0, NULL, NULL);
+        if (ret)
+            healed_sinks[i] = 0;
+    }
+    ret = 0;
 
-        if (call_count == 0)
-                afr_sh_metadata_erase_pending (frame, this);
+out:
+    loc_wipe(&loc);
+    if (xattr)
+        dict_unref(xattr);
+    if (old_xattr)
+        dict_unref(old_xattr);
 
-        return 0;
+    return ret;
 }
 
-
-int
-afr_sh_metadata_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *preop, struct iatt *postop, dict_t *xdata)
+static uint64_t
+mtime_ns(struct iatt *ia)
 {
-        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+    uint64_t ret;
 
-        return 0;
-}
+    ret = (((uint64_t)(ia->ia_mtime)) * 1000000000) +
+          (uint64_t)(ia->ia_mtime_nsec);
 
-
-int
-afr_sh_metadata_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-
-        return 0;
+    return ret;
 }
 
-
-int
-afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
+/*
+ * When directory content is modified, [mc]time is updated. On
+ * Linux, the filesystem does it, while at least on NetBSD, the
+ * kernel file-system independent code does it. This means that
+ * when entries are added while bricks are down, the kernel sends
+ * a SETATTR [mc]time which will cause metadata split brain for
+ * the directory. In this case, clear the split brain by finding
+ * the source with the most recent modification date.
+ */
+static int
+afr_dirtime_splitbrain_source(call_frame_t *frame, xlator_t *this,
+                              struct afr_reply *replies,
+                              unsigned char *locked_on)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-        int              active_sinks = 0;
-        int              call_count = 0;
-        int              i = 0;
-
-        struct iatt      stbuf = {0,};
-        int32_t          valid = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-        active_sinks = sh->active_sinks;
-
-        /*
-         * 2 calls per sink - setattr, setxattr
-         */
-        if (xattr)
-                call_count = active_sinks * 2;
-        else
-                call_count = active_sinks;
-
-        local->call_count = call_count;
-
-        stbuf.ia_atime = sh->buf[source].ia_atime;
-        stbuf.ia_atime_nsec = sh->buf[source].ia_atime_nsec;
-        stbuf.ia_mtime = sh->buf[source].ia_mtime;
-        stbuf.ia_mtime_nsec = sh->buf[source].ia_mtime_nsec;
-
-        stbuf.ia_uid = sh->buf[source].ia_uid;
-        stbuf.ia_gid = sh->buf[source].ia_gid;
-
-        stbuf.ia_type = sh->buf[source].ia_type;
-        stbuf.ia_prot = sh->buf[source].ia_prot;
-
-        valid = GF_SET_ATTR_MODE  |
-                GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
-                GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (call_count == 0) {
-                        break;
-                }
-                if (sh->sources[i] || !local->child_up[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "self-healing metadata of %s from %s to %s",
-                        local->loc.path, priv->children[source]->name,
-                        priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_setattr_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &local->loc, &stbuf, valid, NULL);
-
-                call_count--;
-
-                if (!xattr)
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_xattr_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->setxattr,
-                                   &local->loc, xattr, 0, NULL);
-                call_count--;
-        }
-
-        return 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    struct iatt source_ia;
+    struct iatt child_ia;
+    uint64_t mtime = 0;
+    int i;
+    int ret = -1;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!locked_on[i])
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        if (mtime_ns(&replies[i].poststat) <= mtime)
+            continue;
+
+        mtime = mtime_ns(&replies[i].poststat);
+        source = i;
+    }
+
+    if (source == -1)
+        goto out;
+
+    source_ia = replies[source].poststat;
+    if (source_ia.ia_type != IA_IFDIR)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source)
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        child_ia = replies[i].poststat;
+
+        if (!IA_EQUAL(source_ia, child_ia, gfid) ||
+            !IA_EQUAL(source_ia, child_ia, type) ||
+            !IA_EQUAL(source_ia, child_ia, prot) ||
+            !IA_EQUAL(source_ia, child_ia, uid) ||
+            !IA_EQUAL(source_ia, child_ia, gid) ||
+            !afr_xattrs_are_equal(replies[source].xdata, replies[i].xdata))
+            goto out;
+    }
+
+    /*
+     * Metadata split brain is just about [amc]time
+     * We return our source.
+     */
+    ret = source;
+out:
+    return ret;
 }
 
-
-int
-afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                              dict_t *xdata)
+static int
+__afr_selfheal_metadata_mark_pending_xattrs(call_frame_t *frame, xlator_t *this,
+                                            inode_t *inode,
+                                            struct afr_reply *replies,
+                                            unsigned char *sources)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-
-        int i;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "getxattr of %s failed on subvolume %s (%s). proceeding without xattr",
-                        local->loc.path, priv->children[source]->name,
-                        strerror (op_errno));
-
-                afr_sh_metadata_sync (frame, this, NULL);
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        dict_del (xattr, priv->pending_key[i]);
-                }
-
-                afr_sh_metadata_sync (frame, this, xattr);
+    int ret = 0;
+    int i = 0;
+    int m_idx = 0;
+    afr_private_t *priv = NULL;
+    int raw[AFR_NUM_CHANGE_LOGS] = {0};
+    dict_t *xattr = NULL;
+
+    priv = this->private;
+    m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
+    raw[m_idx] = 1;
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i])
+            continue;
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], raw,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            ret = -1;
+            goto out;
         }
-
-        return 0;
-}
-
-
-int
-afr_sh_metadata_sync_prepare (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-
-        afr_sh_mark_source_sinks (frame, this);
-        if (sh->active_sinks == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no active sinks for performing self-heal on file %s",
-                        local->loc.path);
-                afr_sh_metadata_finish (frame, this);
-                return 0;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        ret = afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, AFR_MSG_SELF_HEAL_INFO,
+                   "Failed to set pending metadata xattr on child %d for %s", i,
+                   uuid_utoa(inode->gfid));
+            goto out;
         }
+    }
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "syncing metadata of %s from subvolume %s to %d active sinks",
-                local->loc.path, priv->children[source]->name,
-                sh->active_sinks);
-
-        STACK_WIND (frame, afr_sh_metadata_getxattr_cbk,
-                    priv->children[source],
-                    priv->children[source]->fops->getxattr,
-                    &local->loc, NULL, NULL);
+    afr_replies_wipe(replies, priv->child_count);
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
 
-        return 0;
+out:
+    if (xattr)
+        dict_unref(xattr);
+    return ret;
 }
 
-
-void
-afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+/*
+ * Look for mismatching uid/gid or mode or user xattrs even if
+ * AFR xattrs don't say so, and pick one arbitrarily as winner. */
+
+static int
+__afr_selfheal_metadata_finalize_source(call_frame_t *frame, xlator_t *this,
+                                        inode_t *inode, unsigned char *sources,
+                                        unsigned char *sinks,
+                                        unsigned char *healed_sinks,
+                                        unsigned char *undid_pending,
+                                        unsigned char *locked_on,
+                                        struct afr_reply *replies)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              nsources = 0;
-        int              source = 0;
-        int              i = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0) {
-                sh->op_failed = 1;
-                afr_sh_set_error (sh, op_errno);
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      AFR_METADATA_TRANSACTION, NULL, _gf_false);
-        if (nsources == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_metadata_finish (frame, this);
-                goto out;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct iatt srcstat = {
+        0,
+    };
+    int source = -1;
+    int sources_count = 0;
+    int ret = 0;
+
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count) {
+        source = afr_mark_split_brain_source_sinks(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, AFR_METADATA_TRANSACTION);
+        if (source >= 0) {
+            _afr_fav_child_reset_sink_xattrs(
+                frame, this, inode, source, healed_sinks, undid_pending,
+                AFR_METADATA_TRANSACTION, locked_on, replies);
+            goto out;
         }
 
-        if ((nsources == -1)
-            && (priv->favorite_child != -1)
-            && (sh->child_errno[priv->favorite_child] == 0)) {
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Picking favorite child %s as authentic source to resolve conflicting metadata of %s",
-                        priv->children[priv->favorite_child]->name,
-                        local->loc.path);
-
-                sh->sources[priv->favorite_child] = 1;
-
-                nsources = afr_sh_source_count (sh->sources,
-                                                priv->child_count);
+        /* If this is a directory mtime/ctime only split brain
+           use the most recent */
+        source = afr_dirtime_splitbrain_source(frame, this, replies, locked_on);
+        if (source != -1) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SPLIT_BRAIN,
+                   "clear time "
+                   "split brain on %s",
+                   uuid_utoa(replies[source].poststat.ia_gfid));
+            sources[source] = 1;
+            healed_sinks[source] = 0;
+            goto out;
         }
 
-        if (nsources == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to self-heal permissions/ownership of '%s' "
-                        "(possible split-brain). Please fix the file on "
-                        "all backend volumes", local->loc.path);
-
-                local->govinda_gOvinda = 1;
-
-                afr_sh_metadata_finish (frame, this);
-                goto out;
+        if (!priv->metadata_splitbrain_forced_heal) {
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;"
+                     "type=metadata;file=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(inode->gfid));
+            return -EIO;
         }
 
-        source = afr_sh_select_source (sh->sources, priv->child_count);
-
-        if (source == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No active sources found.");
-
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-
-        sh->source = source;
-
-        /* detect changes not visible through pending flags -- JIC */
+        /* Metadata split brain, select one subvol
+           arbitrarily */
         for (i = 0; i < priv->child_count; i++) {
-                if (i == source || sh->child_errno[i])
-                        continue;
-
-                if (PERMISSION_DIFFERS (&sh->buf[i], &sh->buf[source]))
-                        sh->sources[i] = 0;
-
-                if (OWNERSHIP_DIFFERS (&sh->buf[i], &sh->buf[source]))
-                        sh->sources[i] = 0;
+            if (locked_on[i] && healed_sinks[i]) {
+                sources[i] = 1;
+                healed_sinks[i] = 0;
+                break;
+            }
         }
-
-        if ((!IA_ISREG (sh->buf[source].ia_type)) &&
-            (!IA_ISDIR (sh->buf[source].ia_type))) {
-                afr_reset_children (sh->fresh_children, priv->child_count);
-                afr_get_fresh_children (sh->success_children, sh->sources,
-                                        sh->fresh_children, priv->child_count);
-                afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                        sh->fresh_children);
+    }
+
+    /* No split brain at this point. If we were called from
+     * afr_heal_splitbrain_file(), abort.*/
+    if (afr_dict_contains_heal_op(frame))
+        return -EIO;
+
+    source = afr_choose_source_by_policy(priv, sources,
+                                         AFR_METADATA_TRANSACTION);
+    srcstat = replies[source].poststat;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] || i == source)
+            continue;
+        if (!IA_EQUAL(srcstat, replies[i].poststat, type) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, uid) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, gid) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, prot)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: iatt mismatch "
+                         "for source(%d) vs (%d)",
+                         uuid_utoa(replies[source].poststat.ia_gfid), source,
+                         i);
+            sources[i] = 0;
+            healed_sinks[i] = 1;
         }
-
-        afr_sh_metadata_sync_prepare (frame, this);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] || i == source)
+            continue;
+        if (!afr_xattrs_are_equal(replies[source].xdata, replies[i].xdata)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: xattr mismatch "
+                         "for source(%d) vs (%d)",
+                         uuid_utoa(replies[source].poststat.ia_gfid), source,
+                         i);
+            sources[i] = 0;
+            healed_sinks[i] = 1;
+        }
+    }
+    if ((sources_count == priv->child_count) && (source > -1) &&
+        (AFR_COUNT(healed_sinks, priv->child_count) != 0)) {
+        ret = __afr_selfheal_metadata_mark_pending_xattrs(frame, this, inode,
+                                                          replies, sources);
+        if (ret < 0)
+            return ret;
+    }
 out:
-        return;
+    afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+    return source;
 }
 
 int
-afr_sh_metadata_post_nonblocking_inodelk_cbk (call_frame_t *frame,
-                                              xlator_t *this)
+__afr_selfheal_metadata_prepare(call_frame_t *frame, xlator_t *this,
+                                inode_t *inode, unsigned char *locked_on,
+                                unsigned char *sources, unsigned char *sinks,
+                                unsigned char *healed_sinks,
+                                unsigned char *undid_pending,
+                                struct afr_reply *replies, unsigned char *pflag)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Non Blocking metadata "
-                        "inodelks failed for %s.", local->loc.path);
-                gf_log (this->name, GF_LOG_ERROR, "Metadata self-heal "
-                        "failed for %s.", local->loc.path);
-                afr_sh_metadata_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking metadata "
-                        "inodelks done for %s. Proceeding to FOP",
-                        local->loc.path);
-                afr_sh_common_lookup (frame, this, &local->loc,
-                                      afr_sh_metadata_fix, NULL,
-                                      AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
-                                      NULL);
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
+    if (ret)
+        return ret;
+
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_METADATA_TRANSACTION, locked_on,
+                                      sources, sinks, witness, pflag);
+    if (ret)
+        return ret;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    /* If any source has witness, pick first
+     * witness source and make everybody else sinks */
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i] && witness[i]) {
+            source = i;
+            break;
         }
+    }
 
-        return 0;
-}
-
-int
-afr_sh_metadata_lock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_METADATA_SELF_HEAL_LK;
-
-        afr_set_lock_number (frame, this);
+    if (source != -1) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (i != source && sources[i]) {
+                sources[i] = 0;
+                healed_sinks[i] = 1;
+            }
+        }
+    }
 
-        int_lock->lk_flock.l_start = LLONG_MAX - 1;
-        int_lock->lk_flock.l_len   = 0;
-        int_lock->lk_flock.l_type  = F_WRLCK;
-        int_lock->lock_cbk         = afr_sh_metadata_post_nonblocking_inodelk_cbk;
+    source = __afr_selfheal_metadata_finalize_source(
+        frame, this, inode, sources, sinks, healed_sinks, undid_pending,
+        locked_on, replies);
 
-        afr_nonblocking_inodelk (frame, this);
+    if (source < 0)
+        return -EIO;
 
-        return 0;
+    return source;
 }
 
-
 int
-afr_self_heal_metadata (call_frame_t *frame, xlator_t *this)
+afr_selfheal_metadata(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv = this->private;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    gf_boolean_t did_sh = _gf_true;
+    int source = -1;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, inode, this->name, LLONG_MAX - 1, 0,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            goto unlock;
+        }
 
+        ret = __afr_selfheal_metadata_prepare(
+            frame, this, inode, data_lock, sources, sinks, healed_sinks,
+            undid_pending, locked_replies, NULL);
+        if (ret < 0)
+            goto unlock;
 
-        local = frame->local;
+        source = ret;
 
-        if (local->self_heal.do_metadata_self_heal && priv->metadata_self_heal) {
-                afr_sh_metadata_lock (frame, this);
-        } else {
-                afr_sh_metadata_done (frame, this);
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
         }
 
-        return 0;
+        ret = __afr_selfheal_metadata_do(frame, this, inode, source,
+                                         healed_sinks, locked_replies);
+        if (ret)
+            goto unlock;
+
+        afr_selfheal_restore_time(frame, this, inode, source, healed_sinks,
+                                  locked_replies);
+
+        ret = afr_selfheal_undo_pending(
+            frame, this, inode, sources, sinks, healed_sinks, undid_pending,
+            AFR_METADATA_TRANSACTION, locked_replies, data_lock);
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, inode, this->name, LLONG_MAX - 1, 0,
+                           data_lock);
+
+    if (did_sh)
+        afr_log_selfheal(inode->gfid, this, ret, "metadata", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
+
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
+    return ret;
+}
+
+int
+afr_selfheal_metadata_by_stbuf(xlator_t *this, struct iatt *stbuf)
+{
+    inode_t *inode = NULL;
+    inode_t *link_inode = NULL;
+    call_frame_t *frame = NULL;
+    int ret = 0;
+
+    if (gf_uuid_is_null(stbuf->ia_gfid)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    inode = inode_new(this->itable);
+    if (!inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    link_inode = inode_link(inode, NULL, NULL, stbuf);
+    if (!link_inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = afr_selfheal_metadata(frame, this, link_inode);
+out:
+    if (inode)
+        inode_unref(inode);
+    if (link_inode)
+        inode_unref(link_inode);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-name.c b/xlators/cluster/afr/src/afr-self-heal-name.c
new file mode 100644
index 00000000000..834aac86d48
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-self-heal-name.c
@@ -0,0 +1,616 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <glusterfs/events.h>
+#include "afr.h"
+#include "afr-self-heal.h"
+#include "afr-messages.h"
+
+int
+__afr_selfheal_assign_gfid(xlator_t *this, inode_t *parent, uuid_t pargfid,
+                           const char *bname, inode_t *inode,
+                           struct afr_reply *replies, void *gfid,
+                           unsigned char *locked_on, int source,
+                           unsigned char *sources, gf_boolean_t is_gfid_absent,
+                           int *gfid_idx)
+{
+    int ret = 0;
+    int up_count = 0;
+    int locked_count = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    gf_uuid_copy(parent->gfid, pargfid);
+
+    if (is_gfid_absent) {
+        /* Ensure all children of AFR are up before performing gfid heal, to
+         * guard against the possibility of gfid split brain. */
+
+        up_count = AFR_COUNT(priv->child_up, priv->child_count);
+        if (up_count != priv->child_count) {
+            ret = -EIO;
+            goto out;
+        }
+
+        locked_count = AFR_COUNT(locked_on, priv->child_count);
+        if (locked_count != priv->child_count) {
+            ret = -EIO;
+            goto out;
+        }
+    }
+
+    ret = afr_lookup_and_heal_gfid(this, parent, bname, inode, replies, source,
+                                   sources, gfid, gfid_idx);
+
+out:
+    return ret;
+}
+
+int
+__afr_selfheal_name_impunge(call_frame_t *frame, xlator_t *this,
+                            inode_t *parent, uuid_t pargfid, const char *bname,
+                            inode_t *inode, struct afr_reply *replies,
+                            int gfid_idx)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
+    unsigned char *sources = NULL;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+
+    gf_uuid_copy(parent->gfid, pargfid);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[gfid_idx].poststat.ia_gfid) == 0) {
+            sources[i] = 1;
+            continue;
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i])
+            continue;
+
+        ret |= afr_selfheal_recreate_entry(frame, i, gfid_idx, sources, parent,
+                                           bname, inode, replies);
+    }
+
+    return ret;
+}
+
+int
+__afr_selfheal_name_expunge(xlator_t *this, inode_t *parent, uuid_t pargfid,
+                            const char *bname, inode_t *inode,
+                            struct afr_reply *replies)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret)
+            continue;
+
+        ret |= afr_selfheal_entry_delete(this, parent, bname, inode, i,
+                                         replies);
+    }
+
+    return ret;
+}
+
+static gf_boolean_t
+afr_selfheal_name_need_heal_check(xlator_t *this, struct afr_reply *replies)
+{
+    int i = 0;
+    int first_idx = -1;
+    gf_boolean_t need_heal = _gf_false;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA))
+            need_heal = _gf_true;
+
+        if (first_idx == -1) {
+            first_idx = i;
+            continue;
+        }
+
+        if (replies[i].op_ret != replies[first_idx].op_ret)
+            need_heal = _gf_true;
+
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[first_idx].poststat.ia_gfid))
+            need_heal = _gf_true;
+
+        if ((replies[i].op_ret == 0) &&
+            (gf_uuid_is_null(replies[i].poststat.ia_gfid)))
+            need_heal = _gf_true;
+    }
+
+    return need_heal;
+}
+
+static int
+afr_selfheal_name_type_mismatch_check(xlator_t *this, struct afr_reply *replies,
+                                      int source, unsigned char *sources,
+                                      uuid_t pargfid, const char *bname)
+{
+    int i = 0;
+    int type_idx = -1;
+    ia_type_t inode_type = IA_INVAL;
+    ia_type_t inode_type1 = IA_INVAL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+
+        if (replies[i].poststat.ia_type == IA_INVAL)
+            continue;
+
+        if (inode_type == IA_INVAL) {
+            inode_type = replies[i].poststat.ia_type;
+            type_idx = i;
+            continue;
+        }
+        inode_type1 = replies[i].poststat.ia_type;
+        if (sources[i] || source == -1) {
+            if ((sources[type_idx] || source == -1) &&
+                (inode_type != inode_type1)) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Type mismatch for <gfid:%s>/%s: "
+                       "%s on %s and %s on %s",
+                       uuid_utoa(pargfid), bname,
+                       gf_inode_type_to_str(inode_type1),
+                       priv->children[i]->name,
+                       gf_inode_type_to_str(inode_type),
+                       priv->children[type_idx]->name);
+                gf_event(EVENT_AFR_SPLIT_BRAIN,
+                         "client-pid=%d;"
+                         "subvol=%s;type=file;"
+                         "file=<gfid:%s>/%s;count=2;"
+                         "child-%d=%s;type-%d=%s;child-%d=%s;"
+                         "type-%d=%s",
+                         this->ctx->cmd_args.client_pid, this->name,
+                         uuid_utoa(pargfid), bname, i, priv->children[i]->name,
+                         i, gf_inode_type_to_str(inode_type1), type_idx,
+                         priv->children[type_idx]->name, type_idx,
+                         gf_inode_type_to_str(inode_type));
+                return -EIO;
+            }
+            inode_type = replies[i].poststat.ia_type;
+            type_idx = i;
+        }
+    }
+    return 0;
+}
+
+static int
+afr_selfheal_name_gfid_mismatch_check(xlator_t *this, struct afr_reply *replies,
+                                      int source, unsigned char *sources,
+                                      int *gfid_idx, uuid_t pargfid,
+                                      const char *bname, inode_t *inode,
+                                      unsigned char *locked_on, dict_t *xdata)
+{
+    int i = 0;
+    int gfid_idx_iter = -1;
+    int ret = -1;
+    void *gfid = NULL;
+    void *gfid1 = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+
+        if (gf_uuid_is_null(replies[i].poststat.ia_gfid))
+            continue;
+
+        if (!gfid) {
+            gfid = &replies[i].poststat.ia_gfid;
+            gfid_idx_iter = i;
+            continue;
+        }
+
+        gfid1 = &replies[i].poststat.ia_gfid;
+        if (sources[i] || source == -1) {
+            if ((sources[gfid_idx_iter] || source == -1) &&
+                gf_uuid_compare(gfid, gfid1)) {
+                ret = afr_gfid_split_brain_source(this, replies, inode, pargfid,
+                                                  bname, gfid_idx_iter, i,
+                                                  locked_on, gfid_idx, xdata);
+                if (!ret && *gfid_idx >= 0) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   "GFID split-brain resolved");
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error setting gfid-"
+                               "heal-msg dict");
+                }
+                return ret;
+            }
+            gfid = &replies[i].poststat.ia_gfid;
+            gfid_idx_iter = i;
+        }
+    }
+
+    *gfid_idx = gfid_idx_iter;
+    return 0;
+}
+
+static gf_boolean_t
+afr_selfheal_name_source_empty_check(xlator_t *this, struct afr_reply *replies,
+                                     unsigned char *sources, int source)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    gf_boolean_t source_is_empty = _gf_true;
+
+    priv = this->private;
+
+    if (source == -1) {
+        source_is_empty = _gf_false;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+
+        if (replies[i].op_ret == -1 && replies[i].op_errno == ENOENT)
+            continue;
+
+        source_is_empty = _gf_false;
+        break;
+    }
+out:
+    return source_is_empty;
+}
+
+int
+__afr_selfheal_name_do(call_frame_t *frame, xlator_t *this, inode_t *parent,
+                       uuid_t pargfid, const char *bname, inode_t *inode,
+                       unsigned char *sources, unsigned char *sinks,
+                       unsigned char *healed_sinks, int source,
+                       unsigned char *locked_on, struct afr_reply *replies,
+                       void *gfid_req, dict_t *xdata)
+{
+    int gfid_idx = -1;
+    int ret = -1;
+    void *gfid = NULL;
+    gf_boolean_t source_is_empty = _gf_true;
+    gf_boolean_t need_heal = _gf_false;
+    gf_boolean_t is_gfid_absent = _gf_false;
+
+    need_heal = afr_selfheal_name_need_heal_check(this, replies);
+    if (!need_heal)
+        return 0;
+
+    source_is_empty = afr_selfheal_name_source_empty_check(this, replies,
+                                                           sources, source);
+    if (source_is_empty) {
+        ret = __afr_selfheal_name_expunge(this, parent, pargfid, bname, inode,
+                                          replies);
+        if (ret == -EIO)
+            ret = -1;
+        return ret;
+    }
+
+    ret = afr_selfheal_name_type_mismatch_check(this, replies, source, sources,
+                                                pargfid, bname);
+    if (ret)
+        return ret;
+
+    ret = afr_selfheal_name_gfid_mismatch_check(this, replies, source, sources,
+                                                &gfid_idx, pargfid, bname,
+                                                inode, locked_on, xdata);
+    if (ret)
+        return ret;
+
+    if (gfid_idx == -1) {
+        if (!gfid_req || gf_uuid_is_null(gfid_req))
+            return -1;
+        gfid = gfid_req;
+    } else {
+        gfid = &replies[gfid_idx].poststat.ia_gfid;
+        if (source == -1)
+            /* Either entry split-brain or dirty xattrs are present on parent.*/
+            source = gfid_idx;
+    }
+
+    is_gfid_absent = (gfid_idx == -1) ? _gf_true : _gf_false;
+    ret = __afr_selfheal_assign_gfid(this, parent, pargfid, bname, inode,
+                                     replies, gfid, locked_on, source, sources,
+                                     is_gfid_absent, &gfid_idx);
+    if (ret || (gfid_idx < 0))
+        return ret;
+
+    ret = __afr_selfheal_name_impunge(frame, this, parent, pargfid, bname,
+                                      inode, replies, gfid_idx);
+    if (ret == -EIO)
+        ret = -1;
+
+    return ret;
+}
+
+int
+__afr_selfheal_name_finalize_source(xlator_t *this, unsigned char *sources,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on, uint64_t *witness)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count || afr_does_witness_exist(this, witness)) {
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+        afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+        return -1;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
+        }
+    }
+
+    return source;
+}
+
+int
+__afr_selfheal_name_prepare(call_frame_t *frame, xlator_t *this,
+                            inode_t *parent, uuid_t pargfid,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            int *source_p)
+{
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+
+    ret = afr_selfheal_unlocked_discover(frame, parent, pargfid, replies);
+    if (ret)
+        goto out;
+
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_ENTRY_TRANSACTION, locked_on, sources,
+                                      sinks, witness, NULL);
+    if (ret)
+        goto out;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    source = __afr_selfheal_name_finalize_source(this, sources, healed_sinks,
+                                                 locked_on, witness);
+    if (source < 0) {
+        /* If source is < 0 (typically split-brain), we perform a
+           conservative merge of entries rather than erroring out */
+    }
+    *source_p = source;
+
+out:
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+
+    return ret;
+}
+
+int
+afr_selfheal_name_do(call_frame_t *frame, xlator_t *this, inode_t *parent,
+                     uuid_t pargfid, const char *bname, void *gfid_req,
+                     dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *locked_on = NULL;
+    int source = -1;
+    struct afr_reply *replies = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+    dict_t *xattr = NULL;
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+
+    ret = dict_set_int32_sizen(xattr, GF_GFIDLESS_LOOKUP, 1);
+    if (ret) {
+        dict_unref(xattr);
+        return -1;
+    }
+
+    priv = this->private;
+
+    locked_on = alloca0(priv->child_count);
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+
+    ret = afr_selfheal_entrylk(frame, this, parent, this->name, bname,
+                               locked_on);
+    {
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_name_prepare(frame, this, parent, pargfid,
+                                          locked_on, sources, sinks,
+                                          healed_sinks, &source);
+        if (ret)
+            goto unlock;
+
+        inode = afr_selfheal_unlocked_lookup_on(frame, parent, bname, replies,
+                                                locked_on, xattr);
+        if (!inode) {
+            ret = -ENOMEM;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_name_do(frame, this, parent, pargfid, bname, inode,
+                                     sources, sinks, healed_sinks, source,
+                                     locked_on, replies, gfid_req, xdata);
+    }
+unlock:
+    afr_selfheal_unentrylk(frame, this, parent, this->name, bname, locked_on,
+                           NULL);
+    if (inode)
+        inode_unref(inode);
+
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
+
+    return ret;
+}
+
+int
+afr_selfheal_name_unlocked_inspect(call_frame_t *frame, xlator_t *this,
+                                   inode_t *parent, uuid_t pargfid,
+                                   const char *bname, gf_boolean_t *need_heal)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    struct afr_reply *replies = NULL;
+    inode_t *inode = NULL;
+    int first_idx = -1;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+
+    inode = afr_selfheal_unlocked_lookup_on(frame, parent, bname, replies,
+                                            local->child_up, NULL);
+    if (!inode)
+        return -ENOMEM;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA)) {
+            *need_heal = _gf_true;
+            break;
+        }
+
+        if (first_idx == -1) {
+            first_idx = i;
+            continue;
+        }
+
+        if (replies[i].op_ret != replies[first_idx].op_ret) {
+            *need_heal = _gf_true;
+            break;
+        }
+
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[first_idx].poststat.ia_gfid)) {
+            *need_heal = _gf_true;
+            break;
+        }
+    }
+
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    return 0;
+}
+
+int
+afr_selfheal_name(xlator_t *this, uuid_t pargfid, const char *bname,
+                  void *gfid_req, dict_t *xdata)
+{
+    inode_t *parent = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    gf_boolean_t need_heal = _gf_false;
+
+    parent = afr_inode_find(this, pargfid);
+    if (!parent)
+        goto out;
+
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        goto out;
+
+    ret = afr_selfheal_name_unlocked_inspect(frame, this, parent, pargfid,
+                                             bname, &need_heal);
+    if (ret)
+        goto out;
+
+    if (need_heal) {
+        ret = afr_selfheal_name_do(frame, this, parent, pargfid, bname,
+                                   gfid_req, xdata);
+        if (ret)
+            goto out;
+    }
+
+    ret = 0;
+out:
+    if (parent)
+        inode_unref(parent);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+
+    return ret;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index 2efc1116d33..48e6dbcfb18 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,43 +8,370 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef __AFR_SELF_HEAL_H__
-#define __AFR_SELF_HEAL_H__
+#ifndef _AFR_SELFHEAL_H
+#define _AFR_SELFHEAL_H
 
-#include <sys/stat.h>
+/* Perform fop on all UP subvolumes and wait for all callbacks to return */
 
-#define FILETYPE_DIFFERS(buf1,buf2) ((buf1)->ia_type != (buf2)->ia_type)
-#define PERMISSION_DIFFERS(buf1,buf2) (st_mode_from_ia ((buf1)->ia_prot, (buf1)->ia_type) != st_mode_from_ia ((buf2)->ia_prot, (buf2)->ia_type))
-#define OWNERSHIP_DIFFERS(buf1,buf2) (((buf1)->ia_uid != (buf2)->ia_uid) || ((buf1)->ia_gid != (buf2)->ia_gid))
-#define SIZE_DIFFERS(buf1,buf2) ((buf1)->ia_size != (buf2)->ia_size)
+#define AFR_ONALL(frame, rfn, fop, args...)                                    \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0, __count = 0;                                              \
+        unsigned char *__child_up = alloca(__priv->child_count);               \
+                                                                               \
+        memcpy(__child_up, __priv->child_up,                                   \
+               sizeof(*__child_up) * __priv->child_count);                     \
+        __count = AFR_COUNT(__child_up, __priv->child_count);                  \
+                                                                               \
+        __local->barrier.waitfor = __count;                                    \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__child_up[__i])                                              \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+        }                                                                      \
+        syncbarrier_wait(&__local->barrier, __count);                          \
+    } while (0)
 
-#define SIZE_GREATER(buf1,buf2) ((buf1)->ia_size > (buf2)->ia_size)
+/* Perform fop on all subvolumes represented by list[] array and wait
+   for all callbacks to return */
+
+#define AFR_ONLIST(list, frame, rfn, fop, args...)                             \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0;                                                           \
+        int __count = 0;                                                       \
+        unsigned char *__list = alloca(__priv->child_count);                   \
+                                                                               \
+        memcpy(__list, list, sizeof(*__list) * __priv->child_count);           \
+        __count = AFR_COUNT(__list, __priv->child_count);                      \
+        __local->barrier.waitfor = __count;                                    \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__list[__i])                                                  \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+        }                                                                      \
+        syncbarrier_wait(&__local->barrier, __count);                          \
+    } while (0)
+
+#define AFR_SEQ(frame, rfn, fop, args...)                                      \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0;                                                           \
+                                                                               \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__priv->child_up[__i])                                        \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+            syncbarrier_wait(&__local->barrier, 1);                            \
+        }                                                                      \
+    } while (0)
+
+#define ALLOC_MATRIX(n, type)                                                  \
+    ({                                                                         \
+        int __i;                                                               \
+        type **__ptr = alloca(n * sizeof(type *));                             \
+                                                                               \
+        for (__i = 0; __i < n; __i++)                                          \
+            __ptr[__i] = alloca0(n * sizeof(type));                            \
+        __ptr;                                                                 \
+    })
+
+#define IA_EQUAL(f, s, field)                                                  \
+    (memcmp(&(f.ia_##field), &(s.ia_##field), sizeof(s.ia_##field)) == 0)
+
+#define SBRAIN_HEAL_NO_GO_MSG                                                  \
+    "Failed to obtain replies from all bricks of "                             \
+    "the replica (are they up?). Cannot resolve split-brain."
+#define SFILE_NOT_IN_SPLIT_BRAIN "File not in split-brain"
+#define SNO_BIGGER_FILE "No bigger file"
+#define SNO_DIFF_IN_MTIME "No difference in mtime"
+#define SUSE_SOURCE_BRICK_TO_HEAL                                              \
+    "Use source-brick option to heal metadata"                                 \
+    " split-brain"
+#define SINVALID_BRICK_NAME "Invalid brick name"
+#define SBRICK_IS_NOT_UP "Brick is not up"
+#define SBRICK_NOT_CONNECTED "Brick is not connected"
+#define SLESS_THAN2_BRICKS_in_REP "< 2 bricks in replica are up"
+#define SBRICK_IS_REMOTE "Brick is remote"
+#define SSTARTED_SELF_HEAL "Started self-heal"
+#define SOP_NOT_SUPPORTED "Operation Not Supported"
+#define SFILE_NOT_UNDER_DATA                                                   \
+    "The file is not under data or metadata "                                  \
+    "split-brain"
+#define SFILE_NOT_IN_SPLIT_BRAIN "File not in split-brain"
+#define SALL_BRICKS_UP_TO_RESOLVE                                              \
+    "All the bricks should be up to resolve the"                               \
+    " gfid split brain"
+#define SERROR_GETTING_SRC_BRICK "Error getting the source brick"
+int
+afr_selfheal(xlator_t *this, uuid_t gfid);
+
+gf_boolean_t
+afr_throttled_selfheal(call_frame_t *frame, xlator_t *this);
+
+int
+afr_selfheal_name(xlator_t *this, uuid_t gfid, const char *name, void *gfid_req,
+                  dict_t *xdata);
+
+int
+afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd);
+
+int
+afr_selfheal_metadata(call_frame_t *frame, xlator_t *this, inode_t *inode);
+
+int
+afr_selfheal_entry(call_frame_t *frame, xlator_t *this, inode_t *inode);
+
+int
+afr_lookup_and_heal_gfid(xlator_t *this, inode_t *parent, const char *name,
+                         inode_t *inode, struct afr_reply *replies, int source,
+                         unsigned char *sources, void *gfid, int *gfid_idx);
+
+int
+afr_selfheal_inodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, off_t off, size_t size,
+                     unsigned char *locked_on);
+
+int
+afr_selfheal_tryinodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, off_t off, size_t size,
+                        unsigned char *locked_on);
+
+int
+afr_selfheal_tie_breaker_inodelk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, off_t off,
+                                 size_t size, unsigned char *locked_on);
+
+int
+afr_selfheal_uninodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, off_t off, size_t size,
+                       const unsigned char *locked_on);
+
+int
+afr_selfheal_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, const char *name, unsigned char *locked_on);
+
+int
+afr_selfheal_tryentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, const char *name, unsigned char *locked_on);
+
+int
+afr_selfheal_tie_breaker_entrylk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, const char *name,
+                                 unsigned char *locked_on);
+
+int
+afr_selfheal_unentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, const char *name, unsigned char *locked_on,
+                       dict_t *xdata);
+
+int
+afr_selfheal_unlocked_discover(call_frame_t *frame, inode_t *inode, uuid_t gfid,
+                               struct afr_reply *replies);
+
+int
+afr_selfheal_unlocked_discover_on(call_frame_t *frame, inode_t *inode,
+                                  uuid_t gfid, struct afr_reply *replies,
+                                  unsigned char *discover_on, dict_t *dict);
+inode_t *
+afr_selfheal_unlocked_lookup_on(call_frame_t *frame, inode_t *parent,
+                                const char *name, struct afr_reply *replies,
+                                unsigned char *lookup_on, dict_t *xattr);
+
+int
+afr_selfheal_find_direction(call_frame_t *frame, xlator_t *this,
+                            struct afr_reply *replies,
+                            afr_transaction_type type, unsigned char *locked_on,
+                            unsigned char *sources, unsigned char *sinks,
+                            uint64_t *witness, unsigned char *flag);
+int
+afr_selfheal_fill_matrix(xlator_t *this, int **matrix, int subvol, int idx,
+                         dict_t *xdata);
+
+int
+afr_selfheal_extract_xattr(xlator_t *this, struct afr_reply *replies,
+                           afr_transaction_type type, int *dirty, int **matrix);
 
 int
-afr_sh_has_metadata_pending (dict_t *xattr, xlator_t *this);
+afr_sh_generic_fop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata);
+
 int
-afr_sh_has_entry_pending (dict_t *xattr, xlator_t *this);
+afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          int source, unsigned char *healed_sinks,
+                          struct afr_reply *replies);
 int
-afr_sh_has_data_pending (dict_t *xattr, xlator_t *this);
+afr_selfheal_undo_pending(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          unsigned char *sources, unsigned char *sinks,
+                          unsigned char *healed_sinks,
+                          unsigned char *undid_pending,
+                          afr_transaction_type type, struct afr_reply *replies,
+                          unsigned char *locked_on);
 
 int
-afr_self_heal_entry (call_frame_t *frame, xlator_t *this);
+afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source,
+                            unsigned char *sources, inode_t *dir,
+                            const char *name, inode_t *inode,
+                            struct afr_reply *replies);
 
 int
-afr_self_heal_data (call_frame_t *frame, xlator_t *this);
+afr_selfheal_post_op(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     int subvol, dict_t *xattr, dict_t *xdata);
+
+call_frame_t *
+afr_frame_create(xlator_t *this, int32_t *op_errno);
+
+inode_t *
+afr_inode_find(xlator_t *this, uuid_t gfid);
 
 int
-afr_self_heal_metadata (call_frame_t *frame, xlator_t *this);
+afr_selfheal_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata, struct iatt *parbuf);
+void
+afr_reply_copy(struct afr_reply *dst, struct afr_reply *src);
+
+void
+afr_replies_copy(struct afr_reply *dst, struct afr_reply *src, int count);
+
+int
+afr_selfheal_newentry_mark(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, struct afr_reply *replies,
+                           unsigned char *sources, unsigned char *newentry);
+
+unsigned int
+afr_success_count(struct afr_reply *replies, unsigned int count);
+
+void
+afr_log_selfheal(uuid_t gfid, xlator_t *this, int ret, char *type, int source,
+                 unsigned char *sources, unsigned char *healed_sinks);
+
+void
+afr_mark_largest_file_as_source(xlator_t *this, unsigned char *sources,
+                                struct afr_reply *replies);
+void
+afr_mark_active_sinks(xlator_t *this, unsigned char *sources,
+                      unsigned char *locked_on, unsigned char *sinks);
+
+gf_boolean_t
+afr_dict_contains_heal_op(call_frame_t *frame);
 
+gf_boolean_t
+afr_can_decide_split_brain_source_sinks(struct afr_reply *replies,
+                                        int child_count);
 int
-afr_self_heal_get_source (xlator_t *this, afr_local_t *local, dict_t **xattr);
+afr_mark_split_brain_source_sinks(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type);
 
 int
-afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_sh_get_fav_by_policy(xlator_t *this, struct afr_reply *replies,
+                         inode_t *inode, char **policy_str);
+
+int
+_afr_fav_child_reset_sink_xattrs(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, int source,
+                                 unsigned char *healed_sinks,
+                                 unsigned char *undid_pending,
+                                 afr_transaction_type type,
+                                 unsigned char *locked_on,
+                                 struct afr_reply *replies);
+
+int
+afr_get_child_index_from_name(xlator_t *this, char *name);
+
+gf_boolean_t
+afr_does_witness_exist(xlator_t *this, uint64_t *witness);
+
+int
+__afr_selfheal_data_prepare(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            unsigned char *undid_pending,
+                            struct afr_reply *replies, unsigned char *flag);
+
+int
+__afr_selfheal_metadata_prepare(call_frame_t *frame, xlator_t *this,
+                                inode_t *inode, unsigned char *locked_on,
+                                unsigned char *sources, unsigned char *sinks,
+                                unsigned char *healed_sinks,
+                                unsigned char *undid_pending,
+                                struct afr_reply *replies, unsigned char *flag);
+int
+__afr_selfheal_entry_prepare(call_frame_t *frame, xlator_t *this,
+                             inode_t *inode, unsigned char *locked_on,
+                             unsigned char *sources, unsigned char *sinks,
+                             unsigned char *healed_sinks,
+                             struct afr_reply *replies, int *source_p,
+                             unsigned char *flag);
 
 int
-afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
-                                          dict_t **xattr,
-                                          afr_transaction_type txn_type,
-                                          uuid_t gfid);
-#endif /* __AFR_SELF_HEAL_H__ */
+afr_selfheal_unlocked_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                              inode_t **link_inode, gf_boolean_t *data_selfheal,
+                              gf_boolean_t *metadata_selfheal,
+                              gf_boolean_t *entry_selfheal,
+                              struct afr_reply *replies);
+
+int
+afr_selfheal_do(call_frame_t *frame, xlator_t *this, uuid_t gfid);
+
+int
+afr_selfheal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata);
+
+int
+afr_locked_fill(call_frame_t *frame, xlator_t *this, unsigned char *locked_on);
+int
+afr_choose_source_by_policy(afr_private_t *priv, unsigned char *sources,
+                            afr_transaction_type type);
+
+int
+afr_selfheal_metadata_by_stbuf(xlator_t *this, struct iatt *stbuf);
+
+int
+afr_sh_fav_by_size(xlator_t *this, struct afr_reply *replies, inode_t *inode);
+int
+afr_sh_fav_by_mtime(xlator_t *this, struct afr_reply *replies, inode_t *inode);
+int
+afr_sh_fav_by_ctime(xlator_t *this, struct afr_reply *replies, inode_t *inode);
+
+int
+afr_gfid_split_brain_source(xlator_t *this, struct afr_reply *replies,
+                            inode_t *inode, uuid_t pargfid, const char *bname,
+                            int src_idx, int child_idx,
+                            unsigned char *locked_on, int *src, dict_t *xdata);
+int
+afr_mark_source_sinks_if_file_empty(xlator_t *this, unsigned char *sources,
+                                    unsigned char *sinks,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on,
+                                    struct afr_reply *replies,
+                                    afr_transaction_type type);
+
+gf_boolean_t
+afr_is_file_empty_on_all_children(afr_private_t *priv,
+                                  struct afr_reply *replies);
+
+int
+afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name,
+                          inode_t *inode, int child, struct afr_reply *replies);
+int
+afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode);
+#endif /* !_AFR_SELFHEAL_H */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index 6184036b278..109fd4b7421 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,1209 +8,1709 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
 #include "afr.h"
-#include "syncop.h"
+#include "afr-self-heal.h"
 #include "afr-self-heald.h"
-#include "afr-self-heal-common.h"
 #include "protocol-common.h"
-#include "event-history.h"
-
-typedef enum {
-        STOP_CRAWL_ON_SINGLE_SUBVOL = 1
-} afr_crawl_flags_t;
-
-typedef enum {
-        HEAL = 1,
-        INFO
-} shd_crawl_op;
-
-typedef struct shd_dump {
-        dict_t   *dict;
-        xlator_t *this;
-        int      child;
-} shd_dump_t;
+#include <glusterfs/syncop-utils.h>
+#include "afr-messages.h"
+#include <glusterfs/byte-order.h>
+
+#define AFR_EH_SPLIT_BRAIN_LIMIT 1024
+#define AFR_STATISTICS_HISTORY_SIZE 50
+
+#define ASSERT_LOCAL(this, healer)                                             \
+    if (!afr_shd_is_subvol_local(this, healer->subvol)) {                      \
+        healer->local = _gf_false;                                             \
+        if (safe_break(healer)) {                                              \
+            break;                                                             \
+        } else {                                                               \
+            continue;                                                          \
+        }                                                                      \
+    } else {                                                                   \
+        healer->local = _gf_true;                                              \
+    }
+
+#define NTH_INDEX_HEALER(this, n)                                              \
+    &((((afr_private_t *)this->private))->shd.index_healers[n])
+#define NTH_FULL_HEALER(this, n)                                               \
+    &((((afr_private_t *)this->private))->shd.full_healers[n])
+
+char *
+afr_subvol_name(xlator_t *this, int subvol)
+{
+    afr_private_t *priv = NULL;
 
-typedef struct shd_event_ {
-        int     child;
-        char    *path;
-} shd_event_t;
+    priv = this->private;
+    if (subvol < 0 || subvol > priv->child_count)
+        return NULL;
 
-typedef struct shd_pos_ {
-        int     child;
-        xlator_t *this;
-        afr_child_pos_t pos;
-} shd_pos_t;
+    return priv->children[subvol]->name;
+}
 
-typedef int
-(*afr_crawl_done_cbk_t)  (int ret, call_frame_t *sync_frame, void *crawl_data);
+void
+afr_destroy_crawl_event_data(void *data)
+{
+    return;
+}
 
 void
-afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
-                 process_entry_cbk_t process_entry, void *op_data,
-                 gf_boolean_t exclusive, int crawl_flags,
-                 afr_crawl_done_cbk_t crawl_done);
+afr_destroy_shd_event_data(void *data)
+{
+    shd_event_t *shd_event = data;
 
-static int
-_crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data);
+    if (!shd_event)
+        return;
+    GF_FREE(shd_event->path);
 
-int
-afr_syncop_find_child_position (void *data);
+    return;
+}
 
-static int
-_loc_assign_gfid_path (loc_t *loc)
+gf_boolean_t
+afr_shd_is_subvol_local(xlator_t *this, int subvol)
 {
-        int  ret = -1;
-        char gfid_path[64] = {0};
-
-        if (loc->inode && !uuid_is_null (loc->inode->gfid)) {
-                ret = inode_path (loc->inode, NULL, (char**)&loc->path);
-        } else if (!uuid_is_null (loc->gfid)) {
-                snprintf (gfid_path, sizeof (gfid_path), "<gfid:%s>",
-                          uuid_utoa (loc->gfid));
-                loc->path = gf_strdup (gfid_path);
-                if (loc->path)
-                        ret = 0;
-        }
-        return ret;
+    afr_private_t *priv = NULL;
+    gf_boolean_t is_local = _gf_false;
+    loc_t loc = {
+        0,
+    };
+
+    loc.inode = this->itable->root;
+    gf_uuid_copy(loc.gfid, loc.inode->gfid);
+    priv = this->private;
+    syncop_is_subvol_local(priv->children[subvol], &loc, &is_local);
+    return is_local;
 }
 
-void
-shd_cleanup_event (void *event)
+int
+__afr_shd_healer_wait(struct subvol_healer *healer)
 {
-        shd_event_t *shd_event = event;
+    afr_private_t *priv = NULL;
+    struct timespec wait_till = {
+        0,
+    };
+    int ret = 0;
 
-        if (!shd_event)
-                goto out;
-        GF_FREE (shd_event->path);
-        GF_FREE (shd_event);
-out:
-        return;
+    priv = healer->this->private;
+
+disabled_loop:
+    wait_till.tv_sec = gf_time() + priv->shd.timeout;
+
+    while (!healer->rerun) {
+        ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till);
+        if (ret == ETIMEDOUT)
+            break;
+    }
+
+    ret = healer->rerun;
+    healer->rerun = 0;
+
+    if (!priv->shd.enabled)
+        goto disabled_loop;
+
+    return ret;
 }
 
 int
-afr_get_local_child (afr_self_heald_t *shd, unsigned int child_count)
+afr_shd_healer_wait(struct subvol_healer *healer)
 {
-        int i = 0;
-        int ret = -1;
-        for (i = 0; i < child_count; i++) {
-                if (shd->pos[i] == AFR_POS_LOCAL) {
-                        ret = i;
-                        break;
-                }
-        }
-        return ret;
+    int ret = 0;
+
+    pthread_mutex_lock(&healer->mutex);
+    {
+        ret = __afr_shd_healer_wait(healer);
+    }
+    pthread_mutex_unlock(&healer->mutex);
+
+    return ret;
 }
 
-static int
-_build_index_loc (xlator_t *this, loc_t *loc, char *name, loc_t *parent)
+gf_boolean_t
+safe_break(struct subvol_healer *healer)
 {
-        int             ret = 0;
-
-        uuid_copy (loc->pargfid, parent->inode->gfid);
-        loc->path = "";
-        loc->name = name;
-        loc->parent = inode_ref (parent->inode);
-        if (!loc->parent) {
-                loc->path = NULL;
-                loc_wipe (loc);
-                ret = -1;
-        }
-        return ret;
+    gf_boolean_t ret = _gf_false;
+
+    pthread_mutex_lock(&healer->mutex);
+    {
+        if (healer->rerun)
+            goto unlock;
+
+        healer->running = _gf_false;
+        ret = _gf_true;
+    }
+unlock:
+    pthread_mutex_unlock(&healer->mutex);
+
+    return ret;
 }
 
-int
-_add_path_to_dict (xlator_t *this, dict_t *output, int child, char *path,
-                   struct timeval *tv, gf_boolean_t dyn)
+inode_t *
+afr_shd_inode_find(xlator_t *this, xlator_t *subvol, uuid_t gfid)
 {
-        //subkey not used for now
-        int             ret = -1;
-        uint64_t        count = 0;
-        char            key[256] = {0};
-        int             xl_id = 0;
+    int ret = 0;
+    uint64_t val = IA_INVAL;
+    dict_t *xdata = NULL;
+    dict_t *rsp_dict = NULL;
+    inode_t *inode = NULL;
+
+    xdata = dict_new();
+    if (!xdata)
+        goto out;
+
+    ret = dict_set_int8(xdata, GF_INDEX_IA_TYPE_GET_REQ, 1);
+    if (ret)
+        goto out;
+
+    ret = syncop_inode_find(this, subvol, gfid, &inode, xdata, &rsp_dict);
+    if (ret < 0)
+        goto out;
+
+    if (rsp_dict) {
+        ret = dict_get_uint64(rsp_dict, GF_INDEX_IA_TYPE_GET_RSP, &val);
+        if (ret)
+            goto out;
+    }
+    ret = inode_ctx_set2(inode, subvol, 0, &val);
+out:
+    if (ret && inode) {
+        inode_unref(inode);
+        inode = NULL;
+    }
+    if (xdata)
+        dict_unref(xdata);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return inode;
+}
 
-        ret = dict_get_int32 (output, this->name, &xl_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
-                goto out;
-        }
+inode_t *
+afr_shd_index_inode(xlator_t *this, xlator_t *subvol, char *vgfid)
+{
+    loc_t rootloc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    int ret = 0;
+    dict_t *xattr = NULL;
+    void *index_gfid = NULL;
 
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
-        ret = dict_get_uint64 (output, key, &count);
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
 
-        snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
-        if (dyn)
-                ret = dict_set_dynstr (output, key, path);
-        else
-                ret = dict_set_str (output, key, path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Could not add to output",
-                        path);
-                goto out;
-        }
+    ret = syncop_getxattr(subvol, &rootloc, &xattr, vgfid, NULL, NULL);
+    if (ret || !xattr) {
+        errno = -ret;
+        goto out;
+    }
 
-        if (!tv)
-                goto inc_count;
-        snprintf (key, sizeof (key), "%d-%d-%"PRIu64"-time", xl_id,
-                  child, count);
-        ret = dict_set_uint32 (output, key, tv->tv_sec);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Could not set time",
-                        path);
-                goto out;
-        }
+    ret = dict_get_ptr(xattr, vgfid, &index_gfid);
+    if (ret)
+        goto out;
+
+    gf_msg_debug(this->name, 0, "%s dir gfid for %s: %s", vgfid, subvol->name,
+                 uuid_utoa(index_gfid));
+
+    inode = afr_shd_inode_find(this, subvol, index_gfid);
 
-inc_count:
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
-        ret = dict_set_uint64 (output, key, count + 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not increment count");
-                goto out;
-        }
-        ret = 0;
 out:
-        return ret;
+    loc_wipe(&rootloc);
+
+    if (xattr)
+        dict_unref(xattr);
+
+    return inode;
 }
 
 int
-_get_path_from_gfid_loc (xlator_t *this, xlator_t *readdir_xl, loc_t *child,
-                         char **fpath)
+afr_shd_entry_purge(xlator_t *subvol, inode_t *inode, char *name,
+                    ia_type_t type)
 {
-        dict_t          *xattr = NULL;
-        char            *path = NULL;
-        int             ret = -1;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
 
-        ret = syncop_getxattr (readdir_xl, child, &xattr,
-                               GFID_TO_PATH_KEY);
-        if (ret)
-                goto out;
-        ret = dict_get_str (xattr, GFID_TO_PATH_KEY, &path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get path for "
-                        "gfid %s", uuid_utoa (child->gfid));
-                goto out;
-        }
-        path = gf_strdup (path);
-        if (!path) {
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
-out:
-        if (!ret)
-                *fpath = path;
-        if (xattr)
-                dict_unref (xattr);
-        return ret;
+    loc.parent = inode_ref(inode);
+    loc.name = name;
+
+    if (IA_ISDIR(type))
+        ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL);
+    else
+        ret = syncop_unlink(subvol, &loc, NULL, NULL);
+
+    loc_wipe(&loc);
+    return ret;
 }
 
-int
-_add_event_to_dict (circular_buffer_t *cb, void *data)
+void
+afr_shd_zero_xattrop(xlator_t *this, uuid_t gfid)
 {
-        int               ret = 0;
-        shd_dump_t        *dump_data = NULL;
-        shd_event_t       *shd_event = NULL;
+    call_frame_t *frame = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int ret = 0;
+    int i = 0;
+    int raw[AFR_NUM_CHANGE_LOGS] = {0};
+
+    priv = this->private;
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        goto out;
+    inode = afr_inode_find(this, gfid);
+    if (!inode)
+        goto out;
+    xattr = dict_new();
+    if (!xattr)
+        goto out;
+    ret = dict_set_static_bin(xattr, AFR_DIRTY, raw,
+                              sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], raw,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret)
+            goto out;
+    }
+
+    /*Send xattrop to all bricks. Doing a lookup to see if bricks are up or
+     * has valid repies for this gfid seems a bit of an overkill.*/
+    for (i = 0; i < priv->child_count; i++)
+        afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
 
-        dump_data = data;
-        shd_event = cb->data;
-        if (shd_event->child != dump_data->child)
-                goto out;
-        ret = _add_path_to_dict (dump_data->this, dump_data->dict,
-                                 dump_data->child, shd_event->path, &cb->tv,
-                                 _gf_false);
 out:
-        return ret;
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    if (inode)
+        inode_unref(inode);
+    if (xattr)
+        dict_unref(xattr);
+    return;
 }
 
 int
-_add_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict, int child)
+afr_shd_selfheal_name(struct subvol_healer *healer, int child, uuid_t parent,
+                      const char *bname)
 {
-        shd_dump_t dump_data = {0};
+    int ret = -1;
 
-        dump_data.this = this;
-        dump_data.dict = dict;
-        dump_data.child = child;
-        eh_dump (eh, &dump_data, _add_event_to_dict);
-        return 0;
+    ret = afr_selfheal_name(THIS, parent, bname, NULL, NULL);
+
+    return ret;
 }
 
 int
-_add_summary_to_dict (xlator_t *this, afr_crawl_data_t *crawl_data,
-                      gf_dirent_t *entry,
-                      loc_t *childloc, loc_t *parentloc, struct iatt *iattr)
+afr_shd_selfheal(struct subvol_healer *healer, int child, uuid_t gfid)
 {
-        dict_t          *output = NULL;
-        xlator_t        *readdir_xl = NULL;
-        int             ret = -1;
-        char            *path = NULL;
+    int ret = 0;
+    eh_t *eh = NULL;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    shd_event_t *shd_event = NULL;
+    char *path = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    crawl_event_t *crawl_event = NULL;
+
+    this = healer->this;
+    priv = this->private;
+    shd = &priv->shd;
+    crawl_event = &healer->crawl_event;
+
+    subvol = priv->children[child];
+
+    // If this fails with ENOENT/ESTALE index is stale
+    ret = syncop_gfid_to_path(this->itable, subvol, gfid, &path);
+    if (ret < 0)
+        return ret;
 
-        if (uuid_is_null (childloc->gfid))
-                goto out;
+    ret = afr_selfheal(this, gfid);
+
+    LOCK(&priv->lock);
+    {
+        if (ret == -EIO) {
+            eh = shd->split_brain;
+            crawl_event->split_brain_count++;
+        } else if (ret < 0) {
+            crawl_event->heal_failed_count++;
+        } else if (ret == 0) {
+            crawl_event->healed_count++;
+        }
+    }
+    UNLOCK(&priv->lock);
 
-        output = crawl_data->op_data;
-        readdir_xl = crawl_data->readdir_xl;
+    if (eh) {
+        shd_event = GF_CALLOC(1, sizeof(*shd_event), gf_afr_mt_shd_event_t);
+        if (!shd_event)
+            goto out;
 
-        ret = _get_path_from_gfid_loc (this, readdir_xl, childloc, &path);
-        if (ret)
-                goto out;
+        shd_event->child = child;
+        shd_event->path = path;
 
-        ret = _add_path_to_dict (this, output, crawl_data->child, path, NULL,
-                                 _gf_true);
+        if (eh_save_history(eh, shd_event) < 0)
+            goto out;
+
+        shd_event = NULL;
+        path = NULL;
+    }
 out:
-        if (ret && path)
-                GF_FREE (path);
-        return ret;
+    GF_FREE(shd_event);
+    GF_FREE(path);
+    return ret;
 }
 
 void
-_remove_stale_index (xlator_t *this, xlator_t *readdir_xl,
-                     loc_t *parent, char *fname)
+afr_shd_sweep_prepare(struct subvol_healer *healer)
 {
-        int              ret = 0;
-        loc_t            index_loc = {0};
+    crawl_event_t *event = NULL;
 
-        ret = _build_index_loc (this, &index_loc, fname, parent);
-        if (ret)
-                goto out;
-        gf_log (this->name, GF_LOG_INFO, "Removing stale index "
-                "for %s on %s", index_loc.name, readdir_xl->name);
-        ret = syncop_unlink (readdir_xl, &index_loc);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to remove"
-                        " index on %s - %s", index_loc.name,
-                        readdir_xl->name, strerror (errno));
-        }
-        index_loc.path = NULL;
-        loc_wipe (&index_loc);
-out:
-        return;
+    event = &healer->crawl_event;
+
+    event->healed_count = 0;
+    event->split_brain_count = 0;
+    event->heal_failed_count = 0;
+
+    event->start_time = gf_time();
+    event->end_time = 0;
+    _mask_cancellation();
 }
 
 void
-_crawl_post_sh_action (xlator_t *this, loc_t *parent, loc_t *child,
-                       int32_t op_ret, int32_t op_errno, dict_t *xattr_rsp,
-                       afr_crawl_data_t *crawl_data)
+afr_shd_sweep_done(struct subvol_healer *healer)
 {
-        int              ret = 0;
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        eh_t             *eh = NULL;
-        char             *path = NULL;
-        shd_event_t      *event = NULL;
-        int32_t          sh_failed = 0;
-        gf_boolean_t     split_brain = 0;
-
-        priv = this->private;
-        shd  = &priv->shd;
-        if (crawl_data->crawl == INDEX) {
-                if ((op_ret < 0) && (op_errno == ENOENT)) {
-                        _remove_stale_index (this, crawl_data->readdir_xl,
-                                             parent, uuid_utoa (child->gfid));
-                        goto out;
-                }
-                ret = _get_path_from_gfid_loc (this, crawl_data->readdir_xl,
-                                               child, &path);
-                if (ret)
-                        goto out;
-        } else {
-                path = gf_strdup (child->path);
-                if (!path) {
-                        ret = -1;
-                        goto out;
-                }
-        }
+    crawl_event_t *event = NULL;
+    crawl_event_t *history = NULL;
+    afr_self_heald_t *shd = NULL;
 
-        if (xattr_rsp)
-                ret = dict_get_int32 (xattr_rsp, "sh-failed", &sh_failed);
-        split_brain = afr_is_split_brain (this, child->inode);
-        if ((op_ret < 0 && op_errno == EIO) || split_brain)
-                eh = shd->split_brain;
-        else if ((op_ret < 0) || sh_failed)
-                eh = shd->heal_failed;
-        else
-                eh = shd->healed;
-        ret = -1;
-        event = GF_CALLOC (1, sizeof (*event), gf_afr_mt_shd_event_t);
-        if (!event)
-                goto out;
-        event->child = crawl_data->child;
-        event->path = path;
-        ret = eh_save_history (eh, event);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "%s:Failed to save to "
-                        "event history, (%d, %s)", path, op_ret, strerror (op_errno));
-                goto out;
-        }
-        ret = 0;
-out:
-        if (ret && path)
-                GF_FREE (path);
+    event = &healer->crawl_event;
+    shd = &(((afr_private_t *)healer->this->private)->shd);
+
+    event->end_time = gf_time();
+    history = gf_memdup(event, sizeof(*event));
+    event->start_time = 0;
+
+    if (!history)
         return;
+
+    if (eh_save_history(shd->statistics[healer->subvol], history) < 0)
+        GF_FREE(history);
+    _unmask_cancellation();
 }
 
 int
-_self_heal_entry (xlator_t *this, afr_crawl_data_t *crawl_data, gf_dirent_t *entry,
-                  loc_t *child, loc_t *parent, struct iatt *iattr)
+afr_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                   void *data)
 {
-        struct iatt      parentbuf = {0};
-        int              ret = 0;
-        dict_t           *xattr_rsp = NULL;
-
-        gf_log (this->name, GF_LOG_DEBUG, "lookup %s", child->path);
-
-        ret = syncop_lookup (this, child, NULL,
-                             iattr, &xattr_rsp, &parentbuf);
-        _crawl_post_sh_action (this, parent, child, ret, errno, xattr_rsp,
-                               crawl_data);
-        if (xattr_rsp)
-                dict_unref (xattr_rsp);
-        return ret;
-}
+    struct subvol_healer *healer = data;
+    afr_private_t *priv = NULL;
+    uuid_t gfid = {0};
+    int ret = 0;
+    uint64_t val = IA_INVAL;
 
-static int
-afr_crawl_done  (int ret, call_frame_t *sync_frame, void *data)
-{
-        GF_FREE (data);
-        STACK_DESTROY (sync_frame->root);
+    priv = healer->this->private;
+    if (!priv->shd.enabled)
+        return -EBUSY;
+
+    gf_msg_debug(healer->this->name, 0, "got entry: %s from %s", entry->d_name,
+                 priv->children[healer->subvol]->name);
+
+    ret = gf_uuid_parse(entry->d_name, gfid);
+    if (ret)
         return 0;
-}
 
-void
-_do_self_heal_on_subvol (xlator_t *this, int child, afr_crawl_type_t crawl)
-{
-        afr_start_crawl (this, child, crawl, _self_heal_entry,
-                         NULL, _gf_true, STOP_CRAWL_ON_SINGLE_SUBVOL,
-                         afr_crawl_done);
+    inode_ctx_get2(parent->inode, subvol, NULL, &val);
+
+    ret = afr_shd_selfheal(healer, healer->subvol, gfid);
+
+    if (ret == -ENOENT || ret == -ESTALE)
+        afr_shd_entry_purge(subvol, parent->inode, entry->d_name, val);
+
+    if (ret == 2)
+        /* If bricks crashed in pre-op after creating indices/xattrop
+         * link but before setting afr changelogs, we end up with stale
+         * xattrop links but zero changelogs. Remove such entries by
+         * sending a post-op with zero changelogs.
+         */
+        afr_shd_zero_xattrop(healer->this, gfid);
+
+    return 0;
 }
 
-gf_boolean_t
-_crawl_proceed (xlator_t *this, int child, int crawl_flags, char **reason)
+int
+afr_shd_index_sweep(struct subvol_healer *healer, char *vgfid)
 {
-        afr_private_t           *priv = NULL;
-        afr_self_heald_t        *shd = NULL;
-        gf_boolean_t            proceed = _gf_false;
-        char                    *msg = NULL;
-
-        priv = this->private;
-        shd  = &priv->shd;
-        if (!shd->enabled) {
-                msg = "Self-heal daemon is not enabled";
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
-        if (!priv->child_up[child]) {
-                gf_log (this->name, GF_LOG_ERROR, "Stopping crawl for %s , "
-                        "subvol went down", priv->children[child]->name);
-                msg = "Brick is Not connected";
-                goto out;
-        }
+    loc_t loc = {0};
+    afr_private_t *priv = NULL;
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    dict_t *xdata = NULL;
+    call_frame_t *frame = NULL;
+
+    priv = healer->this->private;
+    subvol = priv->children[healer->subvol];
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    loc.inode = afr_shd_index_inode(healer->this, subvol, vgfid);
+    if (!loc.inode) {
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_INDEX_DIR_GET_FAILED, "unable to get index-dir on %s",
+               subvol->name);
+        ret = -errno;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata || dict_set_int32_sizen(xdata, "get-gfid-type", 1)) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_mt_dir_scan(frame, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
+                             healer, afr_shd_index_heal, xdata,
+                             priv->shd.max_threads, priv->shd.wait_qlength);
+
+    if (ret == 0)
+        ret = healer->crawl_event.healed_count;
 
-        if (crawl_flags & STOP_CRAWL_ON_SINGLE_SUBVOL) {
-                if (afr_up_children_count (priv->child_up,
-                                           priv->child_count) < 2) {
-                        gf_log (this->name, GF_LOG_ERROR, "Stopping crawl as "
-                                "< 2 children are up");
-                        msg = "< 2 bricks in replica are running";
-                        goto out;
-                }
-        }
-        proceed = _gf_true;
 out:
-        if (reason)
-                *reason = msg;
-        return proceed;
+    loc_wipe(&loc);
+
+    if (xdata)
+        dict_unref(xdata);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-_do_crawl_op_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl,
-                               shd_crawl_op op, dict_t *output)
+afr_shd_index_sweep_all(struct subvol_healer *healer)
 {
-        afr_private_t       *priv = NULL;
-        char                *status = NULL;
-        char                *subkey = NULL;
-        char                key[256] = {0};
-        shd_pos_t           pos_data = {0};
-        int                 op_ret = -1;
-        int                 xl_id = -1;
-        int                 i = 0;
-        int                 ret = 0;
-        int                 crawl_flags = 0;
-
-        priv = this->private;
-        if (op == HEAL)
-                crawl_flags |= STOP_CRAWL_ON_SINGLE_SUBVOL;
-
-        ret = dict_get_int32 (output, this->name, &xl_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid input, "
-                        "translator-id is not available");
-                goto out;
-        }
-        pos_data.this = this;
-        subkey = "status";
-        for (i = 0; i < priv->child_count; i++) {
-                if (_crawl_proceed (this, i, crawl_flags, &status)) {
-                        pos_data.child = i;
-                        ret = synctask_new (this->ctx->env,
-                                            afr_syncop_find_child_position,
-                                            NULL, NULL, &pos_data);
-                        if (ret) {
-                                status = "Not able to find brick location";
-                        } else if (pos_data.pos == AFR_POS_REMOTE) {
-                                status = "brick is remote";
-                        } else {
-                                op_ret = 0;
-                                if (op == HEAL) {
-                                        status = "Started self-heal";
-                                        _do_self_heal_on_subvol (this, i,
-                                                                 crawl);
-                                } else {
-                                        status = "";
-                                        afr_start_crawl (this, i, INDEX,
-                                                         _add_summary_to_dict,
-                                                         output, _gf_false, 0,
-                                                         NULL);
-                                }
-                        }
-                        snprintf (key, sizeof (key), "%d-%d-%s", xl_id,
-                                  i, subkey);
-                        ret = dict_set_str (output, key, status);
-                        if (!op_ret && (crawl == FULL))
-                                break;
-                }
-                snprintf (key, sizeof (key), "%d-%d-%s", xl_id, i, subkey);
-                ret = dict_set_str (output, key, status);
-        }
+    int ret = 0;
+    int count = 0;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_INDEX_GFID);
+    if (ret < 0)
+        goto out;
+    count = ret;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_DIRTY_GFID);
+    if (ret < 0)
+        goto out;
+    count += ret;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_ENTRY_CHANGES_GFID);
+    if (ret < 0)
+        goto out;
+    count += ret;
 out:
-        return op_ret;
+    if (ret < 0)
+        return ret;
+    else
+        return count;
 }
 
 int
-_do_self_heal_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl,
-                                dict_t *output)
+afr_shd_full_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                  void *data)
 {
-        return _do_crawl_op_on_local_subvols (this, crawl, HEAL, output);
+    struct subvol_healer *healer = data;
+    xlator_t *this = healer->this;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (this->cleanup_starting) {
+        return -ENOTCONN;
+    }
+
+    if (!priv->shd.enabled)
+        return -EBUSY;
+
+    afr_shd_selfheal_name(healer, healer->subvol, parent->inode->gfid,
+                          entry->d_name);
+
+    afr_shd_selfheal(healer, healer->subvol, entry->d_stat.ia_gfid);
+
+    return 0;
 }
 
 int
-_get_index_summary_on_local_subvols (xlator_t *this, dict_t *output)
+afr_shd_full_sweep(struct subvol_healer *healer, inode_t *inode)
 {
-        return _do_crawl_op_on_local_subvols (this, INDEX, INFO, output);
+    afr_private_t *priv = NULL;
+    loc_t loc = {0};
+
+    priv = healer->this->private;
+    loc.inode = inode;
+    return syncop_ftw(priv->children[healer->subvol], &loc,
+                      GF_CLIENT_PID_SELF_HEALD, healer, afr_shd_full_heal);
 }
 
 int
-_add_all_subvols_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict)
+afr_shd_fill_ta_loc(xlator_t *this, loc_t *loc)
 {
-        afr_private_t           *priv = NULL;
-        afr_self_heald_t        *shd = NULL;
-        int                     i = 0;
+    afr_private_t *priv = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = -1;
+
+    priv = this->private;
+    loc->parent = inode_ref(this->itable->root);
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
+    loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
+    loc->inode = inode_new(loc->parent->table);
+    GF_CHECK_ALLOC(loc->inode, ret, out);
+
+    if (!gf_uuid_is_null(priv->ta_gfid))
+        goto assign_gfid;
+
+    ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], loc, &stbuf,
+                        0, 0, 0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed lookup on file %s.", loc->name);
+        goto out;
+    }
+
+    gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
+
+assign_gfid:
+    gf_uuid_copy(loc->gfid, priv->ta_gfid);
+    ret = 0;
 
-        priv = this->private;
-        shd = &priv->shd;
+out:
+    if (ret)
+        loc_wipe(loc);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (shd->pos[i] != AFR_POS_LOCAL)
-                        continue;
-                _add_eh_to_dict (this, eh, dict, i);
-        }
-        return 0;
+    return ret;
 }
 
 int
-afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
+_afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata)
 {
-        gf_xl_afr_op_t   op = GF_AFR_OP_INVALID;
-        int              ret = 0;
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        int              xl_id = 0;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int raw[AFR_NUM_CHANGE_LOGS] = {
+        0,
+    };
+    int ret = -1;
+    int i = 0;
+
+    priv = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_GET_FAILED,
+               "Failed to create dict.");
+        goto out;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], &raw,
+                                  AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret)
+            goto out;
+    }
 
-        priv = this->private;
-        shd = &priv->shd;
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, xdata, NULL);
+    if (ret || !(*xdata)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Xattrop failed on %s.", loc->name);
+    }
 
-        ret = dict_get_int32 (input, "xl-op", (int32_t*)&op);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (input, this->name, &xl_id);
-        if (ret)
-                goto out;
-        ret = dict_set_int32 (output, this->name, xl_id);
-        if (ret)
-                goto out;
-        switch (op) {
-        case GF_AFR_OP_HEAL_INDEX:
-                ret = _do_self_heal_on_local_subvols (this, INDEX, output);
-                break;
-        case GF_AFR_OP_HEAL_FULL:
-                ret = _do_self_heal_on_local_subvols (this, FULL, output);
-                break;
-        case GF_AFR_OP_INDEX_SUMMARY:
-                (void)_get_index_summary_on_local_subvols (this, output);
-                ret = 0;
-                break;
-        case GF_AFR_OP_HEALED_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->healed, output);
-                break;
-        case GF_AFR_OP_HEAL_FAILED_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->heal_failed,
-                                                   output);
-                break;
-        case GF_AFR_OP_SPLIT_BRAIN_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->split_brain,
-                                                   output);
-                break;
-        default:
-                gf_log (this->name, GF_LOG_ERROR, "Unknown set op %d", op);
-                break;
-        }
 out:
-        dict_del (output, this->name);
-        return ret;
+    if (xattr)
+        dict_unref(xattr);
+
+    return ret;
 }
 
 void
-afr_poll_self_heal (void *data)
+afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, struct subvol_healer *healer,
+                      dict_t **xdata)
 {
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        struct timeval   timeout = {0};
-        xlator_t         *this = NULL;
-        long             child = (long)data;
-        gf_timer_t       *old_timer = NULL;
-        gf_timer_t       *new_timer = NULL;
-        shd_pos_t        pos_data = {0};
-        int              ret = 0;
-
-        this = THIS;
-        priv = this->private;
-        shd = &priv->shd;
-
-        if (shd->pos[child] == AFR_POS_UNKNOWN) {
-                pos_data.this = this;
-                pos_data.child = child;
-                ret = synctask_new (this->ctx->env,
-                                    afr_syncop_find_child_position,
-                                    NULL, NULL, &pos_data);
-                if (!ret)
-                        shd->pos[child] = pos_data.pos;
-        }
-        if (shd->enabled && (shd->pos[child] == AFR_POS_LOCAL))
-                _do_self_heal_on_subvol (this, child, INDEX);
-        timeout.tv_sec = shd->timeout;
-        timeout.tv_usec = 0;
-        //notify and previous timer should be synchronized.
-        LOCK (&priv->lock);
-        {
-                old_timer = shd->timer[child];
-                if (shd->pos[child] == AFR_POS_REMOTE)
-                        goto unlock;
-                shd->timer[child] = gf_timer_call_after (this->ctx, timeout,
-                                                         afr_poll_self_heal,
-                                                         data);
-                new_timer = shd->timer[child];
-        }
-unlock:
-        UNLOCK (&priv->lock);
-
-        if (old_timer)
-                gf_timer_call_cancel (this->ctx, old_timer);
-        if (!new_timer && (shd->pos[child] != AFR_POS_REMOTE)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Could not create self-heal polling timer for %s",
-                        priv->children[child]->name);
+    int ret = 0;
+
+    loc_wipe(loc);
+    if (afr_shd_fill_ta_loc(this, loc)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc->name);
+        ret = -1;
+        goto out;
+    }
+
+    ret = afr_ta_post_op_lock(this, loc);
+    if (ret)
+        goto out;
+
+    ret = _afr_shd_ta_get_xattrs(this, loc, xdata);
+    if (ret) {
+        if (*xdata) {
+            dict_unref(*xdata);
+            *xdata = NULL;
         }
-        return;
+    }
+
+    afr_ta_post_op_unlock(this, loc);
+
+out:
+    if (ret)
+        healer->rerun = 1;
 }
 
-static int
-afr_local_child_poll_self_heal  (int ret, call_frame_t *sync_frame, void *data)
+int
+afr_shd_ta_unset_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata, int healer)
 {
-        afr_self_heald_t *shd = NULL;
-        shd_pos_t        *pos_data = data;
-        afr_private_t    *priv = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    gf_boolean_t need_xattrop = _gf_false;
+    void *pending_raw = NULL;
+    int *raw = NULL;
+    int pending[AFR_NUM_CHANGE_LOGS] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+    int val = 0;
+    int ret = -1;
+
+    priv = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        raw = GF_CALLOC(AFR_NUM_CHANGE_LOGS, sizeof(int), gf_afr_mt_int32_t);
+        if (!raw) {
+            goto out;
+        }
 
-        if (ret)
-                goto out;
+        ret = dict_get_ptr(*xdata, priv->pending_key[i], &pending_raw);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                   "Error getting value "
+                   "of pending key %s",
+                   priv->pending_key[i]);
+            GF_FREE(raw);
+            goto out;
+        }
+
+        memcpy(pending, pending_raw, sizeof(pending));
+        for (j = 0; j < AFR_NUM_CHANGE_LOGS; j++) {
+            val = ntoh32(pending[j]);
+            if (val) {
+                if (i == healer) {
+                    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_THIN_ARB,
+                           "I am "
+                           "not the good shd. Skipping. "
+                           "SHD = %d.",
+                           healer);
+                    ret = 0;
+                    GF_FREE(raw);
+                    goto out;
+                }
+                need_xattrop = _gf_true;
+                raw[j] = hton32(-val);
+            }
+        }
+
+        ret = dict_set_bin(xattr, priv->pending_key[i], raw,
+                           AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret) {
+            GF_FREE(raw);
+            goto out;
+        }
+
+        if (need_xattrop)
+            break;
+    }
+
+    if (!need_xattrop) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Xattrop failed.");
 
-        priv = pos_data->this->private;
-        shd = &priv->shd;
-        shd->pos[pos_data->child] = pos_data->pos;
-        if (pos_data->pos != AFR_POS_REMOTE)
-                afr_poll_self_heal ((void*)(long)pos_data->child);
 out:
-        GF_FREE (data);
-        return 0;
+    if (xattr)
+        dict_unref(xattr);
+
+    return ret;
 }
 
 void
-afr_proactive_self_heal (void *data)
+afr_shd_ta_check_and_unset_xattrs(xlator_t *this, loc_t *loc,
+                                  struct subvol_healer *healer,
+                                  dict_t *pre_crawl_xdata)
 {
-        xlator_t         *this = NULL;
-        long             child = (long)data;
-        shd_pos_t        *pos_data = NULL;
-        int              ret = 0;
+    int ret_lock = 0;
+    int ret = 0;
+    dict_t *post_crawl_xdata = NULL;
 
-        this = THIS;
+    ret_lock = afr_ta_post_op_lock(this, loc);
+    if (ret_lock)
+        goto unref;
 
-        //Position of brick could have changed and it could be local now.
-        //Compute the position again
-        pos_data = GF_CALLOC (1, sizeof (*pos_data), gf_afr_mt_pos_data_t);
-        if (!pos_data)
-                goto out;
-        pos_data->this = this;
-        pos_data->child = child;
-        ret = synctask_new (this->ctx->env, afr_syncop_find_child_position,
-                            afr_local_child_poll_self_heal, NULL, pos_data);
-        if (ret)
-                goto out;
-out:
-        return;
-}
+    ret = _afr_shd_ta_get_xattrs(this, loc, &post_crawl_xdata);
+    if (ret)
+        goto unref;
 
-static int
-get_pathinfo_host (char *pathinfo, char *hostname, size_t size)
-{
-        char    *start = NULL;
-        char    *end = NULL;
-        int     ret  = -1;
-        int     i    = 0;
+    if (!are_dicts_equal(pre_crawl_xdata, post_crawl_xdata, NULL, NULL)) {
+        ret = -1;
+        goto unref;
+    }
 
-        if (!pathinfo)
-                goto out;
+    ret = afr_shd_ta_unset_xattrs(this, loc, &post_crawl_xdata, healer->subvol);
 
-        start = strchr (pathinfo, ':');
-        if (!start)
-                goto out;
-        end = strrchr (pathinfo, ':');
-        if (start == end)
-                goto out;
+unref:
+    if (post_crawl_xdata) {
+        dict_unref(post_crawl_xdata);
+        post_crawl_xdata = NULL;
+    }
 
-        memset (hostname, 0, size);
-        i = 0;
-        while (++start != end)
-                hostname[i++] = *start;
-        ret = 0;
-out:
-        return ret;
+    if (ret || ret_lock)
+        healer->rerun = 1;
+
+    if (!ret_lock)
+        afr_ta_post_op_unlock(this, loc);
 }
 
-int
-afr_local_pathinfo (char *pathinfo, gf_boolean_t *local)
+gf_boolean_t
+afr_bricks_available_for_heal(afr_private_t *priv)
 {
-        int             ret   = 0;
-        char            pathinfohost[1024] = {0};
-        char            localhost[1024] = {0};
-        xlator_t        *this = THIS;
+    int up_children = 0;
 
-        *local = _gf_false;
-        ret = get_pathinfo_host (pathinfo, pathinfohost, sizeof (pathinfohost));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid pathinfo: %s",
-                        pathinfo);
-                goto out;
-        }
+    up_children = __afr_get_up_children_count(priv);
+    if (up_children < 2) {
+        return _gf_false;
+    }
+    return _gf_true;
+}
 
-        ret = gethostname (localhost, sizeof (localhost));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "gethostname() failed, "
-                        "reason: %s", strerror (errno));
-                goto out;
+static gf_boolean_t
+afr_shd_ta_needs_heal(xlator_t *this, struct subvol_healer *healer)
+{
+    dict_t *xdata = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    int i = 0;
+    gf_boolean_t need_heal = _gf_false;
+
+    priv = this->private;
+
+    ret = afr_shd_fill_ta_loc(this, &loc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        healer->rerun = 1;
+        goto out;
+    }
+
+    if (_afr_shd_ta_get_xattrs(this, &loc, &xdata)) {
+        healer->rerun = 1;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (afr_ta_dict_contains_pending_xattr(xdata, priv, i)) {
+            need_heal = _gf_true;
+            break;
         }
+    }
 
-        if (!strcmp (localhost, pathinfohost))
-                *local = _gf_true;
 out:
-        return ret;
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
+
+    return need_heal;
 }
 
-int
-afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
-                           loc_t *dirloc)
+static int
+afr_shd_anon_inode_cleaner(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                           void *data)
 {
-        afr_private_t *priv = NULL;
-        dict_t        *xattr = NULL;
-        void          *index_gfid = NULL;
-        loc_t         rootloc = {0};
-        struct iatt   iattr = {0};
-        struct iatt   parent = {0};
-        int           ret = 0;
-        xlator_t      *readdir_xl = crawl_data->readdir_xl;
-        inode_t       *link_inode = NULL;
-
-        priv = this->private;
-        if (crawl_data->crawl == FULL) {
-                afr_build_root_loc (this, dirloc);
-        } else {
-                afr_build_root_loc (this, &rootloc);
-                ret = syncop_getxattr (readdir_xl, &rootloc, &xattr,
-                                       GF_XATTROP_INDEX_GFID);
-                if (ret < 0)
-                        goto out;
-                ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get index "
-                                "dir gfid on %s", readdir_xl->name);
-                        goto out;
-                }
-                if (!index_gfid) {
-                        gf_log (this->name, GF_LOG_ERROR, "index gfid empty "
-                                "on %s", readdir_xl->name);
-                        ret = -1;
-                        goto out;
-                }
-                uuid_copy (dirloc->gfid, index_gfid);
-                dirloc->path = "";
-                dirloc->inode = inode_new (priv->root_inode->table);
-                ret = syncop_lookup (readdir_xl, dirloc, NULL,
-                                     &iattr, NULL, &parent);
-                if (ret < 0) {
-                        if (errno != ENOENT) {
-                                gf_log (this->name, GF_LOG_ERROR, "lookup "
-                                        "failed on index dir on %s - (%s)",
-                                        readdir_xl->name, strerror (errno));
-                        }
-                        goto out;
+    struct subvol_healer *healer = data;
+    afr_private_t *priv = healer->this->private;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+    loc_t loc = {0};
+    int count = 0;
+    int i = 0;
+    int op_errno = 0;
+    struct iatt *iatt = NULL;
+    gf_boolean_t multiple_links = _gf_false;
+    unsigned char *gfid_present = alloca0(priv->child_count);
+    unsigned char *entry_present = alloca0(priv->child_count);
+    char *type = "file";
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+    local = frame->local;
+    if (AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) {
+        gf_msg_debug(healer->this->name, 0,
+                     "Not all bricks are up. Skipping "
+                     "cleanup of %s on %s",
+                     entry->d_name, subvol->name);
+        ret = 0;
+        goto out;
+    }
+
+    loc.inode = inode_new(parent->inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    ret = gf_uuid_parse(entry->d_name, loc.gfid);
+    if (ret) {
+        ret = 0;
+        goto out;
+    }
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == 0) {
+            count++;
+            gfid_present[i] = 1;
+            iatt = &local->replies[i].poststat;
+            if (iatt->ia_type == IA_IFDIR) {
+                type = "dir";
+            }
+
+            if (i == healer->subvol) {
+                if (local->replies[i].poststat.ia_nlink > 1) {
+                    multiple_links = _gf_true;
                 }
-                link_inode = inode_link (dirloc->inode, NULL, NULL, &iattr);
-                inode_unref (link_inode);
+            }
+        } else if (local->replies[i].op_errno != ENOENT &&
+                   local->replies[i].op_errno != ESTALE) {
+            /*We don't have complete view. Skip the entry*/
+            gf_msg_debug(healer->this->name, local->replies[i].op_errno,
+                         "Skipping cleanup of %s on %s", entry->d_name,
+                         subvol->name);
+            ret = 0;
+            goto out;
+        }
+    }
+
+    /*Inode is deleted from subvol*/
+    if (count == 1 || (iatt->ia_type != IA_IFDIR && multiple_links)) {
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_EXPUNGING_FILE_OR_DIR, "expunging %s %s/%s on %s", type,
+               priv->anon_inode_name, entry->d_name, subvol->name);
+        ret = afr_shd_entry_purge(subvol, parent->inode, entry->d_name,
+                                  iatt->ia_type);
+        if (ret == -ENOENT || ret == -ESTALE)
+            ret = 0;
+    } else if (count > 1) {
+        loc_wipe(&loc);
+        loc.parent = inode_ref(parent->inode);
+        loc.name = entry->d_name;
+        loc.inode = inode_new(parent->inode->table);
+        if (!loc.inode) {
+            ret = -ENOMEM;
+            goto out;
         }
+        AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup,
+                   &loc, NULL);
+        count = 0;
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->replies[i].op_ret == 0) {
+                count++;
+                entry_present[i] = 1;
+                iatt = &local->replies[i].poststat;
+            } else if (local->replies[i].op_errno != ENOENT &&
+                       local->replies[i].op_errno != ESTALE) {
+                /*We don't have complete view. Skip the entry*/
+                gf_msg_debug(healer->this->name, local->replies[i].op_errno,
+                             "Skipping cleanup of %s on %s", entry->d_name,
+                             subvol->name);
+                ret = 0;
+                goto out;
+            }
+        }
+        for (i = 0; i < priv->child_count; i++) {
+            if (gfid_present[i] && !entry_present[i]) {
+                /*Entry is not anonymous on at least one subvol*/
+                gf_msg_debug(healer->this->name, 0,
+                             "Valid entry present on %s "
+                             "Skipping cleanup of %s on %s",
+                             priv->children[i]->name, entry->d_name,
+                             subvol->name);
+                ret = 0;
+                goto out;
+            }
+        }
+
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "expunging %s %s/%s on all subvols", type, priv->anon_inode_name,
+               entry->d_name);
         ret = 0;
+        for (i = 0; i < priv->child_count; i++) {
+            op_errno = -afr_shd_entry_purge(priv->children[i], loc.parent,
+                                            entry->d_name, iatt->ia_type);
+            if (op_errno != ENOENT && op_errno != ESTALE) {
+                ret |= -op_errno;
+            }
+        }
+    }
+
 out:
-        if (xattr)
-                dict_unref (xattr);
-        loc_wipe (&rootloc);
-        return ret;
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    loc_wipe(&loc);
+    return ret;
 }
 
-int
-afr_crawl_opendir (xlator_t *this, afr_crawl_data_t *crawl_data, fd_t **dirfd,
-                   loc_t *dirloc)
+static void
+afr_cleanup_anon_inode_dir(struct subvol_healer *healer)
 {
-        fd_t          *fd   = NULL;
-        int           ret = 0;
-
-        if (crawl_data->crawl == FULL) {
-                fd = fd_create (dirloc->inode, crawl_data->pid);
-                if (!fd) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to create fd for %s", dirloc->path);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = syncop_opendir (crawl_data->readdir_xl, dirloc, fd);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "opendir failed on %s", dirloc->path);
-                        goto out;
-                }
-        } else {
-                fd = fd_anonymous (dirloc->inode);
-        }
-        ret = 0;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    afr_private_t *priv = healer->this->private;
+    loc_t loc = {0};
+
+    ret = afr_anon_inode_create(healer->this, healer->subvol, &loc.inode);
+    if (ret)
+        goto out;
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = syncop_mt_dir_scan(frame, priv->children[healer->subvol], &loc,
+                             GF_CLIENT_PID_SELF_HEALD, healer,
+                             afr_shd_anon_inode_cleaner, NULL,
+                             priv->shd.max_threads, priv->shd.wait_qlength);
 out:
-        if (!ret)
-                *dirfd = fd;
-        return ret;
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    loc_wipe(&loc);
+    return;
 }
 
-xlator_t*
-afr_crawl_readdir_xl_get (xlator_t *this, afr_crawl_data_t *crawl_data)
+void *
+afr_shd_index_healer(void *data)
 {
-        afr_private_t *priv = this->private;
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    dict_t *pre_crawl_xdata = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    healer = data;
+    THIS = this = healer->this;
+    priv = this->private;
+
+    for (;;) {
+        afr_shd_healer_wait(healer);
+
+        if (!afr_bricks_available_for_heal(priv))
+            continue;
+
+        ASSERT_LOCAL(this, healer);
+        priv->local[healer->subvol] = healer->local;
+
+        if (priv->thin_arbiter_count) {
+            if (afr_shd_ta_needs_heal(this, healer))
+                afr_shd_ta_get_xattrs(this, &loc, healer, &pre_crawl_xdata);
+        }
 
-        if (crawl_data->crawl == FULL) {
-                return this;
-        } else {
-                return priv->children[crawl_data->child];
+        do {
+            gf_msg_debug(this->name, 0, "starting index sweep on subvol %s",
+                         afr_subvol_name(this, healer->subvol));
+
+            afr_shd_sweep_prepare(healer);
+
+            ret = afr_shd_index_sweep_all(healer);
+
+            afr_shd_sweep_done(healer);
+            /*
+              As long as at least one gfid was
+              healed, keep retrying. We may have
+              just healed a directory and thereby
+              created entries for other gfids which
+              could not be healed thus far.
+            */
+
+            gf_msg_debug(this->name, 0, "finished index sweep on subvol %s",
+                         afr_subvol_name(this, healer->subvol));
+            /*
+              Give a pause before retrying to avoid a busy loop
+              in case the only entry in index is because of
+              an ongoing I/O.
+            */
+            sleep(1);
+        } while (ret > 0);
+
+        if (ret == 0) {
+            afr_cleanup_anon_inode_dir(healer);
         }
-        return NULL;
-}
 
-int
-afr_crawl_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent,
-                           gf_dirent_t *entry, afr_crawl_data_t *crawl_data)
-{
-        int           ret = -1;
-        afr_private_t *priv = NULL;
+        if (ret == 0 && pre_crawl_xdata &&
+            !healer->crawl_event.heal_failed_count) {
+            afr_shd_ta_check_and_unset_xattrs(this, &loc, healer,
+                                              pre_crawl_xdata);
+        }
 
-        priv = this->private;
-        if (crawl_data->crawl == FULL) {
-                ret = afr_build_child_loc (this, child, parent, entry->d_name);
-        } else {
-                child->inode = inode_new (priv->root_inode->table);
-                if (!child->inode)
-                        goto out;
-                uuid_parse (entry->d_name, child->gfid);
-                ret = _loc_assign_gfid_path (child);
+        if (pre_crawl_xdata) {
+            dict_unref(pre_crawl_xdata);
+            pre_crawl_xdata = NULL;
         }
-out:
-        return ret;
+    }
+
+    return NULL;
 }
 
-static int
-_process_entries (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
-                  off_t *offset, afr_crawl_data_t *crawl_data)
+void *
+afr_shd_full_healer(void *data)
 {
-        gf_dirent_t      *entry = NULL;
-        gf_dirent_t      *tmp = NULL;
-        int              ret = 0;
-        loc_t            entry_loc = {0};
-        fd_t             *fd = NULL;
-        struct iatt      iattr = {0};
-        inode_t          *link_inode = NULL;
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                if (!_crawl_proceed (this, crawl_data->child,
-                                     crawl_data->crawl_flags, NULL)) {
-                        ret = -1;
-                        goto out;
-                }
-                *offset = entry->d_off;
-                if (IS_ENTRY_CWD (entry->d_name) ||
-                    IS_ENTRY_PARENT (entry->d_name))
-                        continue;
-                if ((crawl_data->crawl == FULL) &&
-                     uuid_is_null (entry->d_stat.ia_gfid)) {
-                        gf_log (this->name, GF_LOG_WARNING, "%s/%s: No "
-                                "gfid present skipping",
-                                parentloc->path, entry->d_name);
-                        continue;
-                }
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    int run = 0;
 
-                loc_wipe (&entry_loc);
-                ret = afr_crawl_build_child_loc (this, &entry_loc, parentloc,
-                                                 entry, crawl_data);
-                if (ret)
-                        goto out;
+    healer = data;
+    THIS = this = healer->this;
 
-                ret = crawl_data->process_entry (this, crawl_data, entry,
-                                                 &entry_loc, parentloc, &iattr);
+    for (;;) {
+        pthread_mutex_lock(&healer->mutex);
+        {
+            run = __afr_shd_healer_wait(healer);
+            if (!run)
+                healer->running = _gf_false;
+        }
+        pthread_mutex_unlock(&healer->mutex);
 
-                if (ret)
-                        continue;
+        if (!run)
+            break;
 
-                link_inode = inode_link (entry_loc.inode, NULL, NULL, &iattr);
-                if (link_inode == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR, "inode link failed "
-                                "on the inode (%s)",entry_loc.path);
-                        ret = -1;
-                        goto out;
-                }
-                inode_unref (link_inode);
-
-                if (crawl_data->crawl == INDEX)
-                        continue;
-
-                if (!IA_ISDIR (iattr.ia_type))
-                        continue;
-                fd = NULL;
-                ret = afr_crawl_opendir (this, crawl_data, &fd, &entry_loc);
-                if (ret)
-                        continue;
-                ret = _crawl_directory (fd, &entry_loc, crawl_data);
-                if (fd)
-                        fd_unref (fd);
-        }
-        ret = 0;
-out:
-        loc_wipe (&entry_loc);
-        return ret;
+        ASSERT_LOCAL(this, healer);
+
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+               "starting full sweep on subvol %s",
+               afr_subvol_name(this, healer->subvol));
+
+        afr_shd_sweep_prepare(healer);
+
+        afr_shd_full_sweep(healer, this->itable->root);
+
+        afr_shd_sweep_done(healer);
+
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+               "finished full sweep on subvol %s",
+               afr_subvol_name(this, healer->subvol));
+    }
+
+    return NULL;
 }
 
-static int
-_crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data)
+int
+afr_shd_healer_init(xlator_t *this, struct subvol_healer *healer)
 {
-        xlator_t        *this = NULL;
-        off_t           offset   = 0;
-        gf_dirent_t     entries;
-        int             ret = 0;
-        gf_boolean_t    free_entries = _gf_false;
-        xlator_t        *readdir_xl = crawl_data->readdir_xl;
-
-        INIT_LIST_HEAD (&entries.list);
-        this = THIS;
-
-        GF_ASSERT (loc->inode);
-
-        if (crawl_data->crawl == FULL)
-                gf_log (this->name, GF_LOG_DEBUG, "crawling %s", loc->path);
-        else
-                gf_log (this->name, GF_LOG_DEBUG, "crawling INDEX %s",
-                        uuid_utoa (loc->gfid));
-
-        while (1) {
-                if (crawl_data->crawl == FULL)
-                        ret = syncop_readdirp (readdir_xl, fd, 131072, offset,
-                                               NULL, &entries);
-                else
-                        ret = syncop_readdir (readdir_xl, fd, 131072, offset,
-                                              &entries);
-                if (ret <= 0)
-                        break;
-                ret = 0;
-                free_entries = _gf_true;
+    int ret = 0;
 
-                if (!_crawl_proceed (this, crawl_data->child,
-                                     crawl_data->crawl_flags, NULL)) {
-                        ret = -1;
-                        goto out;
-                }
-                if (list_empty (&entries.list))
-                        goto out;
+    ret = pthread_mutex_init(&healer->mutex, NULL);
+    if (ret)
+        goto out;
 
-                ret = _process_entries (this, loc, &entries, &offset,
-                                        crawl_data);
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-        }
-        ret = 0;
+    ret = pthread_cond_init(&healer->cond, NULL);
+    if (ret)
+        goto out;
+
+    healer->this = this;
+    healer->running = _gf_false;
+    healer->rerun = _gf_false;
+    healer->local = _gf_false;
 out:
-        if (free_entries)
-                gf_dirent_free (&entries);
-        return ret;
+    return ret;
 }
 
-static char*
-position_str_get (afr_child_pos_t pos)
+int
+afr_shd_healer_spawn(xlator_t *this, struct subvol_healer *healer,
+                     void *(threadfn)(void *))
 {
-        switch (pos) {
-        case AFR_POS_UNKNOWN:
-                return "unknown";
-        case AFR_POS_LOCAL:
-                return "local";
-        case AFR_POS_REMOTE:
-                return "remote";
+    int ret = 0;
+
+    pthread_mutex_lock(&healer->mutex);
+    {
+        if (healer->running) {
+            pthread_cond_signal(&healer->cond);
+        } else {
+            ret = gf_thread_create(&healer->thread, NULL, threadfn, healer,
+                                   "shdheal");
+            if (ret)
+                goto unlock;
+            healer->running = 1;
         }
-        return NULL;
+
+        healer->rerun = 1;
+    }
+unlock:
+    pthread_mutex_unlock(&healer->mutex);
+
+    return ret;
 }
 
 int
-afr_find_child_position (xlator_t *this, int child, afr_child_pos_t *pos)
+afr_shd_full_healer_spawn(xlator_t *this, int subvol)
 {
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd  = NULL;
-        dict_t           *xattr_rsp = NULL;
-        loc_t            loc = {0};
-        int              ret = 0;
-        char             *node_uuid = NULL;
+    return afr_shd_healer_spawn(this, NTH_FULL_HEALER(this, subvol),
+                                afr_shd_full_healer);
+}
 
-        priv = this->private;
-        shd  = &priv->shd;
+int
+afr_shd_index_healer_spawn(xlator_t *this, int subvol)
+{
+    return afr_shd_healer_spawn(this, NTH_INDEX_HEALER(this, subvol),
+                                afr_shd_index_healer);
+}
 
-        afr_build_root_loc (this, &loc);
+int
+afr_shd_dict_add_crawl_event(xlator_t *this, dict_t *output,
+                             crawl_event_t *crawl_event)
+{
+    int ret = 0;
+    uint64_t count = 0;
+    char key[128] = {0};
+    int keylen = 0;
+    char suffix[64] = {0};
+    int xl_id = 0;
+    uint64_t healed_count = 0;
+    uint64_t split_brain_count = 0;
+    uint64_t heal_failed_count = 0;
+    char *start_time_str = 0;
+    char *end_time_str = NULL;
+    char *crawl_type = NULL;
+    int progress = -1;
+    int child = -1;
+
+    child = crawl_event->child;
+    healed_count = crawl_event->healed_count;
+    split_brain_count = crawl_event->split_brain_count;
+    heal_failed_count = crawl_event->heal_failed_count;
+    crawl_type = crawl_event->crawl_type;
+
+    if (!crawl_event->start_time)
+        goto out;
+
+    start_time_str = gf_strdup(ctime(&crawl_event->start_time));
+
+    if (crawl_event->end_time)
+        end_time_str = gf_strdup(ctime(&crawl_event->end_time));
+
+    ret = dict_get_int32(output, this->name, &xl_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+               "xl does not have id");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
+    ret = dict_get_uint64(output, key, &count);
+
+    snprintf(suffix, sizeof(suffix), "%d-%d-%" PRIu64, xl_id, child, count);
+    snprintf(key, sizeof(key), "statistics_healed_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, healed_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_healed_count to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics_sb_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, split_brain_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_split_brain_count to output");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_crawl_type-%s", suffix);
+    ret = dict_set_strn(output, key, keylen, crawl_type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_type to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics_heal_failed_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, heal_failed_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_healed_failed_count to output");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_strt_time-%s", suffix);
+    ret = dict_set_dynstrn(output, key, keylen, start_time_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_start_time to output");
+        goto out;
+    } else {
+        start_time_str = NULL;
+    }
+
+    if (!end_time_str)
+        progress = 1;
+    else
+        progress = 0;
+
+    keylen = snprintf(key, sizeof(key), "statistics_end_time-%s", suffix);
+    if (!end_time_str)
+        end_time_str = gf_strdup("Could not determine the end time");
+    ret = dict_set_dynstrn(output, key, keylen, end_time_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_end_time to output");
+        goto out;
+    } else {
+        end_time_str = NULL;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_inprogress-%s", suffix);
+
+    ret = dict_set_int32n(output, key, keylen, progress);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_inprogress to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
+    ret = dict_set_uint64(output, key, count + 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not increment the counter.");
+        goto out;
+    }
+out:
+    GF_FREE(start_time_str);
+    GF_FREE(end_time_str);
+    return ret;
+}
 
-        ret = syncop_getxattr (priv->children[child], &loc, &xattr_rsp,
-                               GF_XATTR_NODE_UUID_KEY);
+int
+afr_shd_dict_add_path(xlator_t *this, dict_t *output, int child, char *path,
+                      struct timeval *tv)
+{
+    int ret = -1;
+    uint64_t count = 0;
+    char key[64] = {0};
+    int keylen = 0;
+    char xl_id_child_str[32] = {0};
+    int xl_id = 0;
+
+    ret = dict_get_int32(output, this->name, &xl_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+               "xl does not have id");
+        goto out;
+    }
+
+    snprintf(xl_id_child_str, sizeof(xl_id_child_str), "%d-%d", xl_id, child);
+    snprintf(key, sizeof(key), "%s-count", xl_id_child_str);
+    ret = dict_get_uint64(output, key, &count);
+
+    keylen = snprintf(key, sizeof(key), "%s-%" PRIu64, xl_id_child_str, count);
+    ret = dict_set_dynstrn(output, key, keylen, path);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Could not add to output", path);
+        goto out;
+    }
+
+    if (tv) {
+        snprintf(key, sizeof(key), "%s-%" PRIu64 "-time", xl_id_child_str,
+                 count);
+        ret = dict_set_uint32(output, key, tv->tv_sec);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "getxattr failed on %s - "
-                        "(%s)", priv->children[child]->name, strerror (errno));
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "%s: Could not set time", path);
+            goto out;
         }
+    }
 
-        ret = dict_get_str (xattr_rsp, GF_XATTR_NODE_UUID_KEY, &node_uuid);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "node-uuid key not found on "
-                        "child %s", priv->children[child]->name);
-                goto out;
-        }
+    snprintf(key, sizeof(key), "%s-count", xl_id_child_str);
 
-        if (!strcmp (node_uuid, shd->node_uuid))
-                *pos = AFR_POS_LOCAL;
-        else
-                *pos = AFR_POS_REMOTE;
+    ret = dict_set_uint64(output, key, count + 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not increment count");
+        goto out;
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "child %s is %s",
-                priv->children[child]->name, position_str_get (*pos));
+    ret = 0;
 out:
-        if (ret)
-                *pos = AFR_POS_UNKNOWN;
-        loc_wipe (&loc);
-        return ret;
+    return ret;
 }
 
 int
-afr_syncop_find_child_position (void *data)
+afr_add_shd_event(circular_buffer_t *cb, void *data)
 {
-        shd_pos_t *pos_data = data;
-        int       ret = 0;
+    dict_t *output = NULL;
+    xlator_t *this = THIS;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    shd_event_t *shd_event = NULL;
+    char *path = NULL;
+
+    output = data;
+    priv = this->private;
+    shd = &priv->shd;
+    shd_event = cb->data;
+
+    if (!shd->index_healers[shd_event->child].local)
+        return 0;
 
-        ret = afr_find_child_position (pos_data->this, pos_data->child,
-                                       &pos_data->pos);
-        return ret;
+    path = gf_strdup(shd_event->path);
+    if (!path)
+        return -ENOMEM;
+
+    afr_shd_dict_add_path(this, output, shd_event->child, path, &cb->tv);
+    return 0;
 }
 
-static int
-afr_dir_crawl (void *data)
+int
+afr_add_crawl_event(circular_buffer_t *cb, void *data)
 {
-        xlator_t            *this = NULL;
-        int                 ret = -1;
-        xlator_t            *readdir_xl = NULL;
-        fd_t                *fd = NULL;
-        loc_t               dirloc = {0};
-        afr_crawl_data_t    *crawl_data = data;
-
-        this = THIS;
-
-        if (!_crawl_proceed (this, crawl_data->child, crawl_data->crawl_flags,
-                             NULL))
-                goto out;
+    dict_t *output = NULL;
+    xlator_t *this = THIS;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    crawl_event_t *crawl_event = NULL;
+
+    output = data;
+    priv = this->private;
+    shd = &priv->shd;
+    crawl_event = cb->data;
+
+    if (!shd->index_healers[crawl_event->child].local)
+        return 0;
 
-        readdir_xl = afr_crawl_readdir_xl_get (this, crawl_data);
-        if (!readdir_xl)
-                goto out;
-        crawl_data->readdir_xl = readdir_xl;
+    afr_shd_dict_add_crawl_event(this, output, crawl_event);
 
-        ret = afr_crawl_build_start_loc (this, crawl_data, &dirloc);
-        if (ret)
-                goto out;
+    return 0;
+}
 
-        ret = afr_crawl_opendir (this, crawl_data, &fd, &dirloc);
+int
+afr_selfheal_daemon_init(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    int ret = -1;
+    int i = 0;
+
+    priv = this->private;
+    shd = &priv->shd;
+
+    shd->index_healers = GF_CALLOC(sizeof(*shd->index_healers),
+                                   priv->child_count,
+                                   gf_afr_mt_subvol_healer_t);
+    if (!shd->index_healers)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        shd->index_healers[i].subvol = i;
+        ret = afr_shd_healer_init(this, &shd->index_healers[i]);
         if (ret)
-                goto out;
-
-        ret = _crawl_directory (fd, &dirloc, crawl_data);
+            goto out;
+    }
+
+    shd->full_healers = GF_CALLOC(sizeof(*shd->full_healers), priv->child_count,
+                                  gf_afr_mt_subvol_healer_t);
+    if (!shd->full_healers)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        shd->full_healers[i].subvol = i;
+        ret = afr_shd_healer_init(this, &shd->full_healers[i]);
         if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Crawl failed on %s",
-                        readdir_xl->name);
-        else
-                gf_log (this->name, GF_LOG_DEBUG, "Crawl completed "
-                        "on %s", readdir_xl->name);
-        if (crawl_data->crawl == INDEX)
-                dirloc.path = NULL;
+            goto out;
+    }
+
+    shd->split_brain = eh_new(AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
+                              afr_destroy_shd_event_data);
+    if (!shd->split_brain)
+        goto out;
+
+    shd->statistics = GF_CALLOC(sizeof(eh_t *), priv->child_count,
+                                gf_common_mt_eh_t);
+    if (!shd->statistics)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        shd->statistics[i] = eh_new(AFR_STATISTICS_HISTORY_SIZE, _gf_false,
+                                    afr_destroy_crawl_event_data);
+        if (!shd->statistics[i])
+            goto out;
+        shd->full_healers[i].crawl_event.child = i;
+        shd->full_healers[i].crawl_event.crawl_type = "FULL";
+        shd->index_healers[i].crawl_event.child = i;
+        shd->index_healers[i].crawl_event.crawl_type = "INDEX";
+    }
+
+    ret = 0;
 out:
-        if (fd)
-                fd_unref (fd);
-        if (crawl_data->crawl == INDEX)
-                dirloc.path = NULL;
-        loc_wipe (&dirloc);
-        return ret;
+    return ret;
 }
 
-static int
-afr_dir_exclusive_crawl (void *data)
+void
+afr_selfheal_childup(xlator_t *this, afr_private_t *priv)
 {
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        gf_boolean_t     crawl = _gf_false;
-        int              ret = 0;
-        int              child = -1;
-        xlator_t         *this = NULL;
-        afr_crawl_data_t *crawl_data = data;
-
-        this = THIS;
-        priv = this->private;
-        shd = &priv->shd;
-        child = crawl_data->child;
-
-        LOCK (&priv->lock);
-        {
-                if (shd->inprogress[child]) {
-                        if (shd->pending[child] != FULL)
-                                shd->pending[child] = crawl_data->crawl;
-                } else {
-                        shd->inprogress[child] = _gf_true;
-                        crawl = _gf_true;
-                }
-        }
-        UNLOCK (&priv->lock);
+    int subvol = 0;
 
-        if (!crawl) {
-                gf_log (this->name, GF_LOG_INFO, "Another crawl is in progress "
-                        "for %s", priv->children[child]->name);
-                goto out;
-        }
+    if (!priv->shd.iamshd)
+        return;
+    for (subvol = 0; subvol < priv->child_count; subvol++)
+        if (priv->child_up[subvol])
+            afr_shd_index_healer_spawn(this, subvol);
 
-        do {
-                afr_dir_crawl (data);
-                LOCK (&priv->lock);
-                {
-                        if (shd->pending[child] != NONE) {
-                                crawl_data->crawl = shd->pending[child];
-                                shd->pending[child] = NONE;
-                        } else {
-                                shd->inprogress[child] = _gf_false;
-                                crawl = _gf_false;
-                        }
-                }
-                UNLOCK (&priv->lock);
-        } while (crawl);
-out:
-        return ret;
+    return;
 }
 
-void
-afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
-                 process_entry_cbk_t process_entry, void *op_data,
-                 gf_boolean_t exclusive, int crawl_flags,
-                 afr_crawl_done_cbk_t crawl_done)
+int
+afr_shd_get_index_count(xlator_t *this, int i, uint64_t *count)
 {
-        afr_private_t              *priv = NULL;
-        call_frame_t               *frame = NULL;
-        afr_crawl_data_t           *crawl_data = NULL;
-        int                        ret = 0;
-        int (*crawler) (void*) = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t *subvol = NULL;
+    loc_t rootloc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int ret = -1;
 
-        priv = this->private;
+    priv = this->private;
+    subvol = priv->children[i];
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
 
-        afr_set_lk_owner (frame, this, frame->root);
-        afr_set_low_priority (frame);
-        crawl_data = GF_CALLOC (1, sizeof (*crawl_data),
-                                gf_afr_mt_crawl_data_t);
-        if (!crawl_data)
-                goto out;
-        crawl_data->process_entry = process_entry;
-        crawl_data->child = idx;
-        crawl_data->pid = frame->root->pid;
-        crawl_data->crawl = crawl;
-        crawl_data->op_data = op_data;
-        crawl_data->crawl_flags = crawl_flags;
-        gf_log (this->name, GF_LOG_DEBUG, "starting crawl %d for %s",
-                crawl_data->crawl, priv->children[idx]->name);
-
-        if (exclusive)
-                crawler = afr_dir_exclusive_crawl;
-        else
-                crawler = afr_dir_crawl;
-        ret = synctask_new (this->ctx->env, crawler,
-                            crawl_done, frame, crawl_data);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Could not create the "
-                        "task for %d ret %d", idx, ret);
-out:
-        return;
-}
+    ret = syncop_getxattr(subvol, &rootloc, &xattr, GF_XATTROP_INDEX_COUNT,
+                          NULL, NULL);
+    if (ret < 0)
+        goto out;
 
-void
-afr_build_root_loc (xlator_t *this, loc_t *loc)
-{
-        afr_private_t   *priv = NULL;
+    ret = dict_get_uint64(xattr, GF_XATTROP_INDEX_COUNT, count);
+    if (ret)
+        goto out;
+
+    ret = 0;
+
+out:
+    if (xattr)
+        dict_unref(xattr);
+    loc_wipe(&rootloc);
 
-        priv = this->private;
-        loc->path = gf_strdup ("/");
-        loc->name = "";
-        loc->inode = inode_ref (priv->root_inode);
-        uuid_copy (loc->gfid, loc->inode->gfid);
+    return ret;
 }
 
 int
-afr_set_root_gfid (dict_t *dict)
+afr_xl_op(xlator_t *this, dict_t *input, dict_t *output)
 {
-        uuid_t gfid;
-        int ret = 0;
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
+    int ret = 0;
+    int xl_id = 0;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    struct subvol_healer *healer = NULL;
+    int i = 0;
+    char key[64];
+    int keylen = 0;
+    int this_name_len = 0;
+    int op_ret = 0;
+    uint64_t cnt = 0;
+
+#define AFR_SET_DICT_AND_LOG(name, output, key, keylen, dict_str,              \
+                             dict_str_len)                                     \
+    {                                                                          \
+        int ret;                                                               \
+                                                                               \
+        ret = dict_set_nstrn(output, key, keylen, dict_str, dict_str_len);     \
+        if (ret) {                                                             \
+            gf_smsg(name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,         \
+                    "key=%s", key, "value=%s", dict_str, NULL);                \
+        }                                                                      \
+    }
+
+    priv = this->private;
+    shd = &priv->shd;
+
+    ret = dict_get_int32_sizen(input, "xl-op", (int32_t *)&op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                "key=xl-op", NULL);
+        goto out;
+    }
+    this_name_len = strlen(this->name);
+    ret = dict_get_int32n(input, this->name, this_name_len, &xl_id);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                "key=%s", this->name, NULL);
+        goto out;
+    }
+    ret = dict_set_int32n(output, this->name, this_name_len, xl_id);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                "key=%s", this->name, NULL);
+        goto out;
+    }
+    switch (op) {
+        case GF_SHD_OP_HEAL_INDEX:
+            op_ret = 0;
+
+            for (i = 0; i < priv->child_count; i++) {
+                healer = &shd->index_healers[i];
+                keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
+
+                if (!priv->child_up[i]) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                    op_ret = -1;
+                } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SLESS_THAN2_BRICKS_in_REP,
+                                         SLEN(SLESS_THAN2_BRICKS_in_REP));
+                    op_ret = -1;
+                } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_IS_REMOTE,
+                                         SLEN(SBRICK_IS_REMOTE));
+                } else {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SSTARTED_SELF_HEAL,
+                                         SLEN(SSTARTED_SELF_HEAL));
 
-        memset (gfid, 0, 16);
-        gfid[15] = 1;
+                    ret = afr_shd_index_healer_spawn(this, i);
 
-        ret = afr_set_dict_gfid (dict, gfid);
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_HEALER_SPAWN_FAILED, NULL);
+                    }
+                }
+            }
+            break;
+        case GF_SHD_OP_HEAL_FULL:
+            op_ret = -1;
+
+            for (i = 0; i < priv->child_count; i++) {
+                healer = &shd->full_healers[i];
+                keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
+
+                if (!priv->child_up[i]) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SLESS_THAN2_BRICKS_in_REP,
+                                         SLEN(SLESS_THAN2_BRICKS_in_REP));
+                } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_IS_REMOTE,
+                                         SLEN(SBRICK_IS_REMOTE));
+                } else {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SSTARTED_SELF_HEAL,
+                                         SLEN(SSTARTED_SELF_HEAL));
 
-        return ret;
-}
+                    ret = afr_shd_full_healer_spawn(this, i);
 
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_HEALER_SPAWN_FAILED, NULL);
+                    }
+                    op_ret = 0;
+                }
+            }
+            break;
+        case GF_SHD_OP_INDEX_SUMMARY:
+            /* this case has been handled in glfs-heal.c */
+            break;
+        case GF_SHD_OP_SPLIT_BRAIN_FILES:
+            eh_dump(shd->split_brain, output, afr_add_shd_event);
+            break;
+        case GF_SHD_OP_STATISTICS:
+            for (i = 0; i < priv->child_count; i++) {
+                eh_dump(shd->statistics[i], output, afr_add_crawl_event);
+                ret = afr_shd_dict_add_crawl_event(
+                    this, output, &shd->index_healers[i].crawl_event);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                            AFR_MSG_ADD_CRAWL_EVENT_FAILED, NULL);
+                }
+
+                ret = afr_shd_dict_add_crawl_event(
+                    this, output, &shd->full_healers[i].crawl_event);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                            AFR_MSG_ADD_CRAWL_EVENT_FAILED, NULL);
+                }
+            }
+            break;
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT:
+        case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+            op_ret = -1;
+
+            for (i = 0; i < priv->child_count; i++) {
+                if (!priv->child_up[i]) {
+                    keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id,
+                                      i);
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                } else {
+                    snprintf(key, sizeof(key), "%d-%d-hardlinks", xl_id, i);
+                    ret = afr_shd_get_index_count(this, i, &cnt);
+                    if (ret == 0) {
+                        ret = dict_set_uint64(output, key, cnt);
+                    }
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_DICT_SET_FAILED, NULL);
+                    }
+                    op_ret = 0;
+                }
+            }
+
+            break;
+
+        default:
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "op=%d",
+                    op, NULL);
+            break;
+    }
+out:
+    dict_deln(output, this->name, this_name_len);
+    return op_ret;
+
+#undef AFR_SET_DICT_AND_LOG
+}
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
index 32a8aaca50c..18db728ea7b 100644
--- a/xlators/cluster/afr/src/afr-self-heald.h
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,45 +8,68 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef __AFR_SELF_HEALD_H__
-#define __AFR_SELF_HEALD_H__
-#include "xlator.h"
+#ifndef _AFR_SELF_HEALD_H
+#define _AFR_SELF_HEALD_H
 
-#define IS_ROOT_PATH(path) (!strcmp (path, "/"))
-#define IS_ENTRY_CWD(entry) (!strcmp (entry, "."))
-#define IS_ENTRY_PARENT(entry) (!strcmp (entry, ".."))
-#define AFR_ALL_CHILDREN -1
+#include <pthread.h>
 
-typedef struct afr_crawl_data_ {
-        int                 child;
-        pid_t               pid;
-        afr_crawl_type_t    crawl;
-        xlator_t            *readdir_xl;
-        void                *op_data;
-        int                 crawl_flags;
-        int (*process_entry) (xlator_t *this, struct afr_crawl_data_ *crawl_data,
-                              gf_dirent_t *entry, loc_t *child, loc_t *parent,
-                              struct iatt *iattr);
-} afr_crawl_data_t;
+typedef struct {
+    char *path;
+    int child;
+} shd_event_t;
 
-typedef int (*process_entry_cbk_t) (xlator_t *this, afr_crawl_data_t *crawl_data,
-                              gf_dirent_t *entry, loc_t *child, loc_t *parent,
-                              struct iatt *iattr);
+typedef struct {
+    uint64_t healed_count;
+    uint64_t split_brain_count;
+    uint64_t heal_failed_count;
 
-void afr_build_root_loc (xlator_t *this, loc_t *loc);
+    /* If start_time is 0, it means crawler is not in progress
+       and stats are not valid */
+    time_t start_time;
+    /* If start_time is NOT 0 and end_time is 0, it means
+       cralwer is in progress */
+    time_t end_time;
+    char *crawl_type;
+    int child;
+} crawl_event_t;
 
-int afr_set_root_gfid (dict_t *dict);
+struct subvol_healer {
+    xlator_t *this;
+    crawl_event_t crawl_event;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_t thread;
+    int subvol;
+    gf_boolean_t local;
+    gf_boolean_t running;
+    gf_boolean_t rerun;
+};
 
-void
-afr_proactive_self_heal (void *data);
+typedef struct {
+    struct subvol_healer *index_healers;
+    struct subvol_healer *full_healers;
+
+    eh_t *split_brain;
+    eh_t **statistics;
+    int timeout;
+    uint32_t max_threads;
+    uint32_t wait_qlength;
+    uint32_t halo_max_latency_msec;
+    gf_boolean_t iamshd;
+    gf_boolean_t enabled;
+} afr_self_heald_t;
 
 int
-afr_xl_op (xlator_t *this, dict_t *input, dict_t *output);
+afr_selfheal_daemon_init(xlator_t *this);
+
+int
+afr_xl_op(xlator_t *this, dict_t *input, dict_t *output);
+
+int
+afr_shd_gfid_to_path(xlator_t *this, xlator_t *subvol, uuid_t gfid,
+                     char **path_p);
 
-/*
- * In addition to its self-heal use, this is used to find a local default
- * read_child.
- */
 int
-afr_local_pathinfo (char *pathinfo, gf_boolean_t *local);
-#endif /* __AFR_SELF_HEALD_H__ */
+afr_shd_entry_purge(xlator_t *subvol, inode_t *inode, char *name,
+                    ia_type_t type);
+#endif /* !_AFR_SELF_HEALD_H */
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index 2144f483fd0..a51f79b1f43 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -8,1455 +8,2920 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "dict.h"
-#include "byte-order.h"
-#include "common-utils.h"
-#include "timer.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/timer.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
+#include "afr-self-heal.h"
+#include "afr-messages.h"
 
 #include <signal.h>
 
+typedef enum {
+    AFR_TRANSACTION_PRE_OP,
+    AFR_TRANSACTION_POST_OP,
+} afr_xattrop_type_t;
 
-#define LOCKED_NO       0x0        /* no lock held */
-#define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path
-                                      of RENAME */
-#define LOCKED_LOWER    0x2        /* for lower_path of RENAME */
+static void
+afr_lock_resume_shared(struct list_head *list);
 
-afr_fd_ctx_t *
-__afr_fd_ctx_get (fd_t *fd, xlator_t *this)
-{
-        uint64_t       ctx = 0;
-        int            ret = 0;
-        afr_fd_ctx_t  *fd_ctx = NULL;
-        int            i = 0;
-        afr_private_t *priv = NULL;
+static void
+afr_post_op_handle_success(call_frame_t *frame, xlator_t *this);
 
-        priv = this->private;
+static void
+afr_post_op_handle_failure(call_frame_t *frame, xlator_t *this, int op_errno);
 
-        ret = __fd_ctx_get (fd, this, &ctx);
+void
+__afr_transaction_wake_shared(afr_local_t *local, struct list_head *shared);
 
-        if (ret < 0 && fd_is_anonymous (fd)) {
-                ret = __afr_fd_ctx_set (this, fd);
-                if (ret < 0)
-                        goto out;
+void
+afr_changelog_post_op_do(call_frame_t *frame, xlator_t *this);
 
-                ret = __fd_ctx_get (fd, this, &ctx);
-                if (ret < 0)
-                        goto out;
+int
+afr_changelog_post_op_safe(call_frame_t *frame, xlator_t *this);
 
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-                for (i = 0; i < priv->child_count; i++)
-                        fd_ctx->opened_on[i] = AFR_FD_OPENED;
-        }
+gf_boolean_t
+afr_changelog_pre_op_uninherit(call_frame_t *frame, xlator_t *this);
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-out:
-        return fd_ctx;
-}
+gf_boolean_t
+afr_changelog_pre_op_update(call_frame_t *frame, xlator_t *this);
 
+int
+afr_changelog_call_count(afr_transaction_type type,
+                         unsigned char *pre_op_subvols,
+                         unsigned char *failed_subvols,
+                         unsigned int child_count);
+int
+afr_changelog_do(call_frame_t *frame, xlator_t *this, dict_t *xattr,
+                 afr_changelog_resume_t changelog_resume,
+                 afr_xattrop_type_t op);
 
-afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this)
-{
-        afr_fd_ctx_t  *fd_ctx = NULL;
+static void
+afr_ta_decide_post_op_state(call_frame_t *frame, xlator_t *this);
 
-        LOCK(&fd->lock);
-        {
-                fd_ctx = __afr_fd_ctx_get (fd, this);
-        }
-        UNLOCK(&fd->lock);
+static int
+afr_ta_post_op_do(void *opaque);
 
-        return fd_ctx;
-}
+static int
+afr_ta_post_op_synctask(xlator_t *this, afr_local_t *local);
 
+static int
+afr_changelog_post_op_done(call_frame_t *frame, xlator_t *this);
 
 static void
-afr_save_lk_owner (call_frame_t *frame)
-{
-        afr_local_t * local = NULL;
+afr_changelog_post_op_fail(call_frame_t *frame, xlator_t *this, int op_errno);
 
-        local = frame->local;
-
-	local->saved_lk_owner = frame->root->lk_owner;
+void
+afr_ta_locked_priv_invalidate(afr_private_t *priv)
+{
+    priv->ta_bad_child_index = AFR_CHILD_UNKNOWN;
+    priv->release_ta_notify_dom_lock = _gf_false;
+    priv->ta_notify_dom_lock_offset = 0;
 }
 
-
 static void
-afr_restore_lk_owner (call_frame_t *frame)
+afr_ta_process_waitq(xlator_t *this)
 {
-        afr_local_t * local = NULL;
+    afr_local_t *entry = NULL;
+    afr_private_t *priv = this->private;
+    struct list_head waitq = {
+        0,
+    };
+
+    INIT_LIST_HEAD(&waitq);
+    LOCK(&priv->lock);
+    list_splice_init(&priv->ta_waitq, &waitq);
+    UNLOCK(&priv->lock);
+    list_for_each_entry(entry, &waitq, ta_waitq)
+    {
+        afr_ta_decide_post_op_state(entry->transaction.frame, this);
+    }
+}
 
-        local = frame->local;
+int
+afr_ta_lock_release_done(int ret, call_frame_t *ta_frame, void *opaque)
+{
+    afr_ta_process_waitq(ta_frame->this);
+    STACK_DESTROY(ta_frame->root);
+    return 0;
+}
 
-	frame->root->lk_owner = local->saved_lk_owner;
+int
+afr_release_notify_lock_for_ta(void *opaque)
+{
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = -1;
+
+    this = (xlator_t *)opaque;
+    priv = this->private;
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate loc for thin-arbiter.");
+        goto out;
+    }
+    flock.l_type = F_UNLCK;
+    flock.l_start = priv->ta_notify_dom_lock_offset;
+    flock.l_len = 1;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_NOTIFY, &loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_NOTIFY lock.");
+    }
+
+    LOCK(&priv->lock);
+    {
+        afr_ta_locked_priv_invalidate(priv);
+    }
+    UNLOCK(&priv->lock);
+out:
+    loc_wipe(&loc);
+    return ret;
 }
 
+void
+afr_zero_fill_stat(afr_local_t *local)
+{
+    if (!local)
+        return;
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION) {
+        gf_zero_fill_stat(&local->cont.inode_wfop.prebuf);
+        gf_zero_fill_stat(&local->cont.inode_wfop.postbuf);
+    } else if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+               local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+        gf_zero_fill_stat(&local->cont.dir_fop.buf);
+        gf_zero_fill_stat(&local->cont.dir_fop.preparent);
+        gf_zero_fill_stat(&local->cont.dir_fop.postparent);
+        if (local->transaction.type == AFR_ENTRY_TRANSACTION)
+            return;
+        gf_zero_fill_stat(&local->cont.dir_fop.prenewparent);
+        gf_zero_fill_stat(&local->cont.dir_fop.postnewparent);
+    }
+}
 
-static void
-__mark_all_pending (int32_t *pending[], int child_count,
-                    afr_transaction_type type)
+/* In case of errors afr needs to choose which xdata from lower xlators it needs
+ * to unwind with. The way it is done is by checking if there are
+ * any good subvols which failed. Give preference to errnos other than
+ * ENOTCONN even if the child is source */
+void
+afr_pick_error_xdata(afr_local_t *local, afr_private_t *priv, inode_t *inode1,
+                     unsigned char *readable1, inode_t *inode2,
+                     unsigned char *readable2)
 {
-        int i = 0;
-        int j = 0;
+    int s = -1; /*selection*/
+    int i = 0;
+    unsigned char *readable = NULL;
+
+    if (local->xdata_rsp) {
+        dict_unref(local->xdata_rsp);
+        local->xdata_rsp = NULL;
+    }
+
+    readable = alloca0(priv->child_count * sizeof(*readable));
+    if (inode2 && readable2) { /*rename fop*/
+        AFR_INTERSECT(readable, readable1, readable2, priv->child_count);
+    } else {
+        memcpy(readable, readable1, sizeof(*readable) * priv->child_count);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+
+        if (local->replies[i].op_ret >= 0)
+            continue;
+
+        if (local->replies[i].op_errno == ENOTCONN)
+            continue;
+
+        /*Order is important in the following condition*/
+        if ((s < 0) || (!readable[s] && readable[i]))
+            s = i;
+    }
+
+    if (s != -1 && local->replies[s].xdata) {
+        local->xdata_rsp = dict_ref(local->replies[s].xdata);
+    } else if (s == -1) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->replies[i].valid)
+                continue;
 
-        for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
-                pending[i][j] = hton32 (1);
+            if (local->replies[i].op_ret >= 0)
+                continue;
+
+            if (!local->replies[i].xdata)
+                continue;
+            local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            break;
         }
+    }
 }
 
+gf_boolean_t
+afr_needs_changelog_update(afr_local_t *local)
+{
+    if (local->transaction.type == AFR_DATA_TRANSACTION)
+        return _gf_true;
+    if (!local->optimistic_change_log)
+        return _gf_true;
+    return _gf_false;
+}
 
-static void
-__mark_child_dead (int32_t *pending[], int child_count, int child,
-                   afr_transaction_type type)
+gf_boolean_t
+afr_changelog_has_quorum(afr_local_t *local, xlator_t *this)
 {
-        int j = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    unsigned char *success_children = NULL;
 
-        j = afr_index_for_transaction_type (type);
+    priv = this->private;
+    success_children = alloca0(priv->child_count);
 
-        pending[child][j] = 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.failed_subvols[i]) {
+            success_children[i] = 1;
+        }
+    }
+
+    if (afr_has_quorum(success_children, this, NULL)) {
+        return _gf_true;
+    }
+
+    return _gf_false;
 }
 
+gf_boolean_t
+afr_is_write_subvol_valid(call_frame_t *frame, xlator_t *this)
+{
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    uint64_t write_subvol = 0;
+    unsigned char *writable = NULL;
+    uint16_t datamap = 0;
+
+    local = frame->local;
+    priv = this->private;
+    writable = alloca0(priv->child_count);
+
+    write_subvol = afr_write_subvol_get(frame, this);
+    datamap = (write_subvol & 0x00000000ffff0000) >> 16;
+    for (i = 0; i < priv->child_count; i++) {
+        if (datamap & (1 << i))
+            writable[i] = 1;
+
+        if (writable[i] && !local->transaction.failed_subvols[i])
+            return _gf_true;
+    }
+
+    return _gf_false;
+}
 
-static void
-__mark_pre_op_done_on_fd (call_frame_t *frame, xlator_t *this, int child_index)
+int
+afr_transaction_fop(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+    unsigned char *failed_subvols = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    failed_subvols = local->transaction.failed_subvols;
+    call_count = priv->child_count -
+                 AFR_COUNT(failed_subvols, priv->child_count);
+    /* Fail if pre-op did not succeed on quorum no. of bricks. */
+    if (!afr_changelog_has_quorum(local, this) || !call_count) {
+        local->op_ret = -1;
+        /* local->op_errno is already captured in changelog cbk. */
+        afr_transaction_resume(frame, this);
+        return 0;
+    }
+
+    /* Fail if at least one writeable brick isn't up.*/
+    if (local->transaction.type == AFR_DATA_TRANSACTION &&
+        !afr_is_write_subvol_valid(frame, this)) {
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        afr_transaction_resume(frame, this);
+        return 0;
+    }
 
-        local = frame->local;
+    local->call_count = call_count;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] && !failed_subvols[i]) {
+            local->transaction.wind(frame, this, i);
 
-        if (!local->fd)
-                return;
+            if (!--call_count)
+                break;
+        }
+    }
 
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
+    return 0;
+}
 
-        if (!fd_ctx)
-                goto out;
+int
+afr_transaction_done(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t unwind = _gf_false;
+    afr_lock_t *lock = NULL;
+    afr_local_t *lock_local = NULL;
 
-        LOCK (&local->fd->lock);
+    priv = this->private;
+    local = frame->local;
+
+    if (priv->consistent_metadata) {
+        LOCK(&frame->lock);
         {
-                if (local->transaction.type == AFR_DATA_TRANSACTION)
-                        fd_ctx->pre_op_done[child_index]++;
+            unwind = (local->transaction.main_frame != NULL);
         }
-        UNLOCK (&local->fd->lock);
-out:
-        return;
+        UNLOCK(&frame->lock);
+        if (unwind) /*It definitely did post-op*/
+            afr_zero_fill_stat(local);
+    }
+
+    if (local->transaction.do_eager_unlock) {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        LOCK(&local->inode->lock);
+        {
+            lock->acquired = _gf_false;
+            lock->release = _gf_false;
+            list_splice_init(&lock->frozen, &lock->waiting);
+            if (list_empty(&lock->waiting))
+                goto unlock;
+            lock_local = list_entry(lock->waiting.next, afr_local_t,
+                                    transaction.wait_list);
+            list_del_init(&lock_local->transaction.wait_list);
+            list_add(&lock_local->transaction.owner_list, &lock->owners);
+        }
+    unlock:
+        UNLOCK(&local->inode->lock);
+    }
+    if (lock_local) {
+        afr_lock(lock_local->transaction.frame,
+                 lock_local->transaction.frame->this);
+    }
+    local->transaction.unwind(frame, this);
+
+    GF_ASSERT(list_empty(&local->transaction.owner_list));
+    GF_ASSERT(list_empty(&local->transaction.wait_list));
+    AFR_STACK_DESTROY(frame);
+
+    return 0;
 }
 
-
 static void
-__mark_pre_op_undone_on_fd (call_frame_t *frame, xlator_t *this, int child_index)
+afr_lock_fail_shared(afr_local_t *local, struct list_head *list)
 {
-        afr_local_t   *local = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
+    afr_local_t *each = NULL;
+
+    while (!list_empty(list)) {
+        each = list_entry(list->next, afr_local_t, transaction.wait_list);
+        list_del_init(&each->transaction.wait_list);
+        each->op_ret = -1;
+        each->op_errno = local->op_errno;
+        afr_transaction_done(each->transaction.frame,
+                             each->transaction.frame->this);
+    }
+}
 
-        local = frame->local;
+static void
+afr_handle_lock_acquire_failure(afr_local_t *local)
+{
+    struct list_head shared;
+    afr_lock_t *lock = NULL;
 
-        if (!local->fd)
-                return;
+    if (!local->transaction.eager_lock_on)
+        goto out;
 
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
+    lock = &local->inode_ctx->lock[local->transaction.type];
 
-        if (!fd_ctx)
-                goto out;
+    INIT_LIST_HEAD(&shared);
+    LOCK(&local->inode->lock);
+    {
+        lock->release = _gf_true;
+        list_splice_init(&lock->waiting, &shared);
+    }
+    UNLOCK(&local->inode->lock);
 
-        LOCK (&local->fd->lock);
-        {
-                if (local->transaction.type == AFR_DATA_TRANSACTION)
-                        fd_ctx->pre_op_done[child_index]--;
-        }
-        UNLOCK (&local->fd->lock);
+    afr_lock_fail_shared(local, &shared);
+    local->transaction.do_eager_unlock = _gf_true;
 out:
-        return;
+    local->internal_lock.lock_cbk = afr_transaction_done;
+    afr_unlock(local->transaction.frame, local->transaction.frame->this);
 }
 
+call_frame_t *
+afr_transaction_detach_fop_frame(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *fop_frame = NULL;
+
+    local = frame->local;
+
+    afr_handle_inconsistent_fop(frame, &local->op_ret, &local->op_errno);
+    LOCK(&frame->lock);
+    {
+        fop_frame = local->transaction.main_frame;
+        local->transaction.main_frame = NULL;
+    }
+    UNLOCK(&frame->lock);
+
+    return fop_frame;
+}
 
 static void
-__mark_down_children (int32_t *pending[], int child_count,
-                      unsigned char *child_up, afr_transaction_type type)
+afr_save_lk_owner(call_frame_t *frame)
 {
-        int i = 0;
-        int j = 0;
+    afr_local_t *local = NULL;
 
-        for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
+    local = frame->local;
 
-                if (!child_up[i])
-                        pending[i][j] = 0;
-        }
+    local->saved_lk_owner = frame->root->lk_owner;
 }
 
-
 static void
-__mark_all_success (int32_t *pending[], int child_count,
-                    afr_transaction_type type)
+afr_restore_lk_owner(call_frame_t *frame)
 {
-        int i;
-        int j;
+    afr_local_t *local = NULL;
 
-        for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
-                pending[i][j] = hton32 (-1);
-        }
+    local = frame->local;
+
+    frame->root->lk_owner = local->saved_lk_owner;
 }
 
+void
+__mark_all_success(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i;
+
+    local = frame->local;
+    priv = this->private;
 
-static int
-__changelog_enabled (afr_private_t *priv, afr_transaction_type type)
+    for (i = 0; i < priv->child_count; i++) {
+        local->transaction.failed_subvols[i] = 0;
+    }
+}
+
+void
+afr_compute_pre_op_sources(call_frame_t *frame, xlator_t *this)
 {
-        int ret = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_transaction_type type = -1;
+    dict_t *xdata = NULL;
+    int **matrix = NULL;
+    int idx = -1;
+    int i = 0;
+    int j = 0;
+
+    priv = this->private;
+    local = frame->local;
+    type = local->transaction.type;
+    idx = afr_index_for_transaction_type(type);
+    matrix = ALLOC_MATRIX(priv->child_count, int);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.changelog_xdata[i])
+            continue;
+        xdata = local->transaction.changelog_xdata[i];
+        afr_selfheal_fill_matrix(this, matrix, i, idx, xdata);
+    }
+
+    memset(local->transaction.pre_op_sources, 1, priv->child_count);
+
+    /*If lock or pre-op failed on a brick, it is not a source. */
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i])
+            local->transaction.pre_op_sources[i] = 0;
+    }
+
+    /* If brick is blamed by others, it is not a source. */
+    for (i = 0; i < priv->child_count; i++)
+        for (j = 0; j < priv->child_count; j++)
+            if (matrix[i][j] != 0)
+                local->transaction.pre_op_sources[j] = 0;
+}
 
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-                if (priv->data_change_log)
-                        ret = 1;
+void
+afr_txn_arbitrate_fop(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int pre_op_sources_count = 0;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    afr_compute_pre_op_sources(frame, this);
+    pre_op_sources_count = AFR_COUNT(local->transaction.pre_op_sources,
+                                     priv->child_count);
+
+    /* If arbiter is the only source, do not proceed. */
+    if (pre_op_sources_count < 2 &&
+        local->transaction.pre_op_sources[ARBITER_BRICK_INDEX]) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        for (i = 0; i < priv->child_count; i++)
+            local->transaction.failed_subvols[i] = 1;
+    }
+
+    afr_transaction_fop(frame, this);
+
+    return;
+}
 
-                break;
+int
+afr_transaction_perform_fop(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int ret = 0;
+    int failure_count = 0;
+    struct list_head shared;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    INIT_LIST_HEAD(&shared);
+    if (local->transaction.type == AFR_DATA_TRANSACTION &&
+        !local->transaction.inherited) {
+        ret = afr_write_subvol_set(frame, this);
+        if (ret) {
+            /*act as if operation failed on all subvols*/
+            local->op_ret = -1;
+            local->op_errno = -ret;
+            for (i = 0; i < priv->child_count; i++)
+                local->transaction.failed_subvols[i] = 1;
+        }
+    }
+
+    if (local->pre_op_compat)
+        /* old mode, pre-op was done as afr_changelog_do()
+           just now, before OP */
+        afr_changelog_pre_op_update(frame, this);
+
+    if (!local->transaction.eager_lock_on || local->transaction.inherited)
+        goto fop;
+    failure_count = AFR_COUNT(local->transaction.failed_subvols,
+                              priv->child_count);
+    if (failure_count == priv->child_count) {
+        afr_handle_lock_acquire_failure(local);
+        return 0;
+    } else {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        LOCK(&local->inode->lock);
+        {
+            lock->acquired = _gf_true;
+            __afr_transaction_wake_shared(local, &shared);
+        }
+        UNLOCK(&local->inode->lock);
+    }
+
+fop:
+    /*  Perform fops with the lk-owner from top xlator.
+     *  Eg: lk-owner of posix-lk and flush should be same,
+     *  flush cant clear the  posix-lks without that lk-owner.
+     */
+    afr_save_lk_owner(frame);
+    frame->root->lk_owner = local->transaction.main_frame->root->lk_owner;
+
+    if (priv->arbiter_count == 1) {
+        afr_txn_arbitrate_fop(frame, this);
+    } else {
+        afr_transaction_fop(frame, this);
+    }
+
+    afr_lock_resume_shared(&shared);
+    return 0;
+}
 
-        case AFR_METADATA_TRANSACTION:
-                if (priv->metadata_change_log)
-                        ret = 1;
+int
+afr_set_pending_dict(afr_private_t *priv, dict_t *xattr, int **pending)
+{
+    int i = 0;
+    int ret = 0;
 
-                break;
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], pending[i],
+                                  AFR_NUM_CHANGE_LOGS * sizeof(int));
+        /* 3 = data+metadata+entry */
 
-        case AFR_ENTRY_TRANSACTION:
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                if (priv->entry_change_log)
-                        ret = 1;
+        if (ret)
+            break;
+    }
 
+    return ret;
+}
+
+static void
+afr_ta_dom_lock_check_and_release(afr_ta_fop_state_t fop_state, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    unsigned int inmem_count = 0;
+    unsigned int onwire_count = 0;
+    gf_boolean_t release = _gf_false;
+
+    LOCK(&priv->lock);
+    {
+        /*Once we get notify lock release upcall notification,
+         if any of the fop state counters are non-zero, we will
+         not release the lock.
+         */
+        onwire_count = priv->ta_on_wire_txn_count;
+        inmem_count = priv->ta_in_mem_txn_count;
+        switch (fop_state) {
+            case TA_GET_INFO_FROM_TA_FILE:
+                onwire_count = --priv->ta_on_wire_txn_count;
+                break;
+            case TA_INFO_IN_MEMORY_SUCCESS:
+            case TA_INFO_IN_MEMORY_FAILED:
+                inmem_count = --priv->ta_in_mem_txn_count;
+                break;
+            case TA_WAIT_FOR_NOTIFY_LOCK_REL:
+                GF_ASSERT(0);
+                break;
+            case TA_SUCCESS:
                 break;
         }
+        release = priv->release_ta_notify_dom_lock;
+    }
+    UNLOCK(&priv->lock);
 
-        return ret;
-}
+    if (inmem_count != 0 || release == _gf_false || onwire_count != 0)
+        return;
 
+    afr_ta_lock_release_synctask(this);
+}
 
-static int
-__fop_changelog_needed (call_frame_t *frame, xlator_t *this)
+static void
+afr_ta_process_onwireq(afr_ta_fop_state_t fop_state, xlator_t *this)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        int op_ret = 0;
-        afr_transaction_type type = -1;
+    afr_private_t *priv = this->private;
+    afr_local_t *entry = NULL;
+    int bad_child = AFR_CHILD_UNKNOWN;
+
+    struct list_head onwireq = {
+        0,
+    };
+    INIT_LIST_HEAD(&onwireq);
+
+    LOCK(&priv->lock);
+    {
+        bad_child = priv->ta_bad_child_index;
+        if (bad_child == AFR_CHILD_UNKNOWN) {
+            /*The previous on-wire ta_post_op was a failure. Just dequeue
+             *one element to wind on-wire again. */
+            entry = list_entry(priv->ta_onwireq.next, afr_local_t, ta_onwireq);
+            list_del_init(&entry->ta_onwireq);
+        } else {
+            /* Prepare to process all fops based on bad_child_index. */
+            list_splice_init(&priv->ta_onwireq, &onwireq);
+        }
+    }
+    UNLOCK(&priv->lock);
 
-        priv  = this->private;
-        local = frame->local;
-        type  = local->transaction.type;
+    if (entry) {
+        afr_ta_post_op_synctask(this, entry);
+        return;
+    } else {
+        while (!list_empty(&onwireq)) {
+            entry = list_entry(onwireq.next, afr_local_t, ta_onwireq);
+            list_del_init(&entry->ta_onwireq);
+            if (entry->ta_failed_subvol == bad_child) {
+                afr_post_op_handle_success(entry->transaction.frame, this);
+            } else {
+                afr_post_op_handle_failure(entry->transaction.frame, this, EIO);
+            }
+        }
+    }
+}
 
-        if (__changelog_enabled (priv, type)) {
-                switch (local->op) {
+int
+afr_changelog_post_op_done(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+    int_lock = &local->internal_lock;
+
+    if (priv->thin_arbiter_count) {
+        /*fop should not come here with TA_WAIT_FOR_NOTIFY_LOCK_REL state */
+        afr_ta_dom_lock_check_and_release(local->fop_state, this);
+    }
+
+    /* Fail the FOP if post-op did not succeed on quorum no. of bricks. */
+    if (!afr_changelog_has_quorum(local, this)) {
+        local->op_ret = -1;
+        /*local->op_errno is already captured in changelog cbk*/
+    }
+
+    if (local->transaction.resume_stub) {
+        call_resume(local->transaction.resume_stub);
+        local->transaction.resume_stub = NULL;
+    }
+
+    int_lock->lock_cbk = afr_transaction_done;
+    afr_unlock(frame, this);
+
+    return 0;
+}
 
-                case GF_FOP_WRITE:
-                case GF_FOP_FTRUNCATE:
-                        op_ret = 1;
-                        break;
+static void
+afr_changelog_post_op_fail(call_frame_t *frame, xlator_t *this, int op_errno)
+{
+    afr_local_t *local = frame->local;
+    local->op_ret = -1;
+    local->op_errno = op_errno;
 
-                case GF_FOP_FLUSH:
-                        op_ret = 0;
-                        break;
+    gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_THIN_ARB,
+           "Failing %s for gfid %s. Fop state is:%d", gf_fop_list[local->op],
+           uuid_utoa(local->inode->gfid), local->fop_state);
 
-                default:
-                        op_ret = 1;
-                }
-        }
+    afr_changelog_post_op_done(frame, this);
+}
 
-        return op_ret;
+unsigned char *
+afr_locked_nodes_get(afr_transaction_type type, afr_internal_lock_t *int_lock)
+{
+    /*Because same set of subvols participate in all lockee
+     * entities*/
+    return int_lock->lockee[0].locked_nodes;
 }
 
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
-                      int child, afr_xattrop_type_t op)
+afr_changelog_call_count(afr_transaction_type type,
+                         unsigned char *pre_op_subvols,
+                         unsigned char *failed_subvols,
+                         unsigned int child_count)
 {
-        int i = 0;
-        int ret = 0;
+    int i = 0;
+    int call_count = 0;
 
-        if (op == LOCAL_FIRST) {
-                ret = dict_set_static_bin (xattr, priv->pending_key[child],
-                                           pending[child],
-                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
-                if (ret)
-                        goto out;
+    for (i = 0; i < child_count; i++) {
+        if (pre_op_subvols[i] && !failed_subvols[i]) {
+            call_count++;
         }
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == child)
-                        continue;
-                ret = dict_set_static_bin (xattr, priv->pending_key[i],
-                                           pending[i],
-                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
-                /* 3 = data+metadata+entry */
+    }
 
-                if (ret < 0)
-                        goto out;
-        }
-        if (op == LOCAL_LAST) {
-                ret = dict_set_static_bin (xattr, priv->pending_key[child],
-                                           pending[child],
-                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
-                if (ret)
-                        goto out;
-        }
-out:
-        return ret;
+    if (type == AFR_ENTRY_RENAME_TRANSACTION)
+        call_count *= 2;
+
+    return call_count;
 }
 
-int
-afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
+gf_boolean_t
+afr_txn_nothing_failed(call_frame_t *frame, xlator_t *this)
 {
-        int ret = 0;
-
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-                ret = priv->child_count;
-                break;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (priv->thin_arbiter_count) {
+        /* We need to perform post-op even if 1 data brick was down
+         * before the txn started.*/
+        if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count))
+            return _gf_false;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] &&
+            local->transaction.failed_subvols[i])
+            return _gf_false;
+    }
+
+    return _gf_true;
+}
 
-        case AFR_METADATA_TRANSACTION:
-                ret = priv->child_count;
-                break;
+void
+afr_handle_symmetric_errors(call_frame_t *frame, xlator_t *this)
+{
+    if (afr_is_symmetric_error(frame, this))
+        __mark_all_success(frame, this);
+}
 
-        case AFR_ENTRY_TRANSACTION:
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                ret = priv->child_count;
-                break;
+gf_boolean_t
+afr_has_quorum(unsigned char *subvols, xlator_t *this, call_frame_t *frame)
+{
+    unsigned int quorum_count = 0;
+    afr_private_t *priv = NULL;
+    unsigned int up_children_count = 0;
+
+    priv = this->private;
+    up_children_count = AFR_COUNT(subvols, priv->child_count);
+
+    if (afr_lookup_has_quorum(frame, up_children_count))
+        return _gf_true;
+
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        /*
+         * Special case for auto-quorum with an even number of nodes.
+         *
+         * A replica set with even count N can only handle the same
+         * number of failures as odd N-1 before losing "vanilla"
+         * quorum, and the probability of more simultaneous failures is
+         * actually higher.  For example, with a 1% chance of failure
+         * we'd have a 0.03% chance of two simultaneous failures with
+         * N=3 but a 0.06% chance with N=4.  However, the special case
+         * is necessary for N=2 because there's no real quorum in that
+         * case (i.e. can't normally survive *any* failures).  In that
+         * case, we treat the first node as a tie-breaker, allowing
+         * quorum to be retained in some cases while still honoring the
+         * all-important constraint that there can not simultaneously
+         * be two partitioned sets of nodes each believing they have
+         * quorum.  Of two equally sized sets, the one without that
+         * first node will lose.
+         *
+         * It turns out that the special case is beneficial for higher
+         * values of N as well.  Continuing the example above, the
+         * probability of losing quorum with N=4 and this type of
+         * quorum is (very) slightly lower than with N=3 and vanilla
+         * quorum.  The difference becomes even more pronounced with
+         * higher N.  Therefore, even though such replica counts are
+         * unlikely to be seen in practice, we might as well use the
+         * "special" quorum then as well.
+         */
+        if ((up_children_count * 2) == priv->child_count) {
+            return subvols[0];
         }
+    }
+
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        quorum_count = priv->child_count / 2 + 1;
+    } else {
+        quorum_count = priv->quorum_count;
+    }
+
+    if (up_children_count >= quorum_count)
+        return _gf_true;
 
-        return ret;
+    return _gf_false;
 }
 
-/* {{{ pending */
+static gf_boolean_t
+afr_has_fop_quorum(call_frame_t *frame)
+{
+    xlator_t *this = frame->this;
+    afr_local_t *local = frame->local;
+    unsigned char *locked_nodes = NULL;
+
+    locked_nodes = afr_locked_nodes_get(local->transaction.type,
+                                        &local->internal_lock);
+    return afr_has_quorum(locked_nodes, this, NULL);
+}
 
-int32_t
-afr_changelog_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                           dict_t *xdata)
+static gf_boolean_t
+afr_has_fop_cbk_quorum(call_frame_t *frame)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int                  call_count = -1;
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    unsigned char *success = alloca0(priv->child_count);
+    int i = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i])
+            if (!local->transaction.failed_subvols[i])
+                success[i] = 1;
+    }
+
+    return afr_has_quorum(success, this, NULL);
+}
 
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+gf_boolean_t
+afr_need_dirty_marking(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = NULL;
+    gf_boolean_t need_dirty = _gf_false;
 
-        LOCK (&frame->lock);
-        {
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
+    local = frame->local;
 
-        if (call_count == 0) {
-                if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-                        local->transaction.done (frame, this);
-                } else {
-                        int_lock->lock_cbk = local->transaction.done;
-                        afr_unlock (frame, this);
-                }
-        }
+    if (!priv->quorum_count || !local->optimistic_change_log)
+        return _gf_false;
 
-        return 0;
-}
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION)
+        return _gf_false;
 
+    if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count) ==
+        priv->child_count)
+        return _gf_false;
 
-void
-afr_transaction_rm_stale_children (call_frame_t *frame, xlator_t *this,
-                                   inode_t *inode, afr_transaction_type type)
-{
-        int             i = -1;
-        int             count = 0;
-        int             read_child = -1;
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local = NULL;
-        int             **pending = NULL;
-        int             idx = 0;
-        int32_t         *stale_children = NULL;
-        int32_t         *fresh_children = NULL;
-        gf_boolean_t    rm_stale_children = _gf_false;
-
-        idx = afr_index_for_transaction_type (type);
-
-        priv = this->private;
-        local = frame->local;
-        pending = local->pending;
-
-        if (local->op_ret < 0)
-                goto out;
-        fresh_children = local->fresh_children;
-        read_child = afr_inode_get_read_ctx (this, inode, fresh_children);
-        if (read_child < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "Possible split-brain "
-                        "for %s", uuid_utoa (inode->gfid));
-                goto out;
-        }
+    if (!afr_has_fop_cbk_quorum(frame))
+        need_dirty = _gf_true;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!afr_is_child_present (fresh_children,
-                                           priv->child_count, i))
-                        continue;
-                if (pending[i][idx])
-                        continue;
-                /* child is down or op failed on it */
-                if (!stale_children)
-                        stale_children = afr_children_create (priv->child_count);
-                if (!stale_children)
-                        goto out;
+    return need_dirty;
+}
 
-                rm_stale_children = _gf_true;
-                stale_children[count++] = i;
-                gf_log (this->name, GF_LOG_DEBUG, "Removing stale child "
-                        "%d for %s", i, uuid_utoa (inode->gfid));
-        }
+void
+afr_handle_quorum(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    const char *file = NULL;
+    uuid_t gfid = {0};
 
-        if (!rm_stale_children)
-                goto out;
+    local = frame->local;
+    priv = frame->this->private;
 
-        afr_inode_rm_stale_children (this, inode, stale_children);
-out:
-        GF_FREE (stale_children);
+    if (priv->quorum_count == 0)
         return;
-}
 
-unsigned char*
-afr_locked_nodes_get (afr_transaction_type type, afr_internal_lock_t *int_lock)
-{
-        unsigned char *locked_nodes = NULL;
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                locked_nodes = int_lock->inode_locked_nodes;
-        break;
+    /* If the fop already failed return right away to preserve errno */
+    if (local->op_ret == -1)
+        return;
 
+    /*
+     * Network split may happen just after the fops are unwound, so check
+     * if the fop succeeded in a way it still follows quorum. If it doesn't,
+     * mark the fop as failure, mark the changelogs so it reflects that
+     * failure.
+     *
+     * Scenario:
+     * There are 3 mounts on 3 machines(node1, node2, node3) all writing to
+     * single file. Network split happened in a way that node1 can't see
+     * node2, node3. Node2, node3 both of them can't see node1. Now at the
+     * time of sending write all the bricks are up. Just after write fop is
+     * wound on node1, network split happens. Node1 thinks write fop failed
+     * on node2, node3 so marks pending changelog for those 2 extended
+     * attributes on node1. Node2, node3 thinks writes failed on node1 so
+     * they mark pending changelog for node1. When the network is stable
+     * again the file already is in split-brain. These checks prevent
+     * marking pending changelog on other subvolumes if the fop doesn't
+     * succeed in a way it is still following quorum. So with this fix what
+     * is happening is, node1 will have all pending changelog(FOOL) because
+     * the write succeeded only on node1 but failed on node2, node3 so
+     * instead of marking pending changelogs on node2, node3 it just treats
+     * the fop as failure and goes into DIRTY state. Where as node2, node3
+     * say they are sources and have pending changelog to node1 so there is
+     * no split-brain with the fix. The problem is eliminated completely.
+     */
+
+    if (afr_has_fop_cbk_quorum(frame))
+        return;
+
+    if (afr_need_dirty_marking(frame, this))
+        goto set_response;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i])
+            afr_transaction_fop_failed(frame, frame->this, i);
+    }
+
+set_response:
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    if (local->op_errno == 0)
+        local->op_errno = afr_quorum_errno(priv);
+
+    if (local->fd) {
+        gf_uuid_copy(gfid, local->fd->inode->gfid);
+        file = uuid_utoa(gfid);
+    } else {
+        loc_path(&local->loc, local->loc.name);
+        file = local->loc.path;
+    }
+
+    gf_msg(frame->this->name, GF_LOG_WARNING, local->op_errno,
+           AFR_MSG_QUORUM_FAIL, "%s: Failing %s as quorum is not met", file,
+           gf_fop_list[local->op]);
+
+    switch (local->transaction.type) {
         case AFR_ENTRY_TRANSACTION:
         case AFR_ENTRY_RENAME_TRANSACTION:
-                locked_nodes = int_lock->entry_locked_nodes;
-        break;
-        }
-        return locked_nodes;
+            afr_pick_error_xdata(local, priv, local->parent, local->readable,
+                                 local->parent2, local->readable2);
+            break;
+        default:
+            afr_pick_error_xdata(local, priv, local->inode, local->readable,
+                                 NULL, NULL);
+            break;
+    }
 }
 
 int
-afr_changelog_pre_op_call_count (afr_transaction_type type,
-                                 afr_internal_lock_t *int_lock,
-                                 unsigned int child_count)
+afr_fill_ta_loc(xlator_t *this, loc_t *loc, gf_boolean_t is_gfid_based_fop)
+{
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    loc->parent = inode_ref(priv->root_inode);
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
+    loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
+    if (is_gfid_based_fop && gf_uuid_is_null(priv->ta_gfid)) {
+        /* Except afr_ta_id_file_check() which is path based, all other gluster
+         * FOPS need gfid.*/
+        return -EINVAL;
+    }
+    gf_uuid_copy(loc->gfid, priv->ta_gfid);
+    loc->inode = inode_new(loc->parent->table);
+    if (!loc->inode) {
+        loc_wipe(loc);
+        return -ENOMEM;
+    }
+    return 0;
+}
+
+static int
+afr_ta_post_op_done(int ret, call_frame_t *frame, void *opaque)
 {
-        int           call_count = 0;
-        unsigned char *locked_nodes = NULL;
+    xlator_t *this = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *txn_frame = NULL;
+    afr_ta_fop_state_t fop_state;
+
+    local = (afr_local_t *)opaque;
+    fop_state = local->fop_state;
+    txn_frame = local->transaction.frame;
+    this = frame->this;
+
+    if (ret == 0) {
+        /*Mark pending xattrs on the up data brick.*/
+        afr_post_op_handle_success(txn_frame, this);
+    } else {
+        afr_post_op_handle_failure(txn_frame, this, -ret);
+    }
+
+    STACK_DESTROY(frame->root);
+    afr_ta_process_onwireq(fop_state, this);
+
+    return 0;
+}
 
-        locked_nodes = afr_locked_nodes_get (type, int_lock);
-        GF_ASSERT (locked_nodes);
+int **
+afr_set_changelog_xattr(afr_private_t *priv, unsigned char *pending,
+                        dict_t *xattr, afr_local_t *local)
+{
+    int **changelog = NULL;
+    int idx = 0;
+    int ret = 0;
+    int i;
+
+    if (local->is_new_entry == _gf_true) {
+        changelog = afr_mark_pending_changelog(priv, pending, xattr,
+                                               local->cont.dir_fop.buf.ia_type);
+    } else {
+        idx = afr_index_for_transaction_type(local->transaction.type);
+        changelog = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+        if (!changelog) {
+            goto out;
+        }
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->transaction.failed_subvols[i])
+                changelog[i][idx] = hton32(1);
+        }
+        ret = afr_set_pending_dict(priv, xattr, changelog);
+        if (ret < 0) {
+            afr_matrix_cleanup(changelog, priv->child_count);
+            return NULL;
+        }
+    }
 
-        call_count = afr_locked_children_count (locked_nodes, child_count);
-        if (type == AFR_ENTRY_RENAME_TRANSACTION)
-                call_count *= 2;
+out:
+    return changelog;
+}
 
-        return call_count;
+static void
+afr_ta_locked_xattrop_validate(afr_private_t *priv, afr_local_t *local,
+                               gf_boolean_t *valid)
+{
+    if (priv->ta_event_gen > local->ta_event_gen) {
+        /* We can't trust the ta's response anymore.*/
+        afr_ta_locked_priv_invalidate(priv);
+        *valid = _gf_false;
+        return;
+    }
+    return;
 }
 
-int
-afr_changelog_post_op_call_count (afr_transaction_type type,
-                                  unsigned char *pre_op,
-                                  unsigned int child_count)
+static int
+afr_ta_post_op_do(void *opaque)
 {
-        int           call_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    dict_t *xattr = NULL;
+    unsigned char *pending = NULL;
+    int **changelog = NULL;
+    int failed_subvol = -1;
+    int success_subvol = -1;
+    loc_t loc = {
+        0,
+    };
+    int i = 0;
+    int ret = 0;
+    gf_boolean_t valid = _gf_true;
+
+    local = (afr_local_t *)opaque;
+    this = local->transaction.frame->this;
+    priv = this->private;
+
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate loc for thin-arbiter.");
+        goto out;
+    }
+
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    pending = alloca0(priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i]) {
+            pending[i] = 1;
+            failed_subvol = i;
+        } else {
+            success_subvol = i;
+        }
+    }
+
+    changelog = afr_set_changelog_xattr(priv, pending, xattr, local);
+
+    if (!changelog) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = afr_ta_post_op_lock(this, &loc);
+    if (ret)
+        goto out;
+
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Post-op on thin-arbiter id file %s failed for gfid %s.",
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX],
+               uuid_utoa(local->inode->gfid));
+    }
+    LOCK(&priv->lock);
+    {
+        if (ret == 0) {
+            priv->ta_bad_child_index = failed_subvol;
+        } else if (ret == -EINVAL) {
+            priv->ta_bad_child_index = success_subvol;
+            ret = -EIO; /* TA failed the fop. Return EIO to application. */
+        }
+
+        afr_ta_locked_xattrop_validate(priv, local, &valid);
+    }
+    UNLOCK(&priv->lock);
+    if (valid == _gf_false) {
+        gf_msg(this->name, GF_LOG_ERROR, EIO, AFR_MSG_THIN_ARB,
+               "Post-op on thin-arbiter id file %s for gfid %s invalidated due "
+               "to event-gen mismatch.",
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX],
+               uuid_utoa(local->inode->gfid));
+        ret = -EIO;
+    }
+
+    afr_ta_post_op_unlock(this, &loc);
+out:
+    if (xattr)
+        dict_unref(xattr);
+
+    if (changelog)
+        afr_matrix_cleanup(changelog, priv->child_count);
 
-        call_count = afr_pre_op_done_children_count (pre_op, child_count);
-        if (type == AFR_ENTRY_RENAME_TRANSACTION)
-                call_count *= 2;
+    loc_wipe(&loc);
 
-        return call_count;
+    return ret;
 }
 
-void
-afr_compute_txn_changelog (afr_local_t *local, afr_private_t *priv)
+static int
+afr_ta_post_op_synctask(xlator_t *this, afr_local_t *local)
 {
-        int             i = 0;
-        int             index = 0;
-        int32_t         postop = 0;
-        int32_t         preop = 1;
-        int32_t         **txn_changelog = NULL;
+    call_frame_t *ta_frame = NULL;
+    int ret = 0;
+
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        goto err;
+    }
+    ret = synctask_new(this->ctx->env, afr_ta_post_op_do, afr_ta_post_op_done,
+                       ta_frame, local);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to launch post-op on thin arbiter for gfid %s",
+               uuid_utoa(local->inode->gfid));
+        STACK_DESTROY(ta_frame->root);
+        goto err;
+    }
+
+    return ret;
+err:
+    afr_changelog_post_op_fail(local->transaction.frame, this, ENOMEM);
+    return ret;
+}
 
-        txn_changelog = local->transaction.txn_changelog;
-        index = afr_index_for_transaction_type (local->transaction.type);
-        for (i = 0; i < priv->child_count; i++) {
-                postop = ntoh32 (local->pending[i][index]);
-                txn_changelog[i][index] = hton32 (postop + preop);
+static void
+afr_ta_set_fop_state(afr_private_t *priv, afr_local_t *local,
+                     int *on_wire_count)
+{
+    LOCK(&priv->lock);
+    {
+        if (priv->release_ta_notify_dom_lock == _gf_true) {
+            /* Put the fop in waitq until notify dom lock is released.*/
+            local->fop_state = TA_WAIT_FOR_NOTIFY_LOCK_REL;
+            list_add_tail(&local->ta_waitq, &priv->ta_waitq);
+        } else if (priv->ta_bad_child_index == AFR_CHILD_UNKNOWN) {
+            /* Post-op on thin-arbiter to decide success/failure. */
+            local->fop_state = TA_GET_INFO_FROM_TA_FILE;
+            *on_wire_count = ++priv->ta_on_wire_txn_count;
+            if (*on_wire_count > 1) {
+                /*Avoid sending multiple on-wire post-ops on TA*/
+                list_add_tail(&local->ta_onwireq, &priv->ta_onwireq);
+            }
+        } else if (local->ta_failed_subvol == priv->ta_bad_child_index) {
+            /* Post-op on TA not needed as the fop failed on the in-memory bad
+             * brick. Just mark pending xattrs on the good data brick.*/
+            local->fop_state = TA_INFO_IN_MEMORY_SUCCESS;
+            priv->ta_in_mem_txn_count++;
+        } else {
+            /* Post-op on TA not needed as the fop succeeded only on the
+             * in-memory bad data brick and not the good one. Fail the fop.*/
+            local->fop_state = TA_INFO_IN_MEMORY_FAILED;
+            priv->ta_in_mem_txn_count++;
         }
+    }
+    UNLOCK(&priv->lock);
 }
 
-afr_xattrop_type_t
-afr_get_postop_xattrop_type (int32_t **pending, int optimized, int child,
-                             afr_transaction_type type)
+static void
+afr_ta_fill_failed_subvol(afr_private_t *priv, afr_local_t *local)
 {
-        int                     index = 0;
-        afr_xattrop_type_t      op = LOCAL_LAST;
+    int i = 0;
 
-        index = afr_index_for_transaction_type (type);
-        if (optimized && !pending[child][index])
-                op = LOCAL_FIRST;
-        return op;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i]) {
+            local->ta_failed_subvol = i;
+            break;
+        }
+    }
 }
 
-void
-afr_set_postop_dict (afr_local_t *local, xlator_t *this, dict_t *xattr,
-                     int optimized, int child)
-{
-        int32_t                 **txn_changelog = NULL;
-        int32_t                 **changelog = NULL;
-        afr_private_t           *priv = NULL;
-        int                     ret = 0;
-        afr_xattrop_type_t      op = LOCAL_LAST;
-
-        priv = this->private;
-        txn_changelog = local->transaction.txn_changelog;
-        op = afr_get_postop_xattrop_type (local->pending, optimized, child,
-                                          local->transaction.type);
-        if (optimized)
-                changelog = txn_changelog;
-        else
-                changelog = local->pending;
-        ret = afr_set_pending_dict (priv, xattr, changelog, child, op);
-        if (ret < 0)
-                gf_log (this->name, GF_LOG_INFO,
-                        "failed to set pending entry");
+static void
+afr_post_op_handle_success(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+
+    local = frame->local;
+    if (local->is_new_entry == _gf_true) {
+        afr_mark_new_entry_changelog(frame, this);
+    }
+    afr_changelog_post_op_do(frame, this);
+
+    return;
 }
 
-int
-afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
+static void
+afr_post_op_handle_failure(call_frame_t *frame, xlator_t *this, int op_errno)
 {
-        afr_private_t * priv = this->private;
-        afr_internal_lock_t *int_lock = NULL;
-        int i          = 0;
-        int call_count = 0;
+    afr_changelog_post_op_fail(frame, this, op_errno);
 
-        afr_local_t *  local = NULL;
-        afr_fd_ctx_t  *fdctx = NULL;
-        dict_t        **xattr = NULL;
-        int            piggyback = 0;
-        int            index = 0;
-        int            nothing_failed = 1;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    return;
+}
 
-        __mark_down_children (local->pending, priv->child_count,
-                              local->child_up, local->transaction.type);
+static void
+afr_ta_decide_post_op_state(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int on_wire_count = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    afr_ta_set_fop_state(priv, local, &on_wire_count);
+
+    switch (local->fop_state) {
+        case TA_GET_INFO_FROM_TA_FILE:
+            if (on_wire_count == 1)
+                afr_ta_post_op_synctask(this, local);
+            /*else, fop is queued in ta_onwireq.*/
+            break;
+        case TA_WAIT_FOR_NOTIFY_LOCK_REL:
+            /*Post releasing the notify lock, we will act on this queue*/
+            break;
+        case TA_INFO_IN_MEMORY_SUCCESS:
+            afr_post_op_handle_success(frame, this);
+            break;
+        case TA_INFO_IN_MEMORY_FAILED:
+            afr_post_op_handle_failure(frame, this, EIO);
+            break;
+        default:
+            break;
+    }
+    return;
+}
 
-        if (local->fd)
-                afr_transaction_rm_stale_children (frame, this,
-                                                   local->fd->inode,
-                                                   local->transaction.type);
+static void
+afr_handle_failure_using_thin_arbiter(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+
+    afr_ta_fill_failed_subvol(priv, local);
+    gf_msg_debug(this->name, 0,
+                 "Fop failed on data brick (%s) for gfid=%s. "
+                 "ta info needed to decide fop result.",
+                 priv->children[local->ta_failed_subvol]->name,
+                 uuid_utoa(local->inode->gfid));
+    afr_ta_decide_post_op_state(frame, this);
+}
 
-        xattr = alloca (priv->child_count * sizeof (*xattr));
-        memset (xattr, 0, (priv->child_count * sizeof (*xattr)));
-        for (i = 0; i < priv->child_count; i++) {
-                xattr[i] = dict_new ();
+void
+afr_changelog_post_op_do(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = NULL;
+    dict_t *xattr = NULL;
+    int i = 0;
+    int ret = 0;
+    int idx = 0;
+    int nothing_failed = 1;
+    gf_boolean_t need_undirty = _gf_false;
+
+    afr_handle_quorum(frame, this);
+    local = frame->local;
+    idx = afr_index_for_transaction_type(local->transaction.type);
+
+    xattr = dict_new();
+    if (!xattr) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    nothing_failed = afr_txn_nothing_failed(frame, this);
+
+    if (afr_changelog_pre_op_uninherit(frame, this))
+        need_undirty = _gf_false;
+    else
+        need_undirty = _gf_true;
+
+    if (local->op_ret < 0 && !nothing_failed) {
+        if (afr_need_dirty_marking(frame, this)) {
+            local->dirty[idx] = hton32(1);
+            goto set_dirty;
         }
 
-        call_count = afr_changelog_post_op_call_count (local->transaction.type,
-                                                       local->transaction.pre_op,
-                                                       priv->child_count);
-        local->call_count = call_count;
-
-        if (local->fd)
-                fdctx = afr_fd_ctx_get (local->fd, this);
+        afr_changelog_post_op_done(frame, this);
+        goto out;
+    }
+
+    if (nothing_failed && !need_undirty) {
+        afr_changelog_post_op_done(frame, this);
+        goto out;
+    }
+
+    if (local->transaction.in_flight_sb) {
+        afr_changelog_post_op_fail(frame, this,
+                                   local->transaction.in_flight_sb_errno);
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i])
+            local->pending[i][idx] = hton32(1);
+    }
+
+    ret = afr_set_pending_dict(priv, xattr, local->pending);
+    if (ret < 0) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    if (need_undirty)
+        local->dirty[idx] = hton32(-1);
+    else
+        local->dirty[idx] = hton32(0);
+
+set_dirty:
+    ret = dict_set_static_bin(xattr, AFR_DIRTY, local->dirty,
+                              sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    afr_changelog_do(frame, this, xattr, afr_changelog_post_op_done,
+                     AFR_TRANSACTION_POST_OP);
+out:
+    if (xattr)
+        dict_unref(xattr);
 
-        if (call_count == 0) {
-                /* no child is up */
-                int_lock->lock_cbk = local->transaction.done;
-                afr_unlock (frame, this);
-                goto out;
-        }
+    return;
+}
 
-        /* check if something has failed, to handle piggybacking */
-        nothing_failed = 1;
-        index = afr_index_for_transaction_type (local->transaction.type);
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->pending[i][index] == 0) {
-                        nothing_failed = 0;
-                        break;
-                }
+static int
+afr_changelog_post_op_now(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int failed_count = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (priv->thin_arbiter_count) {
+        failed_count = AFR_COUNT(local->transaction.failed_subvols,
+                                 priv->child_count);
+        if (failed_count == 1) {
+            afr_handle_failure_using_thin_arbiter(frame, this);
+            return 0;
+        } else {
+            /* Txn either succeeded or failed on both data bricks. Let
+             * post_op_do handle it as the case might be. */
         }
+    }
 
-        afr_compute_txn_changelog (local , priv);
+    afr_changelog_post_op_do(frame, this);
+    return 0;
+}
 
+gf_boolean_t
+afr_changelog_pre_op_uninherit(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_inode_ctx_t *ctx = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
+
+    local = frame->local;
+    priv = this->private;
+    ctx = local->inode_ctx;
+
+    type = afr_index_for_transaction_type(local->transaction.type);
+    if (type != AFR_DATA_TRANSACTION)
+        return !local->transaction.dirtied;
+
+    if (local->transaction.no_uninherit)
+        return _gf_false;
+
+    /* This function must be idempotent. So check if we
+       were called before and return the same answer again.
+
+       It is important to keep this function idempotent for
+       the call in afr_changelog_post_op_safe() to not have
+       side effects on the call from afr_changelog_post_op_now()
+    */
+    if (local->transaction.uninherit_done)
+        return local->transaction.uninherit_value;
+
+    LOCK(&local->inode->lock);
+    {
         for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.pre_op[i])
-                        continue;
-
-                if (local->transaction.type != AFR_DATA_TRANSACTION)
-                        afr_set_postop_dict (local, this, xattr[i],
-                                             local->optimistic_change_log, i);
-                switch (local->transaction.type) {
-                case AFR_DATA_TRANSACTION:
-                {
-                        if (!fdctx) {
-                                afr_set_postop_dict (local, this, xattr[i],
-                                                     0, i);
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i],
-                                            NULL);
-                                break;
-                        }
-
-                        LOCK (&local->fd->lock);
-                        {
-                                piggyback = 0;
-                                if (fdctx->pre_op_piggyback[i]) {
-                                        fdctx->pre_op_piggyback[i]--;
-                                        piggyback = 1;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
-
-                        afr_set_postop_dict (local, this, xattr[i],
-                                             piggyback, i);
-
-                        if (nothing_failed && piggyback) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i], NULL);
-                        } else {
-                                __mark_pre_op_undone_on_fd (frame, this, i);
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_post_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                        }
-                }
-                break;
-                case AFR_METADATA_TRANSACTION:
-                {
-                        if (nothing_failed && local->optimistic_change_log) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i],
-                                                           NULL);
-                                break;
-                        }
-
-                        if (local->fd)
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->fxattrop,
-                                            local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i],
-                                            NULL);
-                        else
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i],
-                                            NULL);
-                }
-                break;
-
-                case AFR_ENTRY_RENAME_TRANSACTION:
-                {
-                        if (nothing_failed && local->optimistic_change_log) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i],
-                                                           NULL);
-                        } else {
-                                STACK_WIND_COOKIE (frame, afr_changelog_post_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &local->transaction.new_parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                        }
-                        call_count--;
-                }
+            if (local->transaction.pre_op[i] != ctx->pre_op_done[type][i]) {
+                ret = !local->transaction.dirtied;
+                goto unlock;
+            }
+        }
 
-                /*
-                  set it again because previous stack_wind
-                  might have already returned (think of case
-                  where subvolume is posix) and would have
-                  used the dict as placeholder for return
-                  value
-                */
+        if (ctx->inherited[type]) {
+            ret = _gf_true;
+            ctx->inherited[type]--;
+        } else if (ctx->on_disk[type]) {
+            ret = _gf_false;
+            ctx->on_disk[type]--;
+        } else {
+            /* ASSERT */
+            ret = _gf_false;
+        }
 
-                afr_set_postop_dict (local, this, xattr[i],
-                                     local->optimistic_change_log, i);
+        if (!ctx->inherited[type] && !ctx->on_disk[type]) {
+            for (i = 0; i < priv->child_count; i++)
+                ctx->pre_op_done[type][i] = 0;
+        }
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
 
-                /* fall through */
+    local->transaction.uninherit_done = _gf_true;
+    local->transaction.uninherit_value = ret;
 
-                case AFR_ENTRY_TRANSACTION:
-                {
-                        if (nothing_failed && local->optimistic_change_log) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i],
-                                                           NULL);
-                                break;
-                        }
-
-                        if (local->fd)
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->fxattrop,
-                                            local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i],
-                                            NULL);
-                        else
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->transaction.parent_loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i],
-                                            NULL);
-                }
-                break;
-                }
+    return ret;
+}
 
-                if (!--call_count)
-                        break;
+gf_boolean_t
+afr_changelog_pre_op_inherit(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (local->transaction.type != AFR_DATA_TRANSACTION)
+        return _gf_false;
+
+    type = afr_index_for_transaction_type(local->transaction.type);
+
+    LOCK(&local->inode->lock);
+    {
+        if (!local->inode_ctx->on_disk[type]) {
+            /* nothing to inherit yet */
+            ret = _gf_false;
+            goto unlock;
         }
 
-out:
         for (i = 0; i < priv->child_count; i++) {
-                dict_unref (xattr[i]);
+            if (local->transaction.pre_op[i] !=
+                local->inode_ctx->pre_op_done[type][i]) {
+                /* either inherit exactly, or don't */
+                ret = _gf_false;
+                goto unlock;
+            }
         }
 
-        return 0;
-}
+        local->inode_ctx->inherited[type]++;
 
+        ret = _gf_true;
+
+        local->transaction.inherited = _gf_true;
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
+
+    return ret;
+}
 
-int32_t
-afr_changelog_pre_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                          dict_t *xdata)
+gf_boolean_t
+afr_changelog_pre_op_update(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = this->private;
-        int call_count  = -1;
-        int child_index = (long) cookie;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
 
-        local = frame->local;
+    local = frame->local;
+    priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                switch (op_ret) {
-                case 0:
-                        __mark_pre_op_done_on_fd (frame, this, child_index);
-                        //fallthrough we need to mark the pre_op
-                case 1:
-                        local->transaction.pre_op[child_index] = 1;
-                        /* special op_ret for piggyback */
-                        break;
-                case -1:
-                        if (op_errno == ENOTSUP) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "xattrop not supported by %s",
-                                        priv->children[child_index]->name);
-                                local->op_ret = -1;
-
-                        } else if (!child_went_down (op_ret, op_errno)) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "xattrop failed on child %s: %s",
-                                        priv->children[child_index]->name,
-                                        strerror (op_errno));
-                        }
-                        local->op_errno = op_errno;
-                        break;
-                }
+    if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+        local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION)
+        return _gf_false;
 
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
+    if (local->transaction.inherited)
+        /* was already inherited in afr_changelog_pre_op */
+        return _gf_false;
 
-        if (call_count == 0) {
-                if ((local->op_ret == -1) &&
-                    (local->op_errno == ENOTSUP)) {
-                        local->transaction.resume (frame, this);
-                } else {
-                        __mark_all_success (local->pending, priv->child_count,
-                                            local->transaction.type);
-
-			/*  Perform fops with the lk-owner from top xlator.
-			 *  Eg: lk-owner of posix-lk and flush should be same,
-			 *  flush cant clear the  posix-lks without that lk-owner.
-			 */
-			afr_save_lk_owner (frame);
-			frame->root->lk_owner =
-				local->transaction.main_frame->root->lk_owner;
-
-                        local->transaction.fop (frame, this);
+    if (!local->transaction.dirtied)
+        return _gf_false;
+
+    if (!afr_txn_nothing_failed(frame, this))
+        return _gf_false;
+
+    type = afr_index_for_transaction_type(local->transaction.type);
+
+    ret = _gf_false;
+
+    LOCK(&local->inode->lock);
+    {
+        if (!local->inode_ctx->on_disk[type]) {
+            for (i = 0; i < priv->child_count; i++)
+                local->inode_ctx->pre_op_done[type][i] =
+                    (!local->transaction.failed_subvols[i]);
+        } else {
+            for (i = 0; i < priv->child_count; i++)
+                if (local->inode_ctx->pre_op_done[type][i] !=
+                    (!local->transaction.failed_subvols[i])) {
+                    local->transaction.no_uninherit = 1;
+                    goto unlock;
                 }
         }
+        local->inode_ctx->on_disk[type]++;
 
-        return 0;
+        ret = _gf_true;
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
+
+    return ret;
 }
 
 int
-afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
+afr_changelog_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        afr_private_t * priv = this->private;
-        int i = 0;
-        int ret = 0;
-        int call_count = 0;
-        dict_t **xattr = NULL;
-        afr_fd_ctx_t *fdctx = NULL;
-        afr_local_t *local = NULL;
-        int          piggyback = 0;
-        afr_internal_lock_t *int_lock = NULL;
-        unsigned char       *locked_nodes = NULL;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
 
-        xattr = alloca (priv->child_count * sizeof (*xattr));
-        memset (xattr, 0, (priv->child_count * sizeof (*xattr)));
+    local = frame->local;
+    child_index = (long)cookie;
 
-        for (i = 0; i < priv->child_count; i++) {
-                xattr[i] = dict_new ();
-        }
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
 
-        call_count = afr_changelog_pre_op_call_count (local->transaction.type,
-                                                      int_lock,
-                                                      priv->child_count);
-        if (call_count == 0) {
-                local->internal_lock.lock_cbk =
-                        local->transaction.done;
-                afr_unlock (frame, this);
-                goto out;
-        }
+    if (xattr)
+        local->transaction.changelog_xdata[child_index] = dict_ref(xattr);
 
-        local->call_count = call_count;
+    call_count = afr_frame_return(frame);
 
-        __mark_all_pending (local->pending, priv->child_count,
-                            local->transaction.type);
+    if (call_count == 0) {
+        local->transaction.changelog_resume(frame, this);
+    }
 
-        if (local->fd)
-                fdctx = afr_fd_ctx_get (local->fd, this);
+    return 0;
+}
 
-        locked_nodes = afr_locked_nodes_get (local->transaction.type, int_lock);
-        for (i = 0; i < priv->child_count; i++) {
-                if (!locked_nodes[i])
+void
+afr_changelog_populate_xdata(call_frame_t *frame, afr_xattrop_type_t op,
+                             dict_t **xdata, dict_t **newloc_xdata)
+{
+    int i = 0;
+    int ret = 0;
+    char *key = NULL;
+    int keylen = 0;
+    const char *name = NULL;
+    dict_t *xdata1 = NULL;
+    dict_t *xdata2 = NULL;
+    xlator_t *this = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t need_entry_key_set = _gf_true;
+
+    local = frame->local;
+    this = THIS;
+    priv = this->private;
+
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION)
+        goto out;
+
+    if (!priv->esh_granular)
+        goto out;
+
+    xdata1 = dict_new();
+    if (!xdata1)
+        goto out;
+
+    name = local->loc.name;
+    if (local->op == GF_FOP_LINK)
+        name = local->newloc.name;
+
+    switch (op) {
+        case AFR_TRANSACTION_PRE_OP:
+            key = GF_XATTROP_ENTRY_IN_KEY;
+            break;
+        case AFR_TRANSACTION_POST_OP:
+            if (afr_txn_nothing_failed(frame, this)) {
+                key = GF_XATTROP_ENTRY_OUT_KEY;
+                for (i = 0; i < priv->child_count; i++) {
+                    if (!local->transaction.failed_subvols[i])
                         continue;
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending,
-                                            i, LOCAL_FIRST);
-
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
-
-
-                switch (local->transaction.type) {
-                case AFR_DATA_TRANSACTION:
-                {
-                        if (!fdctx) {
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                                break;
-                        }
-
-                        LOCK (&local->fd->lock);
-                        {
-                                piggyback = 0;
-                                if (fdctx->pre_op_done[i]) {
-                                        fdctx->pre_op_piggyback[i]++;
-                                        piggyback = 1;
-                                        fdctx->hit++;
-                                } else {
-                                        fdctx->miss++;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
-
-			afr_set_delayed_post_op (frame, this);
-
-                        if (piggyback)
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i],
-                                                          NULL);
-                        else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                }
-                break;
-                case AFR_METADATA_TRANSACTION:
-                {
-                        if (local->optimistic_change_log) {
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i],
-                                                          NULL);
-                                break;
-                        }
-
-                        if (local->fd)
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                        else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
+                    need_entry_key_set = _gf_false;
+                    break;
                 }
-                break;
+                /* If the transaction itself did not fail and there
+                 * are no failed subvolumes, check whether the fop
+                 * failed due to a symmetric error. If it did, do
+                 * not set the ENTRY_OUT xattr which would end up
+                 * deleting a name index which was created possibly by
+                 * an earlier entry txn that may have failed on some
+                 * of the sub-volumes.
+                 */
+                if (local->op_ret)
+                    need_entry_key_set = _gf_false;
+            } else {
+                key = GF_XATTROP_ENTRY_IN_KEY;
+            }
+            break;
+    }
+
+    if (need_entry_key_set) {
+        keylen = strlen(key);
+        ret = dict_set_strn(xdata1, key, keylen, (char *)name);
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                   "%s/%s: Could not set %s key during xattrop",
+                   uuid_utoa(local->loc.pargfid), local->loc.name, key);
+        if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+            xdata2 = dict_new();
+            if (!xdata2)
+                goto out;
 
-                case AFR_ENTRY_RENAME_TRANSACTION:
-                {
-                        if (local->optimistic_change_log) {
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i],
-                                                          NULL);
-                        } else {
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &local->transaction.new_parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                        }
-
-                        call_count--;
-                }
+            ret = dict_set_strn(xdata2, key, keylen,
+                                (char *)local->newloc.name);
+            if (ret)
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                       "%s/%s: Could not set %s key during "
+                       "xattrop",
+                       uuid_utoa(local->newloc.pargfid), local->newloc.name,
+                       key);
+        }
+    }
 
+    *xdata = xdata1;
+    *newloc_xdata = xdata2;
+    xdata1 = xdata2 = NULL;
+out:
+    if (xdata1)
+        dict_unref(xdata1);
+    return;
+}
 
-                /*
-                  set it again because previous stack_wind
-                  might have already returned (think of case
-                  where subvolume is posix) and would have
-                  used the dict as placeholder for return
-                  value
-                */
+int
+afr_changelog_prepare(xlator_t *this, call_frame_t *frame, int *call_count,
+                      afr_changelog_resume_t changelog_resume,
+                      afr_xattrop_type_t op, dict_t **xdata,
+                      dict_t **newloc_xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending,
-                                            i, LOCAL_FIRST);
+    local = frame->local;
+    priv = this->private;
 
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
+    *call_count = afr_changelog_call_count(
+        local->transaction.type, local->transaction.pre_op,
+        local->transaction.failed_subvols, priv->child_count);
 
-                /* fall through */
+    if (*call_count == 0) {
+        changelog_resume(frame, this);
+        return -1;
+    }
 
-                case AFR_ENTRY_TRANSACTION:
-                {
-                        if (local->optimistic_change_log) {
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i],
-                                                          NULL);
-                                break;
-                        }
-
-                        if (local->fd)
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                        else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &local->transaction.parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                }
-                break;
-                }
+    afr_changelog_populate_xdata(frame, op, xdata, newloc_xdata);
+    local->call_count = *call_count;
 
-                if (!--call_count)
-                        break;
-        }
-out:
-        for (i = 0; i < priv->child_count; i++) {
-                dict_unref (xattr[i]);
+    local->transaction.changelog_resume = changelog_resume;
+    return 0;
+}
+
+int
+afr_changelog_do(call_frame_t *frame, xlator_t *this, dict_t *xattr,
+                 afr_changelog_resume_t changelog_resume, afr_xattrop_type_t op)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    dict_t *newloc_xdata = NULL;
+    int i = 0;
+    int call_count = 0;
+    int ret = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.changelog_xdata[i]) {
+            dict_unref(local->transaction.changelog_xdata[i]);
+            local->transaction.changelog_xdata[i] = NULL;
         }
+    }
+
+    ret = afr_changelog_prepare(this, frame, &call_count, changelog_resume, op,
+                                &xdata, &newloc_xdata);
 
+    if (ret)
         return 0;
-}
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.pre_op[i] ||
+            local->transaction.failed_subvols[i])
+            continue;
 
-int
-afr_post_blocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+        switch (local->transaction.type) {
+            case AFR_DATA_TRANSACTION:
+            case AFR_METADATA_TRANSACTION:
+                if (!local->fd) {
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->xattrop,
+                        &local->loc, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                } else {
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->fxattrop,
+                        local->fd, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                }
+                break;
+            case AFR_ENTRY_RENAME_TRANSACTION:
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+                STACK_WIND_COOKIE(frame, afr_changelog_cbk, (void *)(long)i,
+                                  priv->children[i],
+                                  priv->children[i]->fops->xattrop,
+                                  &local->transaction.new_parent_loc,
+                                  GF_XATTROP_ADD_ARRAY, xattr, newloc_xdata);
+                call_count--;
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Blocking inodelks failed.");
-                local->transaction.done (frame, this);
-        } else {
+                /* fall through */
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Blocking inodelks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
+            case AFR_ENTRY_TRANSACTION:
+                if (local->fd)
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->fxattrop,
+                        local->fd, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                else
+                    STACK_WIND_COOKIE(frame, afr_changelog_cbk, (void *)(long)i,
+                                      priv->children[i],
+                                      priv->children[i]->fops->xattrop,
+                                      &local->transaction.parent_loc,
+                                      GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                break;
         }
 
-        return 0;
-}
+        if (!--call_count)
+            break;
+    }
 
+    if (xdata)
+        dict_unref(xdata);
+    if (newloc_xdata)
+        dict_unref(newloc_xdata);
+    return 0;
+}
 
-int
-afr_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
+static void
+afr_init_optimistic_changelog_for_txn(xlator_t *this, afr_local_t *local)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    int locked_count = 0;
+    afr_private_t *priv = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    priv = this->private;
 
-        /* Initiate blocking locks if non-blocking has failed */
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking inodelks failed. Proceeding to blocking");
-                int_lock->lock_cbk = afr_post_blocking_inodelk_cbk;
-                afr_blocking_lock (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking inodelks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
-        }
+    locked_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
+    if (priv->optimistic_change_log && locked_count == priv->child_count)
+        local->optimistic_change_log = 1;
 
-        return 0;
+    return;
 }
 
-
 int
-afr_post_blocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
+afr_changelog_pre_op(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_private_t *priv = this->private;
+    int i = 0;
+    int ret = 0;
+    int call_count = 0;
+    int op_errno = 0;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    unsigned char *locked_nodes = NULL;
+    int idx = -1;
+    gf_boolean_t pre_nop = _gf_true;
+    dict_t *xdata_req = NULL;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    idx = afr_index_for_transaction_type(local->transaction.type);
+
+    locked_nodes = afr_locked_nodes_get(local->transaction.type, int_lock);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (locked_nodes[i]) {
+            local->transaction.pre_op[i] = 1;
+            call_count++;
+        } else {
+            local->transaction.failed_subvols[i] = 1;
+        }
+    }
+
+    afr_init_optimistic_changelog_for_txn(this, local);
+
+    if (afr_changelog_pre_op_inherit(frame, this))
+        goto next;
+
+    /* This condition should not be met with present code, as
+     * transaction.done will be called if locks are not acquired on even a
+     * single node.
+     */
+    if (call_count == 0) {
+        op_errno = ENOTCONN;
+        goto err;
+    }
+
+    /* Check if the fop can be performed on at least
+     * quorum number of nodes.
+     */
+    if (priv->quorum_count && !afr_has_fop_quorum(frame)) {
+        op_errno = int_lock->lock_op_errno;
+        if (op_errno == 0)
+            op_errno = afr_quorum_errno(priv);
+        goto err;
+    }
+
+    xdata_req = dict_new();
+    if (!xdata_req) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (call_count < priv->child_count)
+        pre_nop = _gf_false;
+
+    /* Set an all-zero pending changelog so that in the cbk, we can get the
+     * current on-disk values. In a replica 3 volume with arbiter enabled,
+     * these values are needed to arrive at a go/ no-go of the fop phase to
+     * avoid ending up in split-brain.*/
+
+    ret = afr_set_pending_dict(priv, xdata_req, local->pending);
+    if (ret < 0) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (afr_needs_changelog_update(local)) {
+        local->dirty[idx] = hton32(1);
+
+        ret = dict_set_static_bin(xdata_req, AFR_DIRTY, local->dirty,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto err;
+        }
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+        pre_nop = _gf_false;
+        local->transaction.dirtied = 1;
+    }
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Blocking entrylks failed.");
-                local->transaction.done (frame, this);
-        } else {
+    if (pre_nop)
+        goto next;
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Blocking entrylks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
-        }
+    if (!local->pre_op_compat) {
+        dict_copy(xdata_req, local->xdata_req);
+        goto next;
+    }
 
-        return 0;
-}
+    afr_changelog_do(frame, this, xdata_req, afr_transaction_perform_fop,
+                     AFR_TRANSACTION_PRE_OP);
 
+    if (xdata_req)
+        dict_unref(xdata_req);
 
-int
-afr_post_nonblocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    return 0;
+next:
+    afr_transaction_perform_fop(frame, this);
 
-        local = frame->local;
-        int_lock = &local->internal_lock;
+    if (xdata_req)
+        dict_unref(xdata_req);
 
-        /* Initiate blocking locks if non-blocking has failed */
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks failed. Proceeding to blocking");
-                int_lock->lock_cbk = afr_post_blocking_entrylk_cbk;
-                afr_blocking_lock (frame, this);
-        } else {
+    return 0;
+err:
+    local->internal_lock.lock_cbk = afr_transaction_done;
+    local->op_ret = -1;
+    local->op_errno = op_errno;
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
-        }
+    afr_handle_lock_acquire_failure(local);
 
-        return 0;
+    if (xdata_req)
+        dict_unref(xdata_req);
+
+    return 0;
 }
 
+int
+afr_post_nonblocking_lock_cbk(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+
+    /* Initiate blocking locks if non-blocking has failed */
+    if (int_lock->lock_op_ret < 0) {
+        gf_msg_debug(this->name, 0,
+                     "Non blocking locks failed. Proceeding to blocking");
+        int_lock->lock_cbk = afr_internal_lock_finish;
+        afr_blocking_lock(frame, this);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Non blocking locks done. Proceeding to FOP");
+
+        afr_internal_lock_finish(frame, this);
+    }
+
+    return 0;
+}
 
 int
-afr_post_blocking_rename_cbk (call_frame_t *frame, xlator_t *this)
+afr_post_blocking_rename_cbk(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Blocking entrylks failed.");
-                local->transaction.done (frame, this);
-        } else {
+    if (int_lock->lock_op_ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INTERNAL_LKS_FAILED,
+               "Blocking entrylks failed.");
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Blocking entrylks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
-        }
-        return 0;
-}
+        afr_transaction_done(frame, this);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Blocking entrylks done. Proceeding to FOP");
 
+        afr_internal_lock_finish(frame, this);
+    }
+    return 0;
+}
 
 int
-afr_post_lower_unlock_cbk (call_frame_t *frame, xlator_t *this)
+afr_post_lower_unlock_cbk(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        GF_ASSERT (!int_lock->higher_locked);
+    GF_ASSERT(!int_lock->higher_locked);
 
-        int_lock->lock_cbk = afr_post_blocking_rename_cbk;
-        afr_blocking_lock (frame, this);
+    int_lock->lock_cbk = afr_post_blocking_rename_cbk;
+    afr_blocking_lock(frame, this);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_set_transaction_flock (afr_local_t *local)
+afr_set_transaction_flock(xlator_t *this, afr_local_t *local,
+                          afr_lockee_t *lockee)
 {
-        afr_internal_lock_t *int_lock = NULL;
-
-        int_lock = &local->internal_lock;
-
-        int_lock->lk_flock.l_len   = local->transaction.len;
-        int_lock->lk_flock.l_start = local->transaction.start;
-        int_lock->lk_flock.l_type  = F_WRLCK;
-
-        return 0;
+    afr_private_t *priv = NULL;
+    struct gf_flock *flock = NULL;
+
+    priv = this->private;
+    flock = &lockee->flock;
+
+    if ((priv->arbiter_count || local->transaction.eager_lock_on ||
+         priv->full_lock) &&
+        local->transaction.type == AFR_DATA_TRANSACTION) {
+        /*Lock entire file to avoid network split brains.*/
+        flock->l_len = 0;
+        flock->l_start = 0;
+    } else {
+        flock->l_len = local->transaction.len;
+        flock->l_start = local->transaction.start;
+    }
+    flock->l_type = F_WRLCK;
+
+    return 0;
 }
 
 int
-afr_lock_rec (call_frame_t *frame, xlator_t *this)
+afr_lock(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        int_lock->transaction_lk_type = AFR_TRANSACTION_LK;
+    int_lock->lock_cbk = afr_post_nonblocking_lock_cbk;
+    int_lock->domain = this->name;
 
-        switch (local->transaction.type) {
+    switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
-                afr_set_transaction_flock (local);
-
-                int_lock->lock_cbk = afr_post_nonblocking_inodelk_cbk;
+            for (i = 0; i < int_lock->lockee_count; i++) {
+                afr_set_transaction_flock(this, local, &int_lock->lockee[i]);
+            }
 
-                afr_nonblocking_inodelk (frame, this);
-                break;
+            break;
 
+        case AFR_ENTRY_TRANSACTION:
+            int_lock->lk_basename = local->transaction.basename;
+            if (local->transaction.parent_loc.path)
+                int_lock->lk_loc = &local->transaction.parent_loc;
+            else
+                GF_ASSERT(local->fd);
+            break;
         case AFR_ENTRY_RENAME_TRANSACTION:
+            break;
+    }
+    afr_lock_nonblocking(frame, this);
 
-                int_lock->lock_cbk = afr_post_blocking_rename_cbk;
-                afr_blocking_lock (frame, this);
-                break;
+    return 0;
+}
 
-        case AFR_ENTRY_TRANSACTION:
-                int_lock->lk_basename = local->transaction.basename;
-                if (&local->transaction.parent_loc)
-                        int_lock->lk_loc = &local->transaction.parent_loc;
-                else
-                        GF_ASSERT (local->fd);
+static gf_boolean_t
+afr_locals_overlap(afr_local_t *local1, afr_local_t *local2)
+{
+    uint64_t start1 = local1->transaction.start;
+    uint64_t start2 = local2->transaction.start;
+    uint64_t end1 = 0;
+    uint64_t end2 = 0;
+
+    if (local1->transaction.len)
+        end1 = start1 + local1->transaction.len - 1;
+    else
+        end1 = ULLONG_MAX;
+
+    if (local2->transaction.len)
+        end2 = start2 + local2->transaction.len - 1;
+    else
+        end2 = ULLONG_MAX;
+
+    return ((end1 >= start2) && (end2 >= start1));
+}
 
-                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
-                afr_nonblocking_entrylk (frame, this);
-                break;
+gf_boolean_t
+afr_has_lock_conflict(afr_local_t *local, gf_boolean_t waitlist_check)
+{
+    afr_local_t *each = NULL;
+    afr_lock_t *lock = NULL;
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    /*
+     * Once full file lock is acquired in eager-lock phase, overlapping
+     * writes do not compete for inode-locks, instead are transferred to the
+     * next writes. Because of this overlapping writes are not ordered.
+     * This can cause inconsistencies in replication.
+     * Example:
+     * Two overlapping writes w1, w2 are sent in parallel on same fd
+     * in two threads t1, t2.
+     * Both threads can execute afr_writev_wind in the following manner.
+     * t1 winds w1 on brick-0
+     * t2 winds w2 on brick-0
+     * t2 winds w2 on brick-1
+     * t1 winds w1 on brick-1
+     *
+     * This check makes sure the locks are not transferred for
+     * overlapping writes.
+     */
+    list_for_each_entry(each, &lock->owners, transaction.owner_list)
+    {
+        if (afr_locals_overlap(each, local)) {
+            return _gf_true;
         }
+    }
+
+    if (!waitlist_check)
+        return _gf_false;
+    list_for_each_entry(each, &lock->waiting, transaction.wait_list)
+    {
+        if (afr_locals_overlap(each, local)) {
+            return _gf_true;
+        }
+    }
+    return _gf_false;
+}
 
-        return 0;
+/* }}} */
+static void
+afr_copy_inodelk_vars(afr_internal_lock_t *dst, afr_internal_lock_t *src,
+                      xlator_t *this, int lockee_num)
+{
+    afr_private_t *priv = this->private;
+    afr_lockee_t *sl = &src->lockee[lockee_num];
+    afr_lockee_t *dl = &dst->lockee[lockee_num];
+
+    dst->domain = src->domain;
+    dl->flock.l_len = sl->flock.l_len;
+    dl->flock.l_start = sl->flock.l_start;
+    dl->flock.l_type = sl->flock.l_type;
+    dl->locked_count = sl->locked_count;
+    memcpy(dl->locked_nodes, sl->locked_nodes,
+           priv->child_count * sizeof(*dl->locked_nodes));
 }
 
+void
+__afr_transaction_wake_shared(afr_local_t *local, struct list_head *shared)
+{
+    gf_boolean_t conflict = _gf_false;
+    afr_local_t *each = NULL;
+    afr_lock_t *lock = &local->inode_ctx->lock[local->transaction.type];
+
+    while (!conflict) {
+        if (list_empty(&lock->waiting))
+            return;
+        each = list_entry(lock->waiting.next, afr_local_t,
+                          transaction.wait_list);
+        if (afr_has_lock_conflict(each, _gf_false)) {
+            conflict = _gf_true;
+        }
+        if (conflict && !list_empty(&lock->owners))
+            return;
+        afr_copy_inodelk_vars(&each->internal_lock, &local->internal_lock,
+                              each->transaction.frame->this, 0);
+        list_move_tail(&each->transaction.wait_list, shared);
+        list_add_tail(&each->transaction.owner_list, &lock->owners);
+    }
+}
+
+static void
+afr_lock_resume_shared(struct list_head *list)
+{
+    afr_local_t *each = NULL;
+
+    while (!list_empty(list)) {
+        each = list_entry(list->next, afr_local_t, transaction.wait_list);
+        list_del_init(&each->transaction.wait_list);
+        afr_changelog_pre_op(each->transaction.frame,
+                             each->transaction.frame->this);
+    }
+}
 
 int
-afr_lock (call_frame_t *frame, xlator_t *this)
+afr_internal_lock_finish(call_frame_t *frame, xlator_t *this)
 {
-        afr_set_lock_number (frame, this);
+    afr_local_t *local = frame->local;
+    afr_lock_t *lock = NULL;
+
+    local->internal_lock.lock_cbk = NULL;
+    if (!local->transaction.eager_lock_on) {
+        if (local->internal_lock.lock_op_ret < 0) {
+            afr_transaction_done(frame, this);
+            return 0;
+        }
+        afr_changelog_pre_op(frame, this);
+    } else {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        if (local->internal_lock.lock_op_ret < 0) {
+            afr_handle_lock_acquire_failure(local);
+        } else {
+            lock->event_generation = local->event_generation;
+            afr_changelog_pre_op(frame, this);
+        }
+    }
 
-        return afr_lock_rec (frame, this);
+    return 0;
 }
 
+gf_boolean_t
+afr_are_conflicting_ops_waiting(afr_local_t *local, xlator_t *this)
+{
+    afr_lock_t *lock = NULL;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+
+    /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
+     * is taken mount2 opened the same file, it won't be able to
+     * perform any {meta,}data operations until mount1 releases eager-lock.
+     * To avoid such scenario do not enable eager-lock for this transaction
+     * if open-fd-count is > 1 for metadata transactions and if num-inodelks > 1
+     * for data transactions
+     */
+
+    if (local->transaction.type == AFR_METADATA_TRANSACTION) {
+        if (local->inode_ctx->open_fd_count > 1) {
+            return _gf_true;
+        }
+    } else if (local->transaction.type == AFR_DATA_TRANSACTION) {
+        if (lock->num_inodelks > 1) {
+            return _gf_true;
+        }
+    }
 
-/* }}} */
+    return _gf_false;
+}
 
-int
-afr_internal_lock_finish (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_delayed_changelog_post_op_needed(call_frame_t *frame, xlator_t *this,
+                                        int delay)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+    gf_boolean_t res = _gf_false;
+
+    local = frame->local;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+
+    if (!afr_txn_nothing_failed(frame, this)) {
+        lock->release = _gf_true;
+        goto out;
+    }
+
+    if (afr_are_conflicting_ops_waiting(local, this)) {
+        lock->release = _gf_true;
+        goto out;
+    }
+
+    if (!list_empty(&lock->owners))
+        goto out;
+    else
+        GF_ASSERT(list_empty(&lock->waiting));
+
+    if (lock->release) {
+        goto out;
+    }
+
+    if (!delay) {
+        goto out;
+    }
+
+    if (local->transaction.disable_delayed_post_op) {
+        goto out;
+    }
+
+    if ((local->op != GF_FOP_WRITE) && (local->op != GF_FOP_FXATTROP) &&
+        (local->op != GF_FOP_FSYNC)) {
+        /*Only allow writes/fsyncs but shard does [f]xattrops on writes, so
+         * they are fine too*/
+        goto out;
+    }
+
+    res = _gf_true;
+out:
+    return res;
+}
 
-        priv  = this->private;
-        local = frame->local;
+void
+afr_delayed_changelog_wake_up_cbk(void *data)
+{
+    afr_lock_t *lock = NULL;
+    afr_local_t *local = data;
+    afr_local_t *timer_local = NULL;
+    struct list_head shared;
+
+    INIT_LIST_HEAD(&shared);
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        timer_local = list_entry(lock->post_op.next, afr_local_t,
+                                 transaction.owner_list);
+        if (list_empty(&lock->owners) && (local == timer_local)) {
+            GF_ASSERT(list_empty(&lock->waiting));
+            /*Last owner*/
+            lock->release = _gf_true;
+            lock->delay_timer = NULL;
+        }
+    }
+    UNLOCK(&local->inode->lock);
+    afr_changelog_post_op_now(local->transaction.frame,
+                              local->transaction.frame->this);
+}
 
-        if (__fop_changelog_needed (frame, this)) {
-                afr_changelog_pre_op (frame, this);
-        } else {
-                __mark_all_success (local->pending, priv->child_count,
-                                    local->transaction.type);
+/* SET operation */
+int
+afr_fd_report_unstable_write(xlator_t *this, afr_local_t *local)
+{
+    LOCK(&local->inode->lock);
+    {
+        local->inode_ctx->witnessed_unstable_write = _gf_true;
+    }
+    UNLOCK(&local->inode->lock);
+
+    return 0;
+}
 
+/* TEST and CLEAR operation */
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write(xlator_t *this, inode_t *inode)
+{
+    afr_inode_ctx_t *ctx = NULL;
+    gf_boolean_t witness = _gf_false;
 
-		/*  Perform fops with the lk-owner from top xlator.
-		 *  Eg: lk-owner of posix-lk and flush should be same,
-		 *  flush cant clear the  posix-lks without that lk-owner.
-		 */
-		afr_save_lk_owner (frame);
-		frame->root->lk_owner =
-			local->transaction.main_frame->root->lk_owner;
+    LOCK(&inode->lock);
+    {
+        (void)__afr_inode_ctx_get(this, inode, &ctx);
 
-                local->transaction.fop (frame, this);
+        if (ctx->witnessed_unstable_write) {
+            witness = _gf_true;
+            ctx->witnessed_unstable_write = _gf_false;
         }
+    }
+    UNLOCK(&inode->lock);
 
-        return 0;
+    return witness;
 }
 
-
-void
-afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this)
+int
+afr_changelog_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *pre,
+                        struct iatt *post, dict_t *xdata)
 {
-	afr_local_t    *local = NULL;
-	afr_private_t  *priv = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_local_t *local = NULL;
 
-	/* call this function from any of the related optimizations
-	   which benefit from delaying post op are enabled, namely:
+    priv = this->private;
+    local = frame->local;
 
-	   - changelog piggybacking
-	   - eager locking
-	*/
+    if (op_ret != 0) {
+        /* Failure of fsync() is as good as failure of previous
+           write(). So treat it like one.
+        */
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, AFR_MSG_FSYNC_FAILED,
+               "fsync(%s) failed on subvolume %s. Transaction was %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name, gf_fop_list[local->op]);
 
-	priv = this->private;
-	if (!priv)
-		return;
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
 
-	if (!priv->post_op_delay_secs)
-		return;
+    call_count = afr_frame_return(frame);
 
-	local = frame->local;
-	if (!local)
-		return;
+    if (call_count == 0)
+        afr_changelog_post_op_now(frame, this);
 
-	if (!local->fd)
-		return;
-
-	if (local->op == GF_FOP_WRITE)
-		local->delayed_post_op = _gf_true;
+    return 0;
 }
 
-
-gf_boolean_t
-is_afr_delayed_changelog_post_op_needed (call_frame_t *frame, xlator_t *this)
+int
+afr_changelog_fsync(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t      *local = NULL;
-	gf_boolean_t      res = _gf_false;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    GF_UNUSED int ret = -1;
 
-	local = frame->local;
-	if (!local)
-		goto out;
+    local = frame->local;
+    priv = this->private;
 
-	if (!local->delayed_post_op)
-		goto out;
+    call_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
 
-	res = _gf_true;
-out:
-	return res;
-}
+    if (!call_count) {
+        /* will go straight to unlock */
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
 
+    local->call_count = call_count;
 
-void
-afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd);
+    xdata = dict_new();
+    if (xdata) {
+        ret = dict_set_int32_sizen(xdata, "batch-fsync", 1);
+        ret = dict_set_str(xdata, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    }
 
-void
-afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.pre_op[i])
+            continue;
 
-void
-afr_delayed_changelog_wake_up_cbk (void *data)
-{
-	fd_t           *fd = NULL;
+        STACK_WIND_COOKIE(frame, afr_changelog_fsync_cbk, (void *)(long)i,
+                          priv->children[i], priv->children[i]->fops->fsync,
+                          local->fd, 1, xdata);
+        if (!--call_count)
+            break;
+    }
 
-	fd = data;
+    if (xdata)
+        dict_unref(xdata);
 
-	afr_delayed_changelog_wake_up (THIS, fd);
+    return 0;
 }
 
+int
+afr_changelog_post_op_safe(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-void
-afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd)
-{
-	afr_fd_ctx_t      *fd_ctx = NULL;
-	call_frame_t      *prev_frame = NULL;
-	struct timeval     delta = {0, };
-	afr_private_t     *priv = NULL;
-
-	priv = this->private;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		return;
-
-	delta.tv_sec = priv->post_op_delay_secs;
-	delta.tv_usec = 0;
-
-	pthread_mutex_lock (&fd_ctx->delay_lock);
-	{
-		prev_frame = fd_ctx->delay_frame;
-		fd_ctx->delay_frame = NULL;
-		if (fd_ctx->delay_timer)
-			gf_timer_call_cancel (this->ctx, fd_ctx->delay_timer);
-		fd_ctx->delay_timer = NULL;
-		if (!frame)
-			goto unlock;
-		fd_ctx->delay_timer = gf_timer_call_after (this->ctx, delta,
-							   afr_delayed_changelog_wake_up_cbk,
-							   fd);
-		fd_ctx->delay_frame = frame;
-	}
-unlock:
-	pthread_mutex_unlock (&fd_ctx->delay_lock);
+    local = frame->local;
+    priv = this->private;
 
-	if (prev_frame) {
-		afr_changelog_post_op_now (prev_frame, this);
-	}
+    if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
+
+    if (afr_changelog_pre_op_uninherit(frame, this) &&
+        afr_txn_nothing_failed(frame, this)) {
+        /* just detected that this post-op is about to
+           be optimized away as a new write() has
+           already piggybacked on this frame's changelog.
+           */
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
+
+    /* Calling afr_changelog_post_op_now() now will result in
+       issuing ->[f]xattrop().
+
+       Performing a hard POST-OP (->[f]xattrop() FOP) is a more
+       responsible operation that what it might appear on the surface.
+
+       The changelog of a file (in the xattr of the file on the server)
+       stores information (pending count) about the state of the file
+       on the OTHER server. This changelog is blindly trusted, and must
+       therefore be updated in such a way it remains trustworthy. This
+       implies that decrementing the pending count (essentially "clearing
+       the dirty flag") must be done STRICTLY after we are sure that the
+       operation on the other server has reached stable storage.
+
+       While the backend filesystem on that server will eventually flush
+       it to stable storage, we (being in userspace) have no mechanism
+       to get notified when the write became "stable".
+
+       This means we need take matter into our own hands and issue an
+       fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
+       and get an acknowledgement for it. And we need to wait for the
+       fsync() acknowledgement before initiating the hard POST-OP.
+
+       However if the FD itself was opened in O_SYNC or O_DSYNC then
+       we are already guaranteed that the writes were made stable as
+       part of the FOP itself. The same holds true for NFS stable
+       writes which happen on an anonymous FD with O_DSYNC or O_SYNC
+       flag set in the writev() @flags param. For all other write types,
+       mark a flag in the fdctx whenever an unstable write is witnessed.
+       */
+
+    if (!afr_fd_has_witnessed_unstable_write(this, local->inode)) {
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
+
+    /* Check whether users want durability and perform fsync/post-op
+     * accordingly.
+     */
+    if (priv->ensure_durability) {
+        /* Time to fsync() */
+        afr_changelog_fsync(frame, this);
+    } else {
+        afr_changelog_post_op_now(frame, this);
+    }
+
+    return 0;
 }
 
-
 void
-afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+afr_changelog_post_op(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t  *local = NULL;
+    struct timespec delta = {
+        0,
+    };
+    afr_private_t *priv = NULL;
+    afr_local_t *local = frame->local;
+    afr_lock_t *lock = NULL;
+    gf_boolean_t post_op = _gf_true;
+    struct list_head shared;
+
+    priv = this->private;
+    delta.tv_sec = priv->post_op_delay_secs;
+    delta.tv_nsec = 0;
+
+    INIT_LIST_HEAD(&shared);
+    if (!local->transaction.eager_lock_on)
+        goto out;
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        list_del_init(&local->transaction.owner_list);
+        list_add(&local->transaction.owner_list, &lock->post_op);
+        __afr_transaction_wake_shared(local, &shared);
+
+        if (!afr_is_delayed_changelog_post_op_needed(frame, this,
+                                                     delta.tv_sec)) {
+            if (list_empty(&lock->owners))
+                lock->release = _gf_true;
+            goto unlock;
+        }
+
+        GF_ASSERT(lock->delay_timer == NULL);
+        lock->delay_timer = gf_timer_call_after(
+            this->ctx, delta, afr_delayed_changelog_wake_up_cbk, local);
+        if (!lock->delay_timer) {
+            lock->release = _gf_true;
+        } else {
+            post_op = _gf_false;
+        }
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
 
-	local = frame->local;
+    if (!list_empty(&shared)) {
+        afr_lock_resume_shared(&shared);
+    }
 
-	if (is_afr_delayed_changelog_post_op_needed (frame, this))
-		afr_delayed_changelog_post_op (this, frame, local->fd);
-	else
-		afr_changelog_post_op_now (frame, this);
+out:
+    if (post_op) {
+        if (!local->transaction.eager_lock_on || lock->release) {
+            afr_changelog_post_op_safe(frame, this);
+        } else {
+            afr_changelog_post_op_now(frame, this);
+        }
+    }
 }
 
-
-void
-afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd)
+int
+afr_transaction_resume(call_frame_t *frame, xlator_t *this)
 {
-	afr_delayed_changelog_post_op (this, NULL, fd);
-}
+    afr_local_t *local = NULL;
 
+    local = frame->local;
 
-int
-afr_transaction_resume (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-	fd_t                *fd = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-	fd       = local->fd;
-
-	if (fd)
-		/* The wake up needs to happen independent of
-		   what type of fop arrives here. If it was
-		   a write, then it has already inherited the
-		   lock and changelog. If it was not a write,
-		   then the presumption of the optimization (of
-		   optimizing for successive write operations)
-		   fails.
-		*/
-		afr_delayed_changelog_wake_up (this, fd);
-
-	afr_restore_lk_owner (frame);
-
-        if (__fop_changelog_needed (frame, this)) {
-                afr_changelog_post_op (frame, this);
-        } else {
-                if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-                        local->transaction.done (frame, this);
-                } else {
-                        int_lock->lock_cbk = local->transaction.done;
-                        afr_unlock (frame, this);
-                }
-        }
+    afr_restore_lk_owner(frame);
 
-        return 0;
-}
+    afr_handle_symmetric_errors(frame, this);
 
+    if (!local->pre_op_compat)
+        /* new mode, pre-op was done along
+           with OP */
+        afr_changelog_pre_op_update(frame, this);
+
+    afr_changelog_post_op(frame, this);
+
+    return 0;
+}
 
 /**
  * afr_transaction_fop_failed - inform that an fop failed
  */
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this, int child_index)
+afr_transaction_fop_failed(call_frame_t *frame, xlator_t *this, int child_index)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
 
-        __mark_child_dead (local->pending, priv->child_count,
-                           child_index, local->transaction.type);
+    local->transaction.failed_subvols[child_index] = 1;
 }
 
-int
-afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type)
+static gf_boolean_t
+__need_previous_lock_unlocked(afr_local_t *local)
+{
+    afr_lock_t *lock = NULL;
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    if (!lock->acquired)
+        return _gf_false;
+    if (lock->acquired && lock->event_generation != local->event_generation)
+        return _gf_true;
+    return _gf_false;
+}
+
+void
+__afr_eager_lock_handle(afr_local_t *local, gf_boolean_t *take_lock,
+                        gf_boolean_t *do_pre_op, afr_local_t **timer_local)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
+    afr_lock_t *lock = NULL;
+    afr_local_t *owner_local = NULL;
+    xlator_t *this = local->transaction.frame->this;
+
+    local->transaction.eager_lock_on = _gf_true;
+    afr_set_lk_owner(local->transaction.frame, this, local->inode);
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    if (__need_previous_lock_unlocked(local)) {
+        if (!list_empty(&lock->owners)) {
+            lock->release = _gf_true;
+        } else if (lock->delay_timer) {
+            lock->release = _gf_true;
+            if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+                /* It will be put in frozen list
+                 * in the code flow below*/
+            } else {
+                *timer_local = list_entry(lock->post_op.next, afr_local_t,
+                                          transaction.owner_list);
+                lock->delay_timer = NULL;
+            }
+        }
+    }
+
+    if (lock->release) {
+        list_add_tail(&local->transaction.wait_list, &lock->frozen);
+        *take_lock = _gf_false;
+        goto out;
+    }
+
+    if (lock->delay_timer) {
+        *take_lock = _gf_false;
+        if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+            list_add_tail(&local->transaction.wait_list, &lock->frozen);
+        } else {
+            *timer_local = list_entry(lock->post_op.next, afr_local_t,
+                                      transaction.owner_list);
+            afr_copy_inodelk_vars(&local->internal_lock,
+                                  &(*timer_local)->internal_lock, this, 0);
+            lock->delay_timer = NULL;
+            *do_pre_op = _gf_true;
+            list_add_tail(&local->transaction.owner_list, &lock->owners);
+        }
+        goto out;
+    }
+
+    if (!list_empty(&lock->owners)) {
+        if (!lock->acquired || afr_has_lock_conflict(local, _gf_true)) {
+            list_add_tail(&local->transaction.wait_list, &lock->waiting);
+            *take_lock = _gf_false;
+            goto out;
+        }
+        owner_local = list_entry(lock->owners.next, afr_local_t,
+                                 transaction.owner_list);
+        afr_copy_inodelk_vars(&local->internal_lock,
+                              &owner_local->internal_lock, this, 0);
+        *take_lock = _gf_false;
+        *do_pre_op = _gf_true;
+    }
+
+    if (lock->acquired)
+        GF_ASSERT(!(*take_lock));
+    list_add_tail(&local->transaction.owner_list, &lock->owners);
+out:
+    return;
+}
+
+void
+afr_transaction_start(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    gf_boolean_t take_lock = _gf_true;
+    gf_boolean_t do_pre_op = _gf_false;
+    afr_local_t *timer_local = NULL;
+
+    priv = this->private;
+
+    if (local->transaction.type != AFR_DATA_TRANSACTION &&
+        local->transaction.type != AFR_METADATA_TRANSACTION)
+        goto lock_phase;
+
+    if (!priv->eager_lock)
+        goto lock_phase;
+
+    LOCK(&local->inode->lock);
+    {
+        __afr_eager_lock_handle(local, &take_lock, &do_pre_op, &timer_local);
+    }
+    UNLOCK(&local->inode->lock);
+lock_phase:
+    if (!local->transaction.eager_lock_on) {
+        afr_set_lk_owner(local->transaction.frame, this,
+                         local->transaction.frame->root);
+    }
+
+    if (take_lock) {
+        afr_lock(local->transaction.frame, this);
+    } else if (do_pre_op) {
+        afr_changelog_pre_op(local->transaction.frame, this);
+    }
+    /*Always call delayed_changelog_wake_up_cbk after calling pre-op above
+     * so that any inheriting can happen*/
+    if (timer_local)
+        afr_delayed_changelog_wake_up_cbk(timer_local);
+}
 
-        local = frame->local;
-        priv  = this->private;
+int
+afr_write_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
+{
+    afr_local_t *local = frame->local;
+
+    if (err) {
+        AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, err);
+        goto fail;
+    }
+
+    afr_transaction_start(local, this);
+    return 0;
+fail:
+    local->transaction.unwind(frame, this);
+    AFR_STACK_DESTROY(frame);
+    return 0;
+}
 
-	if (local->fd && priv->eager_lock &&
-	    local->transaction.type == AFR_DATA_TRANSACTION)
-		afr_set_lk_owner (frame, this, local->fd);
-	else
-		afr_set_lk_owner (frame, this, frame->root);
+int
+afr_transaction_lockee_init(call_frame_t *frame)
+{
+    afr_local_t *local = frame->local;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_private_t *priv = frame->this->private;
+    int ret = 0;
 
-        afr_transaction_local_init (local, this);
+    switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            ret = afr_add_inode_lockee(local, priv->child_count);
+            break;
 
-        local->transaction.resume = afr_transaction_resume;
-        local->transaction.type   = type;
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            ret = afr_add_entry_lockee(local, &local->transaction.parent_loc,
+                                       local->transaction.basename,
+                                       priv->child_count);
+            if (ret) {
+                goto out;
+            }
+            if (local->op == GF_FOP_RENAME) {
+                ret = afr_add_entry_lockee(
+                    local, &local->transaction.new_parent_loc,
+                    local->transaction.new_basename, priv->child_count);
+                if (ret) {
+                    goto out;
+                }
 
-        if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-                afr_internal_lock_finish (frame, this);
-        } else {
-                afr_lock (frame, this);
-        }
+                if (local->newloc.inode &&
+                    IA_ISDIR(local->newloc.inode->ia_type)) {
+                    ret = afr_add_entry_lockee(local, &local->newloc, NULL,
+                                               priv->child_count);
+                    if (ret) {
+                        goto out;
+                    }
+                }
+            } else if (local->op == GF_FOP_RMDIR) {
+                ret = afr_add_entry_lockee(local, &local->loc, NULL,
+                                           priv->child_count);
+                if (ret) {
+                    goto out;
+                }
+            }
+
+            if (int_lock->lockee_count > 1) {
+                qsort(int_lock->lockee, int_lock->lockee_count,
+                      sizeof(*int_lock->lockee), afr_entry_lockee_cmp);
+            }
+            break;
+    }
+out:
+    return ret;
+}
 
-        return 0;
+int
+afr_transaction(call_frame_t *frame, xlator_t *this, afr_transaction_type type)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    int event_generation = 0;
+
+    local = frame->local;
+    priv = this->private;
+    local->transaction.frame = frame;
+
+    local->transaction.type = type;
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        ret = -afr_quorum_errno(priv);
+        goto out;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &ret)) {
+        ret = -ret; /*op_errno to ret conversion*/
+        goto out;
+    }
+
+    if (priv->thin_arbiter_count && !afr_ta_has_quorum(priv, local)) {
+        ret = -afr_quorum_errno(priv);
+        goto out;
+    }
+
+    ret = afr_transaction_local_init(local, this);
+    if (ret < 0)
+        goto out;
+
+    ret = afr_transaction_lockee_init(frame);
+    if (ret)
+        goto out;
+
+    if (type != AFR_METADATA_TRANSACTION) {
+        goto txn_start;
+    }
+
+    ret = afr_inode_get_readable(frame, local->inode, this, local->readable,
+                                 &event_generation, type);
+    if (ret < 0 ||
+        afr_is_inode_refresh_reqd(local->inode, this, priv->event_generation,
+                                  event_generation)) {
+        afr_inode_refresh(frame, this, local->inode, local->loc.gfid,
+                          afr_write_txn_refresh_done);
+        ret = 0;
+        goto out;
+    }
+
+txn_start:
+    ret = 0;
+    afr_transaction_start(local, this);
+out:
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-transaction.h b/xlators/cluster/afr/src/afr-transaction.h
index 102599633e3..beefa26f4a6 100644
--- a/xlators/cluster/afr/src/afr-transaction.h
+++ b/xlators/cluster/afr/src/afr-transaction.h
@@ -11,26 +11,65 @@
 #ifndef __TRANSACTION_H__
 #define __TRANSACTION_H__
 
-typedef enum {
-        LOCAL_FIRST = 1,
-        LOCAL_LAST = 2
-} afr_xattrop_type_t;
+#include "afr.h"
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
-			    int child_index);
+afr_transaction_fop_failed(call_frame_t *frame, xlator_t *this,
+                           int child_index);
+
+int32_t
+afr_transaction(call_frame_t *frame, xlator_t *this, afr_transaction_type type);
 
 int
-afr_lock_server_count (afr_private_t *priv, afr_transaction_type type);
+afr_set_pending_dict(afr_private_t *priv, dict_t *xattr, int32_t **pending);
 
-int32_t
-afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type);
+void
+afr_delayed_changelog_wake_up(xlator_t *this, fd_t *fd);
+
+void
+__mark_all_success(call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+afr_txn_nothing_failed(call_frame_t *frame, xlator_t *this);
 
-afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
-                      int child, afr_xattrop_type_t op);
+afr_read_txn(call_frame_t *frame, xlator_t *this, inode_t *inode,
+             afr_read_txn_wind_t readfn, afr_transaction_type type);
+
+int
+afr_read_txn_continue(call_frame_t *frame, xlator_t *this, int subvol);
+
 void
-afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this);
+afr_pending_read_increment(afr_private_t *priv, int child_index);
+
+void
+afr_pending_read_decrement(afr_private_t *priv, int child_index);
+
+call_frame_t *
+afr_transaction_detach_fop_frame(call_frame_t *frame);
+gf_boolean_t
+afr_has_quorum(unsigned char *subvols, xlator_t *this, call_frame_t *frame);
+gf_boolean_t
+afr_needs_changelog_update(afr_local_t *local);
+void
+afr_zero_fill_stat(afr_local_t *local);
+
+void
+afr_pick_error_xdata(afr_local_t *local, afr_private_t *priv, inode_t *inode1,
+                     unsigned char *readable1, inode_t *inode2,
+                     unsigned char *readable2);
+int
+afr_transaction_resume(call_frame_t *frame, xlator_t *this);
+
+int
+afr_lock(call_frame_t *frame, xlator_t *this);
+
+void
+afr_delayed_changelog_wake_up_cbk(void *data);
+
+int
+afr_release_notify_lock_for_ta(void *opaque);
+
+int
+afr_ta_lock_release_done(int ret, call_frame_t *ta_frame, void *opaque);
 #endif /* __TRANSACTION_H__ */
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 846f985de51..df7366f0a65 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -15,660 +15,1330 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
 #include "afr-common.c"
-
-#define SHD_INODE_LRU_LIMIT          2048
-#define AFR_EH_HEALED_LIMIT          1024
-#define AFR_EH_HEAL_FAIL_LIMIT       1024
-#define AFR_EH_SPLIT_BRAIN_LIMIT     1024
+#include "afr-messages.h"
 
 struct volume_options options[];
 
+static char *afr_favorite_child_policies[AFR_FAV_CHILD_POLICY_MAX + 1] = {
+    [AFR_FAV_CHILD_NONE] = "none",
+    [AFR_FAV_CHILD_BY_SIZE] = "size",
+    [AFR_FAV_CHILD_BY_CTIME] = "ctime",
+    [AFR_FAV_CHILD_BY_MTIME] = "mtime",
+    [AFR_FAV_CHILD_BY_MAJORITY] = "majority",
+    [AFR_FAV_CHILD_POLICY_MAX] = NULL,
+};
+
 int32_t
-notify (xlator_t *this, int32_t event,
-        void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int ret = -1;
-        va_list         ap;
-        void *data2 = NULL;
+    int ret = -1;
+    va_list ap;
+    void *data2 = NULL;
 
-        va_start (ap, data);
-        data2 = va_arg (ap, dict_t*);
-        va_end (ap);
-        ret = afr_notify (this, event, data, data2);
+    va_start(ap, data);
+    data2 = va_arg(ap, dict_t *);
+    va_end(ap);
+    ret = afr_notify(this, event, data, data2);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
+
+    if (!this)
+        return ret;
+
+    ret = xlator_mem_acct_init(this, gf_afr_mt_end + 1);
 
-        if (!this)
-                return ret;
+    if (ret != 0) {
+        return ret;
+    }
 
-        ret = xlator_mem_acct_init (this, gf_afr_mt_end + 1);
+    return ret;
+}
 
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       "failed");
-                return ret;
+int
+xlator_subvolume_index(xlator_t *this, xlator_t *subvol)
+{
+    int index = -1;
+    int i = 0;
+    xlator_list_t *list = NULL;
+
+    list = this->children;
+
+    while (list) {
+        if (subvol == list->xlator ||
+            strcmp(subvol->name, list->xlator->name) == 0) {
+            index = i;
+            break;
         }
+        list = list->next;
+        i++;
+    }
 
-        return ret;
+    return index;
 }
 
+static void
+fix_quorum_options(xlator_t *this, afr_private_t *priv, char *qtype,
+                   dict_t *options)
+{
+    if (dict_get_sizen(options, "quorum-type") == NULL) {
+        /* If user doesn't configure anything enable auto-quorum if the
+         * replica has more than two subvolumes */
+        if (priv->child_count > 2)
+            qtype = "auto";
+    }
+
+    if (priv->quorum_count && strcmp(qtype, "fixed")) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_OVERRIDE,
+               "quorum-type %s overriding quorum-count %u", qtype,
+               priv->quorum_count);
+    }
+
+    if (!strcmp(qtype, "none")) {
+        priv->quorum_count = 0;
+    } else if (!strcmp(qtype, "auto")) {
+        priv->quorum_count = AFR_QUORUM_AUTO;
+    }
+}
 
 int
-xlator_subvolume_index (xlator_t *this, xlator_t *subvol)
+afr_set_favorite_child_policy(afr_private_t *priv, char *policy)
 {
-        int index = -1;
-        int i = 0;
-        xlator_list_t *list = NULL;
-
-        list = this->children;
-
-        while (list) {
-                if (subvol == list->xlator ||
-                    strcmp (subvol->name, list->xlator->name) == 0) {
-                        index = i;
-                        break;
-                }
-                list = list->next;
-                i++;
-        }
+    int index = -1;
+
+    index = gf_get_index_by_elem(afr_favorite_child_policies, policy);
+    if (index < 0 || index >= AFR_FAV_CHILD_POLICY_MAX)
+        return -1;
+
+    priv->fav_child_policy = index;
+
+    return 0;
+}
 
-        return index;
+static void
+set_data_self_heal_algorithm(afr_private_t *priv, char *algo)
+{
+    if (!algo) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DYNAMIC;
+    } else if (strcmp(algo, "full") == 0) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_FULL;
+    } else if (strcmp(algo, "diff") == 0) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DIFF;
+    } else {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DYNAMIC;
+    }
 }
 
 void
-fix_quorum_options (xlator_t *this, afr_private_t *priv, char *qtype)
+afr_handle_anon_inode_options(afr_private_t *priv, dict_t *options)
 {
-        if (priv->quorum_count && strcmp(qtype,"fixed")) {
-                gf_log(this->name,GF_LOG_WARNING,
-                       "quorum-type %s overriding quorum-count %u",
-                       qtype, priv->quorum_count);
-        }
-        if (!strcmp(qtype,"none")) {
-                priv->quorum_count = 0;
-        }
-        else if (!strcmp(qtype,"auto")) {
-                priv->quorum_count = AFR_QUORUM_AUTO;
-        }
+    char *volfile_id_str = NULL;
+    uuid_t anon_inode_gfid = {0};
+
+    /*If volume id is not present don't enable anything*/
+    if (dict_get_str(options, "volume-id", &volfile_id_str))
+        return;
+    GF_ASSERT(strlen(AFR_ANON_DIR_PREFIX) + strlen(volfile_id_str) <= NAME_MAX);
+    /*anon_inode_name is not supposed to change once assigned*/
+    if (!priv->anon_inode_name[0]) {
+        snprintf(priv->anon_inode_name, sizeof(priv->anon_inode_name), "%s-%s",
+                 AFR_ANON_DIR_PREFIX, volfile_id_str);
+        gf_uuid_parse(volfile_id_str, anon_inode_gfid);
+        /*Flip a bit to make sure volfile-id and anon-gfid are not same*/
+        anon_inode_gfid[0] ^= 1;
+        uuid_utoa_r(anon_inode_gfid, priv->anon_gfid_str);
+    }
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        afr_private_t *priv        = NULL;
-        xlator_t      *read_subvol = NULL;
-        int            read_subvol_index = -1;
-        int            ret         = -1;
-        int            index       = -1;
-        char          *qtype       = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t *read_subvol = NULL;
+    int read_subvol_index = -1;
+    int timeout_old = 0;
+    int ret = -1;
+    int index = -1;
+    char *qtype = NULL;
+    char *fav_child_policy = NULL;
+    char *data_self_heal = NULL;
+    char *data_self_heal_algorithm = NULL;
+    char *locking_scheme = NULL;
+    gf_boolean_t consistent_io = _gf_false;
+    gf_boolean_t choose_local_old = _gf_false;
+    gf_boolean_t enabled_old = _gf_false;
 
-        priv = this->private;
+    priv = this->private;
 
-        GF_OPTION_RECONF ("background-self-heal-count",
-                          priv->background_self_heal_count, options, uint32,
-                          out);
+    GF_OPTION_RECONF("metadata-splitbrain-forced-heal",
+                     priv->metadata_splitbrain_forced_heal, options, bool, out);
 
-        GF_OPTION_RECONF ("metadata-self-heal",
-                          priv->metadata_self_heal, options, bool, out);
+    GF_OPTION_RECONF("background-self-heal-count",
+                     priv->background_self_heal_count, options, uint32, out);
 
-        GF_OPTION_RECONF ("data-self-heal", priv->data_self_heal, options, str,
-                          out);
+    GF_OPTION_RECONF("heal-wait-queue-length", priv->heal_wait_qlen, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF ("entry-self-heal", priv->entry_self_heal, options,
-                          bool, out);
+    GF_OPTION_RECONF("metadata-self-heal", priv->metadata_self_heal, options,
+                     bool, out);
 
-        GF_OPTION_RECONF ("strict-readdir", priv->strict_readdir, options, bool,
-                          out);
+    GF_OPTION_RECONF("data-self-heal", data_self_heal, options, str, out);
+    if (gf_string2boolean(data_self_heal, &priv->data_self_heal) == -1)
+        goto out;
 
-        GF_OPTION_RECONF ("data-self-heal-window-size",
-                          priv->data_self_heal_window_size, options,
-                          uint32, out);
+    GF_OPTION_RECONF("entry-self-heal", priv->entry_self_heal, options, bool,
+                     out);
 
-        GF_OPTION_RECONF ("data-change-log", priv->data_change_log, options,
-                          bool, out);
+    GF_OPTION_RECONF("data-self-heal-window-size",
+                     priv->data_self_heal_window_size, options, uint32, out);
 
-        GF_OPTION_RECONF ("metadata-change-log",
-                          priv->metadata_change_log, options, bool, out);
+    GF_OPTION_RECONF("data-self-heal-algorithm", data_self_heal_algorithm,
+                     options, str, out);
+    set_data_self_heal_algorithm(priv, data_self_heal_algorithm);
 
-        GF_OPTION_RECONF ("entry-change-log", priv->entry_change_log, options,
-                          bool, out);
+    GF_OPTION_RECONF("halo-enabled", priv->halo_enabled, options, bool, out);
 
-        GF_OPTION_RECONF ("data-self-heal-algorithm",
-                          priv->data_self_heal_algorithm, options, str, out);
+    GF_OPTION_RECONF("halo-shd-max-latency", priv->shd.halo_max_latency_msec,
+                     options, uint32, out);
 
-        GF_OPTION_RECONF ("self-heal-daemon", priv->shd.enabled, options, bool, out);
+    GF_OPTION_RECONF("halo-nfsd-max-latency", priv->nfsd.halo_max_latency_msec,
+                     options, uint32, out);
 
-        GF_OPTION_RECONF ("read-subvolume", read_subvol, options, xlator, out);
+    GF_OPTION_RECONF("halo-max-latency", priv->halo_max_latency_msec, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF ("read-hash-mode", priv->hash_mode,
-                          options, uint32, out);
+    GF_OPTION_RECONF("halo-max-replicas", priv->halo_max_replicas, options,
+                     uint32, out);
 
-        if (read_subvol) {
-                index = xlator_subvolume_index (this, read_subvol);
-                if (index == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
-                                read_subvol->name);
-                        goto out;
-                }
-                priv->read_child = index;
-        }
+    GF_OPTION_RECONF("halo-min-replicas", priv->halo_min_replicas, options,
+                     uint32, out);
+
+    GF_OPTION_RECONF("read-subvolume", read_subvol, options, xlator, out);
 
-        GF_OPTION_RECONF ("read-subvolume-index",read_subvol_index, options,int32,out);
+    choose_local_old = priv->choose_local;
+    GF_OPTION_RECONF("choose-local", priv->choose_local, options, bool, out);
 
-        if (read_subvol_index >-1) {
-                index=read_subvol_index;
-                if (index >= priv->child_count) {
-                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
-                                index);
-                        goto out;
-                }
-                priv->read_child = index;
+    if (choose_local_old != priv->choose_local) {
+        priv->read_child = -1;
+        if (choose_local_old == _gf_false)
+            priv->did_discovery = _gf_false;
+    }
+
+    GF_OPTION_RECONF("read-hash-mode", priv->hash_mode, options, uint32, out);
+
+    if (read_subvol) {
+        index = xlator_subvolume_index(this, read_subvol);
+        if (index == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%s not a subvolume", read_subvol->name);
+            goto out;
         }
+        priv->read_child = index;
+    }
+
+    GF_OPTION_RECONF("read-subvolume-index", read_subvol_index, options, int32,
+                     out);
+
+    if (read_subvol_index > -1) {
+        index = read_subvol_index;
+        if (index >= priv->child_count) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%d not a subvolume-index", index);
+            goto out;
+        }
+        priv->read_child = index;
+    }
+
+    GF_OPTION_RECONF("pre-op-compat", priv->pre_op_compat, options, bool, out);
+    GF_OPTION_RECONF("locking-scheme", locking_scheme, options, str, out);
+    priv->granular_locks = (strcmp(locking_scheme, "granular") == 0);
+    GF_OPTION_RECONF("full-lock", priv->full_lock, options, bool, out);
+    GF_OPTION_RECONF("granular-entry-heal", priv->esh_granular, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("eager-lock", priv->eager_lock, options, bool, out);
+    GF_OPTION_RECONF("optimistic-change-log", priv->optimistic_change_log,
+                     options, bool, out);
+    GF_OPTION_RECONF("quorum-type", qtype, options, str, out);
+    GF_OPTION_RECONF("quorum-count", priv->quorum_count, options, uint32, out);
+    fix_quorum_options(this, priv, qtype, options);
+    if (priv->quorum_count && !afr_has_quorum(priv->child_up, this, NULL))
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
+               "Client-quorum is not met");
+
+    GF_OPTION_RECONF("post-op-delay-secs", priv->post_op_delay_secs, options,
+                     uint32, out);
+
+    GF_OPTION_RECONF(AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, options,
+                     size_uint64, out);
+    /* Reset this so we re-discover in case the topology changed.  */
+    GF_OPTION_RECONF("ensure-durability", priv->ensure_durability, options,
+                     bool, out);
+
+    enabled_old = priv->shd.enabled;
+    GF_OPTION_RECONF("self-heal-daemon", priv->shd.enabled, options, bool, out);
+
+    GF_OPTION_RECONF("iam-self-heal-daemon", priv->shd.iamshd, options, bool,
+                     out);
+
+    timeout_old = priv->shd.timeout;
+    GF_OPTION_RECONF("heal-timeout", priv->shd.timeout, options, int32, out);
+
+    GF_OPTION_RECONF("consistent-metadata", priv->consistent_metadata, options,
+                     bool, out);
+
+    GF_OPTION_RECONF("shd-max-threads", priv->shd.max_threads, options, uint32,
+                     out);
+
+    GF_OPTION_RECONF("shd-wait-qlength", priv->shd.wait_qlength, options,
+                     uint32, out);
+
+    GF_OPTION_RECONF("favorite-child-policy", fav_child_policy, options, str,
+                     out);
+    if (afr_set_favorite_child_policy(priv, fav_child_policy) == -1)
+        goto out;
+
+    priv->did_discovery = _gf_false;
+
+    GF_OPTION_RECONF("consistent-io", consistent_io, options, bool, out);
+    if (priv->quorum_count != 0)
+        consistent_io = _gf_false;
+    priv->consistent_io = consistent_io;
+
+    afr_handle_anon_inode_options(priv, options);
+
+    GF_OPTION_RECONF("use-anonymous-inode", priv->use_anon_inode, options, bool,
+                     out);
+    if (priv->shd.enabled) {
+        if ((priv->shd.enabled != enabled_old) ||
+            (timeout_old != priv->shd.timeout))
+            afr_selfheal_childup(this, priv);
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
 
-        GF_OPTION_RECONF ("eager-lock", priv->eager_lock, options, bool, out);
-        GF_OPTION_RECONF ("quorum-type", qtype, options, str, out);
-        GF_OPTION_RECONF ("quorum-count", priv->quorum_count, options,
-                          uint32, out);
-        fix_quorum_options(this,priv,qtype);
-        GF_OPTION_RECONF ("heal-timeout", priv->shd.timeout, options,
-                          int32, out);
+static int
+afr_pending_xattrs_init(afr_private_t *priv, xlator_t *this)
+{
+    int ret = -1;
+    int i = 0;
+    char *ptr = NULL;
+    char *ptr1 = NULL;
+    char *xattrs_list = NULL;
+    xlator_list_t *trav = NULL;
+    int child_count = -1;
+
+    trav = this->children;
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        /* priv->pending_key[THIN_ARBITER_BRICK_INDEX] is used as the
+         * name of the thin arbiter file for persistence across add/
+         * removal of DHT subvols.*/
+        child_count++;
+    }
+
+    GF_OPTION_INIT("afr-pending-xattr", xattrs_list, str, out);
+    priv->pending_key = GF_CALLOC(sizeof(*priv->pending_key), child_count,
+                                  gf_afr_mt_char);
+    if (!priv->pending_key) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    if (!xattrs_list) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_NO_CHANGELOG,
+               "Unable to fetch afr-pending-xattr option from volfile."
+               " Falling back to using client translator names. ");
 
-	GF_OPTION_RECONF ("post-op-delay-secs", priv->post_op_delay_secs, options,
-			  uint32, out);
+        while (i < child_count) {
+            ret = gf_asprintf(&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
+                              trav->xlator->name);
+            if (ret == -1) {
+                ret = -ENOMEM;
+                goto out;
+            }
+            trav = trav->next;
+            i++;
+        }
+        ret = 0;
+        goto out;
+    }
 
-        /* Reset this so we re-discover in case the topology changed.  */
-        priv->did_discovery = _gf_false;
+    ptr = ptr1 = gf_strdup(xattrs_list);
+    if (!ptr) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    for (i = 0, ptr = strtok(ptr, ","); ptr; ptr = strtok(NULL, ",")) {
+        ret = gf_asprintf(&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
+                          ptr);
+        if (ret == -1) {
+            ret = -ENOMEM;
+            goto out;
+        }
+        i++;
+    }
+    ret = 0;
 
-        ret = 0;
 out:
-        return ret;
-
+    GF_FREE(ptr1);
+    return ret;
 }
 
-
-static const char *favorite_child_warning_str = "You have specified subvolume '%s' "
-        "as the 'favorite child'. This means that if a discrepancy in the content "
-        "or attributes (ownership, permission, etc.) of a file is detected among "
-        "the subvolumes, the file on '%s' will be considered the definitive "
-        "version and its contents will OVERWRITE the contents of the file on other "
-        "subvolumes. All versions of the file except that on '%s' "
-        "WILL BE LOST.";
-
+void
+afr_ta_init(afr_private_t *priv)
+{
+    priv->thin_arbiter_count = 1;
+    priv->child_count--;
+    priv->ta_child_up = 0;
+    priv->ta_bad_child_index = AFR_CHILD_UNKNOWN;
+    priv->ta_notify_dom_lock_offset = 0;
+    priv->ta_in_mem_txn_count = 0;
+    priv->ta_on_wire_txn_count = 0;
+    priv->release_ta_notify_dom_lock = _gf_false;
+    INIT_LIST_HEAD(&priv->ta_waitq);
+    INIT_LIST_HEAD(&priv->ta_onwireq);
+    gf_uuid_clear(priv->ta_gfid);
+}
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        afr_private_t *priv        = NULL;
-        int            child_count = 0;
-        xlator_list_t *trav        = NULL;
-        int            i           = 0;
-        int            ret         = -1;
-        GF_UNUSED int  op_errno    = 0;
-        xlator_t      *read_subvol = NULL;
-        int            read_subvol_index = -1;
-        xlator_t      *fav_child   = NULL;
-        char          *qtype       = NULL;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "replicate translator needs more than one "
-                        "subvolume defined.");
-                return -1;
+    afr_private_t *priv = NULL;
+    int child_count = 0;
+    xlator_list_t *trav = NULL;
+    int i = 0;
+    int ret = -1;
+    GF_UNUSED int op_errno = 0;
+    xlator_t *read_subvol = NULL;
+    int read_subvol_index = -1;
+    char *qtype = NULL;
+    char *fav_child_policy = NULL;
+    char *thin_arbiter = NULL;
+    char *data_self_heal = NULL;
+    char *locking_scheme = NULL;
+    char *data_self_heal_algorithm = NULL;
+
+    if (!this->children) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_CHILD_MISCONFIGURED,
+               "replicate translator needs more than one "
+               "subvolume defined.");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_VOL_MISCONFIGURED,
+               "Volume is dangling.");
+    }
+
+    this->private = GF_CALLOC(1, sizeof(afr_private_t),
+                              gf_afr_mt_afr_private_t);
+    if (!this->private)
+        goto out;
+
+    priv = this->private;
+    INIT_LIST_HEAD(&priv->saved_locks);
+    INIT_LIST_HEAD(&priv->lk_healq);
+    LOCK_INIT(&priv->lock);
+
+    child_count = xlator_subvolume_count(this);
+
+    priv->child_count = child_count;
+
+    priv->read_child = -1;
+
+    GF_OPTION_INIT("arbiter-count", priv->arbiter_count, uint32, out);
+    GF_OPTION_INIT("thin-arbiter", thin_arbiter, str, out);
+    if (thin_arbiter && strlen(thin_arbiter) > 0) {
+        afr_ta_init(priv);
+    }
+    INIT_LIST_HEAD(&priv->healing);
+    INIT_LIST_HEAD(&priv->heal_waiting);
+
+    priv->spb_choice_timeout = AFR_DEFAULT_SPB_CHOICE_TIMEOUT;
+
+    GF_OPTION_INIT("afr-dirty-xattr", priv->afr_dirty, str, out);
+
+    GF_OPTION_INIT("metadata-splitbrain-forced-heal",
+                   priv->metadata_splitbrain_forced_heal, bool, out);
+
+    GF_OPTION_INIT("read-subvolume", read_subvol, xlator, out);
+    if (read_subvol) {
+        priv->read_child = xlator_subvolume_index(this, read_subvol);
+        if (priv->read_child == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%s not a subvolume", read_subvol->name);
+            goto out;
         }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Volume is dangling.");
+    }
+    GF_OPTION_INIT("read-subvolume-index", read_subvol_index, int32, out);
+    if (read_subvol_index > -1) {
+        if (read_subvol_index >= priv->child_count) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%d not a subvolume-index", read_subvol_index);
+            goto out;
         }
+        priv->read_child = read_subvol_index;
+    }
+    GF_OPTION_INIT("choose-local", priv->choose_local, bool, out);
 
-	this->private = GF_CALLOC (1, sizeof (afr_private_t),
-                                   gf_afr_mt_afr_private_t);
-        if (!this->private)
-                goto out;
-
-        priv = this->private;
-        LOCK_INIT (&priv->lock);
-        LOCK_INIT (&priv->read_child_lock);
-        //lock recovery is not done in afr
-        pthread_mutex_init (&priv->mutex, NULL);
-        INIT_LIST_HEAD (&priv->saved_fds);
+    priv->pending_reads = GF_CALLOC(sizeof(*priv->pending_reads),
+                                    priv->child_count, gf_afr_mt_atomic_t);
 
-        child_count = xlator_subvolume_count (this);
+    GF_OPTION_INIT("read-hash-mode", priv->hash_mode, uint32, out);
 
-        priv->child_count = child_count;
-
-        priv->read_child = -1;
+    priv->favorite_child = -1;
 
-        GF_OPTION_INIT ("read-subvolume", read_subvol, xlator, out);
-        if (read_subvol) {
-                priv->read_child = xlator_subvolume_index (this, read_subvol);
-                if (priv->read_child == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
-                                read_subvol->name);
-                        goto out;
-                }
-        }
-        GF_OPTION_INIT ("read-subvolume-index",read_subvol_index,int32,out);
-        if (read_subvol_index > -1) {
-                if (read_subvol_index >= priv->child_count) {
-                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
-                                read_subvol_index);
-                        goto out;
-                }
-                priv->read_child = read_subvol_index;
-        }
-        GF_OPTION_INIT ("choose-local", priv->choose_local, bool, out);
-
-        GF_OPTION_INIT ("read-hash-mode", priv->hash_mode, uint32, out);
-
-        priv->favorite_child = -1;
-        GF_OPTION_INIT ("favorite-child", fav_child, xlator, out);
-        if (fav_child) {
-                priv->favorite_child = xlator_subvolume_index (this, fav_child);
-                if (priv->favorite_child == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
-                                fav_child->name);
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_WARNING,
-                        favorite_child_warning_str, fav_child->name,
-                        fav_child->name, fav_child->name);
-        }
+    GF_OPTION_INIT("favorite-child-policy", fav_child_policy, str, out);
+    if (afr_set_favorite_child_policy(priv, fav_child_policy) == -1)
+        goto out;
 
+    GF_OPTION_INIT("shd-max-threads", priv->shd.max_threads, uint32, out);
 
-        GF_OPTION_INIT ("background-self-heal-count",
-                        priv->background_self_heal_count, uint32, out);
+    GF_OPTION_INIT("shd-wait-qlength", priv->shd.wait_qlength, uint32, out);
 
-        GF_OPTION_INIT ("data-self-heal", priv->data_self_heal, str, out);
+    GF_OPTION_INIT("background-self-heal-count",
+                   priv->background_self_heal_count, uint32, out);
 
-        GF_OPTION_INIT ("data-self-heal-algorithm",
-                        priv->data_self_heal_algorithm, str, out);
+    GF_OPTION_INIT("heal-wait-queue-length", priv->heal_wait_qlen, uint32, out);
 
-        GF_OPTION_INIT ("data-self-heal-window-size",
-                        priv->data_self_heal_window_size, uint32, out);
+    GF_OPTION_INIT("data-self-heal", data_self_heal, str, out);
+    if (gf_string2boolean(data_self_heal, &priv->data_self_heal) == -1)
+        goto out;
 
-        GF_OPTION_INIT ("metadata-self-heal", priv->metadata_self_heal, bool,
-                        out);
+    GF_OPTION_INIT("data-self-heal-algorithm", data_self_heal_algorithm, str,
+                   out);
+    set_data_self_heal_algorithm(priv, data_self_heal_algorithm);
 
-        GF_OPTION_INIT ("entry-self-heal", priv->entry_self_heal, bool, out);
+    GF_OPTION_INIT("data-self-heal-window-size",
+                   priv->data_self_heal_window_size, uint32, out);
 
-        GF_OPTION_INIT ("self-heal-daemon", priv->shd.enabled, bool, out);
+    GF_OPTION_INIT("metadata-self-heal", priv->metadata_self_heal, bool, out);
 
-        GF_OPTION_INIT ("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
+    GF_OPTION_INIT("entry-self-heal", priv->entry_self_heal, bool, out);
 
-        GF_OPTION_INIT ("data-change-log", priv->data_change_log, bool, out);
+    GF_OPTION_INIT("halo-shd-max-latency", priv->shd.halo_max_latency_msec,
+                   uint32, out);
 
-        GF_OPTION_INIT ("metadata-change-log", priv->metadata_change_log, bool,
-                        out);
+    GF_OPTION_INIT("halo-max-latency", priv->halo_max_latency_msec, uint32,
+                   out);
+    GF_OPTION_INIT("halo-max-replicas", priv->halo_max_replicas, uint32, out);
+    GF_OPTION_INIT("halo-min-replicas", priv->halo_min_replicas, uint32, out);
 
-        GF_OPTION_INIT ("entry-change-log", priv->entry_change_log, bool, out);
+    GF_OPTION_INIT("halo-enabled", priv->halo_enabled, bool, out);
 
-        GF_OPTION_INIT ("optimistic-change-log", priv->optimistic_change_log,
-                        bool, out);
+    GF_OPTION_INIT("halo-nfsd-max-latency", priv->nfsd.halo_max_latency_msec,
+                   uint32, out);
 
-        GF_OPTION_INIT ("inodelk-trace", priv->inodelk_trace, bool, out);
+    GF_OPTION_INIT("iam-nfs-daemon", priv->nfsd.iamnfsd, bool, out);
 
-        GF_OPTION_INIT ("entrylk-trace", priv->entrylk_trace, bool, out);
+    GF_OPTION_INIT("optimistic-change-log", priv->optimistic_change_log, bool,
+                   out);
 
-        GF_OPTION_INIT ("strict-readdir", priv->strict_readdir, bool, out);
+    GF_OPTION_INIT("pre-op-compat", priv->pre_op_compat, bool, out);
+    GF_OPTION_INIT("locking-scheme", locking_scheme, str, out);
+    priv->granular_locks = (strcmp(locking_scheme, "granular") == 0);
+    GF_OPTION_INIT("full-lock", priv->full_lock, bool, out);
+    GF_OPTION_INIT("granular-entry-heal", priv->esh_granular, bool, out);
 
-        GF_OPTION_INIT ("eager-lock", priv->eager_lock, bool, out);
-        GF_OPTION_INIT ("quorum-type", qtype, str, out);
-        GF_OPTION_INIT ("quorum-count", priv->quorum_count, uint32, out);
-        fix_quorum_options(this,priv,qtype);
+    GF_OPTION_INIT("eager-lock", priv->eager_lock, bool, out);
+    GF_OPTION_INIT("quorum-type", qtype, str, out);
+    GF_OPTION_INIT("quorum-count", priv->quorum_count, uint32, out);
+    GF_OPTION_INIT(AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size_uint64,
+                   out);
+    fix_quorum_options(this, priv, qtype, this->options);
 
-	GF_OPTION_INIT ("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
+    GF_OPTION_INIT("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
+    GF_OPTION_INIT("ensure-durability", priv->ensure_durability, bool, out);
 
-        priv->wait_count = 1;
+    GF_OPTION_INIT("self-heal-daemon", priv->shd.enabled, bool, out);
 
-        priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
-                                    gf_afr_mt_char);
-        if (!priv->child_up) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    GF_OPTION_INIT("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
+    GF_OPTION_INIT("heal-timeout", priv->shd.timeout, int32, out);
 
-        for (i = 0; i < child_count; i++)
-                priv->child_up[i] = -1; /* start with unknown state.
-                                           this initialization needed
-                                           for afr_notify() to work
-                                           reliably
-                                        */
+    GF_OPTION_INIT("consistent-metadata", priv->consistent_metadata, bool, out);
+    GF_OPTION_INIT("consistent-io", priv->consistent_io, bool, out);
+    afr_handle_anon_inode_options(priv, this->options);
 
-        priv->children = GF_CALLOC (sizeof (xlator_t *), child_count,
-                                    gf_afr_mt_xlator_t);
-        if (!priv->children) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    GF_OPTION_INIT("use-anonymous-inode", priv->use_anon_inode, bool, out);
+    if (priv->quorum_count != 0)
+        priv->consistent_io = _gf_false;
 
-        priv->pending_key = GF_CALLOC (sizeof (*priv->pending_key),
-                                       child_count,
-                                       gf_afr_mt_char);
-        if (!priv->pending_key) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    priv->wait_count = 1;
 
-        trav = this->children;
-        i = 0;
-        while (i < child_count) {
-                priv->children[i] = trav->xlator;
-
-                ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
-                                   AFR_XATTR_PREFIX,
-                                   trav->xlator->name);
-                if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "asprintf failed to set pending key");
-                        ret = -ENOMEM;
-                        goto out;
-                }
-
-                trav = trav->next;
-                i++;
-        }
+    priv->local = GF_CALLOC(sizeof(unsigned char), child_count, gf_afr_mt_char);
+    if (!priv->local) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
-                                      gf_afr_mt_int32_t);
-        if (!priv->last_event) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    priv->anon_inode = GF_CALLOC(sizeof(unsigned char), child_count,
+                                 gf_afr_mt_char);
 
-        /* keep more local here as we may need them for self-heal etc */
-        this->local_pool = mem_pool_new (afr_local_t, 512);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
+    priv->child_up = GF_CALLOC(sizeof(unsigned char), child_count,
+                               gf_afr_mt_char);
 
-        priv->first_lookup = 1;
-        priv->root_inode = NULL;
+    priv->child_latency = GF_MALLOC(sizeof(*priv->child_latency) * child_count,
+                                    gf_afr_mt_child_latency_t);
+    priv->halo_child_up = GF_CALLOC(sizeof(unsigned char), child_count,
+                                    gf_afr_mt_char);
 
-        if (!priv->shd.iamshd) {
-                ret = 0;
-                goto out;
-        }
+    if (!priv->child_up || !priv->child_latency || !priv->halo_child_up ||
+        !priv->anon_inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    /*Initialize to -ve ping timeout so that they are not considered
+     * in child-up events until ping-event comes*/
+    for (i = 0; i < child_count; i++)
+        priv->child_latency[i] = -1;
+
+    priv->children = GF_CALLOC(sizeof(xlator_t *), child_count,
+                               gf_afr_mt_xlator_t);
+    if (!priv->children) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = afr_pending_xattrs_init(priv, this);
+    if (ret)
+        goto out;
+
+    trav = this->children;
+    i = 0;
+    while (i < child_count) {
+        priv->children[i] = trav->xlator;
+        trav = trav->next;
+        i++;
+    }
+
+    ret = gf_asprintf(&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT, this->name);
+    if (-1 == ret) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
+    priv->last_event = GF_CALLOC(child_count, sizeof(*priv->last_event),
+                                 gf_afr_mt_int32_t);
+    if (!priv->last_event) {
         ret = -ENOMEM;
-        priv->shd.pos = GF_CALLOC (sizeof (*priv->shd.pos), child_count,
-                                   gf_afr_mt_brick_pos_t);
-        if (!priv->shd.pos)
-                goto out;
+        goto out;
+    }
 
-        priv->shd.pending = GF_CALLOC (sizeof (*priv->shd.pending), child_count,
-                                       gf_afr_mt_int32_t);
-        if (!priv->shd.pending)
-                goto out;
+    this->itable = inode_table_new(SHD_INODE_LRU_LIMIT, this);
+    if (!this->itable) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    if (priv->shd.iamshd) {
+        ret = afr_selfheal_daemon_init(this);
+        if (ret) {
+            ret = -ENOMEM;
+            goto out;
+        }
+    }
 
-        priv->shd.inprogress = GF_CALLOC (sizeof (*priv->shd.inprogress),
-                                          child_count, gf_afr_mt_shd_bool_t);
-        if (!priv->shd.inprogress)
-                goto out;
-        priv->shd.timer = GF_CALLOC (sizeof (*priv->shd.timer), child_count,
-                                     gf_afr_mt_shd_timer_t);
-        if (!priv->shd.timer)
-                goto out;
+    /* keep more local here as we may need them for self-heal etc */
+    this->local_pool = mem_pool_new(afr_local_t, 512);
+    if (!this->local_pool) {
+        ret = -1;
+        goto out;
+    }
 
-        priv->shd.healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false);
-        if (!priv->shd.healed)
-                goto out;
+    priv->root_inode = NULL;
 
-        priv->shd.heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false);
-        if (!priv->shd.heal_failed)
-                goto out;
+    ret = 0;
+out:
+    return ret;
+}
+void
+afr_destroy_healer_object(xlator_t *this, struct subvol_healer *healer)
+{
+    int ret = -1;
+
+    if (!healer)
+        return;
+
+    if (healer->running) {
+        /*
+         * If there are any resources to cleanup, We need
+         * to do that gracefully using pthread_cleanup_push
+         */
+        ret = gf_thread_cleanup_xint(healer->thread);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SELF_HEAL_FAILED,
+                   "Failed to clean up healer threads.");
+        healer->thread = 0;
+    }
+    pthread_cond_destroy(&healer->cond);
+    pthread_mutex_destroy(&healer->mutex);
+}
 
-        priv->shd.split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false);
-        if (!priv->shd.split_brain)
-                goto out;
+void
+afr_selfheal_daemon_fini(xlator_t *this)
+{
+    struct subvol_healer *healer = NULL;
+    afr_self_heald_t *shd = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+    if (!priv)
+        return;
+
+    shd = &priv->shd;
+    if (!shd->iamshd)
+        return;
+
+    for (i = 0; i < priv->child_count; i++) {
+        healer = &shd->index_healers[i];
+        afr_destroy_healer_object(this, healer);
+
+        healer = &shd->full_healers[i];
+        afr_destroy_healer_object(this, healer);
+
+        if (shd->statistics[i])
+            eh_destroy(shd->statistics[i]);
+    }
+    GF_FREE(shd->index_healers);
+    GF_FREE(shd->full_healers);
+    GF_FREE(shd->statistics);
+    if (shd->split_brain)
+        eh_destroy(shd->split_brain);
+}
+void
+fini(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
 
-        this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
-        if (!this->itable)
-                goto out;
-        priv->root_inode = inode_ref (this->itable->root);
-        GF_OPTION_INIT ("node-uuid", priv->shd.node_uuid, str, out);
-        GF_OPTION_INIT ("heal-timeout", priv->shd.timeout, int32, out);
+    priv = this->private;
 
-        ret = 0;
-out:
-        return ret;
-}
+    afr_selfheal_daemon_fini(this);
+    GF_ASSERT(list_empty(&priv->saved_locks));
 
+    LOCK(&priv->lock);
+    if (priv->timer != NULL) {
+        gf_timer_call_cancel(this->ctx, priv->timer);
+        priv->timer = NULL;
+    }
+    UNLOCK(&priv->lock);
 
-int
-fini (xlator_t *this)
-{
-        afr_private_t *priv = NULL;
+    if (this->local_pool != NULL) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
 
-        priv = this->private;
-        this->private = NULL;
-        afr_priv_destroy (priv);
-        if (this->itable);//I dont see any destroy func
+    this->private = NULL;
+    afr_priv_destroy(priv);
+    if (this->itable) {
+        inode_table_destroy(this->itable);
+        this->itable = NULL;
+    }
 
-        return 0;
+    return;
 }
 
-
 struct xlator_fops fops = {
-        .lookup      = afr_lookup,
-        .open        = afr_open,
-        .lk          = afr_lk,
-        .flush       = afr_flush,
-        .statfs      = afr_statfs,
-        .fsync       = afr_fsync,
-        .fsyncdir    = afr_fsyncdir,
-        .xattrop     = afr_xattrop,
-        .fxattrop    = afr_fxattrop,
-        .inodelk     = afr_inodelk,
-        .finodelk    = afr_finodelk,
-        .entrylk     = afr_entrylk,
-        .fentrylk    = afr_fentrylk,
-
-        /* inode read */
-        .access      = afr_access,
-        .stat        = afr_stat,
-        .fstat       = afr_fstat,
-        .readlink    = afr_readlink,
-        .getxattr    = afr_getxattr,
-        .fgetxattr   = afr_fgetxattr,
-        .readv       = afr_readv,
-
-        /* inode write */
-        .writev      = afr_writev,
-        .truncate    = afr_truncate,
-        .ftruncate   = afr_ftruncate,
-        .setxattr    = afr_setxattr,
-        .fsetxattr   = afr_fsetxattr,
-        .setattr     = afr_setattr,
-        .fsetattr    = afr_fsetattr,
-        .removexattr = afr_removexattr,
-        .fremovexattr = afr_fremovexattr,
-
-        /* dir read */
-        .opendir     = afr_opendir,
-        .readdir     = afr_readdir,
-        .readdirp    = afr_readdirp,
-
-        /* dir write */
-        .create      = afr_create,
-        .mknod       = afr_mknod,
-        .mkdir       = afr_mkdir,
-        .unlink      = afr_unlink,
-        .rmdir       = afr_rmdir,
-        .link        = afr_link,
-        .symlink     = afr_symlink,
-        .rename      = afr_rename,
+    .lookup = afr_lookup,
+    .lk = afr_lk,
+    .flush = afr_flush,
+    .statfs = afr_statfs,
+    .fsyncdir = afr_fsyncdir,
+    .inodelk = afr_inodelk,
+    .finodelk = afr_finodelk,
+    .entrylk = afr_entrylk,
+    .fentrylk = afr_fentrylk,
+    .ipc = afr_ipc,
+    .lease = afr_lease,
+
+    /* inode read */
+    .access = afr_access,
+    .stat = afr_stat,
+    .fstat = afr_fstat,
+    .readlink = afr_readlink,
+    .getxattr = afr_getxattr,
+    .fgetxattr = afr_fgetxattr,
+    .readv = afr_readv,
+    .seek = afr_seek,
+
+    /* inode write */
+    .writev = afr_writev,
+    .truncate = afr_truncate,
+    .ftruncate = afr_ftruncate,
+    .setxattr = afr_setxattr,
+    .fsetxattr = afr_fsetxattr,
+    .setattr = afr_setattr,
+    .fsetattr = afr_fsetattr,
+    .removexattr = afr_removexattr,
+    .fremovexattr = afr_fremovexattr,
+    .fallocate = afr_fallocate,
+    .discard = afr_discard,
+    .zerofill = afr_zerofill,
+    .xattrop = afr_xattrop,
+    .fxattrop = afr_fxattrop,
+    .fsync = afr_fsync,
+
+    /*inode open*/
+    .opendir = afr_opendir,
+    .open = afr_open,
+
+    /* dir read */
+    .readdir = afr_readdir,
+    .readdirp = afr_readdirp,
+
+    /* dir write */
+    .create = afr_create,
+    .mknod = afr_mknod,
+    .mkdir = afr_mkdir,
+    .unlink = afr_unlink,
+    .rmdir = afr_rmdir,
+    .link = afr_link,
+    .symlink = afr_symlink,
+    .rename = afr_rename,
 };
 
-
 struct xlator_dumpops dumpops = {
-        .priv       = afr_priv_dump,
+    .priv = afr_priv_dump,
 };
 
-
 struct xlator_cbks cbks = {
-        .release     = afr_release,
-        .releasedir  = afr_releasedir,
-        .forget      = afr_forget,
+    .release = afr_release,
+    .releasedir = afr_releasedir,
+    .forget = afr_forget,
 };
 
-
 struct volume_options options[] = {
-        { .key  = {"read-subvolume" },
-          .type = GF_OPTION_TYPE_XLATOR
-        },
-        { .key  = {"read-subvolume-index" },
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "-1",
-        },
-        { .key = {"read-hash-mode" },
-          .type = GF_OPTION_TYPE_INT,
-          .min = 0,
-          .max = 2,
-          .default_value = "0",
-          .description = "0 = first responder, "
-                         "1 = hash by GFID (all clients use same subvolume), "
-                         "2 = hash by GFID and client PID",
-        },
-        { .key  = {"choose-local" },
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "true",
-          .description = "Choose a local subvolume to read from if "
-                         "read-subvolume is not explicitly set.",
-        },
-        { .key  = {"favorite-child"},
-          .type = GF_OPTION_TYPE_XLATOR
-        },
-        { .key  = {"background-self-heal-count"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
-          .default_value = "16",
-          .validate = GF_OPT_VALIDATE_MIN,
-        },
-        { .key  = {"data-self-heal"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"1", "on", "yes", "true", "enable",
-                    "0", "off", "no", "false", "disable",
-                    "open"},
-          .default_value = "on",
-        },
-        { .key  = {"data-self-heal-algorithm"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "",
-          .description   = "Select between \"full\", \"diff\". The "
-                           "\"full\" algorithm copies the entire file from "
-                           "source to sink. The \"diff\" algorithm copies to "
-                           "sink only those blocks whose checksums don't match "
-                           "with those of source.",
-          .value = { "diff", "full", "" }
-        },
-        { .key  = {"data-self-heal-window-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 1,
-          .max  = 1024,
-          .default_value = "1",
-          .description = "Maximum number blocks per file for which self-heal "
-                         "process would be applied simultaneously."
-        },
-        { .key  = {"metadata-self-heal"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key  = {"entry-self-heal"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key  = {"data-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key  = {"metadata-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key  = {"entry-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key  = {"optimistic-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key  = {"strict-readdir"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"inodelk-trace"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"entrylk-trace"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"eager-lock"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key = {"self-heal-daemon"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"iam-self-heal-daemon"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"quorum-type"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { "none", "auto", "fixed", "" },
-          .default_value = "none",
-          .description = "If value is \"fixed\" only allow writes if "
-                         "quorum-count bricks are present.  If value is "
-                         "\"auto\" only allow writes if more than half of "
-                         "bricks, or exactly half including the first, are "
-                         "present.",
-        },
-        { .key = {"quorum-count"},
-          .type = GF_OPTION_TYPE_INT,
-          .min = 1,
-          .max = INT_MAX,
-          .default_value = 0,
-          .description = "If quorum-type is \"fixed\" only allow writes if "
-                         "this many bricks or present.  Other quorum types "
-                         "will OVERWRITE this value.",
-        },
-        { .key  = {"node-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Local glusterd uuid string",
-        },
-        { .key  = {"heal-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 60,
-          .max  = INT_MAX,
-          .default_value = "600",
-          .description = "Poll timeout for checking the need to self-heal"
-        },
-        { .key  = {"post-op-delay-secs"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
-          .max  = INT_MAX,
-          .default_value = "1",
-          .description = "Time interval induced artificially before "
-	                 "post-operation phase of the transaction to "
-                         "enhance overlap of adjacent write operations.",
-        },
-        { .key  = {NULL} },
+    {.key = {"read-subvolume"},
+     .type = GF_OPTION_TYPE_XLATOR,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "inode-read fops happen only on one of the bricks in "
+                    "replicate. Afr will prefer the one specified using "
+                    "this option if it is not stale. Option value must be "
+                    "one of the xlator names of the children. "
+                    "Ex: <volname>-client-0 till "
+                    "<volname>-client-<number-of-bricks - 1>"},
+    {.key = {"read-subvolume-index"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "-1",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "inode-read fops happen only on one of the bricks in "
+                    "replicate. AFR will prefer the one specified using "
+                    "this option if it is not stale. allowed options"
+                    " include -1 till replica-count - 1"},
+    {.key = {"read-hash-mode"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 5,
+     .default_value = "1",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description =
+         "inode-read fops happen only on one of the bricks in "
+         "replicate. AFR will prefer the one computed using "
+         "the method specified using this option.\n"
+         "0 = first readable child of AFR, starting from 1st child.\n"
+         "1 = hash by GFID of file (all clients use "
+         "same subvolume).\n"
+         "2 = hash by GFID of file and client PID.\n"
+         "3 = brick having the least outstanding read requests.\n"
+         "4 = brick having the least network ping latency.\n"
+         "5 = Hybrid mode between 3 and 4, ie least value among "
+         "network-latency multiplied by outstanding-read-requests."},
+    {
+        .key = {"choose-local"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Choose a local subvolume (i.e. Brick) to read from"
+                       " if read-subvolume is not explicitly set.",
+    },
+    {.key = {"background-self-heal-count"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 256,
+     .default_value = "8",
+     .validate = GF_OPT_VALIDATE_MIN,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This specifies the number of per client self-heal "
+                    "jobs that can perform parallel heals in the "
+                    "background."},
+    {.key = {"halo-shd-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "99999",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for shd halo replication in msec."},
+    {.key = {"halo-enabled"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "False",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Enable Halo (geo) replication mode."},
+    {.key = {"halo-nfsd-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "5",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for nfsd halo replication in msec."},
+    {.key = {"halo-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = AFR_HALO_MAX_LATENCY,
+     .default_value = "5",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for halo replication in msec."},
+    {.key = {"halo-max-replicas"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "99999",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "The maximum number of halo replicas; replicas"
+                    " beyond this value will be written asynchronously"
+                    "via the SHD."},
+    {.key = {"halo-min-replicas"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "2",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "The minimmum number of halo replicas, before adding "
+                    "out of region replicas."},
+    {.key = {"heal-wait-queue-length"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 10000, /*Around 100MB with sizeof(afr_local_t)= 10496 bytes*/
+     .default_value = "128",
+     .validate = GF_OPT_VALIDATE_MIN,
+     .op_version = {GD_OP_VERSION_3_7_10},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This specifies the number of heals that can be queued"
+                    " for the parallel background self heal jobs."},
+    {.key = {"data-self-heal"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"1", "on", "yes", "true", "enable", "0", "off", "no", "false",
+               "disable", "open"},
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Using this option we can enable/disable data "
+                    "self-heal on the file. \"open\" means data "
+                    "self-heal action will only be triggered by file "
+                    "open operations."},
+    {.key = {"data-self-heal-algorithm"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Select between \"full\", \"diff\". The "
+                    "\"full\" algorithm copies the entire file from "
+                    "source to sink. The \"diff\" algorithm copies to "
+                    "sink only those blocks whose checksums don't match "
+                    "with those of source. If no option is configured "
+                    "the option is chosen dynamically as follows: "
+                    "If the file does not exist on one of the sinks "
+                    "or empty file exists or if the source file size is "
+                    "about the same as page size the entire file will "
+                    "be read and written i.e \"full\" algo, "
+                    "otherwise \"diff\" algo is chosen.",
+     .value = {"diff", "full"}},
+    {.key = {"data-self-heal-window-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 1024,
+     .default_value = "1",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Maximum number blocks per file for which self-heal "
+                    "process would be applied simultaneously."},
+    {.key = {"metadata-self-heal"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     /*.validate_fn = validate_replica*/
+     .description = "Using this option we can enable/disable metadata "
+                    "i.e. Permissions, ownerships, xattrs self-heal on "
+                    "the file/directory."},
+    {.key = {"entry-self-heal"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     /*.validate_fn = validate_replica*/
+     .description = "Using this option we can enable/disable entry "
+                    "self-heal on the directory."},
+    {.key = {"data-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"metadata-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"entry-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"optimistic-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Entry/Metadata fops will not perform "
+                    "pre fop changelog operations in afr transaction "
+                    "if this option is enabled."},
+    {.key = {"inodelk-trace"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Enabling this option logs inode lock/unlocks"},
+    {.key = {"entrylk-trace"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Enabling this option logs entry lock/unlocks"},
+    {.key = {"pre-op-compat"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Use separate pre-op xattrop() FOP rather than "
+                    "overloading xdata of the OP"},
+    {.key = {"eager-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description =
+         "Enable/Disable eager lock for replica volume. "
+         "Lock phase of a transaction has two sub-phases. "
+         "First is an attempt to acquire locks in parallel by "
+         "broadcasting non-blocking lock requests. If lock "
+         "acquisition fails on any server, then the held locks "
+         "are unlocked and we revert to a blocking locks mode "
+         "sequentially on one server after another.  If this "
+         "option is enabled the initial broadcasting lock "
+         "request attempts to acquire a full lock on the entire file. "
+         "If this fails, we revert back to the sequential "
+         "\"regional\" blocking locks as before. In the case "
+         "where such an \"eager\" lock is granted in the "
+         "non-blocking phase, it gives rise to an opportunity "
+         "for optimization. i.e, if the next write transaction "
+         "on the same FD arrives before the unlock phase of "
+         "the first transaction, it \"takes over\" the full "
+         "file lock. Similarly if yet another data transaction "
+         "arrives before the unlock phase of the \"optimized\" "
+         "transaction, that in turn \"takes over\" the lock as "
+         "well. The actual unlock now happens at the end of "
+         "the last \"optimized\" transaction."
+
+    },
+    {.key = {"self-heal-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     /*.validate_fn   = validate_replica_heal_enable_disable*/
+     .description = "This option applies to only self-heal-daemon. "
+                    "Index directory crawl and automatic healing of files "
+                    "will not be performed if this option is turned off."},
+    {.key = {"iam-self-heal-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option differentiates if the replicate "
+                    "translator is running as part of self-heal-daemon "
+                    "or not."},
+    {.key = {"iam-nfs-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option differentiates if the replicate "
+                    "translator is running as part of an NFS daemon "
+                    "or not."},
+    {
+        .key = {"quorum-type"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"none", "auto", "fixed"},
+        .default_value = "none",
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        /*.option = quorum-type*/
+        .description = "If value is \"fixed\" only allow writes if "
+                       "quorum-count bricks are present.  If value is "
+                       "\"auto\" only allow writes if more than half of "
+                       "bricks, or exactly half including the first, are "
+                       "present.",
+    },
+    {
+        .key = {"quorum-count"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .max = INT_MAX,
+        .default_value = 0,
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        /*.option = quorum-count*/
+        /*.validate_fn = validate_quorum_count*/
+        .description = "If quorum-type is \"fixed\" only allow writes if "
+                       "this many bricks are present.  Other quorum types "
+                       "will OVERWRITE this value.",
+    },
+    {
+        .key = {"quorum-reads"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option has been removed. Reads are not allowed "
+                       "if quorum is not met.",
+    },
+    {
+        .key = {"node-uuid"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Local glusterd uuid string, used in starting "
+                       "self-heal-daemon so that it can crawl only on "
+                       "local index directories.",
+    },
+    {
+        .key = {"post-op-delay-secs"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0,
+        .max = INT_MAX,
+        .default_value = "1",
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Time interval induced artificially before "
+                       "post-operation phase of the transaction to "
+                       "enhance overlap of adjacent write operations.",
+    },
+    {
+        .key = {AFR_SH_READDIR_SIZE_KEY},
+        .type = GF_OPTION_TYPE_SIZET,
+        .description = "readdirp size for performing entry self-heal",
+        .min = 1024,
+        .max = 131072,
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+        .tags = {"replicate"},
+        .default_value = "1KB",
+    },
+    {
+        .key = {"ensure-durability"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .op_version = {3},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Afr performs fsyncs for transactions if this "
+                       "option is on to make sure the changelogs/data is "
+                       "written to the disk",
+        .default_value = "on",
+    },
+    {
+        .key = {"afr-dirty-xattr"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = AFR_DIRTY_DEFAULT,
+    },
+    {.key = {"afr-pending-xattr"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Comma separated list of xattrs that are used to  "
+                    "capture information on pending heals."},
+    {
+        .key = {"metadata-splitbrain-forced-heal"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+    },
+    {.key = {"heal-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 5,
+     .max = INT_MAX,
+     .default_value = "600",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "time interval for checking the need to self-heal "
+                    "in self-heal-daemon"},
+    {
+        .key = {"consistent-metadata"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is enabled, readdirp will force "
+                       "lookups on those entries read whose read child is "
+                       "not the same as that of the parent. This will "
+                       "guarantee that all read operations on a file serve "
+                       "attributes from the same subvol as long as it holds "
+                       " a good copy of the file/dir.",
+    },
+    {.key = {"arbiter-count"},
+     .type = GF_OPTION_TYPE_INT,
+     .description = "subset of child_count. Has to be 0 or 1."},
+    {
+        .key = {"thin-arbiter"},
+        .type = GF_OPTION_TYPE_STR,
+        .op_version = {GD_OP_VERSION_4_1_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"replicate"},
+        .description = "contains host:path of thin abriter brick",
+    },
+    {.key = {"shd-max-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 64,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_3_7_12},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Maximum number of parallel heals SHD can do per "
+                    "local brick. This can substantially lower heal times"
+                    ", but can also crush your bricks if you don't have "
+                    "the storage hardware to support this."},
+    {
+        .key = {"shd-wait-qlength"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .max = 655536,
+        .default_value = "1024",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option can be used to control number of heals"
+                       " that can wait in SHD per subvolume",
+    },
+    {
+        .key = {"locking-scheme"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"full", "granular"},
+        .default_value = "full",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is set to granular, self-heal will "
+                       "stop being compatible with afr-v1, which helps afr "
+                       "be more granular while self-healing",
+    },
+    {.key = {"full-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "yes",
+     .op_version = {GD_OP_VERSION_3_13_2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     .description = "If this option is disabled, then the IOs will take "
+                    "range locks same as versions till 3.13.1."},
+    {
+        .key = {"granular-entry-heal"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_8_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is enabled, self-heal will resort to "
+                       "granular way of recording changelogs and doing entry "
+                       "self-heal.",
+    },
+    {
+        .key = {"favorite-child-policy"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"none", "size", "ctime", "mtime", "majority"},
+        .default_value = "none",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option can be used to automatically resolve "
+                       "split-brains using various policies without user "
+                       "intervention. \"size\" picks the file with the "
+                       "biggest size as the source. \"ctime\" and \"mtime\" "
+                       "pick the file with the latest ctime and mtime "
+                       "respectively as the source. \"majority\" picks a file"
+                       " with identical mtime and size in more than half the "
+                       "number of bricks in the replica.",
+    },
+    {
+        .key = {"consistent-io"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .description = "If this option is enabled, i/o will fail even if "
+                       "one of the bricks is down in the replicas",
+    },
+    {.key = {"use-compound-fops"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "no",
+     .op_version = {GD_OP_VERSION_3_8_4},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"use-anonymous-inode"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "no",
+     .op_version = {GD_OP_VERSION_8_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     .description = "Setting this option heals directory renames efficiently"},
+
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "replicate",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index abb95533449..d62f9a9caf2 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -8,1065 +8,1416 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __AFR_H__
 #define __AFR_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "call-stub.h"
-#include "compat-errno.h"
+#include <glusterfs/call-stub.h>
+#include <glusterfs/compat-errno.h>
 #include "afr-mem-types.h"
-#include "afr-self-heal-algorithm.h"
 
 #include "libxlator.h"
-#include "timer.h"
+#include <glusterfs/timer.h>
+#include <glusterfs/syncop.h>
+
+#include "afr-self-heald.h"
+#include "afr-messages.h"
 
-#define AFR_XATTR_PREFIX "trusted.afr"
+#define SHD_INODE_LRU_LIMIT 1
 #define AFR_PATHINFO_HEADER "REPLICATE:"
+#define AFR_SH_READDIR_SIZE_KEY "self-heal-readdir-size"
+#define AFR_SH_DATA_DOMAIN_FMT "%s:self-heal"
+#define AFR_DIRTY_DEFAULT AFR_XATTR_PREFIX ".dirty"
+#define AFR_DIRTY (((afr_private_t *)(THIS->private))->afr_dirty)
+
+#define AFR_LOCKEE_COUNT_MAX 3
+#define AFR_DOM_COUNT_MAX 3
+#define AFR_NUM_CHANGE_LOGS 3              /*data + metadata + entry*/
+#define AFR_DEFAULT_SPB_CHOICE_TIMEOUT 300 /*in seconds*/
+
+#define ARBITER_BRICK_INDEX 2
+#define THIN_ARBITER_BRICK_INDEX 2
+#define AFR_TA_DOM_NOTIFY "afr.ta.dom-notify"
+#define AFR_TA_DOM_MODIFY "afr.ta.dom-modify"
+
+#define AFR_LK_HEAL_DOM "afr.lock-heal.domain"
+
+#define AFR_HALO_MAX_LATENCY 99999
+#define AFR_ANON_DIR_PREFIX ".glusterfs-anonymous-inode"
+
+#define PFLAG_PENDING (1 << 0)
+#define PFLAG_SBRAIN (1 << 1)
+
+typedef int (*afr_lock_cbk_t)(call_frame_t *frame, xlator_t *this);
+
+typedef int (*afr_read_txn_wind_t)(call_frame_t *frame, xlator_t *this,
+                                   int subvol);
+
+typedef int (*afr_inode_refresh_cbk_t)(call_frame_t *frame, xlator_t *this,
+                                       int err);
+
+typedef int (*afr_changelog_resume_t)(call_frame_t *frame, xlator_t *this);
+
+#define AFR_COUNT(array, max)                                                  \
+    ({                                                                         \
+        int __i;                                                               \
+        int __res = 0;                                                         \
+        for (__i = 0; __i < max; __i++)                                        \
+            if (array[__i])                                                    \
+                __res++;                                                       \
+        __res;                                                                 \
+    })
+#define AFR_INTERSECT(dst, src1, src2, max)                                    \
+    ({                                                                         \
+        int __i;                                                               \
+        for (__i = 0; __i < max; __i++)                                        \
+            dst[__i] = src1[__i] && src2[__i];                                 \
+    })
+#define AFR_CMP(a1, a2, len)                                                   \
+    ({                                                                         \
+        int __cmp = 0;                                                         \
+        int __i;                                                               \
+        for (__i = 0; __i < len; __i++)                                        \
+            if (a1[__i] != a2[__i]) {                                          \
+                __cmp = 1;                                                     \
+                break;                                                         \
+            }                                                                  \
+        __cmp;                                                                 \
+    })
+#define AFR_IS_ARBITER_BRICK(priv, index)                                      \
+    ((priv->arbiter_count == 1) && (index == ARBITER_BRICK_INDEX))
+
+#define AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(ret, errnum)                       \
+    do {                                                                       \
+        local->op_ret = ret;                                                   \
+        local->op_errno = errnum;                                              \
+        if (local->op_errno == EIO)                                            \
+            gf_msg(this->name, GF_LOG_ERROR, local->op_errno,                  \
+                   AFR_MSG_SPLIT_BRAIN,                                        \
+                   "Failing %s on gfid %s: "                                   \
+                   "split-brain observed.",                                    \
+                   gf_fop_list[local->op], uuid_utoa(local->inode->gfid));     \
+    } while (0)
+
+#define AFR_ERROR_OUT_IF_FDCTX_INVALID(__fd, __this, __error, __label)         \
+    do {                                                                       \
+        afr_fd_ctx_t *__fd_ctx = NULL;                                         \
+        __fd_ctx = afr_fd_ctx_get(__fd, __this);                               \
+        if (__fd_ctx && __fd_ctx->is_fd_bad) {                                 \
+            __error = EBADF;                                                   \
+            goto __label;                                                      \
+        }                                                                      \
+    } while (0)
 
-struct _pump_private;
+typedef enum {
+    AFR_READ_POLICY_FIRST_UP,
+    AFR_READ_POLICY_GFID_HASH,
+    AFR_READ_POLICY_GFID_PID_HASH,
+    AFR_READ_POLICY_LESS_LOAD,
+    AFR_READ_POLICY_LEAST_LATENCY,
+    AFR_READ_POLICY_LOAD_LATENCY_HYBRID,
+} afr_read_hash_mode_t;
 
-typedef int (*afr_expunge_done_cbk_t) (call_frame_t *frame, xlator_t *this,
-                                       int child, int32_t op_error,
-                                       int32_t op_errno);
+typedef enum {
+    AFR_FAV_CHILD_NONE,
+    AFR_FAV_CHILD_BY_SIZE,
+    AFR_FAV_CHILD_BY_CTIME,
+    AFR_FAV_CHILD_BY_MTIME,
+    AFR_FAV_CHILD_BY_MAJORITY,
+    AFR_FAV_CHILD_POLICY_MAX,
+} afr_favorite_child_policy;
 
-typedef int (*afr_impunge_done_cbk_t) (call_frame_t *frame, xlator_t *this,
-                                       int32_t op_error, int32_t op_errno);
-typedef int (*afr_post_remove_call_t) (call_frame_t *frame, xlator_t *this);
+typedef enum {
+    AFR_SELFHEAL_DATA_FULL = 0,
+    AFR_SELFHEAL_DATA_DIFF,
+    AFR_SELFHEAL_DATA_DYNAMIC,
+} afr_data_self_heal_type_t;
 
-typedef int (*afr_lock_cbk_t) (call_frame_t *frame, xlator_t *this);
-typedef void (*afr_lookup_done_cbk_t) (call_frame_t *frame, xlator_t *this,
-                                      int32_t op_ret, int32_t op_errno);
+typedef enum {
+    AFR_CHILD_UNKNOWN = -1,
+    AFR_CHILD_ZERO,
+    AFR_CHILD_ONE,
+    AFR_CHILD_THIN_ARBITER,
+} afr_child_index;
 
 typedef enum {
-        AFR_POS_UNKNOWN,
-        AFR_POS_LOCAL,
-        AFR_POS_REMOTE
-} afr_child_pos_t;
+    TA_WAIT_FOR_NOTIFY_LOCK_REL, /*FOP came after notify domain lock upcall
+                                   notification and waiting for its release.*/
+    TA_GET_INFO_FROM_TA_FILE,    /*FOP needs post-op on ta file to get
+                                  *info about which brick is bad.*/
+    TA_INFO_IN_MEMORY_SUCCESS,   /*Bad brick info is in memory and fop failed
+                                  *on BAD brick - Success*/
+    TA_INFO_IN_MEMORY_FAILED,    /*Bad brick info is in memory and fop failed
+                                  *on GOOD brick - Failed*/
+    TA_SUCCESS,                  /*FOP succeeded on both data bricks.*/
+} afr_ta_fop_state_t;
+
+struct afr_nfsd {
+    uint32_t halo_max_latency_msec;
+    gf_boolean_t iamnfsd;
+};
+
+typedef struct _afr_lk_heal_info {
+    fd_t *fd;
+    int32_t cmd;
+    struct gf_flock flock;
+    dict_t *xdata_req;
+    unsigned char *locked_nodes;
+    struct list_head pos;
+    gf_lkowner_t lk_owner;
+    pid_t pid;
+    int32_t *child_up_event_gen;
+    int32_t *child_down_event_gen;
+} afr_lk_heal_info_t;
+
+typedef struct _afr_private {
+    gf_lock_t lock;             /* to guard access to child_count, etc */
+    unsigned int child_count;   /* total number of children   */
+    unsigned int arbiter_count; /*subset of child_count.
+                                  Has to be 0 or 1.*/
+
+    xlator_t **children;
+
+    inode_t *root_inode;
+
+    int favorite_child; /* subvolume to be preferred in resolving
+                                    split-brain cases */
+    /* For thin-arbiter. */
+    uuid_t ta_gfid;
+    unsigned int thin_arbiter_count; /* 0 or 1 at the moment.*/
+    int ta_bad_child_index;
+    int ta_event_gen;
+    unsigned int ta_in_mem_txn_count;
+    unsigned int ta_on_wire_txn_count;
+    struct list_head ta_waitq;
+    struct list_head ta_onwireq;
+
+    unsigned char *anon_inode;
+    unsigned char *child_up;
+    unsigned char *halo_child_up;
+    int64_t *child_latency;
+    unsigned char *local;
+
+    char **pending_key;
+
+    afr_data_self_heal_type_t data_self_heal_algorithm;
+    unsigned int data_self_heal_window_size; /* max number of pipelined
+                                                read/writes */
+
+    struct list_head heal_waiting; /*queue for files that need heal*/
+    uint32_t heal_wait_qlen; /*configurable queue length for heal_waiting*/
+    int32_t heal_waiters;    /* No. of elements currently in wait queue.*/
+
+    struct list_head healing;            /* queue for files that are undergoing
+                                            background heal*/
+    uint32_t background_self_heal_count; /*configurable queue length for
+                                           healing queue*/
+    int32_t healers; /* No. of elements currently undergoing background
+                      heal*/
+
+    gf_boolean_t release_ta_notify_dom_lock;
+
+    gf_boolean_t metadata_self_heal; /* on/off */
+    gf_boolean_t entry_self_heal;    /* on/off */
+
+    gf_boolean_t metadata_splitbrain_forced_heal; /* on/off */
+    int read_child;                               /* read-subvolume */
+    gf_atomic_t *pending_reads; /*No. of pending read cbks per child.*/
+
+    gf_timer_t *timer; /* launched when parent up is received */
+
+    unsigned int wait_count; /* # of servers to wait for success */
+
+    unsigned char ta_child_up;
+    gf_boolean_t optimistic_change_log;
+    gf_boolean_t eager_lock;
+    gf_boolean_t pre_op_compat; /* on/off */
+    uint32_t post_op_delay_secs;
+    unsigned int quorum_count;
+
+    off_t ta_notify_dom_lock_offset;
+    afr_favorite_child_policy fav_child_policy; /*Policy to use for automatic
+                                              resolution of split-brains.*/
+    afr_read_hash_mode_t hash_mode; /* for when read_child is not set */
+
+    int32_t *last_event;
+
+    /* @event_generation: Keeps count of number of events received which can
+       potentially impact consistency decisions. The events are CHILD_UP
+       and CHILD_DOWN, when we have to recalculate the freshness/staleness
+       of copies to detect if changes had happened while the other server
+       was down. CHILD_DOWN and CHILD_UP can also be received on network
+       disconnect/reconnects and not necessarily server going down/up.
+       Recalculating freshness/staleness on network events is equally
+       important as we might have had a network split brain.
+    */
+    uint32_t event_generation;
+    char vol_uuid[UUID_SIZE + 1];
+
+    gf_boolean_t choose_local;
+    gf_boolean_t did_discovery;
+    gf_boolean_t ensure_durability;
+    gf_boolean_t halo_enabled;
+    gf_boolean_t consistent_metadata;
+    gf_boolean_t need_heal;
+    gf_boolean_t granular_locks;
+    uint64_t sh_readdir_size;
+    char *sh_domain;
+    char *afr_dirty;
+
+    uint64_t spb_choice_timeout;
+
+    afr_self_heald_t shd;
+    struct afr_nfsd nfsd;
+
+    uint32_t halo_max_latency_msec;
+    uint32_t halo_max_replicas;
+    uint32_t halo_min_replicas;
+
+    gf_boolean_t full_lock;
+    gf_boolean_t esh_granular;
+    gf_boolean_t consistent_io;
+    gf_boolean_t data_self_heal; /* on/off */
+    gf_boolean_t use_anon_inode;
+
+    /*For lock healing.*/
+    struct list_head saved_locks;
+    struct list_head lk_healq;
+
+    /*For anon-inode handling */
+    char anon_inode_name[NAME_MAX + 1];
+    char anon_gfid_str[UUID_SIZE + 1];
+} afr_private_t;
 
 typedef enum {
-        SPLIT_BRAIN = 1,
-        ALL_FOOLS = 2
-} afr_subvol_status_t;
+    AFR_DATA_TRANSACTION,         /* truncate, write, ... */
+    AFR_METADATA_TRANSACTION,     /* chmod, chown, ... */
+    AFR_ENTRY_TRANSACTION,        /* create, rmdir, ... */
+    AFR_ENTRY_RENAME_TRANSACTION, /* rename */
+} afr_transaction_type;
+
+/*
+  xattr format: trusted.afr.volume = [x y z]
+  x - data pending
+  y - metadata pending
+  z - entry pending
+*/
+
+static inline int
+afr_index_for_transaction_type(afr_transaction_type type)
+{
+    switch (type) {
+        case AFR_DATA_TRANSACTION:
+            return 0;
+
+        case AFR_METADATA_TRANSACTION:
+            return 1;
+
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            return 2;
+    }
+
+    return -1; /* make gcc happy */
+}
+
+static inline int
+afr_index_from_ia_type(ia_type_t type)
+{
+    switch (type) {
+        case IA_IFDIR:
+            return afr_index_for_transaction_type(AFR_ENTRY_TRANSACTION);
+        case IA_IFREG:
+            return afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+        default:
+            return -1;
+    }
+}
+
+typedef struct {
+    struct gf_flock flock;
+    loc_t loc;
+    fd_t *fd;
+    char *basename;
+    unsigned char *locked_nodes;
+    int locked_count;
+
+} afr_lockee_t;
+
+int
+afr_entry_lockee_cmp(const void *l1, const void *l2);
+
+typedef struct {
+    loc_t *lk_loc;
+
+    afr_lockee_t lockee[AFR_LOCKEE_COUNT_MAX];
+
+    const char *lk_basename;
+    const char *lower_basename;
+    const char *higher_basename;
+
+    unsigned char *lower_locked_nodes;
+
+    afr_lock_cbk_t lock_cbk;
+
+    int lockee_count;
+
+    int32_t lk_call_count;
+    int32_t lk_expected_count;
+    int32_t lk_attempted_count;
+
+    int32_t lock_op_ret;
+    int32_t lock_op_errno;
+    char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
+    int32_t lock_count;
+    char lower_locked;
+    char higher_locked;
+} afr_internal_lock_t;
+
+struct afr_reply {
+    int valid;
+    int32_t op_ret;
+    dict_t *xattr; /*For xattrop*/
+    dict_t *xdata;
+    struct iatt poststat;
+    struct iatt postparent;
+    struct iatt prestat;
+    struct iatt preparent;
+    struct iatt preparent2;
+    struct iatt postparent2;
+    int32_t op_errno;
+    /* For rchecksum */
+    uint8_t checksum[SHA256_DIGEST_LENGTH];
+    gf_boolean_t buf_has_zeroes;
+    gf_boolean_t fips_mode_rchecksum;
+    /* For lookup */
+    int8_t need_heal;
+};
 
 typedef enum {
-        AFR_INODE_SET_READ_CTX = 1,
-        AFR_INODE_RM_STALE_CHILDREN,
-        AFR_INODE_SET_OPENDIR_DONE,
-        AFR_INODE_SET_SPLIT_BRAIN,
-        AFR_INODE_GET_READ_CTX,
-        AFR_INODE_GET_OPENDIR_DONE,
-        AFR_INODE_GET_SPLIT_BRAIN,
-} afr_inode_op_t;
-
-typedef struct afr_inode_params_ {
-        afr_inode_op_t op;
-        union {
-                gf_boolean_t value;
-                struct {
-                        int32_t read_child;
-                        int32_t *children;
-                } read_ctx;
-        } u;
-} afr_inode_params_t;
-
-typedef struct afr_inode_ctx_ {
-        uint64_t masks;
-        int32_t  *fresh_children;//increasing order of latency
-} afr_inode_ctx_t;
+    AFR_FD_NOT_OPENED,
+    AFR_FD_OPENED,
+    AFR_FD_OPENING
+} afr_fd_open_status_t;
+
+typedef struct {
+    afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
+    int flags;
+
+    /* the subvolume on which the latest sequence of readdirs (starting
+       at offset 0) has begun. Till the next readdir request with 0 offset
+       arrives, we continue to read off this subvol.
+    */
+    int readdir_subvol;
+    /* lock-healing related members. */
+    gf_boolean_t is_fd_bad;
+    afr_lk_heal_info_t *lk_heal_info;
+
+} afr_fd_ctx_t;
 
 typedef enum {
-        NONE,
-        INDEX,
-        FULL,
-} afr_crawl_type_t;
-
-typedef struct afr_self_heald_ {
-        gf_boolean_t     enabled;
-        gf_boolean_t     iamshd;
-        afr_crawl_type_t *pending;
-        gf_boolean_t     *inprogress;
-        afr_child_pos_t  *pos;
-        gf_timer_t       **timer;
-        eh_t             *healed;
-        eh_t             *heal_failed;
-        eh_t             *split_brain;
-        char             *node_uuid;
-        int              timeout;
-} afr_self_heald_t;
+    AFR_FOP_LOCK_PARALLEL,
+    AFR_FOP_LOCK_SERIAL,
+    AFR_FOP_LOCK_QUORUM_FAILED,
+} afr_fop_lock_state_t;
+
+typedef struct _afr_inode_lock_t {
+    /* @num_inodelks:
+       Number of inodelks queried from the server, as queried through
+       xdata in FOPs. Currently, used to decide if eager-locking must be
+       temporarily disabled.
+    */
+    int32_t num_inodelks;
+    unsigned int event_generation;
+    gf_timer_t *delay_timer;
+    struct list_head owners;  /*Transactions that are performing fop*/
+    struct list_head post_op; /*Transactions that are done with the fop
+                               *So can not conflict with the fops*/
+    struct list_head waiting; /*Transaction that are waiting for
+                               *conflicting transactions to complete*/
+    struct list_head frozen;  /*Transactions that need to go as part of
+                               * next batch of eager-lock*/
+    gf_boolean_t release;
+    gf_boolean_t acquired;
+} afr_lock_t;
+
+typedef struct _afr_inode_ctx {
+    uint64_t read_subvol;
+    uint64_t write_subvol;
+    int lock_count;
+    int spb_choice;
+    gf_timer_t *timer;
+    unsigned int *pre_op_done[AFR_NUM_CHANGE_LOGS];
+    int inherited[AFR_NUM_CHANGE_LOGS];
+    int on_disk[AFR_NUM_CHANGE_LOGS];
+    /*Only 2 types of transactions support eager-locks now. DATA/METADATA*/
+    afr_lock_t lock[2];
+
+    /* @open_fd_count:
+       Number of open FDs queried from the server, as queried through
+       xdata in FOPs. Currently, used to decide if eager-locking must be
+       temporarily disabled.
+    */
+    uint32_t open_fd_count;
+    gf_boolean_t need_refresh;
+
+    /* set if any write on this fd was a non stable write
+       (i.e, without O_SYNC or O_DSYNC)
+    */
+    gf_boolean_t witnessed_unstable_write;
+} afr_inode_ctx_t;
 
-typedef struct _afr_private {
-        gf_lock_t lock;               /* to guard access to child_count, etc */
-        unsigned int child_count;     /* total number of children   */
+typedef struct _afr_local {
+    glusterfs_fop_t op;
+    unsigned int call_count;
 
-        unsigned int read_child_rr;   /* round-robin index of the read_child */
-        gf_lock_t read_child_lock;    /* lock to protect above */
+    /* @event_generation: copy of priv->event_generation taken at the
+       time of starting the transaction. The copy is made so that we
+       have a stable value through the various phases of the transaction.
+    */
+    unsigned int event_generation;
 
-        xlator_t **children;
+    uint32_t open_fd_count;
+    int32_t num_inodelks;
 
-        int first_lookup;
-        inode_t *root_inode;
+    int32_t op_ret;
+    int32_t op_errno;
 
-        unsigned char *child_up;
+    int dirty[AFR_NUM_CHANGE_LOGS];
 
-        char **pending_key;
+    int32_t **pending;
 
-        char         *data_self_heal;              /* on/off/open */
-        char *       data_self_heal_algorithm;    /* name of algorithm */
-        unsigned int data_self_heal_window_size;  /* max number of pipelined
-                                                     read/writes */
+    loc_t loc;
+    loc_t newloc;
 
-        unsigned int background_self_heal_count;
-        unsigned int background_self_heals_started;
-        gf_boolean_t metadata_self_heal;   /* on/off */
-        gf_boolean_t entry_self_heal;      /* on/off */
+    fd_t *fd;
+    afr_fd_ctx_t *fd_ctx;
 
-        gf_boolean_t data_change_log;       /* on/off */
-        gf_boolean_t metadata_change_log;   /* on/off */
-        gf_boolean_t entry_change_log;      /* on/off */
+    /* @child_up: copy of priv->child_up taken at the time of transaction
+       start. The copy is taken so that we have a stable child_up array
+       through the phases of the transaction as priv->child_up[i] can keep
+       changing through time.
+    */
+    unsigned char *child_up;
 
-        int read_child;               /* read-subvolume */
-        unsigned int hash_mode;       /* for when read_child is not set */
-        int favorite_child;  /* subvolume to be preferred in resolving
-                                         split-brain cases */
+    /* @read_attempted:
+       array of flags representing subvolumes where read operations of
+       the read transaction have already been attempted. The array is
+       first pre-filled with down subvolumes, and as reads are performed
+       on other subvolumes, those are set as well. This way if the read
+       operation fails we do not retry on that subvolume again.
+    */
+    unsigned char *read_attempted;
 
-        gf_boolean_t inodelk_trace;
-        gf_boolean_t entrylk_trace;
+    /* @readfn:
 
-        gf_boolean_t strict_readdir;
+       pointer to function which will perform the read operation on a given
+       subvolume. Used in read transactions.
+    */
 
-        unsigned int wait_count;      /* # of servers to wait for success */
+    afr_read_txn_wind_t readfn;
 
-        uint64_t up_count;      /* number of CHILD_UPs we have seen */
-        uint64_t down_count;    /* number of CHILD_DOWNs we have seen */
+    /* @inode:
 
-        struct _pump_private *pump_private; /* Set if we are loaded as pump */
-        int                   use_afr_in_pump;
+       the inode on which the read txn is performed on. ref'ed and copied
+       from either fd->inode or loc.inode
+    */
 
-        pthread_mutex_t  mutex;
-        struct list_head saved_fds;   /* list of fds on which locks have succeeded */
-        gf_boolean_t      optimistic_change_log;
-        gf_boolean_t      eager_lock;
-	uint32_t          post_op_delay_secs;
-        unsigned int      quorum_count;
+    inode_t *inode;
 
-        char                   vol_uuid[UUID_SIZE + 1];
-        int32_t                *last_event;
-        afr_self_heald_t       shd;
-        gf_boolean_t           choose_local;
-        gf_boolean_t           did_discovery;
-} afr_private_t;
+    /* @parent[2]:
 
-typedef struct {
-        /* External interface: These are variables (some optional) that
-           are set by whoever has triggered self-heal */
-
-        gf_boolean_t do_data_self_heal;
-        gf_boolean_t do_metadata_self_heal;
-        gf_boolean_t do_entry_self_heal;
-        gf_boolean_t do_gfid_self_heal;
-        gf_boolean_t do_missing_entry_self_heal;
-
-        gf_boolean_t forced_merge;        /* Is this a self-heal triggered to
-                                             forcibly merge the directories? */
-
-        gf_boolean_t background;          /* do self-heal in background
-                                             if possible */
-        ia_type_t type;                   /* st_mode of the entry we're doing
-                                             self-heal on */
-        inode_t   *inode;                 /* inode on which the self-heal is
-                                             performed on */
-        uuid_t  sh_gfid_req;                 /* gfid self-heal needs to be done
-                                             with this gfid if it is not null */
-
-        /* Function to call to unwind. If self-heal is being done in the
-           background, this function will be called as soon as possible. */
-
-        int (*unwind) (call_frame_t *frame, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno, int32_t sh_failed);
-
-        /* End of external interface members */
-
-
-        /* array of stat's, one for each child */
-        struct iatt *buf;
-        struct iatt *parentbufs;
-        struct iatt parentbuf;
-        struct iatt entrybuf;
-
-        afr_expunge_done_cbk_t expunge_done;
-        afr_impunge_done_cbk_t impunge_done;
-
-        /* array of xattr's, one for each child */
-        dict_t **xattr;
-
-        /* array containing if the lookups succeeded in the order of response
-         */
-        int32_t *success_children;
-        int     success_count;
-        /* array containing the fresh children found in the self-heal process */
-        int32_t *fresh_children;
-        /* array containing the fresh children found in the parent lookup */
-        int32_t *fresh_parent_dirs;
-        /* array of errno's, one for each child */
-        int *child_errno;
-        /*loc used for lookup*/
-        loc_t lookup_loc;
-        int32_t lookup_flags;
-        afr_lookup_done_cbk_t lookup_done;
-
-        int32_t **pending_matrix;
-        int32_t **delta_matrix;
-
-        int32_t op_ret;
-        int32_t op_errno;
-
-        int *sources;
-        int source;
-        int active_source;
-        int active_sinks;
-        unsigned char *success;
-        unsigned char *locked_nodes;
-        int lock_count;
-
-        const char *linkname;
-        gf_boolean_t entries_skipped;
-
-        int   op_failed;
-
-        gf_boolean_t sync_done;
-        gf_boolean_t data_lock_held;
-        gf_boolean_t eof_reached;
-        fd_t  *healing_fd;
-        int   file_has_holes;
-        blksize_t block_size;
-        off_t file_size;
-        off_t offset;
-        unsigned char *write_needed;
-        uint8_t *checksum;
-        afr_post_remove_call_t post_remove_call;
+       parent inode[s] on which directory transactions are performed.
+    */
 
-        loc_t parent_loc;
+    inode_t *parent;
+    inode_t *parent2;
 
-        call_frame_t *orig_frame;
-        call_frame_t *old_loop_frame;
-        gf_boolean_t unwound;
+    /* @readable:
 
-        afr_sh_algo_private_t *private;
+       array of flags representing servers from which a read can be
+       performed. This is the output of afr_inode_refresh()
+    */
+    unsigned char *readable;
+    unsigned char *readable2; /*For rename transaction*/
 
-        struct afr_sh_algorithm  *algo;
-        afr_lock_cbk_t data_lock_success_handler;
-        afr_lock_cbk_t data_lock_failure_handler;
-        int (*completion_cbk) (call_frame_t *frame, xlator_t *this);
-        int (*sh_data_algo_start) (call_frame_t *frame, xlator_t *this);
-        int (*algo_completion_cbk) (call_frame_t *frame, xlator_t *this);
-        int (*algo_abort_cbk) (call_frame_t *frame, xlator_t *this);
-        void (*gfid_sh_success_cbk) (call_frame_t *sh_frame, xlator_t *this);
+    afr_inode_refresh_cbk_t refreshfn;
 
-        call_frame_t *sh_frame;
-} afr_self_heal_t;
+    /* @refreshinode:
 
-typedef enum {
-        AFR_DATA_TRANSACTION,          /* truncate, write, ... */
-        AFR_METADATA_TRANSACTION,      /* chmod, chown, ... */
-        AFR_ENTRY_TRANSACTION,         /* create, rmdir, ... */
-        AFR_ENTRY_RENAME_TRANSACTION,  /* rename */
-} afr_transaction_type;
+       Inode currently getting refreshed.
+    */
+    inode_t *refreshinode;
 
-typedef enum {
-        AFR_TRANSACTION_LK,
-        AFR_SELFHEAL_LK,
-} transaction_lk_type_t;
+    dict_t *xattr_req;
 
-typedef enum {
-        AFR_LOCK_OP,
-        AFR_UNLOCK_OP,
-} afr_lock_op_type_t;
+    dict_t *dict;
 
-typedef enum {
-        AFR_DATA_SELF_HEAL_LK,
-        AFR_METADATA_SELF_HEAL_LK,
-        AFR_ENTRY_SELF_HEAL_LK,
-}selfheal_lk_type_t;
+    int read_subvol; /* Current read subvolume */
 
-typedef enum {
-        AFR_INODELK_TRANSACTION,
-        AFR_INODELK_NB_TRANSACTION,
-        AFR_ENTRYLK_TRANSACTION,
-        AFR_ENTRYLK_NB_TRANSACTION,
-        AFR_INODELK_SELFHEAL,
-        AFR_INODELK_NB_SELFHEAL,
-        AFR_ENTRYLK_SELFHEAL,
-        AFR_ENTRYLK_NB_SELFHEAL,
-} afr_lock_call_type_t;
+    int optimistic_change_log;
 
-/*
-  xattr format: trusted.afr.volume = [x y z]
-  x - data pending
-  y - metadata pending
-  z - entry pending
-*/
+    afr_internal_lock_t internal_lock;
 
-static inline int
-afr_index_for_transaction_type (afr_transaction_type type)
-{
-        switch (type) {
+    /*To handle setattr/setxattr on yet to be linked inode from dht*/
+    uuid_t refreshgfid;
 
-        case AFR_DATA_TRANSACTION:
-                return 0;
+    /* @refreshed:
 
-        case AFR_METADATA_TRANSACTION:
-                return 1;
+       the inode was "refreshed" (i.e, pending xattrs from all subvols
+       freshly inspected and inode ctx updated accordingly) as part of
+       this transaction already.
+    */
+    gf_boolean_t refreshed;
 
-        case AFR_ENTRY_TRANSACTION:
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                return 2;
-        }
+    gf_boolean_t update_num_inodelks;
+    gf_boolean_t update_open_fd_count;
 
-        return -1;  /* make gcc happy */
-}
+    /*
+      @pre_op_compat:
 
+      compatibility mode of pre-op. send a separate pre-op and
+      op operations as part of transaction, rather than combining
+    */
 
-typedef struct {
-        loc_t *lk_loc;
-        struct gf_flock lk_flock;
-
-        const char *lk_basename;
-        const char *lower_basename;
-        const char *higher_basename;
-        char lower_locked;
-        char higher_locked;
-
-        unsigned char *locked_nodes;
-        unsigned char *lower_locked_nodes;
-        unsigned char *inode_locked_nodes;
-        unsigned char *entry_locked_nodes;
-
-        selfheal_lk_type_t selfheal_lk_type;
-        transaction_lk_type_t transaction_lk_type;
-
-        int32_t lock_count;
-        int32_t inodelk_lock_count;
-        int32_t entrylk_lock_count;
-
-        uint64_t lock_number;
-        int32_t lk_call_count;
-        int32_t lk_expected_count;
-
-        int32_t lock_op_ret;
-        int32_t lock_op_errno;
-        afr_lock_cbk_t lock_cbk;
-} afr_internal_lock_t;
+    gf_boolean_t pre_op_compat;
 
-typedef struct _afr_locked_fd {
-        fd_t  *fd;
-        struct list_head list;
-} afr_locked_fd_t;
+    /* Is the current writev() going to perform a stable write?
+       i.e, is fd->flags or @flags writev param have O_SYNC or
+       O_DSYNC?
+    */
+    gf_boolean_t stable_write;
 
-typedef struct _afr_local {
-        int     uid;
-        int     gid;
-        unsigned int call_count;
-        unsigned int success_count;
-        unsigned int enoent_count;
+    /* This write appended to the file. Nnot necessarily O_APPEND,
+       just means the offset of write was at the end of file.
+    */
+    gf_boolean_t append_write;
 
+    /*
+      This struct contains the arguments for the "continuation"
+      (scheme-like) of fops
+    */
 
-        unsigned int govinda_gOvinda;
+    struct {
+        struct {
+            struct statvfs buf;
+            unsigned char buf_set;
+        } statfs;
 
-        unsigned int read_child_index;
-        unsigned char read_child_returned;
-        unsigned int first_up_child;
+        struct {
+            fd_t *fd;
+            int32_t flags;
+        } open;
 
-	gf_lkowner_t  saved_lk_owner;
+        struct {
+            struct gf_flock user_flock;
+            struct gf_flock ret_flock;
+            unsigned char *locked_nodes;
+            int32_t cmd;
+            /*For lock healing only.*/
+            unsigned char *dom_locked_nodes;
+            int32_t *dom_lock_op_ret;
+            int32_t *dom_lock_op_errno;
+            struct gf_flock *getlk_rsp;
+        } lk;
+
+        /* inode read */
 
-        int32_t op_ret;
-        int32_t op_errno;
+        struct {
+            int32_t mask;
+            int last_index; /* index of the child we tried previously */
+        } access;
 
-        int32_t **pending;
+        struct {
+            int last_index;
+        } stat;
 
-        loc_t loc;
-        loc_t newloc;
+        struct {
+            int last_index;
+        } fstat;
 
-        fd_t *fd;
-        unsigned char *fd_open_on;
+        struct {
+            size_t size;
+            int last_index;
+        } readlink;
 
-        glusterfs_fop_t fop;
+        struct {
+            char *name;
+            long xattr_len;
+            int last_index;
+        } getxattr;
 
-        unsigned char *child_up;
-        int32_t       *fresh_children; //in the order of response
+        struct {
+            size_t size;
+            off_t offset;
+            int last_index;
+            uint32_t flags;
+        } readv;
 
-        int32_t *child_errno;
+        /* dir read */
 
-        dict_t  *xattr_req;
+        struct {
+            uint32_t *checksum;
+            int success_count;
+            int32_t op_ret;
+            int32_t op_errno;
+        } opendir;
 
-        int32_t  inodelk_count;
-        int32_t  entrylk_count;
+        struct {
+            int32_t op_ret;
+            int32_t op_errno;
+            size_t size;
+            off_t offset;
+            dict_t *dict;
+            int last_index;
+            gf_boolean_t failed;
+        } readdir;
+        /* inode write */
 
-        afr_internal_lock_t internal_lock;
+        struct {
+            struct iatt prebuf;
+            struct iatt postbuf;
+        } inode_wfop;  // common structure for all inode-write-fops
 
-        afr_locked_fd_t *locked_fd;
-        int32_t          source_child;
-        int32_t          lock_recovery_child;
+        struct {
+            struct iovec *vector;
+            struct iobref *iobref;
+            off_t offset;
+            int32_t op_ret;
+            int32_t count;
+            uint32_t flags;
+        } writev;
 
-        dict_t  *dict;
-        int      optimistic_change_log;
-	gf_boolean_t      delayed_post_op;
+        struct {
+            off_t offset;
+        } truncate;
 
-        gf_boolean_t    fop_paused;
-        int (*fop_call_continue) (call_frame_t *frame, xlator_t *this);
+        struct {
+            off_t offset;
+        } ftruncate;
 
-        /*
-          This struct contains the arguments for the "continuation"
-          (scheme-like) of fops
-        */
+        struct {
+            struct iatt in_buf;
+            int32_t valid;
+        } setattr;
+
+        struct {
+            struct iatt in_buf;
+            int32_t valid;
+        } fsetattr;
+
+        struct {
+            dict_t *dict;
+            int32_t flags;
+        } setxattr;
 
-        int   op;
         struct {
-                struct {
-                        unsigned char buf_set;
-                        struct statvfs buf;
-                } statfs;
-
-                struct {
-                        uint32_t parent_entrylk;
-                        uuid_t  gfid_req;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt postparent;
-                        dict_t **xattrs;
-                        dict_t *xattr;
-                        struct iatt *postparents;
-                        struct iatt *bufs;
-                        int32_t read_child;
-                        int32_t *sources;
-                        int32_t *success_children;
-                        gf_boolean_t fresh_lookup;
-                } lookup;
-
-                struct {
-                        int32_t flags;
-                } open;
-
-                struct {
-                        int32_t cmd;
-                        struct gf_flock user_flock;
-                        struct gf_flock ret_flock;
-                        unsigned char *locked_nodes;
-                } lk;
-
-                /* inode read */
-
-                struct {
-                        int32_t mask;
-                        int last_index;  /* index of the child we tried previously */
-                } access;
-
-                struct {
-                        int last_index;
-                } stat;
-
-                struct {
-                        int last_index;
-                } fstat;
-
-                struct {
-                        size_t size;
-                        int last_index;
-                } readlink;
-
-                struct {
-                        char *name;
-                        int last_index;
-                        long xattr_len;
-                } getxattr;
-
-                struct {
-                        size_t size;
-                        off_t offset;
-                        int last_index;
-                        uint32_t flags;
-                } readv;
-
-                /* dir read */
-
-                struct {
-                        int success_count;
-                        int32_t op_ret;
-                        int32_t op_errno;
-
-                        uint32_t *checksum;
-                } opendir;
-
-                struct {
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        size_t size;
-                        off_t offset;
-                        dict_t *dict;
-                        gf_boolean_t failed;
-                        int last_index;
-                } readdir;
-                /* inode write */
-
-                struct {
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-
-                        int32_t op_ret;
-
-                        struct iovec *vector;
-                        struct iobref *iobref;
-                        int32_t count;
-                        off_t offset;
-                        uint32_t flags;
-                } writev;
-
-                struct {
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } fsync;
-
-                struct {
-                        off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } truncate;
-
-                struct {
-                        off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } ftruncate;
-
-                struct {
-                        struct iatt in_buf;
-                        int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
-                } setattr;
-
-                struct {
-                        struct iatt in_buf;
-                        int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
-                } fsetattr;
-
-                struct {
-                        dict_t *dict;
-                        int32_t flags;
-                } setxattr;
-
-                struct {
-                        dict_t *dict;
-                        int32_t flags;
-                } fsetxattr;
-
-                struct {
-                        char *name;
-                } removexattr;
-
-                struct {
-                        dict_t *xattr;
-                } xattrop;
-
-                struct {
-                        dict_t *xattr;
-                } fxattrop;
-
-                /* dir write */
-
-                struct {
-                        fd_t *fd;
-                        dict_t *params;
-                        int32_t flags;
-                        mode_t mode;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                        struct iatt read_child_buf;
-                } create;
-
-                struct {
-                        dev_t dev;
-                        mode_t mode;
-                        dict_t *params;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                        struct iatt read_child_buf;
-                } mknod;
-
-                struct {
-                        int32_t mode;
-                        dict_t *params;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } mkdir;
-
-                struct {
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } unlink;
-
-                struct {
-                        int   flags;
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } rmdir;
-
-                struct {
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preoldparent;
-                        struct iatt prenewparent;
-                        struct iatt postoldparent;
-                        struct iatt postnewparent;
-                } rename;
-
-                struct {
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } link;
-
-                struct {
-                        inode_t *inode;
-                        dict_t *params;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        char *linkpath;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } symlink;
-
-                struct {
-                        int32_t flags;
-                        dir_entry_t *entries;
-                        int32_t count;
-                } setdents;
-        } cont;
+            dict_t *dict;
+            int32_t flags;
+        } fsetxattr;
 
         struct {
-                off_t start, len;
+            char *name;
+        } removexattr;
 
-                int *eager_lock;
+        struct {
+            dict_t *xattr;
+            gf_xattrop_flags_t optype;
+        } xattrop;
 
-                char *basename;
-                char *new_basename;
+        /* dir write */
 
-                loc_t parent_loc;
-                loc_t new_parent_loc;
+        struct {
+            inode_t *inode;
+            struct iatt buf;
+            struct iatt preparent;
+            struct iatt postparent;
+            struct iatt prenewparent;
+            struct iatt postnewparent;
+        } dir_fop;  // common structure for all dir fops
 
-                afr_transaction_type type;
+        struct {
+            fd_t *fd;
+            dict_t *params;
+            int32_t flags;
+            mode_t mode;
+        } create;
 
-                int32_t         **txn_changelog;//changelog after pre+post ops
-                unsigned char   *pre_op;
+        struct {
+            dict_t *params;
+            dev_t dev;
+            mode_t mode;
+        } mknod;
 
-                call_frame_t *main_frame;
+        struct {
+            dict_t *params;
+            int32_t mode;
+        } mkdir;
 
-                int (*fop) (call_frame_t *frame, xlator_t *this);
+        struct {
+            dict_t *params;
+            char *linkpath;
+        } symlink;
 
-                int (*done) (call_frame_t *frame, xlator_t *this);
+        struct {
+            off_t offset;
+            size_t len;
+            int32_t mode;
+        } fallocate;
 
-                int (*resume) (call_frame_t *frame, xlator_t *this);
+        struct {
+            off_t offset;
+            size_t len;
+        } discard;
 
-                int (*unwind) (call_frame_t *frame, xlator_t *this);
+        struct {
+            off_t offset;
+            off_t len;
+            struct iatt prebuf;
+            struct iatt postbuf;
+        } zerofill;
 
-                /* post-op hook */
-        } transaction;
+        struct {
+            char *volume;
+            int32_t cmd;
+            int32_t in_cmd;
+            struct gf_flock in_flock;
+            struct gf_flock flock;
+            void *xdata;
+        } inodelk;
 
-        afr_self_heal_t self_heal;
+        struct {
+            char *volume;
+            char *basename;
+            void *xdata;
+            entrylk_cmd in_cmd;
+            entrylk_cmd cmd;
+            entrylk_type type;
+        } entrylk;
 
-        struct marker_str     marker;
+        struct {
+            off_t offset;
+            gf_seek_what_t what;
+        } seek;
 
-        /* extra data for fops */
-        dict_t         *xdata_req;
-        dict_t         *xdata_rsp;
+        struct {
+            struct gf_lease user_lease;
+            struct gf_lease ret_lease;
+            unsigned char *locked_nodes;
+        } lease;
 
-        mode_t          umask;
-        int             xflag;
-        gf_boolean_t    do_discovery;
-} afr_local_t;
+        struct {
+            int flags;
+        } rmdir;
 
-typedef enum {
-        AFR_FD_NOT_OPENED,
-        AFR_FD_OPENED,
-        AFR_FD_OPENING
-} afr_fd_open_status_t;
+        struct {
+            int32_t datasync;
+        } fsync;
 
-typedef struct {
-        struct list_head call_list;
-        call_frame_t    *frame;
-} afr_fd_paused_call_t;
+        struct {
+            uuid_t gfid_req;
+            gf_boolean_t needs_fresh_lookup;
+        } lookup;
 
-typedef struct {
-        unsigned int *pre_op_done;
-        afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
-        unsigned int *pre_op_piggyback;
+    } cont;
 
-        unsigned int *lock_piggyback;
-        unsigned int *lock_acquired;
+    struct {
+        char *basename;
+        char *new_basename;
 
-        int flags;
-        uint64_t up_count;   /* number of CHILD_UPs this fd has seen */
-        uint64_t down_count; /* number of CHILD_DOWNs this fd has seen */
+        loc_t parent_loc;
+        loc_t new_parent_loc;
 
-        int32_t last_tried;
+        /* stub to resume on destruction
+           of the transaction frame */
+        call_stub_t *resume_stub;
 
-        int  hit, miss;
-        gf_boolean_t failed_over;
-        struct list_head entries; /* needed for readdir failover */
+        struct list_head owner_list;
+        struct list_head wait_list;
 
-        unsigned char *locked_on; /* which subvolumes locks have been successful */
-	struct list_head  paused_calls; /* queued calls while fix_open happens  */
+        unsigned char *pre_op;
 
-	/* used for delayed-post-op optimization */
-	pthread_mutex_t    delay_lock;
-	gf_timer_t        *delay_timer;
-	call_frame_t      *delay_frame;
-} afr_fd_ctx_t;
+        /* Changelog xattr dict for [f]xattrop*/
+        dict_t **changelog_xdata;
+        unsigned char *pre_op_sources;
 
+        /* @failed_subvols: subvolumes on which a pre-op or a
+            FOP failed. */
+        unsigned char *failed_subvols;
 
-/* try alloc and if it fails, goto label */
-#define AFR_LOCAL_ALLOC_OR_GOTO(var, label) do {                    \
-                var = mem_get0 (THIS->local_pool);                  \
-                if (!var) {                                         \
-                        gf_log (this->name, GF_LOG_ERROR,           \
-                                "out of memory :(");                \
-                        op_errno = ENOMEM;                          \
-                        goto label;                                 \
-                }                                                   \
-        } while (0);
+        call_frame_t *main_frame; /*Fop frame*/
+        call_frame_t *frame;      /*Transaction frame*/
 
+        int (*wind)(call_frame_t *frame, xlator_t *this, int subvol);
 
-/* did a call fail due to a child failing? */
-#define child_went_down(op_ret, op_errno) (((op_ret) < 0) &&            \
-                                           ((op_errno == ENOTCONN) ||   \
-                                            (op_errno == EBADFD)))
+        int (*unwind)(call_frame_t *frame, xlator_t *this);
 
-#define afr_fop_failed(op_ret, op_errno) ((op_ret) == -1)
+        off_t start, len;
 
-/* have we tried all children? */
-#define all_tried(i, count)  ((i) == (count) - 1)
+        afr_transaction_type type;
 
-int32_t
-afr_set_dict_gfid (dict_t *dict, uuid_t gfid);
+        int32_t in_flight_sb_errno; /* This is where the cause of the
+                                       failure on the last good copy of
+                                       the file is stored.
+                                       */
 
-int
-pump_command_reply (call_frame_t *frame, xlator_t *this);
+        /* @changelog_resume: function to be called after changlogging
+           (either pre-op or post-op) is done
+        */
+        afr_changelog_resume_t changelog_resume;
 
-int32_t
-afr_notify (xlator_t *this, int32_t event, void *data, void *data2);
+        gf_boolean_t eager_lock_on;
+        gf_boolean_t do_eager_unlock;
+
+        /* @dirtied: flag which indicates whether we set dirty flag
+           in the OP. Typically true when we are performing operation
+           on more than one subvol and optimistic changelog is disabled
+
+           A 'true' value set in @dirtied flag means an 'undirtying'
+           has to be done in POST-OP phase.
+        */
+        gf_boolean_t dirtied;
+
+        /* @inherited: flag which indicates that the dirty flags
+           of the previous transaction were inherited
+        */
+        gf_boolean_t inherited;
+
+        /*
+          @no_uninherit: flag which indicates that a pre_op_uninherit()
+          must _not_ be attempted (and returned as failure) always. This
+          flag is set when a hard pre-op is performed, but not accounted
+          for it in fd_ctx->on_disk[]. Such transactions are "isolated"
+          from the pre-op piggybacking entirely and therefore uninherit
+          must not be attempted.
+        */
+        gf_boolean_t no_uninherit;
 
+        gf_boolean_t in_flight_sb; /* Indicator for occurrence of
+                                      split-brain while in the middle of
+                                      a txn. */
+
+        /* @uninherit_done:
+           @uninherit_value:
+
+           The above pair variables make pre_op_uninherit() idempotent.
+           Both are FALSE initially. The first call to pre_op_uninherit
+           sets @uninherit_done to TRUE and the return value to
+           @uninherit_value. Further calls will check for @uninherit_done
+           to be TRUE and if so will simply return @uninherit_value.
+        */
+        gf_boolean_t uninherit_done;
+        gf_boolean_t uninherit_value;
+
+        gf_boolean_t disable_delayed_post_op;
+    } transaction;
+
+    syncbarrier_t barrier;
+
+    /* extra data for fops */
+    dict_t *xdata_req;
+    dict_t *xdata_rsp;
+
+    dict_t *xattr_rsp; /*for [f]xattrop*/
+
+    mode_t umask;
+    int xflag;
+    struct afr_reply *replies;
+
+    /* For  client side background heals. */
+    struct list_head healer;
+    call_frame_t *heal_frame;
+
+    afr_inode_ctx_t *inode_ctx;
+
+    /*For thin-arbiter transactions.*/
+    int ta_failed_subvol;
+    int ta_event_gen;
+    struct list_head ta_waitq;
+    struct list_head ta_onwireq;
+    afr_ta_fop_state_t fop_state;
+    afr_fop_lock_state_t fop_lock_state;
+    gf_lkowner_t saved_lk_owner;
+    unsigned char read_txn_query_child;
+    unsigned char ta_child_up;
+    gf_boolean_t do_discovery;
+    gf_boolean_t need_full_crawl;
+    gf_boolean_t is_read_txn;
+    gf_boolean_t is_new_entry;
+} afr_local_t;
+
+typedef struct afr_spbc_timeout {
+    call_frame_t *frame;
+    loc_t *loc;
+    int spb_child_index;
+    gf_boolean_t d_spb;
+    gf_boolean_t m_spb;
+} afr_spbc_timeout_t;
+
+typedef struct afr_spb_status {
+    call_frame_t *frame;
+    loc_t *loc;
+} afr_spb_status_t;
+
+typedef struct afr_empty_brick_args {
+    call_frame_t *frame;
+    char *op_type;
+    loc_t loc;
+    int empty_index;
+} afr_empty_brick_args_t;
+
+typedef struct afr_read_subvol_args {
+    ia_type_t ia_type;
+    uuid_t gfid;
+} afr_read_subvol_args_t;
+
+typedef struct afr_granular_esh_args {
+    fd_t *heal_fd;
+    xlator_t *xl;
+    call_frame_t *frame;
+    gf_boolean_t mismatch; /* flag to represent occurrence of type/gfid
+                              mismatch */
+} afr_granular_esh_args_t;
+
+int
+afr_inode_get_readable(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                       unsigned char *readable, int *event_p, int type);
 int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
+afr_inode_read_subvol_get(inode_t *inode, xlator_t *this,
+                          unsigned char *data_subvols,
+                          unsigned char *metadata_subvols,
+                          int *event_generation);
+int
+__afr_inode_read_subvol_get(inode_t *inode, xlator_t *this,
+                            unsigned char *data_subvols,
+                            unsigned char *metadata_subvols,
+                            int *event_generation);
 
 int
-afr_save_locked_fd (xlator_t *this, fd_t *fd);
+__afr_inode_read_subvol_set(inode_t *inode, xlator_t *this,
+                            unsigned char *data_subvols,
+                            unsigned char *metadata_subvol,
+                            int event_generation);
+int
+afr_inode_read_subvol_set(inode_t *inode, xlator_t *this,
+                          unsigned char *data_subvols,
+                          unsigned char *metadata_subvols,
+                          int event_generation);
 
 int
-afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
-                       unsigned char *locked_nodes);
+__afr_inode_need_refresh_set(inode_t *inode, xlator_t *this);
 
-void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner);
+int
+afr_inode_need_refresh_set(inode_t *inode, xlator_t *this);
 
 int
-afr_set_lock_number (call_frame_t *frame, xlator_t *this);
+afr_read_subvol_select_by_policy(inode_t *inode, xlator_t *this,
+                                 unsigned char *readable,
+                                 afr_read_subvol_args_t *args);
 
+int
+afr_inode_read_subvol_type_get(inode_t *inode, xlator_t *this,
+                               unsigned char *readable, int *event_p, int type);
+int
+afr_read_subvol_get(inode_t *inode, xlator_t *this, int *subvol_p,
+                    unsigned char *readables, int *event_p,
+                    afr_transaction_type type, afr_read_subvol_args_t *args);
 
-loc_t *
-lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2);
+#define afr_data_subvol_get(i, t, s, r, e, a)                                  \
+    afr_read_subvol_get(i, t, s, r, e, AFR_DATA_TRANSACTION, a)
 
-int32_t
-afr_unlock (call_frame_t *frame, xlator_t *this);
+#define afr_metadata_subvol_get(i, t, s, r, e, a)                              \
+    afr_read_subvol_get(i, t, s, r, e, AFR_METADATA_TRANSACTION, a)
 
 int
-afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this);
+afr_inode_refresh(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  uuid_t gfid, afr_inode_refresh_cbk_t cbk);
+
+int32_t
+afr_notify(xlator_t *this, int32_t event, void *data, void *data2);
 
 int
-afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this);
+xattr_is_equal(dict_t *this, char *key1, data_t *value1, void *data);
 
 int
-afr_blocking_lock (call_frame_t *frame, xlator_t *this);
+afr_add_entry_lockee(afr_local_t *local, loc_t *loc, char *basename,
+                     int child_count);
 
 int
-afr_internal_lock_finish (call_frame_t *frame, xlator_t *this);
+afr_add_inode_lockee(afr_local_t *local, int child_count);
 
 void
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src,
-                          unsigned int child_count);
-
-int pump_start (call_frame_t *frame, xlator_t *this);
+afr_lockees_cleanup(afr_internal_lock_t *int_lock);
 
 int
-__afr_fd_ctx_set (xlator_t *this, fd_t *fd);
+afr_attempt_lock_recovery(xlator_t *this, int32_t child_index);
 
 int
-afr_fd_ctx_set (xlator_t *this, fd_t *fd);
-
-int32_t
-afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children);
+afr_mark_locked_nodes(xlator_t *this, fd_t *fd, unsigned char *locked_nodes);
 
 void
-afr_inode_set_read_ctx (xlator_t *this, inode_t *inode, int32_t read_child,
-                        int32_t *fresh_children);
+afr_set_lk_owner(call_frame_t *frame, xlator_t *this, void *lk_owner);
 
 int
-afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno);
+afr_set_lock_number(call_frame_t *frame, xlator_t *this);
 
-unsigned int
-afr_up_children_count (unsigned char *child_up, unsigned int child_count);
+int32_t
+afr_unlock(call_frame_t *frame, xlator_t *this);
 
-unsigned int
-afr_locked_children_count (unsigned char *children, unsigned int child_count);
+int
+afr_lock_nonblocking(call_frame_t *frame, xlator_t *this);
 
-unsigned int
-afr_pre_op_done_children_count (unsigned char *pre_op,
-                                unsigned int child_count);
+int
+afr_blocking_lock(call_frame_t *frame, xlator_t *this);
 
-gf_boolean_t
-afr_is_fresh_lookup (loc_t *loc, xlator_t *this);
+int
+afr_internal_lock_finish(call_frame_t *frame, xlator_t *this);
 
-void
-afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent);
+int
+__afr_fd_ctx_set(xlator_t *this, fd_t *fd);
+
+afr_fd_ctx_t *
+afr_fd_ctx_get(fd_t *fd, xlator_t *this);
 
 int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
+afr_build_parent_loc(loc_t *parent, loc_t *child, int32_t *op_errno);
 
-void
-afr_local_cleanup (afr_local_t *local, xlator_t *this);
+int
+afr_locked_nodes_count(unsigned char *locked_nodes, int child_count);
 
 int
-afr_frame_return (call_frame_t *frame);
+afr_replies_interpret(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      gf_boolean_t *start_heal);
 
-gf_boolean_t
-afr_is_split_brain (xlator_t *this, inode_t *inode);
+void
+afr_local_replies_wipe(afr_local_t *local, afr_private_t *priv);
 
 void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set);
+afr_local_cleanup(afr_local_t *local, xlator_t *this);
 
 int
-afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata);
-
-void
-afr_set_opendir_done (xlator_t *this, inode_t *inode);
+afr_frame_return(call_frame_t *frame);
 
-gf_boolean_t
-afr_is_opendir_done (xlator_t *this, inode_t *inode);
+int
+afr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata);
 
 void
-afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this);
+afr_local_transaction_cleanup(afr_local_t *local, xlator_t *this);
 
 int
-afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd);
+afr_cleanup_fd_ctx(xlator_t *this, fd_t *fd);
+
+#define AFR_STACK_UNWIND(fop, frame, op_ret, op_errno, params...)              \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        int32_t __op_ret = 0;                                                  \
+        int32_t __op_errno = 0;                                                \
+                                                                               \
+        __op_ret = op_ret;                                                     \
+        __op_errno = op_errno;                                                 \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            __this = frame->this;                                              \
+            afr_handle_inconsistent_fop(frame, &__op_ret, &__op_errno);        \
+            if (__local && __local->is_read_txn)                               \
+                afr_pending_read_decrement(__this->private,                    \
+                                           __local->read_subvol);              \
+            if (__local && __local->xdata_req &&                               \
+                afr_is_lock_mode_mandatory(__local->xdata_req))                \
+                afr_dom_lock_release(frame);                                   \
+            frame->local = NULL;                                               \
+        }                                                                      \
+                                                                               \
+        STACK_UNWIND_STRICT(fop, frame, __op_ret, __op_errno, params);         \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0)
+
+#define AFR_STACK_DESTROY(frame)                                               \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        __local = frame->local;                                                \
+        __this = frame->this;                                                  \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0);
+
+#define AFR_FRAME_INIT(frame, op_errno)                                        \
+    ({                                                                         \
+        frame->local = mem_get0(THIS->local_pool);                             \
+        if (afr_local_init(frame->local, frame->this->private, &op_errno)) {   \
+            afr_local_cleanup(frame->local, frame->this);                      \
+            mem_put(frame->local);                                             \
+            frame->local = NULL;                                               \
+        };                                                                     \
+        frame->local;                                                          \
+    })
+
+#define AFR_STACK_RESET(frame)                                                 \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        __local = frame->local;                                                \
+        __this = frame->this;                                                  \
+        frame->local = NULL;                                                   \
+        int __opr;                                                             \
+        STACK_RESET(frame->root);                                              \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+        AFR_FRAME_INIT(frame, __opr);                                          \
+    } while (0)
 
-int
-afr_launch_openfd_self_heal (call_frame_t *frame, xlator_t *this, fd_t *fd);
-
-#define AFR_STACK_UNWIND(fop, frame, params ...)                \
-        do {                                                    \
-                afr_local_t *__local = NULL;                    \
-                xlator_t    *__this = NULL;                     \
-                if (frame) {                                    \
-                        __local = frame->local;                 \
-                        __this = frame->this;                   \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                if (__local) {                                  \
-                        afr_local_cleanup (__local, __this);    \
-                        mem_put (__local);                      \
-                }                                               \
-        } while (0)
-
-#define AFR_STACK_DESTROY(frame)                                \
-        do {                                                    \
-                afr_local_t *__local = NULL;                    \
-                xlator_t    *__this = NULL;                     \
-                __local = frame->local;                         \
-                __this = frame->this;                           \
-                frame->local = NULL;                            \
-                STACK_DESTROY (frame->root);                    \
-                if (__local) {                                  \
-                        afr_local_cleanup (__local, __this);    \
-                        mem_put (__local);                      \
-                }                                               \
-        } while (0);
-
-#define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
 /* allocate and return a string that is the basename of argument */
 static inline char *
-AFR_BASENAME (const char *str)
+AFR_BASENAME(const char *str)
 {
-        char *__tmp_str = NULL;
-        char *__basename_str = NULL;
-        __tmp_str = gf_strdup (str);
-        __basename_str = gf_strdup (basename (__tmp_str));
-        GF_FREE (__tmp_str);
-        return __basename_str;
+    char *__tmp_str = NULL;
+    char *__basename_str = NULL;
+    __tmp_str = gf_strdup(str);
+    __basename_str = gf_strdup(basename(__tmp_str));
+    GF_FREE(__tmp_str);
+    return __basename_str;
 }
 
+call_frame_t *
+afr_copy_frame(call_frame_t *base);
+
 int
-afr_transaction_local_init (afr_local_t *local, xlator_t *this);
+afr_transaction_local_init(afr_local_t *local, xlator_t *this);
 
 int32_t
-afr_marker_getxattr (call_frame_t *frame, xlator_t *this,
-                     loc_t *loc, const char *name,afr_local_t *local, afr_private_t *priv );
+afr_marker_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, afr_local_t *local, afr_private_t *priv);
 
-int32_t *
-afr_children_create (int32_t child_count);
+int
+afr_local_init(afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
 
 int
-afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
+afr_internal_lock_init(afr_internal_lock_t *lk, size_t child_count);
 
 int
-afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
-                        transaction_lk_type_t lk_type);
+afr_higher_errno(int32_t old_errno, int32_t new_errno);
 
 int
-afr_first_up_child (unsigned char *child_up, size_t child_count);
+afr_final_errno(afr_local_t *local, afr_private_t *priv);
 
 int
-afr_select_read_child_from_policy (int32_t *fresh_children, int32_t child_count,
-                                   int32_t prev_read_child,
-                                   int32_t config_read_child, int32_t *sources,
-                                   unsigned int hmode, uuid_t gfid);
+afr_xattr_req_prepare(xlator_t *this, dict_t *xattr_req);
 
 void
-afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
-                              int32_t *fresh_children, int32_t prev_read_child,
-                              int32_t config_read_child, uuid_t gfid);
+afr_fix_open(fd_t *fd, xlator_t *this);
 
-int32_t
-afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
-                    int32_t *fresh_children,
-                    int32_t *call_child, int32_t *last_index);
+afr_fd_ctx_t *
+afr_fd_ctx_get(fd_t *fd, xlator_t *this);
 
-int32_t
-afr_next_call_child (int32_t *fresh_children, unsigned char *child_up,
-                     size_t child_count, int32_t *last_index,
-                     int32_t read_child);
 void
-afr_get_fresh_children (int32_t *success_children, int32_t *sources,
-                        int32_t *children, unsigned int child_count);
+afr_set_low_priority(call_frame_t *frame);
+int
+afr_child_fd_ctx_set(xlator_t *this, fd_t *fd, int32_t child, int flags);
+
 void
-afr_children_add_child (int32_t *children, int32_t child,
-                              int32_t child_count);
+afr_matrix_cleanup(int32_t **pending, unsigned int m);
+
+int32_t **
+afr_matrix_create(unsigned int m, unsigned int n);
+
+int **
+afr_mark_pending_changelog(afr_private_t *priv, unsigned char *pending,
+                           dict_t *xattr, ia_type_t iat);
+
 void
-afr_children_rm_child (int32_t *children, int32_t child,
-                             int32_t child_count);
+afr_filter_xattrs(dict_t *xattr);
+
+/*
+ * Special value indicating we should use the "auto" quorum method instead of
+ * a fixed value (including zero to turn off quorum enforcement).
+ */
+#define AFR_QUORUM_AUTO INT_MAX
+
+int
+afr_fd_report_unstable_write(xlator_t *this, afr_local_t *local);
+
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write(xlator_t *this, inode_t *inode);
+
 void
-afr_reset_children (int32_t *children, int32_t child_count);
+afr_reply_wipe(struct afr_reply *reply);
+
+void
+afr_replies_wipe(struct afr_reply *replies, int count);
+
 gf_boolean_t
-afr_error_more_important (int32_t old_errno, int32_t new_errno);
+afr_xattrs_are_equal(dict_t *dict1, dict_t *dict2);
+
+gf_boolean_t
+afr_is_xattr_ignorable(char *key);
+
+int
+afr_get_heal_info(call_frame_t *frame, xlator_t *this, loc_t *loc);
+
 int
-afr_errno_count (int32_t *children, int *child_errno,
-                 unsigned int child_count, int32_t op_errno);
+afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc);
+
+int
+afr_get_split_brain_status(void *opaque);
+
 int
-afr_get_children_count (int32_t *children, unsigned int child_count);
+afr_get_split_brain_status_cbk(int ret, call_frame_t *frame, void *opaque);
+
+int
+afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this,
+                                 int spb_choice);
+int
+afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this,
+                                call_frame_t *frame, int *spb_subvol);
+int
+afr_get_child_index_from_name(xlator_t *this, char *name);
+
+int
+afr_is_split_brain(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb);
+int
+afr_spb_choice_timeout_cancel(xlator_t *this, inode_t *inode);
+
+int
+afr_set_split_brain_choice(int ret, call_frame_t *frame, void *opaque);
+
 gf_boolean_t
-afr_is_child_present (int32_t *success_children, int32_t child_count,
-                      int32_t child);
-void
-afr_update_gfid_from_iatts (uuid_t uuid, struct iatt *bufs,
-                            int32_t *success_children,
-                            unsigned int child_count);
+afr_get_need_heal(xlator_t *this);
+
 void
-afr_reset_xattr (dict_t **xattr, unsigned int child_count);
+afr_set_need_heal(xlator_t *this, afr_local_t *local);
+
+int
+afr_selfheal_data_open(xlator_t *this, inode_t *inode, fd_t **fd);
+
+int
+afr_get_msg_id(char *op_type);
+
+int
+afr_set_in_flight_sb_status(xlator_t *this, call_frame_t *frame,
+                            inode_t *inode);
+
+int32_t
+afr_quorum_errno(afr_private_t *priv);
+
 gf_boolean_t
-afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
-                        unsigned int child_count, const char *path,
-                        const char *xlator_name);
-unsigned int
-afr_gfid_missing_count (const char *xlator_name, int32_t *children,
-                        struct iatt *bufs, unsigned int child_count,
-                        const char *path);
+afr_is_consistent_io_possible(afr_local_t *local, afr_private_t *priv,
+                              int32_t *op_errno);
 void
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req, const char *path);
+afr_handle_inconsistent_fop(call_frame_t *frame, int32_t *op_ret,
+                            int32_t *op_errno);
+
 void
-afr_children_copy (int32_t *dst, int32_t *src, unsigned int child_count);
-afr_transaction_type
-afr_transaction_type_get (ia_type_t ia_type);
-int32_t
-afr_resultant_errno_get (int32_t *children,
-                         int *child_errno, unsigned int child_count);
+afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata);
 void
-afr_inode_rm_stale_children (xlator_t *this, inode_t *inode,
-                             int32_t *stale_children);
+afr_process_post_writev(call_frame_t *frame, xlator_t *this);
+
+void
+afr_writev_unwind(call_frame_t *frame, xlator_t *this);
+
+void
+afr_writev_copy_outvars(call_frame_t *src_frame, call_frame_t *dst_frame);
+
 void
-afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                      gf_boolean_t background, ia_type_t ia_type, char *reason,
-                      void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
-                                                   xlator_t *this),
-                      int (*unwind) (call_frame_t *frame, xlator_t *this,
-                                     int32_t op_ret, int32_t op_errno,
-                                     int32_t sh_failed));
+afr_update_uninodelk(afr_local_t *local, afr_internal_lock_t *int_lock,
+                     int32_t child_index);
+afr_fd_ctx_t *
+__afr_fd_ctx_get(fd_t *fd, xlator_t *this);
+
+gf_boolean_t
+afr_is_inode_refresh_reqd(inode_t *inode, xlator_t *this, int event_gen1,
+                          int event_gen2);
+
 int
-afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
-              int need_open_count, int *need_open);
+afr_serialize_xattrs_with_delimiter(call_frame_t *frame, xlator_t *this,
+                                    char *buf, const char *default_str,
+                                    int32_t *serz_len, char delimiter);
+gf_boolean_t
+afr_is_symmetric_error(call_frame_t *frame, xlator_t *this);
+
 int
-afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop);
+__afr_inode_ctx_get(xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx);
+
+uint64_t
+afr_write_subvol_get(call_frame_t *frame, xlator_t *this);
+
 int
-afr_set_elem_count_get (unsigned char *elems, int child_count);
+afr_write_subvol_set(call_frame_t *frame, xlator_t *this);
 
-afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this);
+int
+afr_write_subvol_reset(call_frame_t *frame, xlator_t *this);
+
+int
+afr_set_inode_local(xlator_t *this, afr_local_t *local, inode_t *inode);
+
+int
+afr_fill_ta_loc(xlator_t *this, loc_t *loc, gf_boolean_t is_gfid_based_fop);
+
+int
+afr_ta_post_op_lock(xlator_t *this, loc_t *loc);
+
+int
+afr_ta_post_op_unlock(xlator_t *this, loc_t *loc);
 
 gf_boolean_t
-afr_open_only_data_self_heal (char *data_self_heal);
+afr_is_pending_set(xlator_t *this, dict_t *xdata, int type);
+
+int
+__afr_get_up_children_count(afr_private_t *priv);
+
+call_frame_t *
+afr_ta_frame_create(xlator_t *this);
 
 gf_boolean_t
-afr_data_self_heal_enabled (char *data_self_heal);
+afr_ta_has_quorum(afr_private_t *priv, afr_local_t *local);
 
 void
-afr_set_low_priority (call_frame_t *frame);
-int
-afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags);
+afr_ta_lock_release_synctask(xlator_t *this);
+
+void
+afr_ta_locked_priv_invalidate(afr_private_t *priv);
 
 gf_boolean_t
-afr_have_quorum (char *logname, afr_private_t *priv);
+afr_lookup_has_quorum(call_frame_t *frame,
+                      const unsigned int up_children_count);
 
 void
-afr_matrix_cleanup (int32_t **pending, unsigned int m);
+afr_mark_new_entry_changelog(call_frame_t *frame, xlator_t *this);
 
-int32_t**
-afr_matrix_create (unsigned int m, unsigned int n);
-/*
- * Special value indicating we should use the "auto" quorum method instead of
- * a fixed value (including zero to turn off quorum enforcement).
- */
-#define AFR_QUORUM_AUTO INT_MAX
+void
+afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this);
 
-/*
- * Having this as a macro will make debugging a bit weirder, but does reduce
- * the probability of functions handling this check inconsistently.
- */
-#define QUORUM_CHECK(_func,_label) do {                                  \
-        if (priv->quorum_count && !afr_have_quorum(this->name,priv)) { \
-                gf_log(this->name,GF_LOG_WARNING,                        \
-                       "failing "#_func" due to lack of quorum");        \
-                op_errno = EROFS;                                        \
-                goto _label;                                             \
-        }                                                                \
-} while (0);
+gf_boolean_t
+afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv,
+                                   int child);
 
+void
+afr_selfheal_childup(xlator_t *this, afr_private_t *priv);
+
+gf_boolean_t
+afr_is_lock_mode_mandatory(dict_t *xdata);
+
+void
+afr_dom_lock_release(call_frame_t *frame);
+
+void
+afr_fill_success_replies(afr_local_t *local, afr_private_t *priv,
+                         unsigned char *replies);
+
+gf_boolean_t
+afr_is_private_directory(afr_private_t *priv, uuid_t pargfid, const char *name,
+                         pid_t pid);
 #endif /* __AFR_H__ */
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
deleted file mode 100644
index ff42d8dc11c..00000000000
--- a/xlators/cluster/afr/src/pump.c
+++ /dev/null
@@ -1,2635 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include <unistd.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <fnmatch.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "afr-common.c"
-#include "defaults.c"
-#include "glusterfs.h"
-
-static uint64_t pump_pid = 0;
-static inline void
-pump_fill_loc_info (loc_t *loc, struct iatt *iatt, struct iatt *parent)
-{
-        afr_update_loc_gfids (loc, iatt, parent);
-        uuid_copy (loc->inode->gfid, iatt->ia_gfid);
-}
-
-static int
-pump_mark_start_pending (xlator_t *this)
-{
-        afr_private_t  *priv      = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        pump_priv->pump_start_pending = 1;
-
-        return 0;
-}
-
-static int
-is_pump_start_pending (xlator_t *this)
-{
-        afr_private_t  *priv      = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        return (pump_priv->pump_start_pending);
-}
-
-static int
-pump_remove_start_pending (xlator_t *this)
-{
-        afr_private_t  *priv      = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        pump_priv->pump_start_pending = 0;
-
-        return 0;
-}
-
-static pump_state_t
-pump_get_state ()
-{
-        xlator_t *this = NULL;
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        pump_state_t ret;
-
-        this = THIS;
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        LOCK (&pump_priv->pump_state_lock);
-        {
-                ret = pump_priv->pump_state;
-        }
-        UNLOCK (&pump_priv->pump_state_lock);
-
-        return ret;
-}
-
-int
-pump_change_state (xlator_t *this, pump_state_t state)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        pump_state_t state_old;
-        pump_state_t state_new;
-
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        GF_ASSERT (pump_priv);
-
-        LOCK (&pump_priv->pump_state_lock);
-        {
-                state_old = pump_priv->pump_state;
-                state_new = state;
-
-                pump_priv->pump_state = state;
-
-        }
-        UNLOCK (&pump_priv->pump_state_lock);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Pump changing state from %d to %d",
-                state_old,
-                state_new);
-
-        return  0;
-}
-
-static int
-pump_set_resume_path (xlator_t *this, const char *path)
-{
-        int ret = 0;
-
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        GF_ASSERT (pump_priv);
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                strncpy (pump_priv->resume_path, path, strlen (path) + 1);
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        return ret;
-}
-
-static int
-pump_save_path (xlator_t *this, const char *path)
-{
-        afr_private_t *priv = NULL;
-        pump_state_t state;
-        dict_t *dict = NULL;
-        loc_t  loc = {0};
-        int dict_ret = 0;
-        int ret = -1;
-
-        state = pump_get_state ();
-        if (state == PUMP_STATE_RESUME)
-                return 0;
-
-        priv = this->private;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-        dict = dict_new ();
-        dict_ret = dict_set_str (dict, PUMP_PATH, (char *)path);
-        if (dict_ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set the key %s", path, PUMP_PATH);
-
-        ret = syncop_setxattr (PUMP_SOURCE_CHILD (this), &loc, dict, 0);
-
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "setxattr failed - could not save path=%s", path);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setxattr succeeded - saved path=%s", path);
-        }
-
-        dict_unref (dict);
-
-        loc_wipe (&loc);
-        return 0;
-}
-
-static int
-pump_check_and_update_status (xlator_t *this)
-{
-        pump_state_t state;
-        int ret = -1;
-
-        state = pump_get_state ();
-
-        switch (state) {
-
-        case PUMP_STATE_RESUME:
-        case PUMP_STATE_RUNNING:
-        {
-                ret = 0;
-                break;
-        }
-        case PUMP_STATE_PAUSE:
-        {
-                ret = -1;
-                break;
-        }
-        case PUMP_STATE_ABORT:
-        {
-                pump_save_path (this, "/");
-                ret = -1;
-                break;
-        }
-        default:
-        {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unknown pump state");
-                ret = -1;
-                break;
-        }
-
-        }
-
-        return ret;
-}
-
-static const char *
-pump_get_resume_path (xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        const char *resume_path = NULL;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        resume_path = pump_priv->resume_path;
-
-        return resume_path;
-}
-
-static int
-pump_update_resume_state (xlator_t *this, const char *path)
-{
-        pump_state_t state;
-        const char *resume_path = NULL;
-
-        state = pump_get_state ();
-
-        if (state == PUMP_STATE_RESUME) {
-                resume_path = pump_get_resume_path (this);
-                if (strcmp (resume_path, "/") == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Reached the resume path (/). Proceeding to change state"
-                                " to running");
-                        pump_change_state (this, PUMP_STATE_RUNNING);
-                } else if (strcmp (resume_path, path) == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Reached the resume path. Proceeding to change state"
-                                " to running");
-                        pump_change_state (this, PUMP_STATE_RUNNING);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Not yet hit the resume path:res-path=%s,path=%s",
-                                resume_path, path);
-                }
-        }
-
-        return 0;
-}
-
-static gf_boolean_t
-is_pump_traversal_allowed (xlator_t *this, const char *path)
-{
-        pump_state_t state;
-        const char *resume_path = NULL;
-        gf_boolean_t ret = _gf_true;
-
-        state = pump_get_state ();
-
-        if (state == PUMP_STATE_RESUME) {
-                resume_path = pump_get_resume_path (this);
-                if (strstr (resume_path, path)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "On the right path to resumption path");
-                        ret = _gf_true;
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Not the right path to resuming=> ignoring traverse");
-                        ret = _gf_false;
-                }
-        }
-
-        return ret;
-}
-
-static int
-pump_save_file_stats (xlator_t *this, const char *path)
-{
-        afr_private_t  *priv        = NULL;
-        pump_private_t *pump_priv   = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                pump_priv->number_files_pumped++;
-
-                strncpy (pump_priv->current_file, path,
-                         PATH_MAX);
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        return 0;
-}
-
-static int
-gf_pump_traverse_directory (loc_t *loc)
-{
-        xlator_t        *this              = NULL;
-        fd_t            *fd                = NULL;
-        off_t           offset             = 0;
-        loc_t           entry_loc          = {0};
-        gf_dirent_t     *entry             = NULL;
-        gf_dirent_t     *tmp               = NULL;
-        gf_dirent_t     entries;
-	struct iatt     iatt               = {0};
-        struct iatt     parent             = {0};
-	dict_t          *xattr_rsp         = NULL;
-        int             ret                = 0;
-        gf_boolean_t    is_directory_empty = _gf_true;
-        gf_boolean_t    free_entries       = _gf_false;
-
-        INIT_LIST_HEAD (&entries.list);
-        this = THIS;
-
-        GF_ASSERT (loc->inode);
-
-	fd = fd_create (loc->inode, pump_pid);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create fd for %s", loc->path);
-                goto out;
-        }
-
-        ret = syncop_opendir (this, loc, fd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "opendir failed on %s", loc->path);
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "pump opendir on %s returned=%d",
-                loc->path, ret);
-
-        while (syncop_readdirp (this, fd, 131072, offset, NULL, &entries)) {
-                free_entries = _gf_true;
-
-                if (list_empty (&entries.list)) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "no more entries in directory");
-                        goto out;
-                }
-
-                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "found readdir entry=%s", entry->d_name);
-
-                        offset = entry->d_off;
-                        if (uuid_is_null (entry->d_stat.ia_gfid)) {
-                                gf_log (this->name, GF_LOG_WARNING, "%s/%s: No "
-                                        "gfid present skipping",
-                                        loc->path, entry->d_name);
-                                continue;
-                        }
-                        loc_wipe (&entry_loc);
-                        ret = afr_build_child_loc (this, &entry_loc, loc,
-                                                   entry->d_name);
-                        if (ret)
-                                goto out;
-
-                        if (!IS_ENTRY_CWD (entry->d_name) &&
-                            !IS_ENTRY_PARENT (entry->d_name)) {
-
-                                    is_directory_empty = _gf_false;
-                                    gf_log (this->name, GF_LOG_DEBUG,
-                                            "lookup %s => %"PRId64,
-                                            entry_loc.path,
-                                            iatt.ia_ino);
-
-                                    ret = syncop_lookup (this, &entry_loc, NULL,
-                                                         &iatt, &xattr_rsp, &parent);
-
-                                    if (ret) {
-                                            gf_log (this->name, GF_LOG_ERROR,
-                                                    "%s: lookup failed",
-                                                    entry_loc.path);
-                                            continue;
-                                    }
-                                    pump_fill_loc_info (&entry_loc, &iatt,
-                                                       &parent);
-
-                                    pump_update_resume_state (this, entry_loc.path);
-
-                                    pump_save_path (this, entry_loc.path);
-                                    pump_save_file_stats (this, entry_loc.path);
-
-                                    ret = pump_check_and_update_status (this);
-                                    if (ret < 0) {
-                                            gf_log (this->name, GF_LOG_DEBUG,
-                                                    "Pump beginning to exit out");
-                                            goto out;
-                                    }
-
-                                    if (IA_ISDIR (iatt.ia_type)) {
-                                            if (is_pump_traversal_allowed (this, entry_loc.path)) {
-                                                    gf_log (this->name, GF_LOG_TRACE,
-                                                            "entering dir=%s",
-                                                            entry->d_name);
-                                                    gf_pump_traverse_directory (&entry_loc);
-                                            }
-                                    }
-                        }
-                }
-
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-                gf_log (this->name, GF_LOG_TRACE,
-                        "offset incremented to %d",
-                        (int32_t ) offset);
-
-        }
-
-        ret = syncop_close (fd);
-        if (ret < 0)
-                gf_log (this->name, GF_LOG_DEBUG, "closing the fd failed");
-
-        if (is_directory_empty && IS_ROOT_PATH (loc->path)) {
-               pump_change_state (this, PUMP_STATE_RUNNING);
-               gf_log (this->name, GF_LOG_INFO, "Empty source brick. "
-                                "Nothing to be done.");
-        }
-
-out:
-        if (entry_loc.path)
-                loc_wipe (&entry_loc);
-        if (free_entries)
-                gf_dirent_free (&entries);
-        return 0;
-}
-
-static int
-pump_update_resume_path (xlator_t *this)
-{
-        const char *resume_path = NULL;
-
-        resume_path = pump_get_resume_path (this);
-
-        if (resume_path) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Found a path to resume from: %s",
-                        resume_path);
-
-        }else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Did not find a path=> setting to '/'");
-                pump_set_resume_path (this, "/");
-        }
-
-        pump_change_state (this, PUMP_STATE_RESUME);
-
-        return 0;
-}
-
-static int32_t
-pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_private_t  *priv      = NULL;
-        loc_t           loc       = {0};
-        int             i         = 0;
-        int             ret       = 0;
-        int             source    = 0;
-        int             sink      = 1;
-
-        priv      = this->private;
-
-        afr_build_root_loc (this, &loc);
-
-        ret = syncop_removexattr (priv->children[source], &loc,
-                                          PUMP_PATH);
-
-        ret = syncop_removexattr (priv->children[sink], &loc,
-                                  PUMP_SINK_COMPLETE);
-
-        for (i = 0; i < priv->child_count; i++) {
-                ret = syncop_removexattr (priv->children[i], &loc,
-                                          PUMP_SOURCE_COMPLETE);
-                if (ret)
-                        gf_log (this->name, GF_LOG_DEBUG, "removexattr "
-                                "failed with %s", strerror (errno));
-        }
-
-        loc_wipe (&loc);
-        return pump_command_reply (frame, this);
-}
-
-static int
-pump_complete_migration (xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-        dict_t *dict = NULL;
-        pump_state_t state;
-        loc_t  loc = {0};
-        int dict_ret = 0;
-        int ret = -1;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-        dict = dict_new ();
-
-        state = pump_get_state ();
-        if (state == PUMP_STATE_RUNNING) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Pump finished pumping");
-
-                pump_priv->pump_finished = _gf_true;
-
-                dict_ret = dict_set_str (dict, PUMP_SOURCE_COMPLETE, "jargon");
-                if (dict_ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to set the key %s",
-                                loc.path, PUMP_SOURCE_COMPLETE);
-
-                ret = syncop_setxattr (PUMP_SOURCE_CHILD (this), &loc, dict, 0);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "setxattr failed - while  notifying source complete");
-                }
-                dict_ret = dict_set_str (dict, PUMP_SINK_COMPLETE, "jargon");
-                if (dict_ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to set the key %s",
-                                loc.path, PUMP_SINK_COMPLETE);
-
-                ret = syncop_setxattr (PUMP_SINK_CHILD (this), &loc, dict, 0);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "setxattr failed - while notifying sink complete");
-                }
-
-                pump_save_path (this, "/");
-
-        } else if (state == PUMP_STATE_ABORT) {
-                gf_log (this->name, GF_LOG_DEBUG, "Starting cleanup "
-                        "of pump internal xattrs");
-                call_resume (pump_priv->cleaner);
-        }
-
-        loc_wipe (&loc);
-        return 0;
-}
-
-static int
-pump_lookup_sink (loc_t *loc)
-{
-        xlator_t *this = NULL;
-	struct iatt iatt, parent;
-	dict_t *xattr_rsp;
-        dict_t *xattr_req = NULL;
-        int ret = 0;
-
-        this = THIS;
-
-        xattr_req = dict_new ();
-
-        ret = afr_set_root_gfid (xattr_req);
-        if (ret)
-                goto out;
-
-        ret = syncop_lookup (PUMP_SINK_CHILD (this), loc,
-                             xattr_req, &iatt, &xattr_rsp, &parent);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Lookup on sink child failed");
-                goto out;
-        }
-
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        return ret;
-}
-
-static int
-pump_task (void *data)
-{
-	xlator_t *this = NULL;
-        afr_private_t *priv = NULL;
-
-
-        loc_t loc = {0};
-	struct iatt iatt, parent;
-	dict_t *xattr_rsp = NULL;
-        dict_t *xattr_req = NULL;
-
-        int ret = -1;
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        afr_set_root_gfid (xattr_req);
-        ret = syncop_lookup (this, &loc, xattr_req,
-                             &iatt, &xattr_rsp, &parent);
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "lookup: path=%s gfid=%s",
-                loc.path, uuid_utoa (loc.inode->gfid));
-
-        ret = pump_check_and_update_status (this);
-        if (ret < 0) {
-                goto out;
-        }
-
-        pump_update_resume_path (this);
-
-        afr_set_root_gfid (xattr_req);
-        ret = pump_lookup_sink (&loc);
-        if (ret) {
-                pump_update_resume_path (this);
-                goto out;
-        }
-
-        gf_pump_traverse_directory (&loc);
-
-        pump_complete_migration (this);
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        loc_wipe (&loc);
-	return 0;
-}
-
-
-static int
-pump_task_completion (int ret, call_frame_t *sync_frame, void *data)
-{
-        xlator_t *this = NULL;
-        afr_private_t *priv = NULL;
-
-        this = THIS;
-
-        priv = this->private;
-
-        inode_unref (priv->root_inode);
-        STACK_DESTROY (sync_frame->root);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Pump xlator exiting");
-	return 0;
-}
-
-int
-pump_start (call_frame_t *pump_frame, xlator_t *this)
-{
-	afr_private_t *priv = NULL;
-	pump_private_t *pump_priv = NULL;
-
-	int ret = -1;
-
-	priv = this->private;
-        pump_priv = priv->pump_private;
-
-        afr_set_lk_owner (pump_frame, this, pump_frame->root);
-	pump_pid = (uint64_t) (unsigned long)pump_frame->root;
-
-	ret = synctask_new (pump_priv->env, pump_task,
-                            pump_task_completion,
-                            pump_frame, NULL);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "starting pump failed");
-                pump_change_state (this, PUMP_STATE_ABORT);
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setting pump as started lk_owner: %s %"PRIu64,
-                lkowner_utoa (&pump_frame->root->lk_owner), pump_pid);
-
-        priv->use_afr_in_pump = 1;
-out:
-	return ret;
-}
-
-static int
-pump_start_synctask (xlator_t *this)
-{
-        call_frame_t *frame = NULL;
-        int ret = 0;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        pump_change_state (this, PUMP_STATE_RUNNING);
-
-        ret = pump_start (frame, this);
-
-out:
-        return ret;
-}
-
-int32_t
-pump_cmd_start_setxattr_cbk (call_frame_t *frame,
-                             void *cookie,
-                             xlator_t *this,
-                             int32_t op_ret,
-                             int32_t op_errno, dict_t *xdata)
-
-{
-        call_frame_t *prev = NULL;
-        afr_local_t *local = NULL;
-        int ret = 0;
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not initiate destination "
-                        "brick connect");
-                ret = op_ret;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Successfully initiated destination "
-                "brick connect");
-
-        pump_mark_start_pending (this);
-
-        /* send the PARENT_UP as pump is ready now */
-        prev = cookie;
-        if (prev && prev->this)
-                prev->this->notify (prev->this, GF_EVENT_PARENT_UP, this);
-
-out:
-        local->op_ret = ret;
-        pump_command_reply (frame, this);
-
-        return 0;
-}
-
-static int
-pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local     = NULL;
-        afr_private_t *priv      = NULL;
-        dict_t        *dict      = NULL;
-        data_t        *data      = NULL;
-        char          *clnt_cmd  = NULL;
-        loc_t loc = {0};
-
-        int ret = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-        data = data_ref (dict_get (local->dict, RB_PUMP_CMD_START));
-        if (!data) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not get destination brick value");
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        clnt_cmd = GF_CALLOC (1, data->len+1, gf_common_mt_char);
-        if (!clnt_cmd) {
-                ret = -1;
-                goto out;
-        }
-
-        memcpy (clnt_cmd, data->data, data->len);
-        clnt_cmd[data->len] = '\0';
-        gf_log (this->name, GF_LOG_DEBUG, "Got destination brick %s\n",
-                        clnt_cmd);
-
-        ret = dict_set_dynstr (dict, CLIENT_CMD_CONNECT, clnt_cmd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not inititiate destination brick "
-                        "connect");
-                goto out;
-        }
-
-	STACK_WIND (frame,
-		    pump_cmd_start_setxattr_cbk,
-		    PUMP_SINK_CHILD(this),
-		    PUMP_SINK_CHILD(this)->fops->setxattr,
-		    &loc,
-		    dict,
-		    0, NULL);
-
-        ret = 0;
-
-out:
-        if (dict)
-                dict_unref (dict);
-
-        if (data)
-                data_unref (data);
-
-        if (ret && clnt_cmd)
-                GF_FREE (clnt_cmd);
-
-        loc_wipe (&loc);
-        return ret;
-}
-
-static int
-is_pump_aborted (xlator_t *this)
-{
-        pump_state_t state;
-
-        state = pump_get_state ();
-
-        return ((state == PUMP_STATE_ABORT));
-}
-
-int32_t
-pump_cmd_start_getxattr_cbk (call_frame_t *frame,
-                             void *cookie,
-                             xlator_t *this,
-                             int32_t op_ret,
-                             int32_t op_errno,
-                             dict_t *dict, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-        char *path = NULL;
-
-        pump_state_t state;
-        int ret = 0;
-        int need_unwind = 0;
-        int dict_ret = -1;
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "getxattr failed - changing pump "
-                        "state to RUNNING with '/'");
-                path = "/";
-                ret = op_ret;
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "getxattr succeeded");
-
-                dict_ret =  dict_get_str (dict, PUMP_PATH, &path);
-                if (dict_ret < 0)
-                        path = "/";
-        }
-
-        state = pump_get_state ();
-        if ((state == PUMP_STATE_RUNNING) ||
-            (state == PUMP_STATE_RESUME)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Pump is already started");
-                ret = -1;
-                goto out;
-        }
-
-        pump_set_resume_path (this, path);
-
-        if (is_pump_aborted (this))
-                /* We're re-starting pump afresh */
-                ret = pump_initiate_sink_connect (frame, this);
-        else {
-                /* We're re-starting pump from a previous
-                   pause */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "about to start synctask");
-                ret = pump_start_synctask (this);
-                need_unwind = 1;
-        }
-
-out:
-        if ((ret < 0) || (need_unwind == 1)) {
-                local->op_ret = ret;
-                pump_command_reply (frame, this);
-        }
-	return 0;
-}
-
-int
-pump_execute_status (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        uint64_t number_files = 0;
-
-        char filename[PATH_MAX];
-        char *dict_str = NULL;
-
-        int32_t op_ret = 0;
-        int32_t op_errno = 0;
-
-        dict_t *dict = NULL;
-        int ret = -1;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                number_files  = pump_priv->number_files_pumped;
-                strncpy (filename, pump_priv->current_file, PATH_MAX);
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        dict_str     = GF_CALLOC (1, PATH_MAX + 256, gf_afr_mt_char);
-        if (!dict_str) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        if (pump_priv->pump_finished) {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"        Migration complete ",
-                  number_files);
-        } else {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"       Current file= %s ",
-                  number_files, filename);
-        }
-
-        dict = dict_new ();
-
-        ret = dict_set_dynstr (dict, RB_PUMP_CMD_STATUS, dict_str);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_set_dynstr returned negative value");
-        } else {
-                dict_str = NULL;
-        }
-
-        op_ret = 0;
-
-out:
-
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
-
-        if (dict)
-                dict_unref (dict);
-
-        GF_FREE (dict_str);
-
-        return 0;
-}
-
-int
-pump_execute_pause (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        pump_change_state (this, PUMP_STATE_PAUSE);
-
-        local->op_ret = 0;
-        pump_command_reply (frame, this);
-
-        return 0;
-}
-
-int
-pump_execute_start (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        afr_local_t   *local = NULL;
-
-        int ret = 0;
-        loc_t loc = {0};
-
-        priv = this->private;
-        local = frame->local;
-
-        if (!priv->root_inode) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Pump xlator cannot be started without an initial "
-                        "lookup");
-                ret = -1;
-                goto out;
-        }
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-	STACK_WIND (frame,
-		    pump_cmd_start_getxattr_cbk,
-		    PUMP_SOURCE_CHILD(this),
-		    PUMP_SOURCE_CHILD(this)->fops->getxattr,
-		    &loc,
-		    PUMP_PATH, NULL);
-
-        ret = 0;
-
-out:
-        if (ret < 0) {
-                local->op_ret = ret;
-                pump_command_reply (frame, this);
-        }
-
-        loc_wipe (&loc);
-	return 0;
-}
-
-static int
-pump_cleanup_helper (void *data) {
-        call_frame_t *frame = data;
-
-        pump_xattr_cleaner (frame, 0, frame->this, 0, 0, NULL);
-
-        return 0;
-}
-
-static int
-pump_cleanup_done (int ret, call_frame_t *sync_frame, void *data)
-{
-        STACK_DESTROY (sync_frame->root);
-
-        return 0;
-}
-
-int
-pump_execute_commit (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t  *priv       = NULL;
-        pump_private_t *pump_priv  = NULL;
-        afr_local_t    *local      = NULL;
-        call_frame_t   *sync_frame = NULL;
-        int             ret        = 0;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-        local     = frame->local;
-
-        local->op_ret = 0;
-        if (pump_priv->pump_finished) {
-                pump_change_state (this, PUMP_STATE_COMMIT);
-                sync_frame = create_frame (this, this->ctx->pool);
-                ret = synctask_new (pump_priv->env, pump_cleanup_helper,
-                                    pump_cleanup_done, sync_frame, frame);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Couldn't create "
-                                "synctask for cleaning up xattrs.");
-                }
-
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "Commit can't proceed. "
-                        "Migration in progress");
-                local->op_ret = -1;
-                local->op_errno = EINPROGRESS;
-                pump_command_reply (frame, this);
-        }
-
-        return 0;
-}
-int
-pump_execute_abort (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t  *priv       = NULL;
-        pump_private_t *pump_priv  = NULL;
-        afr_local_t    *local      = NULL;
-        call_frame_t   *sync_frame = NULL;
-        int             ret        = 0;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-        local     = frame->local;
-
-        pump_change_state (this, PUMP_STATE_ABORT);
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                pump_priv->number_files_pumped = 0;
-                pump_priv->current_file[0] = '\0';
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        local->op_ret = 0;
-        if (pump_priv->pump_finished) {
-                sync_frame = create_frame (this, this->ctx->pool);
-                ret = synctask_new (pump_priv->env, pump_cleanup_helper,
-                                    pump_cleanup_done, sync_frame, frame);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Couldn't create "
-                                "synctask for cleaning up xattrs.");
-                }
-
-        } else {
-                pump_priv->cleaner = fop_setxattr_cbk_stub (frame,
-                                                            pump_xattr_cleaner,
-                                                            0, 0, NULL);
-        }
-
-        return 0;
-}
-
-gf_boolean_t
-pump_command_status (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_STATUS, &cmd);
-        if (dict_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump status command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - status");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_pause (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_PAUSE, &cmd);
-        if (dict_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump pause command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - pause");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_commit (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_COMMIT, &cmd);
-        if (dict_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump commit command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - commit");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_abort (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_ABORT, &cmd);
-        if (dict_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump abort command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - abort");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_start (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_START, &cmd);
-        if (dict_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump start command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - start");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-struct _xattr_key {
-        char *key;
-        struct list_head list;
-};
-
-static void
-__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
-                     void *data)
-{
-        struct list_head *  list  = data;
-        struct _xattr_key * xkey  = NULL;
-
-        if (!strncmp (key, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
-
-                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
-                if (!xkey)
-                        return;
-
-                xkey->key = key;
-                INIT_LIST_HEAD (&xkey->list);
-
-                list_add_tail (&xkey->list, list);
-        }
-}
-
-static void
-__filter_xattrs (dict_t *dict)
-{
-        struct list_head keys;
-
-        struct _xattr_key *key;
-        struct _xattr_key *tmp;
-
-        INIT_LIST_HEAD (&keys);
-
-        dict_foreach (dict, __gather_xattr_keys,
-                      (void *) &keys);
-
-        list_for_each_entry_safe (key, tmp, &keys, list) {
-                dict_del (dict, key->key);
-
-                list_del_init (&key->list);
-
-                GF_FREE (key);
-        }
-}
-
-int32_t
-pump_getxattr_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  dict_t *dict, dict_t *xdata)
-{
-	afr_private_t   *priv           = NULL;
-	afr_local_t     *local          = NULL;
-	xlator_t        **children      = NULL;
-	int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children = NULL;
-
-
-	priv     = this->private;
-	children = priv->children;
-
-	local = frame->local;
-
-        read_child = (long) cookie;
-
-	if (op_ret == -1) {
-		last_index = &local->cont.getxattr.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-		unwind = 0;
-		STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
-				   (void *) (long) read_child,
-				   children[next_call_child],
-				   children[next_call_child]->fops->getxattr,
-				   &local->loc,
-				   local->cont.getxattr.name, NULL);
-	}
-
-out:
-	if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
-
-		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
-	}
-
-	return 0;
-}
-
-int32_t
-pump_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name, dict_t *xdata)
-{
-	afr_private_t *   priv       = NULL;
-	xlator_t **       children   = NULL;
-	int               call_child = 0;
-	afr_local_t       *local     = NULL;
-	int32_t           ret     = -1;
-	int32_t           op_errno   = 0;
-        uint64_t          read_child = 0;
-
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
-
-	children = priv->children;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_getxattr_cbk,
-                            FIRST_CHILD (this),
-                            (FIRST_CHILD (this))->fops->getxattr,
-                            loc, name, xdata);
-                return 0;
-        }
-
-
-	AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-	local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        if (name) {
-                if (!strncmp (name, AFR_XATTR_PREFIX,
-                              strlen (AFR_XATTR_PREFIX))) {
-
-                        op_errno = ENODATA;
-                        goto out;
-                }
-
-                if (!strcmp (name, RB_PUMP_CMD_STATUS)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Hit pump command - status");
-                        pump_execute_status (frame, this);
-                        ret = 0;
-                        goto out;
-                }
-        }
-
-        local->fresh_children = GF_CALLOC (priv->child_count,
-                                          sizeof (*local->fresh_children),
-                                          gf_afr_mt_int32_t);
-        if (!local->fresh_children) {
-                ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        read_child = afr_inode_get_read_ctx (this, loc->inode, local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.getxattr.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-	loc_copy (&local->loc, loc);
-	if (name)
-	  local->cont.getxattr.name       = gf_strdup (name);
-
-	STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->getxattr,
-			   loc, name, xdata);
-
-	ret = 0;
-out:
-	if (ret < 0)
-		AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
-static int
-afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *   local = NULL;
-	call_frame_t   *main_frame = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-		AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno, NULL);
-	}
-	return 0;
-}
-
-static int
-afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count  = -1;
-	int need_unwind = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->child_count) {
-				need_unwind = 1;
-			}
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	if (need_unwind)
-		local->transaction.unwind (frame, this);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-
-	return 0;
-}
-
-static int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->setxattr,
-					   &local->loc,
-					   local->cont.setxattr.dict,
-					   local->cont.setxattr.flags, NULL);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-static int
-afr_setxattr_done (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t * local = frame->local;
-
-	local->transaction.unwind (frame, this);
-
-	AFR_STACK_DESTROY (frame);
-
-	return 0;
-}
-
-int32_t
-pump_setxattr_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno, dict_t *xdata)
-{
-	AFR_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
-	return 0;
-}
-
-int
-pump_command_reply (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        if (local->op_ret < 0)
-                gf_log (this->name, GF_LOG_INFO,
-                        "Command failed");
-        else
-                gf_log (this->name, GF_LOG_INFO,
-                        "Command succeeded");
-
-        AFR_STACK_UNWIND (setxattr,
-                          frame,
-                          local->op_ret,
-                          local->op_errno, NULL);
-
-        return 0;
-}
-
-int
-pump_parse_command (call_frame_t *frame, xlator_t *this,
-                    afr_local_t *local, dict_t *dict)
-{
-
-        int ret = -1;
-
-        if (pump_command_start (this, dict)) {
-                frame->local = local;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_start (frame, this);
-
-        } else if (pump_command_pause (this, dict)) {
-                frame->local = local;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_pause (frame, this);
-
-        } else if (pump_command_abort (this, dict)) {
-                frame->local = local;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_abort (frame, this);
-
-        } else if (pump_command_commit (this, dict)) {
-                frame->local = local;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_commit (frame, this);
-        }
-        return ret;
-}
-
-int
-pump_setxattr (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
-{
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
-        data_pair_t   * trav  = NULL;
-	int ret = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict,
-                                   trav, op_errno, out);
-
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_setxattr_cbk,
-                            FIRST_CHILD (this),
-                            (FIRST_CHILD (this))->fops->setxattr,
-                            loc, dict, flags, xdata);
-                return 0;
-        }
-
-
-	AFR_LOCAL_ALLOC_OR_GOTO (local, out);
-
-	ret = afr_local_init (local, priv, &op_errno);
-	if (ret < 0) {
-                afr_local_cleanup (local, this);
-		goto out;
-        }
-
-        ret = pump_parse_command (frame, this,
-                                  local, dict);
-        if (ret >= 0) {
-                ret = 0;
-                goto out;
-        }
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-                op_errno = ENOMEM;
-                ret = -1;
-                afr_local_cleanup (local, this);
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	local->op_ret = -1;
-
-	local->cont.setxattr.dict  = dict_ref (dict);
-	local->cont.setxattr.flags = flags;
-
-	local->transaction.fop    = afr_setxattr_wind;
-	local->transaction.done   = afr_setxattr_done;
-	local->transaction.unwind = afr_setxattr_unwind;
-
-	loc_copy (&local->loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.start   = LLONG_MAX - 1;
-	local->transaction.len     = 0;
-
-	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-
-	ret = 0;
-out:
-	if (ret < 0) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-	}
-
-	return 0;
-}
-
-/* Defaults */
-static int32_t
-pump_lookup (call_frame_t *frame,
-             xlator_t *this,
-             loc_t *loc,
-             dict_t *xattr_req)
-{
-	afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_lookup_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup,
-                            loc,
-                            xattr_req);
-                return 0;
-        }
-
-        afr_lookup (frame, this, loc, xattr_req);
-        return 0;
-}
-
-
-static int32_t
-pump_truncate (call_frame_t *frame,
-               xlator_t *this,
-               loc_t *loc,
-               off_t offset, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_truncate_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            loc,
-                            offset, xdata);
-                return 0;
-        }
-
-        afr_truncate (frame, this, loc, offset, xdata);
-        return 0;
-}
-
-
-static int32_t
-pump_ftruncate (call_frame_t *frame,
-                xlator_t *this,
-                fd_t *fd,
-                off_t offset, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_ftruncate_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate,
-                            fd,
-                            offset, xdata);
-                return 0;
-        }
-
-        afr_ftruncate (frame, this, fd, offset, xdata);
-        return 0;
-}
-
-
-
-
-int
-pump_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_mknod_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mknod,
-                            loc, mode, rdev, umask, xdata);
-                return 0;
-        }
-        afr_mknod (frame, this, loc, mode, rdev, umask, xdata);
-        return 0;
-
-}
-
-
-
-int
-pump_mkdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_mkdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mkdir,
-                            loc, mode, umask, xdata);
-                return 0;
-        }
-        afr_mkdir (frame, this, loc, mode, umask, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_unlink (call_frame_t *frame,
-             xlator_t *this,
-             loc_t *loc, int xflag, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_unlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink,
-                            loc, xflag, xdata);
-                return 0;
-        }
-        afr_unlink (frame, this, loc, xflag, xdata);
-        return 0;
-
-}
-
-
-static int
-pump_rmdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int flags, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-
-	priv = this->private;
-
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_rmdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rmdir,
-                            loc, flags, xdata);
-                return 0;
-        }
-
-        afr_rmdir (frame, this, loc, flags, xdata);
-        return 0;
-
-}
-
-
-
-int
-pump_symlink (call_frame_t *frame, xlator_t *this,
-              const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_symlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->symlink,
-                            linkpath, loc, umask, xdata);
-                return 0;
-        }
-        afr_symlink (frame, this, linkpath, loc, umask, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_rename (call_frame_t *frame,
-             xlator_t *this,
-             loc_t *oldloc,
-             loc_t *newloc, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_rename_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rename,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
-        afr_rename (frame, this, oldloc, newloc, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_link (call_frame_t *frame,
-           xlator_t *this,
-           loc_t *oldloc,
-           loc_t *newloc, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_link_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->link,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
-        afr_link (frame, this, oldloc, newloc, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_create (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int32_t flags, mode_t mode,
-             mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_create_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->create,
-                            loc, flags, mode, umask, fd, xdata);
-                return 0;
-        }
-        afr_create (frame, this, loc, flags, mode, umask, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_open (call_frame_t *frame,
-           xlator_t *this,
-           loc_t *loc,
-           int32_t flags, fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_open_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open,
-                            loc, flags, fd, xdata);
-                return 0;
-        }
-        afr_open (frame, this, loc, flags, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_writev (call_frame_t *frame,
-             xlator_t *this,
-             fd_t *fd,
-             struct iovec *vector,
-             int32_t count,
-             off_t off, uint32_t flags,
-             struct iobref *iobref, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_writev_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->writev,
-                            fd,
-                            vector,
-                            count,
-                            off, flags,
-                            iobref, xdata);
-                return 0;
-        }
-
-        afr_writev (frame, this, fd, vector, count, off, flags, iobref, xdata);
-        return 0;
-}
-
-
-static int32_t
-pump_flush (call_frame_t *frame,
-            xlator_t *this,
-            fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_flush_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->flush,
-                            fd, xdata);
-                return 0;
-        }
-        afr_flush (frame, this, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_fsync (call_frame_t *frame,
-            xlator_t *this,
-            fd_t *fd,
-            int32_t flags, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fsync_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsync,
-                            fd,
-                            flags, xdata);
-                return 0;
-        }
-        afr_fsync (frame, this, fd, flags, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_opendir (call_frame_t *frame,
-              xlator_t *this,
-              loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_opendir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->opendir,
-                            loc, fd, xdata);
-                return 0;
-        }
-        afr_opendir (frame, this, loc, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_fsyncdir (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               int32_t flags, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fsyncdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsyncdir,
-                            fd,
-                            flags, xdata);
-                return 0;
-        }
-        afr_fsyncdir (frame, this, fd, flags, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_xattrop (call_frame_t *frame,
-              xlator_t *this,
-              loc_t *loc,
-              gf_xattrop_flags_t flags,
-              dict_t *dict, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_xattrop_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->xattrop,
-                            loc,
-                            flags,
-                            dict, xdata);
-                return 0;
-        }
-        afr_xattrop (frame, this, loc, flags, dict, xdata);
-        return 0;
-
-}
-
-static int32_t
-pump_fxattrop (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               gf_xattrop_flags_t flags,
-               dict_t *dict, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fxattrop_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fxattrop,
-                            fd,
-                            flags,
-                            dict, xdata);
-                return 0;
-        }
-        afr_fxattrop (frame, this, fd, flags, dict, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_removexattr (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t *loc,
-                  const char *name, dict_t *xdata)
-{
-        afr_private_t *priv     = NULL;
-        int            op_errno = -1;
-
-        VALIDATE_OR_GOTO (this, out);
-
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.pump*",
-                                 name, op_errno, out);
-
-        op_errno = 0;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_removexattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->removexattr,
-                            loc,
-                            name, xdata);
-                return 0;
-        }
-        afr_removexattr (frame, this, loc, name, xdata);
-
- out:
-        if (op_errno)
-                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
-        return 0;
-
-}
-
-
-
-static int32_t
-pump_readdir (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              size_t size,
-              off_t off, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_readdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdir,
-                            fd, size, off, xdata);
-                return 0;
-        }
-        afr_readdir (frame, this, fd, size, off, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               size_t size, off_t off, dict_t *dict)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_readdirp_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdirp,
-                            fd, size, off, dict);
-                return 0;
-        }
-        afr_readdirp (frame, this, fd, size, off, dict);
-        return 0;
-
-}
-
-
-
-static int32_t
-pump_releasedir (xlator_t *this,
-                 fd_t *fd)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (priv->use_afr_in_pump)
-                afr_releasedir (this, fd);
-	return 0;
-
-}
-
-static int32_t
-pump_release (xlator_t *this,
-              fd_t *fd)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (priv->use_afr_in_pump)
-                afr_release (this, fd);
-	return 0;
-
-}
-
-static int32_t
-pump_forget (xlator_t *this, inode_t *inode)
-{
-        afr_private_t  *priv  = NULL;
-
-        priv = this->private;
-        if (priv->use_afr_in_pump)
-                afr_forget (this, inode);
-
-        return 0;
-}
-
-static int32_t
-pump_setattr (call_frame_t *frame,
-              xlator_t *this,
-              loc_t *loc,
-              struct iatt *stbuf,
-              int32_t valid, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_setattr_cbk,
-                            FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid, xdata);
-                return 0;
-        }
-        afr_setattr (frame, this, loc, stbuf, valid, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_fsetattr (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               struct iatt *stbuf,
-               int32_t valid, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fsetattr_cbk,
-                            FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsetattr,
-                            fd, stbuf, valid, xdata);
-                return 0;
-        }
-        afr_fsetattr (frame, this, fd, stbuf, valid, xdata);
-        return 0;
-
-}
-
-
-/* End of defaults */
-
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_afr_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-static int
-is_xlator_pump_sink (xlator_t *child)
-{
-        return (child == PUMP_SINK_CHILD(THIS));
-}
-
-static int
-is_xlator_pump_source (xlator_t *child)
-{
-        return (child == PUMP_SOURCE_CHILD(THIS));
-}
-
-int32_t
-notify (xlator_t *this, int32_t event,
-	void *data, ...)
-{
-        int ret = -1;
-        xlator_t *child_xl = NULL;
-
-        child_xl = (xlator_t *) data;
-
-        ret = afr_notify (this, event, data, NULL);
-
-	switch (event) {
-	case GF_EVENT_CHILD_DOWN:
-                if (is_xlator_pump_source (child_xl))
-                        pump_change_state (this, PUMP_STATE_ABORT);
-                break;
-
-        case GF_EVENT_CHILD_UP:
-                if (is_xlator_pump_sink (child_xl))
-                        if (is_pump_start_pending (this)) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "about to start synctask");
-                                ret = pump_start_synctask (this);
-                                if (ret < 0)
-                                        gf_log (this->name, GF_LOG_DEBUG,
-                                                "Could not start pump "
-                                                "synctask");
-                                else
-                                        pump_remove_start_pending (this);
-                        }
-        }
-
-        return ret;
-}
-
-int32_t
-init (xlator_t *this)
-{
-	afr_private_t * priv        = NULL;
-        pump_private_t *pump_priv = NULL;
-	int             child_count = 0;
-	xlator_list_t * trav        = NULL;
-	int             i           = 0;
-	int             ret         = -1;
-	GF_UNUSED int   op_errno    = 0;
-
-        int source_child = 0;
-
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"pump translator needs a source and sink"
-                        "subvolumes defined.");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Volume is dangling.");
-	}
-
-	this->private = GF_CALLOC (1, sizeof (afr_private_t),
-                                   gf_afr_mt_afr_private_t);
-        if (!this->private)
-                goto out;
-
-	priv = this->private;
-        LOCK_INIT (&priv->lock);
-        LOCK_INIT (&priv->read_child_lock);
-        //lock recovery is not done in afr
-        pthread_mutex_init (&priv->mutex, NULL);
-        INIT_LIST_HEAD (&priv->saved_fds);
-
-        child_count = xlator_subvolume_count (this);
-        if (child_count != 2) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "There should be exactly 2 children - one source "
-                        "and one sink");
-                return -1;
-        }
-	priv->child_count = child_count;
-
-        priv->read_child = source_child;
-        priv->favorite_child = source_child;
-        priv->background_self_heal_count = 0;
-
-	priv->data_self_heal     = "on";
-	priv->metadata_self_heal = 1;
-	priv->entry_self_heal    = 1;
-
-        priv->data_self_heal_algorithm = "";
-
-        priv->data_self_heal_window_size = 16;
-
-	priv->data_change_log     = 1;
-	priv->metadata_change_log = 1;
-	priv->entry_change_log    = 1;
-        priv->use_afr_in_pump = 1;
-
-	/* Locking options */
-
-        /* Lock server count infact does not matter. Locks are held
-           on all subvolumes, in this case being the source
-           and the sink.
-        */
-
-	priv->strict_readdir = _gf_false;
-
-	priv->wait_count = 1;
-	priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
-                                 gf_afr_mt_char);
-	if (!priv->child_up) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-	priv->children = GF_CALLOC (sizeof (xlator_t *), child_count,
-                                 gf_afr_mt_xlator_t);
-	if (!priv->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-        priv->pending_key = GF_CALLOC (sizeof (*priv->pending_key),
-                                       child_count,
-                                       gf_afr_mt_char);
-        if (!priv->pending_key) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-	trav = this->children;
-	i = 0;
-	while (i < child_count) {
-		priv->children[i] = trav->xlator;
-
-                ret = gf_asprintf (&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
-                                   trav->xlator->name);
-                if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "asprintf failed to set pending key");
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-		trav = trav->next;
-		i++;
-	}
-
-        priv->first_lookup = 1;
-        priv->root_inode = NULL;
-
-        priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
-                                      gf_afr_mt_int32_t);
-        if (!priv->last_event) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-	pump_priv = GF_CALLOC (1, sizeof (*pump_priv),
-                            gf_afr_mt_pump_priv);
-	if (!pump_priv) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-                op_errno = ENOMEM;
-		goto out;
-	}
-
-        LOCK_INIT (&pump_priv->resume_path_lock);
-        LOCK_INIT (&pump_priv->pump_state_lock);
-
-        pump_priv->resume_path = GF_CALLOC (1, PATH_MAX,
-                                            gf_afr_mt_char);
-        if (!pump_priv->resume_path) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-	pump_priv->env = syncenv_new (0);
-        if (!pump_priv->env) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not create new sync-environment");
-                ret = -1;
-                goto out;
-        }
-
-        /* keep more local here as we may need them for self-heal etc */
-        this->local_pool = mem_pool_new (afr_local_t, 128);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
-
-	priv->pump_private = pump_priv;
-
-        pump_change_state (this, PUMP_STATE_ABORT);
-
-	ret = 0;
-out:
-	return ret;
-}
-
-int
-fini (xlator_t *this)
-{
-        afr_private_t * priv        = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        this->private = NULL;
-        if (!priv)
-                goto out;
-
-        pump_priv = priv->pump_private;
-        if (!pump_priv)
-                goto afr_priv;
-
-        if (pump_priv->env)
-                syncenv_destroy (pump_priv->env);
-
-        GF_FREE (pump_priv->resume_path);
-        LOCK_DESTROY (&pump_priv->resume_path_lock);
-        LOCK_DESTROY (&pump_priv->pump_state_lock);
-        GF_FREE (pump_priv);
-afr_priv:
-        afr_priv_destroy (priv);
-out:
-	return 0;
-}
-
-
-struct xlator_fops fops = {
-	.lookup      = pump_lookup,
-	.open        = pump_open,
-	.flush       = pump_flush,
-	.fsync       = pump_fsync,
-	.fsyncdir    = pump_fsyncdir,
-	.xattrop     = pump_xattrop,
-	.fxattrop    = pump_fxattrop,
-        .getxattr    = pump_getxattr,
-
-	/* inode write */
-	.writev      = pump_writev,
-	.truncate    = pump_truncate,
-	.ftruncate   = pump_ftruncate,
-	.setxattr    = pump_setxattr,
-        .setattr     = pump_setattr,
-	.fsetattr    = pump_fsetattr,
-	.removexattr = pump_removexattr,
-
-	/* dir read */
-	.opendir     = pump_opendir,
-	.readdir     = pump_readdir,
-	.readdirp    = pump_readdirp,
-
-	/* dir write */
-	.create      = pump_create,
-	.mknod       = pump_mknod,
-	.mkdir       = pump_mkdir,
-	.unlink      = pump_unlink,
-	.rmdir       = pump_rmdir,
-	.link        = pump_link,
-	.symlink     = pump_symlink,
-	.rename      = pump_rename,
-};
-
-struct xlator_dumpops dumpops = {
-        .priv       = afr_priv_dump,
-};
-
-
-struct xlator_cbks cbks = {
-	.release     = pump_release,
-	.releasedir  = pump_releasedir,
-        .forget      = pump_forget,
-};
-
-struct volume_options options[] = {
-	{ .key  = {NULL} },
-};
diff --git a/xlators/cluster/afr/src/pump.h b/xlators/cluster/afr/src/pump.h
deleted file mode 100644
index bc4c31a78a5..00000000000
--- a/xlators/cluster/afr/src/pump.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __PUMP_H__
-#define __PUMP_H__
-
-#include "syncop.h"
-
-/* FIXME: Needs to be defined in a common file */
-#define CLIENT_CMD_CONNECT "trusted.glusterfs.client-connect"
-#define CLIENT_CMD_DISCONNECT "trusted.glusterfs.client-disconnect"
-
-#define PUMP_SOURCE_COMPLETE "trusted.glusterfs.pump-source-complete"
-#define PUMP_SINK_COMPLETE "trusted.glusterfs.pump-sink-complete"
-
-#define PUMP_PATH "trusted.glusterfs.pump-path"
-
-#define PUMP_SOURCE_CHILD(xl) (xl->children->xlator)
-#define PUMP_SINK_CHILD(xl) (xl->children->next->xlator)
-
-typedef enum {
-        PUMP_STATE_RUNNING,             /* Pump is running and migrating files */
-        PUMP_STATE_RESUME,              /* Pump is resuming from a previous pause */
-        PUMP_STATE_PAUSE,               /* Pump is paused */
-        PUMP_STATE_ABORT,               /* Pump is aborted */
-        PUMP_STATE_COMMIT,              /* Pump is commited */
-} pump_state_t;
-
-typedef struct _pump_private {
-	struct syncenv *env;            /* The env pointer to the pump synctask */
-        char *resume_path;              /* path to resume from the last pause */
-        gf_lock_t resume_path_lock;     /* Synchronize resume_path changes */
-        gf_lock_t pump_state_lock;      /* Synchronize pump_state changes */
-        pump_state_t pump_state;        /* State of pump */
-        char current_file[PATH_MAX];    /* Current file being pumped */
-        uint64_t number_files_pumped;   /* Number of files pumped */
-        gf_boolean_t pump_finished;     /* Boolean to indicate pump termination */
-        char pump_start_pending;        /* Boolean to mark start pending until
-                                           CHILD_UP */
-        call_stub_t *cleaner;
-} pump_private_t;
-
-void
-build_root_loc (inode_t *inode, loc_t *loc);
-int pump_start (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_start (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_start (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_pause (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_pause (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_abort (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_abort (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_status (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_status (call_frame_t *frame, xlator_t *this);
-
-#endif /* __PUMP_H__ */
diff --git a/xlators/cluster/dht/src/Makefile.am b/xlators/cluster/dht/src/Makefile.am
index e35058d65d4..56f1f2ad7c8 100644
--- a/xlators/cluster/dht/src/Makefile.am
+++ b/xlators/cluster/dht/src/Makefile.am
@@ -1,32 +1,37 @@
-
 xlator_LTLIBRARIES = dht.la nufa.la switch.la
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
 dht_common_source = dht-layout.c dht-helper.c dht-linkfile.c dht-rebalance.c \
 	dht-selfheal.c dht-rename.c dht-hashfn.c dht-diskusage.c \
-	dht-common.c dht-inode-write.c dht-inode-read.c \
-	$(top_builddir)/xlators/lib/src/libxlator.c
+	dht-common.c dht-inode-write.c dht-inode-read.c dht-shared.c \
+	dht-lock.c $(top_builddir)/xlators/lib/src/libxlator.c
 
 dht_la_SOURCES = $(dht_common_source) dht.c
 
 nufa_la_SOURCES = $(dht_common_source) nufa.c
 switch_la_SOURCES = $(dht_common_source) switch.c
 
-dht_la_LDFLAGS = -module -avoidversion
+dht_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 dht_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-nufa_la_LDFLAGS = -module -avoidversion
+nufa_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-switch_la_LDFLAGS = -module -avoidversion
+switch_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = dht-common.h dht-mem-types.h \
-	$(top_builddir)/xlators/lib/src/libxlator.h
+noinst_HEADERS = dht-common.h dht-mem-types.h dht-messages.h \
+	dht-lock.h $(top_builddir)/xlators/lib/src/libxlator.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
-	-I$(top_srcdir)/xlators/lib/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/xlators/lib/src \
+	-DDATADIR=\"$(localstatedir)\" \
+	-DLIBDIR=\"$(libdir)\"
 
 CLEANFILES =
 
@@ -35,3 +40,9 @@ uninstall-local:
 
 install-data-hook:
 	ln -sf dht.so $(DESTDIR)$(xlatordir)/distribute.so
+
+if UNITTEST
+CLEANFILES += *.gcda *.gcno *_xunit.xml
+noinst_PROGRAMS =
+TESTS =
+endif
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 5b57552c388..8ba0cc4c732 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -8,90 +8,439 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 /* TODO: add NS locking */
 
-#include "glusterfs.h"
-#include "xlator.h"
 #include "libxlator.h"
 #include "dht-common.h"
-#include "defaults.h"
-#include "byte-order.h"
+#include "dht-lock.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/quota-common-utils.h>
+#include <glusterfs/upcall-utils.h>
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
+#include <glusterfs/common-utils.h>
 
 #include <sys/time.h>
 #include <libgen.h>
+#include <signal.h>
 
-void
-dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
+static int
+dht_rmdir_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata);
+
+static int
+dht_link2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+
+static int
+dht_set_dir_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req);
+
+static int
+dht_lookup_everywhere_done(call_frame_t *frame, xlator_t *this);
+
+static int
+dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, dict_t *xdata);
+
+static int
+dht_rmdir_unlock(call_frame_t *frame, xlator_t *this);
+
+static const char *dht_dbg_vxattrs[] = {DHT_DBG_HASHED_SUBVOL_PATTERN, NULL};
+
+/* Check the xdata to make sure EBADF has been set by client xlator */
+int32_t
+dht_check_remote_fd_failed_error(dht_local_t *local, int op_ret, int op_errno)
 {
-        dict_t  *dst  = NULL;
-        int64_t *ptr  = 0, *size = NULL;
-        int32_t  ret  = -1;
-        data_pair_t  *data_pair = NULL;
+    if (op_ret == -1 && (op_errno == EBADF || op_errno == EBADFD) &&
+        !(local->fd_checked)) {
+        return 1;
+    }
+    return 0;
+}
 
-        dst = data;
+/* Sets the blocks and size values to fixed values. This is to be called
+ * only for dirs. The caller is responsible for checking the type
+ */
+int32_t
+dht_set_fixed_dir_stat(struct iatt *stat)
+{
+    if (stat) {
+        stat->ia_blocks = DHT_DIR_STAT_BLOCKS;
+        stat->ia_size = DHT_DIR_STAT_SIZE;
+        return 0;
+    }
+    return -1;
+}
 
-        if (strcmp (key, GF_XATTR_QUOTA_SIZE_KEY) == 0) {
-                ret = dict_get_bin (dst, key, (void **)&size);
-                if (ret < 0) {
-                        size = GF_CALLOC (1, sizeof (int64_t),
-                                          gf_common_mt_char);
-                        if (size == NULL) {
-                                gf_log ("dht", GF_LOG_WARNING,
-                                        "memory allocation failed");
-                                return;
-                        }
-                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
-                        if (ret < 0) {
-                                gf_log ("dht", GF_LOG_WARNING,
-                                        "dht aggregate dict set failed");
-                                GF_FREE (size);
-                                return;
-                        }
-                }
+/* Return true if key exists in array
+ */
+static gf_boolean_t
+dht_match_xattr(const char *key)
+{
+    char **xattrs_to_heal = get_xattrs_to_heal();
 
-                ptr = data_to_bin (value);
-                if (ptr == NULL) {
-                        gf_log ("dht", GF_LOG_WARNING, "data to bin failed");
-                        return;
-                }
+    return gf_get_index_by_elem(xattrs_to_heal, (char *)key) >= 0;
+}
 
-                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
+static int
+dht_aggregate_quota_xattr(dict_t *dst, char *key, data_t *value)
+{
+    int ret = -1;
+    quota_meta_t *meta_dst = NULL;
+    quota_meta_t *meta_src = NULL;
+    int64_t *size = NULL;
+    int64_t dst_dir_count = 0;
+    int64_t src_dir_count = 0;
+
+    if (value == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DATA_NULL,
+               "data value is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_bin(dst, key, (void **)&meta_dst);
+    if (ret < 0) {
+        meta_dst = GF_CALLOC(1, sizeof(quota_meta_t), gf_common_quota_meta_t);
+        if (meta_dst == NULL) {
+            gf_msg("dht", GF_LOG_WARNING, ENOMEM, DHT_MSG_NO_MEMORY,
+                   "Memory allocation failed");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_bin(dst, key, meta_dst, sizeof(quota_meta_t));
+        if (ret < 0) {
+            gf_msg("dht", GF_LOG_WARNING, EINVAL, DHT_MSG_DICT_SET_FAILED,
+                   "dht aggregate dict set failed");
+            GF_FREE(meta_dst);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (value->len > sizeof(int64_t)) {
+        meta_src = data_to_bin(value);
+
+        meta_dst->size = hton64(ntoh64(meta_dst->size) +
+                                ntoh64(meta_src->size));
+        meta_dst->file_count = hton64(ntoh64(meta_dst->file_count) +
+                                      ntoh64(meta_src->file_count));
+
+        if (value->len > (2 * sizeof(int64_t))) {
+            dst_dir_count = ntoh64(meta_dst->dir_count);
+            src_dir_count = ntoh64(meta_src->dir_count);
+
+            if (src_dir_count > dst_dir_count)
+                meta_dst->dir_count = meta_src->dir_count;
         } else {
-                /* compare user xattrs only */
-                if (!strncmp (key, "user.", strlen ("user."))) {
-                        ret = dict_lookup (dst, key, &data_pair); 
-                        if (!ret && data_pair && value) {
-                                ret = is_data_equal (data_pair->value, value);
-                                if (!ret)
-                                        gf_log ("dht", GF_LOG_DEBUG,
-                                                "xattr mismatch for %s", key);
-                        }
-                }
-                ret = dict_set (dst, key, value);
-                if (ret)
-                        gf_log ("dht", GF_LOG_WARNING, "xattr dict set failed");
+            meta_dst->dir_count = 0;
         }
+    } else {
+        size = data_to_bin(value);
+        meta_dst->size = hton64(ntoh64(meta_dst->size) + ntoh64(*size));
+    }
 
-        return;
+    ret = 0;
+out:
+    return ret;
 }
 
+static int
+add_opt(char **optsp, const char *opt)
+{
+    char *newopts = NULL;
+    unsigned oldsize = 0;
+    unsigned newsize = 0;
+
+    if (*optsp == NULL)
+        newopts = gf_strdup(opt);
+    else {
+        oldsize = strlen(*optsp);
+        newsize = oldsize + 1 + strlen(opt) + 1;
+        newopts = GF_REALLOC(*optsp, newsize);
+        if (newopts)
+            sprintf(newopts + oldsize, ",%s", opt);
+    }
+    if (newopts == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+               "Error to add choices in buffer in add_opt");
+        return -1;
+    }
+    *optsp = newopts;
+    return 0;
+}
 
-void
-dht_aggregate_xattr (dict_t *dst, dict_t *src)
+/* Return Choice list from Split brain status */
+static char *
+getChoices(const char *value)
+{
+    int i = 0;
+    char *ptr = NULL;
+    char *tok = NULL;
+    char *result = NULL;
+    char *newval = NULL;
+
+    ptr = strstr(value, "Choices:");
+    if (!ptr) {
+        result = ptr;
+        goto out;
+    }
+
+    newval = gf_strdup(ptr);
+    if (!newval) {
+        result = newval;
+        goto out;
+    }
+
+    tok = strtok(newval, ":");
+    if (!tok) {
+        result = tok;
+        goto out;
+    }
+
+    while (tok) {
+        i++;
+        if (i == 2)
+            break;
+        tok = strtok(NULL, ":");
+    }
+
+    result = gf_strdup(tok);
+
+out:
+    if (newval)
+        GF_FREE(newval);
+
+    return result;
+}
+
+/* This function prepare a list of choices for key
+   (replica.split-brain-status) in   case of metadata split brain
+   only on the basis of key-value passed to this function.
+   After prepare the list of choices it update the same key in dict
+   with this value to reflect the same in
+   replica.split-brain-status attr for file.
+
+*/
+
+static int
+dht_aggregate_split_brain_xattr(dict_t *dst, char *key, data_t *value)
 {
-        if ((dst == NULL) || (src == NULL)) {
+    int ret = 0;
+    char *oldvalue = NULL;
+    char *old_choice = NULL;
+    char *new_choice = NULL;
+    char *full_choice = NULL;
+    char *status = NULL;
+
+    if (value == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DATA_NULL,
+               "GF_AFR_SBRAIN_STATUS value is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(dst, key, &oldvalue);
+    if (ret)
+        goto out;
+
+    /* skip code that is irrelevant if !oldvalue */
+    if (!oldvalue)
+        goto out;
+
+    if (strstr(oldvalue, "not")) {
+        gf_msg_debug("dht", 0, "Need to update split-brain status in dict");
+        ret = -1;
+        goto out;
+    }
+    if (strstr(oldvalue, "metadata-split-brain:yes") &&
+        (strstr(oldvalue, "data-split-brain:no"))) {
+        if (strstr(value->data, "not")) {
+            gf_msg_debug("dht", 0, "No need to update split-brain status");
+            ret = 0;
+            goto out;
+        }
+        if (strstr(value->data, "yes") &&
+            (strncmp(oldvalue, value->data, strlen(oldvalue)))) {
+            old_choice = getChoices(oldvalue);
+            if (!old_choice) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to get choices");
+                ret = -1;
+                goto out;
+            }
+
+            ret = add_opt(&full_choice, old_choice);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to add choices");
+                ret = -1;
                 goto out;
+            }
+
+            new_choice = getChoices(value->data);
+            if (!new_choice) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to get choices");
+                ret = -1;
+                goto out;
+            }
+
+            ret = add_opt(&full_choice, new_choice);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to add choices ");
+                ret = -1;
+                goto out;
+            }
+            ret = gf_asprintf(&status,
+                              "data-split-brain:%s    "
+                              "metadata-split-brain:%s   Choices:%s",
+                              "no", "yes", full_choice);
+
+            if (-1 == ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to prepare status ");
+                goto out;
+            }
+            ret = dict_set_dynstr(dst, key, status);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                       "Failed to set full choice");
+            }
         }
+    }
 
-        dict_foreach (src, dht_aggregate, dst);
 out:
-        return;
+    if (old_choice)
+        GF_FREE(old_choice);
+    if (new_choice)
+        GF_FREE(new_choice);
+    if (full_choice)
+        GF_FREE(full_choice);
+
+    return ret;
+}
+
+static int
+dht_aggregate(dict_t *this, char *key, data_t *value, void *data)
+{
+    dict_t *dst = NULL;
+    int32_t ret = -1;
+    data_t *dict_data = NULL;
+
+    dst = data;
+
+    /* compare split brain xattr only */
+    if (strcmp(key, GF_AFR_SBRAIN_STATUS) == 0) {
+        ret = dht_aggregate_split_brain_xattr(dst, key, value);
+        if (!ret)
+            goto out;
+    } else if (strcmp(key, QUOTA_SIZE_KEY) == 0) {
+        ret = dht_aggregate_quota_xattr(dst, key, value);
+        if (ret) {
+            gf_msg("dht", GF_LOG_WARNING, 0,
+                   DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
+                   "Failed to aggregate quota xattr");
+        }
+        goto out;
+    } else if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
+        ret = gf_get_min_stime(THIS, dst, key, value);
+        goto out;
+    } else {
+        /* compare user xattrs only */
+        if (!strncmp(key, "user.", SLEN("user."))) {
+            ret = dict_lookup(dst, key, &dict_data);
+            if (!ret && dict_data && value) {
+                ret = is_data_equal(dict_data, value);
+                if (!ret)
+                    gf_msg_debug("dht", 0, "xattr mismatch for %s", key);
+            }
+        }
+    }
+
+    ret = dict_set(dst, key, value);
+    if (ret) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s", key);
+    }
+
+out:
+    return ret;
+}
+
+static void
+dht_aggregate_xattr(dict_t *dst, dict_t *src)
+{
+    if ((dst == NULL) || (src == NULL)) {
+        goto out;
+    }
+
+    dict_foreach(src, dht_aggregate, dst);
+out:
+    return;
+}
+
+/* Code to save hashed subvol on inode ctx as a mds subvol
+ */
+int
+dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol)
+{
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+    uint64_t ctx_int = 0;
+    gf_boolean_t ctx_free = _gf_false;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctx_int);
+        if (ctx_int) {
+            ctx = (dht_inode_ctx_t *)(uintptr_t)ctx_int;
+            ctx->mds_subvol = mds_subvol;
+        } else {
+            ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+            if (!ctx)
+                goto unlock;
+            ctx->mds_subvol = mds_subvol;
+            ctx_free = _gf_true;
+            ctx_int = (long)ctx;
+            ret = __inode_ctx_set(inode, this, &ctx_int);
+        }
+    }
+unlock:
+    UNLOCK(&inode->lock);
+    if (ret && ctx_free)
+        GF_FREE(ctx);
+    return ret;
+}
+
+/*Code to get mds subvol from inode ctx */
+
+int
+dht_inode_ctx_mdsvol_get(inode_t *inode, xlator_t *this, xlator_t **mdsvol)
+{
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+
+    if (!mdsvol)
+        return ret;
+
+    if (__is_root_gfid(inode->gfid)) {
+        (*mdsvol) = FIRST_CHILD(this);
+        return 0;
+    }
+
+    ret = dht_inode_ctx_get(inode, this, &ctx);
+
+    if (!ret && ctx) {
+        if (ctx->mds_subvol) {
+            *mdsvol = ctx->mds_subvol;
+            ret = 0;
+        } else {
+            ret = -1;
+        }
+    }
+
+    return ret;
 }
 
 /* TODO:
@@ -101,4727 +450,10942 @@ out:
    - complete linkfile selfheal
 */
 
-
-int
-dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this,
-                         int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_lookup_selfheal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-	int           ret = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
-        local = frame->local;
-        ret = op_ret;
+    local = frame->local;
+    conf = this->private;
+    ret = op_ret;
 
-        FRAME_SU_UNDO (frame, dht_local_t);
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-        if (ret == 0) {
-                layout = local->selfheal.layout;
-                ret = dht_layout_set (this, local->inode, layout);
-        }
+    if (ret == 0) {
+        layout = local->selfheal.layout;
+        ret = dht_layout_set(this, local->inode, layout);
+    }
 
-        WIPE (&local->postparent);
+    dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, &local->postparent,
+                                  1);
+    }
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    dht_set_fixed_dir_stat(&local->postparent);
+    /* Delete mds xattr at the time of STACK UNWIND */
+    GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
 
-        DHT_STACK_UNWIND (lookup, frame, ret, local->op_errno, local->inode,
-                          &local->stbuf, local->xattr, &local->postparent);
+    DHT_STACK_UNWIND(lookup, frame, ret, local->op_errno, local->inode,
+                     &local->stbuf, local->xattr, &local->postparent);
 
 out:
-        return ret;
+    return ret;
 }
 
-
-int
-dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
+static int
+dht_discover_complete(xlator_t *this, call_frame_t *discover_frame)
 {
-        dht_local_t     *local = NULL;
-        call_frame_t    *main_frame = NULL;
-        int              op_errno = 0;
-        int              ret = -1;
-        dht_layout_t    *layout = NULL;
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *main_frame = NULL;
+    call_frame_t *heal_frame = NULL;
+    int op_errno = 0;
+    int ret = -1;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    uint32_t vol_commit_hash = 0;
+    xlator_t *source = NULL;
+    int heal_path = 0;
+    int error_while_marking_mds = 0;
+    int i = 0;
+    loc_t loc = {0};
+    int8_t is_read_only = 0, layout_anomalies = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+
+    local = discover_frame->local;
+    layout = local->layout;
+    conf = this->private;
+    gf_uuid_unparse(local->gfid, gfid_local);
+
+    LOCK(&discover_frame->lock);
+    {
+        main_frame = local->main_frame;
+        local->main_frame = NULL;
+    }
+    UNLOCK(&discover_frame->lock);
 
-        local = discover_frame->local;
-        layout = local->layout;
+    if (!main_frame)
+        return 0;
 
-        LOCK(&discover_frame->lock);
-        {
-                main_frame = local->main_frame;
-                local->main_frame = NULL;
+    /* Code to update all extended attributed from
+       subvol to local->xattr on that internal xattr has found
+    */
+    if (conf->subvolume_cnt == 1)
+        local->need_xattr_heal = 0;
+    if (local->need_xattr_heal && (local->mds_xattr)) {
+        dht_dir_set_heal_xattr(this, local, local->xattr, local->mds_xattr,
+                               NULL, NULL);
+        dict_unref(local->mds_xattr);
+        local->mds_xattr = NULL;
+    }
+
+    ret = dict_get_int8(local->xattr_req, QUOTA_READ_ONLY_KEY, &is_read_only);
+    if (ret < 0)
+        gf_msg_debug(this->name, 0, "key = %s not present in dict",
+                     QUOTA_READ_ONLY_KEY);
+
+    if (local->file_count && local->dir_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+               "path %s exists as a file on one subvolume "
+               "and directory on another. "
+               "Please fix it manually",
+               local->loc.path);
+        op_errno = EIO;
+        goto out;
+    }
+
+    if (local->cached_subvol) {
+        ret = dht_layout_preset(this, local->cached_subvol, local->inode);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_SET_FAILED,
+                   "failed to set layout for subvolume %s",
+                   local->cached_subvol ? local->cached_subvol->name : "<nil>");
+            op_errno = EINVAL;
+            goto out;
+        }
+    } else {
+        ret = dht_layout_normalize(this, &local->loc, layout);
+        if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
+            /* either the layout is incorrect or the directory is
+             * not found even in one subvolume.
+             */
+            gf_msg_debug(this->name, 0,
+                         "normalizing failed on %s "
+                         "(overlaps/holes present: %s, "
+                         "ENOENT errors: %d)",
+                         local->loc.path, (ret < 0) ? "yes" : "no",
+                         (ret > 0) ? ret : 0);
+            layout_anomalies = 1;
+        } else if (local->inode) {
+            dht_layout_set(this, local->inode, layout);
+        }
+    }
+
+    if (!conf->vch_forced) {
+        ret = dict_get_uint32(local->xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
         }
-        UNLOCK(&discover_frame->lock);
+    }
 
-        if (!main_frame)
+    if (IA_ISDIR(local->stbuf.ia_type) && !is_read_only) {
+        for (i = 0; i < layout->cnt; i++) {
+            if (!source && !layout->list[i].err)
+                source = layout->list[i].xlator;
+            if (layout->list[i].err == ENOENT ||
+                layout->list[i].err == ESTALE) {
+                heal_path = 1;
+            }
+
+            if (source && heal_path)
+                break;
+        }
+    }
+
+    if (IA_ISDIR(local->stbuf.ia_type)) {
+        /* Call function to save hashed subvol on inode ctx if
+           internal mds xattr is not present and all subvols are up
+        */
+        if (!local->op_ret && !__is_root_gfid(local->stbuf.ia_gfid))
+            (void)dht_common_mark_mdsxattr(discover_frame,
+                                           &error_while_marking_mds, 1);
+
+        if (local->need_xattr_heal && !heal_path) {
+            local->need_xattr_heal = 0;
+            ret = dht_dir_xattr_heal(this, local, &op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                       "xattr heal failed for "
+                       "directory  gfid is %s ",
+                       gfid_local);
+            }
+        }
+    }
+
+    if (source && (heal_path || layout_anomalies || error_while_marking_mds)) {
+        gf_uuid_copy(loc.gfid, local->gfid);
+        if (gf_uuid_is_null(loc.gfid)) {
+            goto done;
+        }
+
+        if (local->inode)
+            loc.inode = inode_ref(local->inode);
+        else
+            goto done;
+
+        heal_frame = create_frame(this, this->ctx->pool);
+        if (heal_frame) {
+            heal_local = dht_local_init(heal_frame, &loc, NULL, 0);
+            if (!heal_local)
+                goto cleanup;
+
+            gf_uuid_copy(heal_local->gfid, local->gfid);
+            heal_frame->cookie = source;
+            heal_local->xattr = dict_ref(local->xattr);
+            heal_local->stbuf = local->stbuf;
+            heal_local->postparent = local->postparent;
+            heal_local->inode = inode_ref(loc.inode);
+            heal_local->main_frame = main_frame;
+            FRAME_SU_DO(heal_frame, dht_local_t);
+            ret = synctask_new(this->ctx->env, dht_heal_full_path,
+                               dht_heal_full_path_done, heal_frame, heal_frame);
+            if (!ret) {
+                loc_wipe(&loc);
                 return 0;
+            }
+            /*
+             * Failed to spawn the synctask. Returning
+             * with out doing heal.
+             */
+        cleanup:
+            loc_wipe(&loc);
+            DHT_STACK_DESTROY(heal_frame);
+        }
+    }
+done:
+    dht_set_fixed_dir_stat(&local->postparent);
+    /* Delete mds xattr at the time of STACK UNWIND */
+    if (local->xattr)
+        GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
 
-        if (local->file_count && local->dir_count) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "path %s exists as a file on one subvolume "
-                        "and directory on another. "
-                        "Please fix it manually",
-                        local->loc.path);
-                op_errno = EIO;
-                goto out;
-        }
+    DHT_STACK_UNWIND(lookup, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
+    return 0;
 
-        if (local->cached_subvol) {
-                ret = dht_layout_preset (this, local->cached_subvol,
-                                         local->inode);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set layout for subvolume %s",
-                                local->cached_subvol ? local->cached_subvol->name : "<nil>");
-                        op_errno = EINVAL;
-                        goto out;
-                }
-        } else {
-                ret = dht_layout_normalize (this, &local->loc, layout);
-                if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
-                        /* either the layout is incorrect or the directory is
-                         * not found even in one subvolume.
-                         */
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "normalizing failed on %s "
-                                "(overlaps/holes present: %s, "
-                                "ENOENT errors: %d)", local->loc.path,
-                                (ret < 0) ? "yes" : "no", (ret > 0) ? ret : 0);
-                        op_errno = EINVAL;
-                        goto out;
-                }
+out:
+    DHT_STACK_UNWIND(lookup, main_frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-                dht_layout_set (this, local->inode, layout);
-        }
+    return ret;
+}
 
-        DHT_STACK_UNWIND (lookup, main_frame, local->op_ret, local->op_errno,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
-        return 0;
+static int
+dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = cookie;
+    int ret = -1;
+    dht_conf_t *conf = 0;
+    dht_layout_t *layout = NULL;
+    int32_t mds_heal_fresh_lookup = 0;
+
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+
+    local = frame->local;
+    conf = this->private;
+    layout = local->selfheal.layout;
+    mds_heal_fresh_lookup = local->mds_heal_fresh_lookup;
+
+    if (op_ret) {
+        gf_msg_debug(this->name, op_ret,
+                     "Failed to set %s on the MDS %s for path %s. ",
+                     conf->mds_xattr_key, prev->name, local->loc.path);
+    } else {
+        /* Save mds subvol on inode ctx */
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set mds subvol on inode ctx"
+                   " %s for %s ",
+                   prev->name, local->loc.path);
+        }
+    }
+    if (!local->mds_heal_fresh_lookup && layout) {
+        dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf, 0xffffffff,
+                                 layout);
+    }
 out:
-        DHT_STACK_UNWIND (lookup, main_frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
-
-        return ret;
+    if (mds_heal_fresh_lookup)
+        DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
+static xlator_t *
+dht_inode_get_hashed_subvol(inode_t *inode, xlator_t *this, loc_t *loc)
+{
+    char *path = NULL;
+    loc_t populate_loc = {
+        0,
+    };
+    char *name = NULL;
+    xlator_t *hash_subvol = NULL;
+
+    if (!inode)
+        return hash_subvol;
+
+    if (loc && loc->parent && loc->path) {
+        if (!loc->name) {
+            name = strrchr(loc->path, '/');
+            if (name) {
+                loc->name = name + 1;
+            } else {
+                goto out;
+            }
+        }
+        hash_subvol = dht_subvol_get_hashed(this, loc);
+        goto out;
+    }
 
-int
-dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno,
-                  inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                  struct iatt *postparent)
-{
-        dht_local_t  *local                   = NULL;
-        int           this_call_cnt           = 0;
-        call_frame_t *prev                    = NULL;
-        dht_layout_t *layout                  = NULL;
-        int           ret                     = -1;
-        int           is_dir                  = 0;
-        int           is_linkfile             = 0;
-        int           attempt_unwind          = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+    if (!gf_uuid_is_null(inode->gfid)) {
+        populate_loc.inode = inode_ref(inode);
+        populate_loc.parent = inode_parent(populate_loc.inode, NULL, NULL);
+        inode_path(populate_loc.inode, NULL, &path);
 
-        local = frame->local;
-        prev  = cookie;
+        if (!path)
+            goto out;
 
-        layout = local->layout;
+        populate_loc.path = path;
+        if (!populate_loc.name && populate_loc.path) {
+            name = strrchr(populate_loc.path, '/');
+            if (name) {
+                populate_loc.name = name + 1;
 
-        /* Check if the gfid is different for file from other node */
-        if (!op_ret && uuid_compare (local->gfid, stbuf->ia_gfid)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: gfid different on %s",
-                        local->loc.path, prev->this->name);
+            } else {
+                goto out;
+            }
         }
+        hash_subvol = dht_subvol_get_hashed(this, &populate_loc);
+    }
+out:
+    if (populate_loc.inode)
+        loc_wipe(&populate_loc);
+    return hash_subvol;
+}
 
+/* Common function call by revalidate/selfheal code path to populate
+   internal xattr if it is not present, mark_during_fresh_lookup value
+   determines either function is call by revalidate_cbk(discover_complete)
+   or call by selfheal code path while fresh lookup.
+   Here we do wind a call serially in case of fresh lookup and
+   for other lookup code path we do wind a call parallel.The reason
+   to wind a call serially is at the time of fresh lookup directory is not
+   discovered and at the time of revalidate_lookup directory is
+   already discovered. So, revalidate codepath can race with setxattr
+   codepath and can get into spurious heals because of an ongoing setxattr.
+   This can slow down revalidates, if healing happens in foreground.
+   However, if healing happens in background, there is no direct performance
+   penalty.
+*/
+int
+dht_common_mark_mdsxattr(call_frame_t *frame, int *errst,
+                         int mark_during_fresh_lookup)
+{
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *hashed_subvol = NULL;
+    int ret = 0;
+    int i = 0;
+    dict_t *xattrs = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+    int32_t zero[1] = {0};
+    dht_conf_t *conf = 0;
+    dht_layout_t *layout = NULL;
+    dht_local_t *copy_local = NULL;
+    call_frame_t *xattr_frame = NULL;
+    gf_boolean_t vol_down = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    local = frame->local;
+    conf = this->private;
+    layout = local->selfheal.layout;
+    local->mds_heal_fresh_lookup = mark_during_fresh_lookup;
+
+    gf_uuid_unparse(local->gfid, gfid_local);
+
+    /* Code to update hashed subvol consider as a mds subvol
+       and wind a setxattr call on hashed subvol to update
+       internal xattr
+    */
+    if (!local->xattr || !dict_get(local->xattr, conf->mds_xattr_key)) {
+        /* It means no internal MDS xattr has been set yet
+         */
+        /* Check the status of all subvol are up while call
+           this function call by lookup code path
+        */
+        if (mark_during_fresh_lookup) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (!conf->subvolume_status[i]) {
+                    vol_down = _gf_true;
+                    break;
+                }
+            }
+            if (vol_down) {
+                gf_msg_debug(this->name, 0,
+                             "subvol %s is down. Unable to "
+                             " save mds subvol on inode for "
+                             " path %s gfid is %s ",
+                             conf->subvolumes[i]->name, local->loc.path,
+                             gfid_local);
+                goto out;
+            }
+        }
 
-        LOCK (&frame->lock);
-        {
-                /* TODO: assert equal mode on stbuf->st_mode and
-                   local->stbuf->st_mode
+        /* Calculate hashed subvol based on inode and parent node
+         */
+        hashed_subvol = dht_inode_get_hashed_subvol(local->inode, this,
+                                                    &local->loc);
+        if (!hashed_subvol) {
+            gf_msg(this->name, GF_LOG_DEBUG, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for path %s"
+                   "gfid is %s ",
+                   local->loc.path, gfid_local);
+            if (errst)
+                (*errst) = 1;
+            ret = -1;
+            goto out;
+        }
+        xattrs = dict_new();
+        if (!xattrs) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                   "dict_new failed");
+            ret = -1;
+            goto out;
+        }
+        /* Add internal MDS xattr on disk for hashed subvol
+         */
+        ret = dht_dict_set_array(xattrs, conf->mds_xattr_key, zero, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary"
+                   "  value:key = %s for "
+                   "path %s",
+                   conf->mds_xattr_key, local->loc.path);
+            ret = -1;
+            goto out;
+        }
+        /* Create a new frame to wind a call only while
+           this function call by revalidate_cbk code path
+           To wind a call parallel need to create a new frame
+        */
+        if (mark_during_fresh_lookup) {
+            xattr_frame = create_frame(this, this->ctx->pool);
+            if (!xattr_frame) {
+                ret = -1;
+                goto out;
+            }
+            copy_local = dht_local_init(xattr_frame, &(local->loc), NULL, 0);
+            if (!copy_local) {
+                ret = -1;
+                DHT_STACK_DESTROY(xattr_frame);
+                goto out;
+            }
+            copy_local->stbuf = local->stbuf;
+            copy_local->mds_heal_fresh_lookup = mark_during_fresh_lookup;
+            if (!copy_local->inode)
+                copy_local->inode = inode_ref(local->inode);
+            gf_uuid_copy(copy_local->loc.gfid, local->gfid);
+            FRAME_SU_DO(xattr_frame, dht_local_t);
+            STACK_WIND_COOKIE(xattr_frame, dht_common_mark_mdsxattr_cbk,
+                              hashed_subvol, hashed_subvol,
+                              hashed_subvol->fops->setxattr, &local->loc,
+                              xattrs, 0, NULL);
+        } else {
+            STACK_WIND_COOKIE(frame, dht_common_mark_mdsxattr_cbk,
+                              (void *)hashed_subvol, hashed_subvol,
+                              hashed_subvol->fops->setxattr, &local->loc,
+                              xattrs, 0, NULL);
+        }
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "internal xattr %s is present on subvol"
+                     "on path %s gfid is %s ",
+                     conf->mds_xattr_key, local->loc.path, gfid_local);
+        if (!mark_during_fresh_lookup)
+            dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf,
+                                     0xffffffff, layout);
+    }
 
-                   else mkdir/chmod/chown and fix
-                */
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        op_ret, op_errno, xattr);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to merge layouts", local->loc.path);
+out:
+    if (xattrs)
+        dict_unref(xattrs);
+    return ret;
+}
 
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "lookup of %s on %s returned error (%s)",
-                                local->loc.path, prev->this->name,
-                                strerror (op_errno));
+/* Get the value of key from dict in the bytewise and save in array after
+   convert from network byte order to host byte order
+*/
+static int32_t
+dht_dict_get_array(dict_t *dict, char *key, int32_t value[], int32_t size,
+                   int *errst)
+{
+    void *ptr = NULL;
+    int32_t len = -1;
+    int32_t vindex = -1;
+    int32_t err = -1;
+    int ret = 0;
+
+    if (dict == NULL) {
+        (*errst) = -1;
+        return -EINVAL;
+    }
+    err = dict_get_ptr_and_len(dict, key, &ptr, &len);
+    if (err != 0) {
+        (*errst) = -1;
+        return err;
+    }
+
+    if (len != (size * sizeof(int32_t))) {
+        (*errst) = -1;
+        return -EINVAL;
+    }
+
+    for (vindex = 0; vindex < size; vindex++) {
+        value[vindex] = ntoh32(*((int32_t *)ptr + vindex));
+        if (value[vindex] < 0)
+            ret = -1;
+    }
+
+    return ret;
+}
 
-                        goto unlock;
-                }
+static int
+dht_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *stbuf,
+                 dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int32_t check_mds = 0;
+    int is_linkfile = 0;
+    int attempt_unwind = 0;
+    dht_conf_t *conf = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_node[GF_UUID_BUF_SIZE] = {0};
+    int32_t mds_xattr_val[1] = {0};
+    int errst = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    layout = local->layout;
+
+    /* Check if the gfid is different for file from other node */
+    if (!op_ret && gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
+        gf_uuid_unparse(local->gfid, gfid_local);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on %s, gfid local = %s"
+               "gfid other = %s",
+               local->loc.path, prev->name, gfid_local, gfid_node);
+    }
+
+    LOCK(&frame->lock);
+    {
+        /* TODO: assert equal mode on stbuf->st_mode and
+           local->stbuf->st_mode
+
+           else mkdir/chmod/chown and fix
+        */
+
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                   "%s: failed to merge layouts for subvol %s", local->loc.path,
+                   prev->name);
 
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr);
-                is_dir = check_is_dir (inode, stbuf, xattr);
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            gf_msg_debug(this->name, op_errno,
+                         "lookup of %s on %s returned error", local->loc.path,
+                         prev->name);
 
-                if (is_dir) {
-                        local->dir_count ++;
-                } else {
-                        local->file_count ++;
-
-                        if (!is_linkfile) {
-                                /* real file */
-                                local->cached_subvol = prev->this;
-                                attempt_unwind = 1;
-                        } else {
-                                goto unlock;
-                        }
-                }
+            goto unlock;
+        }
 
-                local->op_ret = 0;
+        is_linkfile = check_is_linkfile(inode, stbuf, xattr,
+                                        conf->link_xattr_name);
+        is_dir = check_is_dir(inode, stbuf, xattr);
 
-                if (local->xattr == NULL) {
-                        local->xattr = dict_ref (xattr);
-                } else {
-                        dht_aggregate_xattr (local->xattr, xattr);
-                }
+        if (is_dir) {
+            local->dir_count++;
+        } else {
+            local->file_count++;
+
+            if (!is_linkfile && !local->cached_subvol) {
+                /* real file */
+                /* Ok, we somehow managed to find a file on
+                 * more than one subvol. ignore this or we
+                 * will end up overwriting information while a
+                 * a thread is potentially unwinding from
+                 * dht_discover_complete
+                 */
+                local->cached_subvol = prev;
+                attempt_unwind = 1;
+            } else {
+                goto unlock;
+            }
+        }
 
-                if (local->inode == NULL)
-                        local->inode = inode_ref (inode);
+        local->op_ret = 0;
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+        if (local->xattr == NULL) {
+            local->xattr = dict_ref(xattr);
+        } else {
+            /* Don't aggregate for files. See BZ#1484709 */
+            if (is_dir)
+                dht_aggregate_xattr(local->xattr, xattr);
         }
+
+        if (local->inode == NULL)
+            local->inode = inode_ref(inode);
+
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
+
+        if (!dict_get(xattr, conf->mds_xattr_key)) {
+            goto unlock;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "internal xattr %s is present on subvol"
+                         "on path %s gfid is %s ",
+                         conf->mds_xattr_key, local->loc.path, gfid_local);
+        }
+        check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                       mds_xattr_val, 1, &errst);
+        /* save mds subvol on inode ctx */
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set hashed subvol for %s vol is %s",
+                   local->loc.path, prev->name);
+        }
+
+        if ((check_mds < 0) && !errst) {
+            local->mds_xattr = dict_ref(xattr);
+            gf_msg_debug(this->name, 0,
+                         "Value of %s is not zero on mds subvol"
+                         "so xattr needs to be healed on non mds"
+                         " path is %s and vol name is %s "
+                         " gfid is %s",
+                         conf->mds_xattr_key, local->loc.path, prev->name,
+                         gfid_local);
+            local->need_xattr_heal = 1;
+            local->mds_subvol = prev;
+        }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 out:
-        this_call_cnt = dht_frame_return (frame);
+    /* Make sure, the thread executing dht_discover_complete is the one
+     * which calls STACK_DESTROY (frame). In the case of "attempt_unwind",
+     * this makes sure that the thread don't call dht_frame_return, till
+     * call to dht_discover_complete is done.
+     */
+    if (attempt_unwind) {
+        dht_discover_complete(this, frame);
+    }
 
-        if (is_last_call (this_call_cnt) || attempt_unwind) {
-                dht_discover_complete (this, frame);
-        }
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_DESTROY (frame);
+    if (is_last_call(this_call_cnt) && !attempt_unwind) {
+        dht_discover_complete(this, frame);
+    }
 
-        return 0;
-}
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_DESTROY(frame);
 
+    return 0;
+}
 
-int
-dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
+static int
+dht_set_file_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        int          ret;
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf = NULL;
-        int          call_cnt = 0;
-        int          op_errno = EINVAL;
-        int          i = 0;
-        call_frame_t *discover_frame = NULL;
+    int ret = -EINVAL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    if (!conf) {
+        goto err;
+    }
+
+    if (!xattr_req) {
+        goto err;
+    }
+
+    /* Used to check whether this is a linkto file.
+     */
+    ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->link_xattr_name, loc->path);
+        goto err;
+    }
+
+    /* This is used to make sure we don't unlink linkto files
+     * which are the target of an ongoing file migration.
+     */
+    ret = dict_set_uint32(xattr_req, GLUSTERFS_OPEN_FD_COUNT, 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               GLUSTERFS_OPEN_FD_COUNT, loc->path);
+        goto err;
+    }
+
+    ret = 0;
+err:
+    return ret;
+}
 
+/* This is a gfid based nameless lookup. Without a name, the hashed subvol
+ * cannot be calculated so a lookup is sent to all subvols.
+ */
+static int
+dht_do_discover(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int ret;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int op_errno = EINVAL;
+    int i = 0;
+    call_frame_t *discover_frame = NULL;
+
+    conf = this->private;
+    local = frame->local;
+
+    /* As we do not know if this is a file or directory, request
+     * both file and directory xattrs
+     */
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        goto err;
+    }
+
+    if (loc_is_root(loc)) {
+        /* Request the DHT commit hash xattr (trusted.glusterfs.dht.commithash)
+         * set on the brick root.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->commithash_xattr_name,
+                              sizeof(uint32_t));
+    }
 
-        conf = this->private;
-        local = frame->local;
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
 
-        ret = dict_set_uint32 (local->xattr_req,
-                               "trusted.glusterfs.dht", 4 * 4);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht' key",
-                        loc->path);
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
 
-        ret = dict_set_uint32 (local->xattr_req,
-                               "trusted.glusterfs.dht.linkto", 256);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht.linkto' key",
-                        loc->path);
+    if (!local->layout) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        call_cnt        = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
+    gf_uuid_copy(local->gfid, loc->gfid);
 
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
+    discover_frame = copy_frame(frame);
+    if (!discover_frame) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        if (!local->layout) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    discover_frame->local = local;
+    frame->local = NULL;
+    local->main_frame = frame;
 
-        uuid_copy (local->gfid, loc->gfid);
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(discover_frame, dht_discover_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, &local->loc,
+                          local->xattr_req);
+    }
 
-        discover_frame = copy_frame (frame);
-        if (!discover_frame) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    return 0;
 
-        discover_frame->local = local;
-        frame->local = NULL;
-        local->main_frame = frame;
+err:
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (discover_frame, dht_discover_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            &local->loc, local->xattr_req);
-        }
+    return 0;
+}
 
-        return 0;
+/* Code to call syntask to heal custom xattr from hashed subvol
+   to non hashed subvol
+*/
+int
+dht_dir_xattr_heal(xlator_t *this, dht_local_t *local, int *op_errno)
+{
+    dht_local_t *copy_local = NULL;
+    call_frame_t *copy = NULL;
+    int ret = -1;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+
+    if (gf_uuid_is_null(local->gfid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DIR_XATTR_HEAL_FAILED,
+               "No gfid exists for path %s "
+               "so healing xattr is not possible",
+               local->loc.path);
+        *op_errno = EIO;
+        goto out;
+    }
+
+    gf_uuid_unparse(local->gfid, gfid_local);
+    copy = create_frame(this, this->ctx->pool);
+    if (copy) {
+        copy_local = dht_local_init(copy, &(local->loc), NULL, 0);
+        if (!copy_local) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM,
+                   DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                   "Memory allocation failed "
+                   "for path %s gfid %s ",
+                   local->loc.path, gfid_local);
+            *op_errno = ENOMEM;
+            DHT_STACK_DESTROY(copy);
+        } else {
+            copy_local->stbuf = local->stbuf;
+            gf_uuid_copy(copy_local->loc.gfid, local->gfid);
+            copy_local->mds_subvol = local->mds_subvol;
+            FRAME_SU_DO(copy, dht_local_t);
+            ret = synctask_new(this->ctx->env, dht_dir_heal_xattrs,
+                               dht_dir_heal_xattrs_done, copy, copy);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM,
+                       DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                       "Synctask creation failed to heal xattr "
+                       "for path %s gfid %s ",
+                       local->loc.path, gfid_local);
+                *op_errno = ENOMEM;
+                DHT_STACK_DESTROY(copy);
+            }
+        }
+    }
+out:
+    return ret;
+}
 
-err:
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
+static int
+dht_needs_selfheal(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int needs_selfheal = 0;
+    int ret = 0;
+
+    local = frame->local;
+    layout = local->layout;
+
+    if (local->need_attrheal || local->need_xattr_heal ||
+        local->need_selfheal) {
+        needs_selfheal = 1;
+    }
+
+    ret = dht_layout_normalize(this, &local->loc, layout);
+
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0, "fixing assignment on %s", local->loc.path);
+        needs_selfheal = 1;
+    }
+    return needs_selfheal;
+}
 
+static int
+is_permission_different(ia_prot_t *prot1, ia_prot_t *prot2)
+{
+    if ((prot1->owner.read != prot2->owner.read) ||
+        (prot1->owner.write != prot2->owner.write) ||
+        (prot1->owner.exec != prot2->owner.exec) ||
+        (prot1->group.read != prot2->group.read) ||
+        (prot1->group.write != prot2->group.write) ||
+        (prot1->group.exec != prot2->group.exec) ||
+        (prot1->other.read != prot2->other.read) ||
+        (prot1->other.write != prot2->other.write) ||
+        (prot1->other.exec != prot2->other.exec) ||
+        (prot1->suid != prot2->suid) || (prot1->sgid != prot2->sgid) ||
+        (prot1->sticky != prot2->sticky)) {
+        return 1;
+    } else {
         return 0;
+    }
 }
 
-
 int
-dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent)
-{
-        dht_local_t  *local                   = NULL;
-        int           this_call_cnt           = 0;
-        call_frame_t *prev                    = NULL;
-        dht_layout_t *layout                  = NULL;
-        int           ret                     = -1;
-        int           is_dir                  = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+dht_lookup_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int32_t check_mds = 0;
+    int errst = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_node[GF_UUID_BUF_SIZE] = {0};
+    int32_t mds_xattr_val[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    layout = local->layout;
+    gf_msg_debug(this->name, op_errno,
+                 "%s: lookup on %s returned with op_ret = %d, op_errno = %d",
+                 local->loc.path, prev->name, op_ret, op_errno);
+
+    /* The first successful lookup*/
+    if (!op_ret && gf_uuid_is_null(local->gfid)) {
+        memcpy(local->gfid, stbuf->ia_gfid, 16);
+    }
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid_local);
+    }
+
+    /* Check if the gfid is different for file from other node */
+    if (!op_ret && gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on %s."
+               " gfid local = %s, gfid subvol = %s",
+               local->loc.path, prev->name, gfid_local, gfid_node);
+    }
+
+    LOCK(&frame->lock);
+    {
+        /* TODO: assert equal mode on stbuf->st_mode and
+           local->stbuf->st_mode
+           else mkdir/chmod/chown and fix
+        */
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
 
-        local = frame->local;
-        prev  = cookie;
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
 
-        layout = local->layout;
+            /* The GFID is missing on this subvol. Force a heal. */
+            if (op_errno == ENODATA) {
+                local->need_lookup_everywhere = 1;
+            }
+            goto unlock;
+        }
+
+        is_dir = check_is_dir(inode, stbuf, xattr);
+        if (!is_dir) {
+            gf_msg_debug(this->name, 0,
+                         "%s: lookup on %s returned non dir 0%o"
+                         "calling lookup_everywhere",
+                         local->loc.path, prev->name, stbuf->ia_type);
 
-        if (!op_ret && uuid_is_null (local->gfid))
-                memcpy (local->gfid, stbuf->ia_gfid, 16);
+            local->need_lookup_everywhere = 1;
+            goto unlock;
+        }
 
-        /* Check if the gfid is different for file from other node */
-        if (!op_ret && uuid_compare (local->gfid, stbuf->ia_gfid)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: gfid different on %s",
-                        local->loc.path, prev->this->name);
+        local->op_ret = 0;
+        if (local->xattr == NULL) {
+            local->xattr = dict_ref(xattr);
+        } else {
+            dht_aggregate_xattr(local->xattr, xattr);
+        }
+
+        if (__is_root_gfid(stbuf->ia_gfid)) {
+            ret = dht_dir_has_layout(xattr, conf->xattr_name);
+            if (ret >= 0) {
+                if (is_greater_time(local->prebuf.ia_ctime,
+                                    local->prebuf.ia_ctime_nsec,
+                                    stbuf->ia_ctime, stbuf->ia_ctime_nsec)) {
+                    /* Choose source */
+                    local->prebuf.ia_gid = stbuf->ia_gid;
+                    local->prebuf.ia_uid = stbuf->ia_uid;
+
+                    local->prebuf.ia_ctime = stbuf->ia_ctime;
+                    local->prebuf.ia_ctime_nsec = stbuf->ia_ctime_nsec;
+                    local->prebuf.ia_prot = stbuf->ia_prot;
+                }
+            }
         }
 
-        LOCK (&frame->lock);
-        {
-                /* TODO: assert equal mode on stbuf->st_mode and
-                   local->stbuf->st_mode
+        if (local->stbuf.ia_type != IA_INVAL) {
+            /* This is not the first subvol to respond
+             * Compare values to see if attrs need to be healed
+             */
+            if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                (local->stbuf.ia_uid != stbuf->ia_uid) ||
+                (is_permission_different(&local->stbuf.ia_prot,
+                                         &stbuf->ia_prot))) {
+                local->need_attrheal = 1;
+            }
+        }
 
-                   else mkdir/chmod/chown and fix
-                */
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        op_ret, op_errno, xattr);
+        if (local->inode == NULL)
+            local->inode = inode_ref(inode);
 
-                if (op_ret == -1) {
-                        local->op_errno = ENOENT;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "lookup of %s on %s returned error (%s)",
-                                local->loc.path, prev->this->name,
-                                strerror (op_errno));
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
 
-                        goto unlock;
-                }
+        if (!dict_get(xattr, conf->mds_xattr_key)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: mds xattr %s is not present "
+                         "on %s(gfid = %s)",
+                         local->loc.path, conf->mds_xattr_key, prev->name,
+                         gfid_local);
+            goto unlock;
+        }
 
-                is_dir = check_is_dir (inode, stbuf, xattr);
-                if (!is_dir) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "lookup of %s on %s returned non dir 0%o",
-                                local->loc.path, prev->this->name,
-                                stbuf->ia_type);
-                        local->need_selfheal = 1;
-                        goto unlock;
-                }
+        /* Save the mds subvol info and stbuf. This is the value that will
+         * be used for healing
+         */
+        local->mds_subvol = prev;
+        local->mds_stbuf = *stbuf;
 
-                local->op_ret = 0;
-                if (local->xattr == NULL) {
-                        local->xattr = dict_ref (xattr);
-                } else {
-                        dht_aggregate_xattr (local->xattr, xattr);
-                }
+        /* Save mds subvol on inode ctx */
 
-                if (local->inode == NULL)
-                        local->inode = inode_ref (inode);
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "%s: Failed to set mds (%s)", local->loc.path, prev->name);
+        }
+        check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                       mds_xattr_val, 1, &errst);
+        if ((check_mds < 0) && !errst) {
+            /* Check if xattrs need to be healed on the directories */
+            local->mds_xattr = dict_ref(xattr);
+            gf_msg_debug(this->name, 0,
+                         "%s: %s is not zero on %s. Xattrs need to be healed."
+                         "(gfid = %s)",
+                         local->loc.path, conf->mds_xattr_key, prev->name,
+                         gfid_local);
+            local->need_xattr_heal = 1;
+        }
+    }
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
-        }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
+    this_call_cnt = dht_frame_return(frame);
 
-        this_call_cnt = dht_frame_return (frame);
+    if (is_last_call(this_call_cnt)) {
+        /* If the mds subvol is not set correctly*/
+        if (!__is_root_gfid(local->gfid) &&
+            (!dict_get(local->xattr, conf->mds_xattr_key))) {
+            local->need_selfheal = 1;
+        }
 
-        if (is_last_call (this_call_cnt)) {
-                if (local->need_selfheal) {
-                        local->need_selfheal = 0;
-                        dht_lookup_everywhere (frame, this, &local->loc);
-                        return 0;
-                }
+        /* No need to call xattr heal code if volume count is 1
+         */
+        if (conf->subvolume_cnt == 1) {
+            local->need_xattr_heal = 0;
+        }
 
-                if (local->op_ret == 0) {
-                        ret = dht_layout_normalize (this, &local->loc, layout);
+        if (local->need_selfheal || local->need_lookup_everywhere) {
+            /* Set the gfid-req so posix will set the GFID*/
+            if (!gf_uuid_is_null(local->gfid)) {
+                /* Ok, this should _never_ happen */
+                ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                          local->gfid, 16);
+            } else {
+                if (!gf_uuid_is_null(local->gfid_req))
+                    ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                              local->gfid_req, 16);
+            }
+        }
 
-                        if (ret != 0) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "fixing assignment on %s",
-                                        local->loc.path);
-                                goto selfheal;
-                        }
+        if (local->need_lookup_everywhere) {
+            local->need_lookup_everywhere = 0;
+            dht_lookup_everywhere(frame, this, &local->loc);
+            return 0;
+        }
 
-                        dht_layout_set (this, local->inode, layout);
-                }
+        if (local->op_ret == 0) {
+            if (dht_needs_selfheal(frame, this)) {
+                goto selfheal;
+            }
+
+            dht_layout_set(this, local->inode, layout);
+            if (local->inode) {
+                dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+            }
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                                  local->inode, &local->stbuf, local->xattr,
-                                  &local->postparent);
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
         }
 
-        return 0;
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        /* Delete mds xattr at the time of STACK UNWIND */
+        if (local->xattr)
+            GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
+
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+    }
+
+    return 0;
 
 selfheal:
-        FRAME_SU_DO (frame, dht_local_t);
-        uuid_copy (local->loc.gfid, local->gfid);
-        ret = dht_selfheal_directory (frame, dht_lookup_selfheal_cbk,
-                                      &local->loc, layout);
+    FRAME_SU_DO(frame, dht_local_t);
+    ret = dht_selfheal_directory(frame, dht_lookup_selfheal_cbk, &local->loc,
+                                 layout);
 out:
-        return ret;
+    return ret;
+}
+
+static int
+dht_lookup_directory(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int call_cnt = 0;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, unwind);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, unwind);
+    GF_VALIDATE_OR_GOTO("dht", this->private, unwind);
+    GF_VALIDATE_OR_GOTO("dht", loc, unwind);
+
+    conf = this->private;
+    local = frame->local;
+
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
+
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!local->layout) {
+        goto unwind;
+    }
+
+    if (local->xattr != NULL) {
+        dict_unref(local->xattr);
+        local->xattr = NULL;
+    }
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        /* use this gfid in order to heal any missing ones */
+        ret = dict_set_gfuuid(local->xattr_req, "gfid-req", local->gfid, true);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "%s: Failed to set dictionary value:"
+                   " key = gfid-req",
+                   local->loc.path);
+    }
+
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(
+            frame, dht_lookup_dir_cbk, conf->subvolumes[i], conf->subvolumes[i],
+            conf->subvolumes[i]->fops->lookup, &local->loc, local->xattr_req);
+    }
+    return 0;
+unwind:
+    DHT_STACK_UNWIND(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+out:
+    return 0;
 }
 
 int
-dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent)
-{
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev          = NULL;
-        dht_layout_t *layout        = NULL;
-        dht_conf_t   *conf          = NULL;
-        int           ret  = -1;
-        int           is_dir = 0;
-        int           is_linkfile = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, err);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, err);
+dht_revalidate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int is_linkfile = 0;
+    int follow_link = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    uint32_t vol_commit_hash = 0;
+    xlator_t *subvol = NULL;
+    int32_t check_mds = 0;
+    int errst = 0, i = 0;
+    int32_t mds_xattr_val[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", cookie, err);
+    GF_VALIDATE_OR_GOTO("dht", this->private, err);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (!conf->vch_forced) {
+        /* Update the commithash value if available
+         */
+        ret = dict_get_uint32(xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
+        }
+    }
 
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
-        if (!conf)
-                goto out;
+    gf_uuid_unparse(local->loc.gfid, gfid);
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-
-                        if ((op_errno != ENOTCONN)
-                            && (op_errno != ENOENT)
-                            && (op_errno != ESTALE)) {
-				gf_log (this->name, GF_LOG_INFO,
-					"subvolume %s for %s returned -1 (%s)",
-					prev->this->name, local->loc.path,
-                                        strerror (op_errno));
-			}
-                        if (op_errno == ESTALE) {
-                                /* propagate the ESTALE to parent.
-                                 * setting local->return_estale would send
-                                 * ESTALE to parent. */
-                                local->return_estale = 1;
-                        }
+    gf_msg_debug(this->name, op_errno,
+                 "%s: revalidate lookup on %s returned op_ret %d",
+                 local->loc.path, prev->name, op_ret);
+
+    LOCK(&frame->lock);
+    {
+        if (gf_uuid_is_null(local->gfid)) {
+            memcpy(local->gfid, local->loc.gfid, 16);
+        }
 
-                        /* if it is ENOENT, we may have to do a
-                         * 'lookup_everywhere()' to make sure
-                         * the file is not migrated */
-                        if (op_errno == ENOENT) {
-                                if (IA_ISREG (local->loc.inode->ia_type)) {
-                                        local->need_lookup_everywhere = 1;
-                                }
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+
+            if ((op_errno != ENOTCONN) && (op_errno != ENOENT) &&
+                (op_errno != ESTALE)) {
+                gf_msg(this->name, GF_LOG_INFO, op_errno,
+                       DHT_MSG_REVALIDATE_CBK_INFO,
+                       "Revalidate: subvolume %s for %s "
+                       "(gfid = %s) returned -1",
+                       prev->name, local->loc.path, gfid);
+            }
+            if (op_errno == ESTALE) {
+                /* propagate the ESTALE to parent.
+                 * setting local->return_estale would send
+                 * ESTALE to parent. */
+                local->return_estale = 1;
+            }
+
+            /* if it is ENOENT, we may have to do a
+             * 'lookup_everywhere()' to make sure
+             * the file is not migrated */
+            if (op_errno == ENOENT) {
+                if (IA_ISREG(local->loc.inode->ia_type)) {
+                    gf_msg_debug(this->name, 0,
+                                 "found ENOENT for %s. "
+                                 "Setting "
+                                 "need_lookup_everywhere"
+                                 " flag to 1",
+                                 local->loc.path);
+
+                    local->need_lookup_everywhere = 1;
+                } else if (IA_ISDIR(local->loc.inode->ia_type)) {
+                    layout = local->layout;
+                    for (i = 0; i < layout->cnt; i++) {
+                        if (layout->list[i].xlator == prev) {
+                            layout->list[i].err = op_errno;
+                            break;
                         }
-                        goto unlock;
+                    }
+
+                    local->need_selfheal = 1;
                 }
+            }
 
-                if (stbuf->ia_type != local->inode->ia_type) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "mismatching filetypes 0%o v/s 0%o for %s",
-                                (stbuf->ia_type), (local->inode->ia_type),
-                                local->loc.path);
+            /* The GFID is missing on this subvol. Lookup everywhere to force a
+             * gfid heal
+             */
+            if ((op_errno == ENODATA) &&
+                (IA_ISDIR(local->loc.inode->ia_type))) {
+                local->need_lookup_everywhere = 1;
+            }
 
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
+            goto unlock;
+        }
 
-                        goto unlock;
-                }
+        if ((!IA_ISINVAL(local->inode->ia_type)) &&
+            stbuf->ia_type != local->inode->ia_type) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+                   "mismatching filetypes 0%o v/s 0%o for %s,"
+                   " gfid = %s",
+                   (stbuf->ia_type), (local->inode->ia_type), local->loc.path,
+                   gfid);
 
-                layout = local->layout;
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
 
-                is_dir = check_is_dir (inode, stbuf, xattr);
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+            goto unlock;
+        }
 
-                if (is_linkfile) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "linkfile found in revalidate for %s",
-                                local->loc.path);
-                        local->return_estale = 1;
+        layout = local->layout;
 
-                        goto unlock;
+        is_dir = check_is_dir(inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile(inode, stbuf, xattr,
+                                        conf->link_xattr_name);
+        if (is_linkfile) {
+            follow_link = 1;
+            goto unlock;
+        }
+        if (is_dir) {
+            ret = dht_dir_has_layout(xattr, conf->xattr_name);
+            if (ret >= 0) {
+                if (is_greater_time(local->prebuf.ia_ctime,
+                                    local->prebuf.ia_ctime_nsec,
+                                    stbuf->ia_ctime, stbuf->ia_ctime_nsec)) {
+                    /* Choose source */
+                    local->prebuf.ia_gid = stbuf->ia_gid;
+                    local->prebuf.ia_uid = stbuf->ia_uid;
+
+                    local->prebuf.ia_ctime = stbuf->ia_ctime;
+                    local->prebuf.ia_ctime_nsec = stbuf->ia_ctime_nsec;
+
+                    if (__is_root_gfid(stbuf->ia_gfid))
+                        local->prebuf.ia_prot = stbuf->ia_prot;
                 }
+            }
+
+            if (local->stbuf.ia_type != IA_INVAL) {
+                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                    (local->stbuf.ia_uid != stbuf->ia_uid) ||
+                    is_permission_different(&local->stbuf.ia_prot,
+                                            &stbuf->ia_prot)) {
+                    local->need_attrheal = 1;
+                }
+            }
+
+            if (!dict_get(xattr, conf->mds_xattr_key)) {
+                gf_msg_debug(this->name, 0,
+                             "%s: internal xattr %s is not present"
+                             " on subvol %s(gfid is %s)",
+                             local->loc.path, conf->mds_xattr_key, prev->name,
+                             gfid);
+            } else {
+                check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                               mds_xattr_val, 1, &errst);
+                local->mds_subvol = prev;
+                local->mds_stbuf.ia_gid = stbuf->ia_gid;
+                local->mds_stbuf.ia_uid = stbuf->ia_uid;
+                local->mds_stbuf.ia_prot = stbuf->ia_prot;
+
+                /* save mds subvol on inode ctx */
+                ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           DHT_MSG_SET_INODE_CTX_FAILED,
+                           "Failed to set MDS subvol for %s vol is %s",
+                           local->loc.path, prev->name);
+                }
+                if ((check_mds < 0) && !errst) {
+                    /* Check if xattrs need to be healed on the directory
+                     */
+                    local->mds_xattr = dict_ref(xattr);
+                    gf_msg_debug(this->name, 0,
+                                 "Value of %s is not zero on "
+                                 "hashed subvol so xattr needs to"
+                                 " be healed on non hashed"
+                                 " path is %s and vol name is %s "
+                                 " gfid is %s",
+                                 conf->mds_xattr_key, local->loc.path,
+                                 prev->name, gfid);
+                    local->need_xattr_heal = 1;
+                }
+            }
+            ret = dht_layout_dir_mismatch(this, layout, prev, &local->loc,
+                                          xattr);
+            if (ret != 0) {
+                /* In memory layout does not match on-disk layout.
+                 */
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_MISMATCH,
+                       "Mismatching layouts for %s, gfid = %s", local->loc.path,
+                       gfid);
 
-                if (is_dir) {
-                        ret = dht_layout_dir_mismatch (this, layout,
-                                                       prev->this, &local->loc,
-                                                       xattr);
-                        if (ret != 0) {
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "mismatching layouts for %s",
-                                        local->loc.path);
-
-                                local->layout_mismatch = 1;
+                local->layout_mismatch = 1;
 
-                                goto unlock;
-                        }
-                }
+                goto unlock;
+            }
+        }
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+        gf_uuid_copy(local->stbuf.ia_gfid, stbuf->ia_gfid);
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
 
-                local->op_ret = 0;
+        local->op_ret = 0;
 
-                if (!local->xattr) {
-                        local->xattr = dict_ref (xattr);
-                } else if (is_dir) {
-                        dht_aggregate_xattr (local->xattr, xattr);
-                }
+        if (!local->xattr) {
+            local->xattr = dict_ref(xattr);
+        } else if (is_dir) {
+            dht_aggregate_xattr(local->xattr, xattr);
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
-out:
-        this_call_cnt = dht_frame_return (frame);
-
-        if (is_last_call (this_call_cnt)) {
-                if (!IA_ISDIR (local->stbuf.ia_type)
-                    && (local->hashed_subvol != local->cached_subvol)
-                    && (local->stbuf.ia_nlink == 1)
-                    && (conf && conf->unhashed_sticky_bit)) {
-                        local->stbuf.ia_prot.sticky = 1;
-                }
-                if (local->layout_mismatch) {
-                        /* Found layout mismatch in the directory, need to
-                           fix this in the inode context */
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-                        dht_lookup_directory (frame, this, &local->loc);
-                        return 0;
-                }
+    UNLOCK(&frame->lock);
 
-                if (local->need_lookup_everywhere) {
-                        /* As the current layout gave ENOENT error, we would
-                           need a new layout */
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-
-                        /* We know that current cached subvol is no more
-                           valid, get the new one */
-                        local->cached_subvol = NULL;
-                        dht_lookup_everywhere (frame, this, &local->loc);
-                        return 0;
-                }
-                if (local->return_estale) {
-                        local->op_ret = -1;
-                        local->op_errno = ESTALE;
-                }
+    if (follow_link) {
+        /* Found a linkto file. Follow it to see if the target file exists
+         */
+        gf_uuid_copy(local->gfid, stbuf->ia_gfid);
 
-                WIPE (&local->postparent);
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+        if (!subvol) {
+            op_errno = ESTALE;
+            local->op_ret = -1;
+        } else {
+            STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                              subvol->fops->lookup, &local->loc,
+                              local->xattr_req);
+            return 0;
+        }
+    }
+
+    this_call_cnt = dht_frame_return(frame);
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                                  local->inode, &local->stbuf, local->xattr,
-                                  &local->postparent);
+    if (is_last_call(this_call_cnt)) {
+        if (!IA_ISDIR(local->stbuf.ia_type) &&
+            (local->hashed_subvol != local->cached_subvol) &&
+            (local->stbuf.ia_nlink == 1) &&
+            (conf && conf->unhashed_sticky_bit)) {
+            local->stbuf.ia_prot.sticky = 1;
         }
+        /* No need to call heal code if volume count is 1
+         */
+        if (conf->subvolume_cnt == 1)
+            local->need_xattr_heal = 0;
+
+        if (IA_ISDIR(local->stbuf.ia_type)) {
+            /* No mds xattr found. Trigger a heal to set it */
+            if (!__is_root_gfid(local->loc.inode->gfid) &&
+                (!dict_get(local->xattr, conf->mds_xattr_key)))
+                local->need_selfheal = 1;
+
+            if (dht_needs_selfheal(frame, this)) {
+                if (!__is_root_gfid(local->loc.inode->gfid)) {
+                    if (local->mds_subvol) {
+                        local->stbuf.ia_gid = local->mds_stbuf.ia_gid;
+                        local->stbuf.ia_uid = local->mds_stbuf.ia_uid;
+                        local->stbuf.ia_prot = local->mds_stbuf.ia_prot;
+                    }
+                } else {
+                    local->stbuf.ia_gid = local->prebuf.ia_gid;
+                    local->stbuf.ia_uid = local->prebuf.ia_uid;
+                    local->stbuf.ia_prot = local->prebuf.ia_prot;
+                }
 
-err:
-        return ret;
-}
+                layout = local->layout;
+                dht_selfheal_directory(frame, dht_lookup_selfheal_cbk,
+                                       &local->loc, layout);
+                return 0;
+            }
+        }
 
+        if (local->layout_mismatch) {
+            /* Found layout mismatch in the directory, need to
+               fix this in the inode context */
+            dht_layout_unref(this, local->layout);
+            local->layout = NULL;
+            dht_lookup_directory(frame, this, &local->loc);
+            return 0;
+        }
 
-int
-dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent,
-                                dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        xlator_t     *cached_subvol = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+        if (local->need_lookup_everywhere) {
+            /* As the current layout gave ENOENT error, we would
+               need a new layout */
+            dht_layout_unref(this, local->layout);
+            local->layout = NULL;
+
+            /* We know that current cached subvol is no longer
+               valid, get the new one */
+            local->cached_subvol = NULL;
+            if (local->xattr_req) {
+                if (!gf_uuid_is_null(local->gfid)) {
+                    ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                              local->gfid, 16);
+                }
+            }
 
-        local = frame->local;
-        cached_subvol = local->cached_subvol;
-        conf = this->private;
+            dht_lookup_everywhere(frame, this, &local->loc);
+            return 0;
+        }
+        if (local->return_estale) {
+            local->op_ret = -1;
+            local->op_errno = ESTALE;
+        }
 
-        ret = dht_layout_preset (this, local->cached_subvol, inode);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set layout for subvolume %s",
-                        cached_subvol ? cached_subvol->name : "<nil>");
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                goto unwind;
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
         }
 
-        local->op_ret = 0;
-        if ((local->stbuf.ia_nlink == 1)
-            && (conf && conf->unhashed_sticky_bit)) {
-                local->stbuf.ia_prot.sticky = 1;
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+
+        /* local->stbuf is updated only from subvols which have a layout
+         * The reason is to avoid choosing attr heal source from newly
+         * added bricks. In case e.g we have only one subvol and for
+         * some reason layout is not present on it, then local->stbuf
+         * will be EINVAL. This is an indication that the subvols
+         * active in the cluster do not have layouts on disk.
+         * Unwind with ESTALE to trigger a fresh lookup */
+        if (is_dir && local->stbuf.ia_type == IA_INVAL) {
+            local->op_ret = -1;
+            local->op_errno = ESTALE;
         }
+        /* Delete mds xattr at the time of STACK UNWIND */
+        if (local->xattr)
+            GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
+
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+    }
+
+err:
+    return ret;
+}
+
+static int
+dht_lookup_linkfile_create_cbk(call_frame_t *frame, void *cooie, xlator_t *this,
+                               int32_t op_ret, int32_t op_errno, inode_t *inode,
+                               struct iatt *stbuf, struct iatt *preparent,
+                               struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+
+    local = frame->local;
+    cached_subvol = local->cached_subvol;
+    conf = this->private;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (local->locked)
+        dht_unlock_namespace(frame, &local->lock[0]);
+
+    ret = dht_layout_preset(this, local->cached_subvol, local->loc.inode);
+    if (ret < 0) {
+        gf_msg_debug(this->name, EINVAL,
+                     "Failed to set layout for subvolume %s, "
+                     "(gfid = %s)",
+                     cached_subvol ? cached_subvol->name : "<nil>", gfid);
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+
+    local->op_ret = 0;
+    if ((local->stbuf.ia_nlink == 1) && (conf && conf->unhashed_sticky_bit)) {
+        local->stbuf.ia_prot.sticky = 1;
+    }
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
 
 unwind:
-        WIPE (&local->postparent);
+    gf_msg_debug(this->name, 0,
+                 "creation of linkto on hashed subvol:%s, "
+                 "returned with op_ret %d and op_errno %d: %s",
+                 local->hashed_subvol->name, op_ret, op_errno,
+                 uuid_utoa(local->loc.gfid));
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
+    if (local->linked == _gf_true)
+        dht_linkfile_attr_heal(frame, this);
+
+    dht_set_fixed_dir_stat(&local->postparent);
+
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
 out:
-        return ret;
+    return ret;
 }
 
+static int
+dht_lookup_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-int
-dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
+    local = (dht_local_t *)frame->local;
+    path = local->loc.path;
+    FRAME_SU_UNDO(frame, dht_local_t);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "lookup_unlink returned with "
+           "op_ret -> %d and op-errno -> %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
+
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_lookup_everywhere_done(frame, this);
+    }
+
+    return 0;
+}
+
+static int
+dht_lookup_unlink_of_false_linkto_cbk(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int op_ret, int op_errno,
+                                      struct iatt *preparent,
+                                      struct iatt *postparent, dict_t *xdata)
 {
-        int           ret = 0;
-        dht_local_t  *local = NULL;
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
-        dht_layout_t *layout = NULL;
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-        local = frame->local;
-        hashed_subvol = local->hashed_subvol;
-        cached_subvol = local->cached_subvol;
+    local = (dht_local_t *)frame->local;
+    path = local->loc.path;
 
-        if (local->file_count && local->dir_count) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "path %s exists as a file on one subvolume "
-                        "and directory on another. "
-                        "Please fix it manually",
-                        local->loc.path);
-                DHT_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL, NULL,
-                                  NULL);
-                return 0;
-        }
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-        if (local->dir_count) {
-                dht_lookup_directory (frame, this, &local->loc);
-                return 0;
-        }
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "lookup_unlink returned with "
+           "op_ret -> %d and op-errno -> %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
 
-        if (!cached_subvol) {
-                DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL, NULL, NULL,
-                                  NULL);
-                return 0;
-        }
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if ((op_ret == 0) || ((op_errno != EBUSY) && (op_errno != ENOTCONN))) {
+            dht_lookup_everywhere_done(frame, this);
+        } else {
+            /*When dht_lookup_everywhere is performed, one cached
+             *and one hashed file was found and hashed file does
+             *not point to the above mentioned cached node. So it
+             *was considered as stale and an unlink was performed.
+             *But unlink fails. So may be rebalance is in progress.
+             *now ideally we have two data-files. One obtained during
+             *lookup_everywhere and one where unlink-failed. So
+             *at this point in time we cannot decide which one to
+             *choose because there are chances of first cached
+             *file is truncated after rebalance and if it is chosen
+             *as cached node, application will fail. So return EIO.*/
+
+            if (op_errno == EBUSY) {
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_UNLINK_FAILED,
+                       "Could not unlink the linkto file as "
+                       "either fd is open and/or linkto xattr "
+                       "is set for %s",
+                       ((path == NULL) ? "null" : path));
+            }
+            DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
+        }
+    }
+
+    return 0;
+}
 
-        if (local->need_lookup_everywhere) {
-                if (uuid_compare (local->gfid, local->inode->gfid)) {
-                        /* GFID different, return error */
-                        DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL,
-                                          NULL, NULL, NULL);
-                        return 0;
-                }
-                local->op_ret = 0;
-                local->op_errno = 0;
-                layout = dht_layout_for_subvol (this, cached_subvol);
-                if (!layout) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%s: no pre-set layout for subvolume %s",
-                                local->loc.path, (cached_subvol ?
-                                                  cached_subvol->name :
-                                                  "<nil>"));
-                }
+static int
+dht_lookup_unlink_stale_linkto_cbk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int op_ret, int op_errno,
+                                   struct iatt *preparent,
+                                   struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-                ret = dht_layout_set (this, local->inode, layout);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%s: failed to set layout for subvol %s",
-                                local->loc.path, (cached_subvol ?
-                                                  cached_subvol->name :
-                                                  "<nil>"));
-                }
+    /* NOTE:
+     * If stale file unlink fails either there is an open-fd or is not an
+     * dht-linkto-file then posix_unlink returns EBUSY, which is overwritten
+     *  to ENOENT
+     */
 
-                WIPE (&local->postparent);
+    local = frame->local;
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
-                                  local->op_errno, local->inode,
-                                  &local->stbuf, local->xattr,
-                                  &local->postparent);
-                return 0;
-        }
+    if (local) {
+        FRAME_SU_UNDO(frame, dht_local_t);
+        if (local->loc.path)
+            path = local->loc.path;
+    }
 
-        if (!hashed_subvol) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "cannot create linkfile file for %s on %s: "
-                        "hashed subvolume cannot be found.",
-                        local->loc.path, cached_subvol->name);
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "Returned with op_ret %d and "
+           "op_errno %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
 
-                local->op_ret = 0;
-                local->op_errno = 0;
+    DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL, NULL);
 
-                ret = dht_layout_preset (frame->this, cached_subvol,
-                                         local->inode);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set layout for subvol %s",
-                                cached_subvol ? cached_subvol->name :
-                                "<nil>");
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
-                }
+    return 0;
+}
 
-                WIPE (&local->postparent);
+static int
+dht_fill_dict_to_avoid_unlink_of_migrating_file(dict_t *dict)
+{
+    int ret = 0;
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
-                                  local->op_errno, local->inode,
-                                  &local->stbuf, local->xattr,
-                                  &local->postparent);
-                return 0;
-        }
+    ret = dict_set_int32_sizen(dict, DHT_SKIP_NON_LINKTO_UNLINK, 1);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "linking file %s existing on %s to %s (hash)",
-                local->loc.path, cached_subvol->name,
-                hashed_subvol->name);
+    if (ret)
+        return -1;
 
-        ret = dht_linkfile_create (frame,
-                                   dht_lookup_linkfile_create_cbk,
-                                   cached_subvol, hashed_subvol, &local->loc);
+    ret = dict_set_int32_sizen(dict, DHT_SKIP_OPEN_FD_UNLINK, 1);
 
-        return ret;
+    if (ret)
+        return -1;
+
+    return 0;
 }
 
+static int32_t
+dht_linkfile_create_lookup_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *buf, dict_t *xdata,
+                               struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int call_cnt = 0, ret = 0;
+    xlator_t *subvol = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    char gfid_str[GF_UUID_BUF_SIZE] = {0};
+
+    subvol = cookie;
+    local = frame->local;
+
+    if (subvol == local->hashed_subvol) {
+        if ((op_ret == 0) || (op_errno != ENOENT))
+            local->dont_create_linkto = _gf_true;
+    } else {
+        if (gf_uuid_is_null(local->gfid))
+            gf_uuid_copy(gfid, local->loc.gfid);
+        else
+            gf_uuid_copy(gfid, local->gfid);
+
+        if ((op_ret == 0) && gf_uuid_compare(gfid, buf->ia_gfid)) {
+            gf_uuid_unparse(gfid, gfid_str);
+            gf_msg_debug(this->name, 0,
+                         "gfid (%s) different on cached subvol "
+                         "(%s) and looked up inode (%s), not "
+                         "creating linkto",
+                         uuid_utoa(buf->ia_gfid), subvol->name, gfid_str);
+            local->dont_create_linkto = _gf_true;
+        } else if (op_ret == -1) {
+            local->dont_create_linkto = _gf_true;
+        }
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->dont_create_linkto)
+            goto no_linkto;
+        else {
+            gf_msg_debug(this->name, 0,
+                         "Creating linkto file on %s(hash) to "
+                         "%s on %s (gfid = %s)",
+                         local->hashed_subvol->name, local->loc.path,
+                         local->cached_subvol->name, gfid_str);
+
+            ret = dht_linkfile_create(frame, dht_lookup_linkfile_create_cbk,
+                                      this, local->cached_subvol,
+                                      local->hashed_subvol, &local->loc);
+
+            if (ret < 0)
+                goto no_linkto;
+        }
+    }
+
+    return 0;
+
+no_linkto:
+    gf_msg_debug(this->name, 0,
+                 "skipped linkto creation (path:%s) (gfid:%s) "
+                 "(hashed-subvol:%s) (cached-subvol:%s)",
+                 local->loc.path, gfid_str, local->hashed_subvol->name,
+                 local->cached_subvol->name);
+
+    dht_lookup_linkfile_create_cbk(frame, NULL, this, 0, 0, local->loc.inode,
+                                   &local->stbuf, &local->preparent,
+                                   &local->postparent, local->xattr);
+    return 0;
+}
 
-int
-dht_lookup_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       struct iatt *preparent, struct iatt *postparent,
-                       dict_t *xdata)
+static int32_t
+dht_call_lookup_linkfile_create(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
 {
-        int  this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int i = 0;
+    xlator_t *subvol = NULL;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_lookup_everywhere_done (frame, this);
-        }
+    local = frame->local;
+    if (gf_uuid_is_null(local->gfid))
+        gf_uuid_unparse(local->loc.gfid, gfid);
+    else
+        gf_uuid_unparse(local->gfid, gfid);
 
-        return 0;
+    if (op_ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "protecting namespace failed, skipping linkto "
+               "creation (path:%s)(gfid:%s)(hashed-subvol:%s)"
+               "(cached-subvol:%s)",
+               local->loc.path, gfid, local->hashed_subvol->name,
+               local->cached_subvol->name);
+        goto err;
+    }
+
+    local->locked = _gf_true;
+
+    local->call_cnt = 2;
+
+    for (i = 0; i < 2; i++) {
+        subvol = (subvol == NULL) ? local->hashed_subvol : local->cached_subvol;
+
+        STACK_WIND_COOKIE(frame, dht_linkfile_create_lookup_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, NULL);
+    }
+
+    return 0;
+
+err:
+    dht_lookup_linkfile_create_cbk(frame, NULL, this, 0, 0, local->loc.inode,
+                                   &local->stbuf, &local->preparent,
+                                   &local->postparent, local->xattr);
+    return 0;
 }
 
+/* Rebalance is performed from cached_node to hashed_node. Initial cached_node
+ * contains a non-linkto file. After migration it is converted to linkto and
+ * then unlinked. And at hashed_subvolume, first a linkto file is present,
+ * then after migration it is converted to a non-linkto file.
+ *
+ * Lets assume a file is present on cached subvolume and a new brick is added
+ * and new brick is the new_hashed subvolume. So fresh lookup on newly added
+ * hashed subvolume will fail and dht_lookup_everywhere gets called.  If just
+ * before sending the dht_lookup_everywhere request rebalance is in progress,
+ *
+ * from cached subvolume it may see: Nonlinkto or linkto or No file
+ * from hashed subvolume it may see: No file or linkto file or non-linkto file
+ *
+ * So this boils down to 9 cases:
+ *   at cached_subvol            at hashed_subvol
+ *   ----------------           -----------------
+ *
+ *a)   No file                     No file
+ *    [request reached after    [Request reached before
+ *       migration]                Migration]
+ *
+ *b)   No file                     Linkto File
+ *
+ *c)   No file                     Non-Linkto File
+ *
+ *d)   Linkto                      No-File
+ *
+ *e)   Linkto                      Linkto
+ *
+ *f)   Linkto                      Non-Linkto
+ *
+ *g)   NonLinkto                   No-File
+ *
+ *h)   NonLinkto                   Linkto
+ *
+ *i)   NonLinkto                   NonLinkto
+ *
+ * dht_lookup_everywhere_done takes decision based on any of the above case
+ */
 
-int
-dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           inode_t *inode, struct iatt *buf, dict_t *xattr,
-                           struct iatt *postparent)
-{
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev          = NULL;
-        int           is_linkfile   = 0;
-        int           is_dir        = 0;
-        xlator_t     *subvol        = NULL;
-        loc_t        *loc           = NULL;
-        xlator_t     *link_subvol   = NULL;
-        int           ret = -1;
-        int32_t       fd_count = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-
-        local  = frame->local;
-        loc    = &local->loc;
-
-        prev   = cookie;
-        subvol = prev->this;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno != ENOENT)
-                                local->op_errno = op_errno;
-                        goto unlock;
-                }
+static int
+dht_lookup_everywhere_done(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    dht_local_t *local = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_layout_t *layout = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    gf_boolean_t found_non_linkto_on_hashed = _gf_false;
+
+    local = frame->local;
+    hashed_subvol = local->hashed_subvol;
+    cached_subvol = local->cached_subvol;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (local->file_count && local->dir_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+               "path %s (gfid = %s)exists as a file on one "
+               "subvolume and directory on another. "
+               "Please fix it manually",
+               local->loc.path, gfid);
+        DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
+        return 0;
+    }
+    if (local->op_ret && local->gfid_missing) {
+        if (gf_uuid_is_null(local->gfid_req)) {
+            DHT_STACK_UNWIND(lookup, frame, -1, ENODATA, NULL, NULL, NULL,
+                             NULL);
+            return 0;
+        }
+        /* A hack */
+        dht_lookup_directory(frame, this, &local->loc);
+        return 0;
+    }
+
+    if (local->dir_count) {
+        dht_lookup_directory(frame, this, &local->loc);
+        return 0;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "STATUS: hashed_subvol %s "
+                 "cached_subvol %s",
+                 (hashed_subvol == NULL) ? "null" : hashed_subvol->name,
+                 (cached_subvol == NULL) ? "null" : cached_subvol->name);
+
+    if (!cached_subvol) {
+        if (local->skip_unlink.handle_valid_link && hashed_subvol) {
+            /*Purpose of "DHT_SKIP_NON_LINKTO_UNLINK":
+             * If this lookup is performed by rebalance and this
+             * rebalance process detected hashed file and by
+             * the time it sends the lookup request to cached node,
+             * file got migrated and now at initial hashed_node,
+             * final migrated file is present. With current logic,
+             * because this process fails to find the cached_node,
+             * it will unlink the file at initial hashed_node.
+             *
+             * So we avoid this by setting key, and checking at the
+             * posix_unlink that unlink the file only if file is a
+             * linkto file and not a migrated_file.
+             */
+
+            ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(
+                local->xattr_req);
+
+            if (ret) {
+                /* If for some reason, setting key in the dict
+                 * fails, return with ENOENT, as with respect to
+                 * this process, it detected only a stale link
+                 * file.
+                 *
+                 * Next lookup will delete it.
+                 *
+                 * Performing deletion of stale link file when
+                 * setting key in dict fails, may cause the data
+                 * loss because of the above mentioned race.
+                 */
+
+                DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL,
+                                 NULL);
+            } else {
+                local->skip_unlink.handle_valid_link = _gf_false;
+
+                gf_msg_debug(this->name, 0,
+                             "No Cached was found and "
+                             "unlink on hashed was skipped"
+                             " so performing now: %s",
+                             local->loc.path);
+                FRAME_SU_DO(frame, dht_local_t);
+                STACK_WIND(frame, dht_lookup_unlink_stale_linkto_cbk,
+                           hashed_subvol, hashed_subvol->fops->unlink,
+                           &local->loc, 0, local->xattr_req);
+            }
+
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "There was no cached file and  "
+                         "unlink on hashed is not skipped %s",
+                         local->loc.path);
+
+            DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL, NULL);
+        }
+        return 0;
+    }
+
+    /* At the time of dht_lookup, no file was found on hashed and that is
+     * why dht_lookup_everywhere is called, but by the time
+     * dht_lookup_everywhere
+     * reached to server, file might have already migrated. In that case we
+     * will find a migrated file at the hashed_node. In this case store the
+     * layout in context and return successfully.
+     */
+
+    if (hashed_subvol || local->need_lookup_everywhere) {
+        if (local->need_lookup_everywhere) {
+            found_non_linkto_on_hashed = _gf_true;
+
+        } else if ((local->file_count == 1) &&
+                   (hashed_subvol == cached_subvol)) {
+            gf_msg_debug(this->name, 0,
+                         "found cached file on hashed subvolume "
+                         "so store in context and return for %s",
+                         local->loc.path);
 
-                if (uuid_is_null (local->gfid))
-                        uuid_copy (local->gfid, buf->ia_gfid);
+            found_non_linkto_on_hashed = _gf_true;
+        }
+
+        if (found_non_linkto_on_hashed)
+            goto preset_layout;
+    }
+
+    if (hashed_subvol) {
+        if (local->skip_unlink.handle_valid_link == _gf_true) {
+            if (cached_subvol == local->skip_unlink.hash_links_to) {
+                if (gf_uuid_compare(local->skip_unlink.cached_gfid,
+                                    local->skip_unlink.hashed_gfid)) {
+                    /*GFID different, return error*/
+                    DHT_STACK_UNWIND(lookup, frame, -1, ESTALE, NULL, NULL,
+                                     NULL, NULL);
 
-                if (uuid_compare (local->gfid, buf->ia_gfid)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: gfid differs on subvolume %s",
-                                loc->path, prev->this->name);
+                    return 0;
                 }
 
-                is_linkfile = check_is_linkfile (inode, buf, xattr);
-                is_dir = check_is_dir (inode, buf, xattr);
-
-                if (is_linkfile) {
-                        link_subvol = dht_linkfile_subvol (this, inode, buf,
-                                                           xattr);
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "found on %s linkfile %s (-> %s)",
-                                subvol->name, loc->path,
-                                link_subvol ? link_subvol->name : "''");
-                        goto unlock;
+                ret = dht_layout_preset(this, cached_subvol, local->loc.inode);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_INFO, 0,
+                           DHT_MSG_LAYOUT_PRESET_FAILED,
+                           "Could not set pre-set layout "
+                           "for subvolume %s",
+                           cached_subvol->name);
                 }
 
-                /* non linkfile GFID takes precedence */
-                uuid_copy (local->gfid, buf->ia_gfid);
+                local->op_ret = (ret == 0) ? ret : -1;
+                local->op_errno = (ret == 0) ? ret : EINVAL;
 
-                if (is_dir) {
-                        local->dir_count++;
+                /* Presence of local->cached_subvol validates
+                 * that lookup from cached node is successful
+                 */
 
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "found on %s directory %s",
-                                subvol->name, loc->path);
-                } else {
-                        local->file_count++;
-
-                        if (!local->cached_subvol) {
-                                /* found one file */
-                                dht_iatt_merge (this, &local->stbuf, buf,
-                                                subvol);
-                                local->xattr = dict_ref (xattr);
-                                local->cached_subvol = subvol;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "found on %s file %s",
-                                        subvol->name, loc->path);
-
-                                dht_iatt_merge (this, &local->postparent,
-                                                postparent, subvol);
-                        } else {
-                                /* This is where we need 'rename' both entries logic */
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "multiple subvolumes (%s and %s) have "
-                                        "file %s (preferably rename the file "
-                                        "in the backend, and do a fresh lookup)",
-                                        local->cached_subvol->name,
-                                        subvol->name, local->loc.path);
-                        }
+                if (!local->op_ret && local->loc.parent) {
+                    dht_inode_ctx_time_update(local->loc.parent, this,
+                                              &local->postparent, 1);
                 }
-        }
-unlock:
-        UNLOCK (&frame->lock);
 
-        if (is_linkfile) {
-                ret = dict_get_int32 (xattr, GLUSTERFS_OPEN_FD_COUNT, &fd_count);
-                /* Delete the linkfile only if there are no open fds on it.
-                   if there is a open-fd, it may be in migration */
-                if (!ret && (fd_count == 0)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "deleting stale linkfile %s on %s",
-                                loc->path, subvol->name);
-                        STACK_WIND (frame, dht_lookup_unlink_cbk,
-                                    subvol, subvol->fops->unlink, loc, 0, NULL);
-                        return 0;
+                gf_msg_debug(this->name, 0,
+                             "Skipped unlinking linkto file "
+                             "on the hashed subvolume. "
+                             "Returning success as it is a "
+                             "valid linkto file. Path:%s",
+                             local->loc.path);
+
+                goto unwind_hashed_and_cached;
+            } else {
+                local->skip_unlink.handle_valid_link = _gf_false;
+
+                gf_msg_debug(this->name, 0,
+                             "Linkto file found on hashed "
+                             "subvol "
+                             "and data file found on cached "
+                             "subvolume. But linkto points to "
+                             "different cached subvolume (%s) "
+                             "path %s",
+                             (local->skip_unlink.hash_links_to
+                                  ? local->skip_unlink.hash_links_to->name
+                                  : " <nil>"),
+                             local->loc.path);
+
+                if (local->skip_unlink.opend_fd_count == 0) {
+                    ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(
+                        local->xattr_req);
+
+                    if (ret) {
+                        DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL,
+                                         NULL, NULL);
+                    } else {
+                        local->call_cnt = 1;
+                        FRAME_SU_DO(frame, dht_local_t);
+                        STACK_WIND(frame, dht_lookup_unlink_of_false_linkto_cbk,
+                                   hashed_subvol, hashed_subvol->fops->unlink,
+                                   &local->loc, 0, local->xattr_req);
+                    }
+
+                    return 0;
                 }
+            }
         }
+    }
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_lookup_everywhere_done (frame, this);
+preset_layout:
+
+    if (found_non_linkto_on_hashed) {
+        if (local->need_lookup_everywhere) {
+            if (gf_uuid_compare(local->gfid, local->inode->gfid)) {
+                /* GFID different, return error */
+                DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL,
+                                 NULL);
+                return 0;
+            }
         }
 
-out:
-        return ret;
-}
+        local->op_ret = 0;
+        local->op_errno = 0;
+        layout = dht_layout_for_subvol(this, cached_subvol);
+        if (!layout) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "%s: no pre-set layout for subvolume %s,"
+                   " gfid = %s",
+                   local->loc.path,
+                   (cached_subvol ? cached_subvol->name : "<nil>"), gfid);
+        }
 
+        ret = dht_layout_set(this, local->inode, layout);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "%s: failed to set layout for subvol %s, "
+                   "gfid = %s",
+                   local->loc.path,
+                   (cached_subvol ? cached_subvol->name : "<nil>"), gfid);
+        }
 
-int
-dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        dht_conf_t     *conf = NULL;
-        dht_local_t    *local = NULL;
-        int             i = 0;
-        int             call_cnt = 0;
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
+        }
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", loc, out);
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+        return 0;
+    }
 
-        conf = this->private;
-        local = frame->local;
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "Cannot create linkfile for %s on %s: "
+                     "hashed subvolume cannot be found, gfid = %s.",
+                     local->loc.path, cached_subvol->name, gfid);
 
-        call_cnt = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
+        local->op_ret = 0;
+        local->op_errno = 0;
 
-        if (!local->inode)
-                local->inode = inode_ref (loc->inode);
+        ret = dht_layout_preset(frame->this, cached_subvol, local->inode);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+                   "Failed to set layout for subvol %s"
+                   ", gfid = %s",
+                   cached_subvol ? cached_subvol->name : "<nil>", gfid);
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
+        }
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_lookup_everywhere_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            loc, local->xattr_req);
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
         }
 
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
         return 0;
-out:
-        DHT_STACK_UNWIND (lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
-err:
-        return -1;
+    }
+
+    if (frame->root->op != GF_FOP_RENAME) {
+        local->current = &local->lock[0];
+        ret = dht_protect_namespace(frame, &local->loc, hashed_subvol,
+                                    &local->current->ns,
+                                    dht_call_lookup_linkfile_create);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Creating linkto file on %s(hash) to %s on %s "
+                     "(gfid = %s)",
+                     hashed_subvol->name, local->loc.path, cached_subvol->name,
+                     gfid);
+
+        ret = dht_linkfile_create(frame, dht_lookup_linkfile_create_cbk, this,
+                                  cached_subvol, hashed_subvol, &local->loc);
+    }
+
+    return ret;
+
+unwind_hashed_and_cached:
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
+    return 0;
 }
 
-
-int
-dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
-{
-        call_frame_t *prev          = NULL;
-        dht_local_t  *local         = NULL;
-        xlator_t     *subvol        = NULL;
-        loc_t        *loc           = NULL;
-        dht_conf_t   *conf          = NULL;
-        int           ret           = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, unwind);
-
-        prev   = cookie;
-        subvol = prev->this;
-        conf   = this->private;
-        local  = frame->local;
-        loc    = &local->loc;
-
+static int
+dht_lookup_everywhere_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xattr,
+                          struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int is_linkfile = 0;
+    int is_dir = 0;
+    loc_t *loc = NULL;
+    xlator_t *link_subvol = NULL;
+    int ret = -1;
+    int32_t fd_count = 0;
+    dht_conf_t *conf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dict_t *dict_req = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+
+    local = frame->local;
+    loc = &local->loc;
+    conf = this->private;
+
+    prev = cookie;
+
+    gf_msg_debug(this->name, 0,
+                 "returned with op_ret %d and op_errno %d (%s) "
+                 "from subvol %s",
+                 op_ret, op_errno, loc->path, prev->name);
+
+    LOCK(&frame->lock);
+    {
         if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "lookup of %s on %s (following linkfile) failed (%s)",
-                        local->loc.path, subvol->name, strerror (op_errno));
-                goto err;
+            if (op_errno != ENOENT)
+                local->op_errno = op_errno;
+            if (op_errno == ENODATA)
+                local->gfid_missing = _gf_true;
+            goto unlock;
         }
 
-        if (check_is_dir (inode, stbuf, xattr)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "lookup of %s on %s (following linkfile) reached dir",
-                        local->loc.path, subvol->name);
-                goto err;
-        }
+        if (gf_uuid_is_null(local->gfid))
+            gf_uuid_copy(local->gfid, buf->ia_gfid);
 
-        if (check_is_linkfile (inode, stbuf, xattr)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "lookup of %s on %s (following linkfile) reached link",
-                        local->loc.path, subvol->name);
-                goto err;
-        }
+        gf_uuid_unparse(local->gfid, gfid);
 
-        if (uuid_compare (local->gfid, stbuf->ia_gfid)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: gfid different on data file on %s",
-                        local->loc.path, subvol->name);
-                goto err;
+        if (gf_uuid_compare(local->gfid, buf->ia_gfid)) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+                   "%s: gfid differs on subvolume %s,"
+                   " gfid local = %s, gfid node = %s",
+                   loc->path, prev->name, gfid, uuid_utoa(buf->ia_gfid));
         }
 
-        if ((stbuf->ia_nlink == 1)
-            && (conf && conf->unhashed_sticky_bit)) {
-                stbuf->ia_prot.sticky = 1;
+        is_linkfile = check_is_linkfile(inode, buf, xattr,
+                                        conf->link_xattr_name);
+
+        if (is_linkfile) {
+            link_subvol = dht_linkfile_subvol(this, inode, buf, xattr);
+            gf_msg_debug(this->name, 0, "found on %s linkfile %s (-> %s)",
+                         prev->name, loc->path,
+                         link_subvol ? link_subvol->name : "''");
+            goto unlock;
         }
 
-        ret = dht_layout_preset (this, prev->this, inode);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "failed to set layout for subvolume %s",
-                        prev->this->name);
-                op_ret   = -1;
-                op_errno = EINVAL;
+        is_dir = check_is_dir(inode, buf, xattr);
+
+        /* non linkfile GFID takes precedence but don't overwrite
+         gfid if we have already found a cached file*/
+        if (!local->cached_subvol)
+            gf_uuid_copy(local->gfid, buf->ia_gfid);
+
+        if (is_dir) {
+            local->dir_count++;
+
+            gf_msg_debug(this->name, 0, "found on %s directory %s", prev->name,
+                         loc->path);
+        } else {
+            local->file_count++;
+
+            gf_msg_debug(this->name, 0, "found cached file on %s for %s",
+                         prev->name, loc->path);
+
+            if (!local->cached_subvol) {
+                /* found one file */
+                dht_iatt_merge(this, &local->stbuf, buf);
+
+                local->xattr = dict_ref(xattr);
+                local->cached_subvol = prev;
+
+                gf_msg_debug(this->name, 0,
+                             "storing cached on %s file"
+                             " %s",
+                             prev->name, loc->path);
+
+                dht_iatt_merge(this, &local->postparent, postparent);
+
+                gf_uuid_copy(local->skip_unlink.cached_gfid, buf->ia_gfid);
+            } else {
+                /* This is where we need 'rename' both entries logic */
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_FILE_ON_MULT_SUBVOL,
+                       "multiple subvolumes (%s and %s) have "
+                       "file %s (preferably rename the file "
+                       "in the backend,and do a fresh lookup)",
+                       local->cached_subvol->name, prev->name, local->loc.path);
+            }
         }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    if (is_linkfile) {
+        ret = dict_get_int32(xattr, GLUSTERFS_OPEN_FD_COUNT, &fd_count);
+
+        /*  Any linkto file found on the non-hashed subvolume should
+         *  be unlinked (performed in the "else if" block below)
+         *
+         *  But if a linkto file is found on hashed subvolume, it may be
+         *  pointing to valid cached node. So unlinking of linkto
+         *  file on hashed subvolume is skipped and inside
+         *  dht_lookup_everywhere_done, checks are performed. If this
+         *  linkto file is found as stale linkto file, it is deleted
+         *  otherwise unlink is skipped.
+         */
 
-unwind:
-        WIPE (postparent);
+        if (local->hashed_subvol && local->hashed_subvol == prev) {
+            local->skip_unlink.handle_valid_link = _gf_true;
+            local->skip_unlink.opend_fd_count = fd_count;
+            local->skip_unlink.hash_links_to = link_subvol;
+            gf_uuid_copy(local->skip_unlink.hashed_gfid, buf->ia_gfid);
+
+            gf_msg_debug(this->name, 0,
+                         "Found"
+                         " one linkto file on hashed subvol %s "
+                         "for %s: Skipping unlinking till "
+                         "everywhere_done",
+                         prev->name, loc->path);
+
+        } else if (!ret && (fd_count == 0)) {
+            dict_req = dict_new();
+
+            ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(dict_req);
+
+            if (ret) {
+                /* Skip unlinking for dict_failure
+                 *File is found as a linkto file on non-hashed,
+                 *subvolume. In the current implementation,
+                 *finding a linkto-file on non-hashed does not
+                 *always implies that it is stale. So deletion
+                 *of file should be done only when both fd is
+                 *closed and linkto-xattr is set. In case of
+                 *dict_set failure, avoid skipping of file.
+                 *NOTE: dht_frame_return should get called for
+                 *      this block.
+                 */
 
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
-                          postparent);
+                dict_unref(dict_req);
+
+            } else {
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                       "attempting deletion of stale linkfile "
+                       "%s on %s (hashed subvol is %s)",
+                       loc->path, prev->name,
+                       (local->hashed_subvol ? local->hashed_subvol->name
+                                             : "<null>"));
+                /* *
+                 * These stale files may be created using root
+                 * user. Hence deletion will work only with
+                 * root.
+                 */
+                FRAME_SU_DO(frame, dht_local_t);
+                STACK_WIND(frame, dht_lookup_unlink_cbk, prev,
+                           prev->fops->unlink, loc, 0, dict_req);
 
-        return 0;
+                dict_unref(dict_req);
+
+                return 0;
+            }
+        }
+    }
+
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_lookup_everywhere_done(frame, this);
+    }
 
-err:
-        dht_lookup_everywhere (frame, this, loc);
 out:
-        return 0;
+    return ret;
 }
 
-
 int
-dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_lookup_everywhere(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-        int           call_cnt = 0;
-        int           i = 0;
-        dht_conf_t   *conf = NULL;
-        dht_local_t  *local = NULL;
-        int           ret = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int i = 0;
+    int call_cnt = 0;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", loc, unwind);
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", loc, out);
 
-        conf = this->private;
-        local = frame->local;
+    conf = this->private;
+    local = frame->local;
 
-        call_cnt        = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
 
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
-        if (!local->layout) {
-                goto unwind;
-        }
+    if (!local->inode)
+        local->inode = inode_ref(loc->inode);
 
-        if (local->xattr != NULL) {
-                dict_unref (local->xattr);
-                local->xattr = NULL;
-        }
+    gf_msg_debug(this->name, 0, "winding lookup call to %d subvols", call_cnt);
 
-        if (!uuid_is_null (local->gfid)) {
-                ret = dict_set_static_bin (local->xattr_req, "gfid-req",
-                                           local->gfid, 16);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to set gfid", local->loc.path);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_lookup_everywhere_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, loc,
+                          local->xattr_req);
+    }
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_lookup_dir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            &local->loc, local->xattr_req);
-        }
-        return 0;
-unwind:
-        DHT_STACK_UNWIND (lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+    return 0;
 out:
-        return 0;
-
+    DHT_STACK_UNWIND(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+err:
+    return -1;
 }
 
-
 int
-dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno,
-                inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                struct iatt *postparent)
-{
-        char          is_linkfile   = 0;
-        char          is_dir        = 0;
-        xlator_t     *subvol        = NULL;
-        dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        loc_t        *loc           = NULL;
-        call_frame_t *prev          = NULL;
-        int           ret           = 0;
-        uint64_t      tmp_layout    = 0;
-        dht_layout_t *parent_layout = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
+dht_lookup_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
+{
+    xlator_t *prev = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    loc_t *loc = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, unwind);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, unwind);
+    GF_VALIDATE_OR_GOTO("dht", this->private, unwind);
+    GF_VALIDATE_OR_GOTO("dht", cookie, unwind);
+
+    prev = cookie;
+    subvol = prev;
+    conf = this->private;
+    local = frame->local;
+    loc = &local->loc;
+
+    gf_uuid_unparse(loc->gfid, gfid);
+
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "Lookup of %s on %s (following linkfile) failed "
+               ",gfid = %s",
+               local->loc.path, subvol->name, gfid);
+
+        /* If cached subvol returned ENOTCONN, do not do
+        lookup_everywhere. We need to make sure linkfile does not get
+        removed, which can take away the namespace, and subvol is
+        anyways down. */
+
+        local->cached_subvol = NULL;
+        if (op_errno != ENOTCONN)
+            goto err;
+        else
+            goto unwind;
+    }
+
+    if (check_is_dir(inode, stbuf, xattr)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "Lookup of %s on %s (following linkfile) reached dir,"
+               " gfid = %s",
+               local->loc.path, subvol->name, gfid);
+        goto err;
+    }
+
+    if (check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "lookup of %s on %s (following linkfile) reached link,"
+               "gfid = %s",
+               local->loc.path, subvol->name, gfid);
+        goto err;
+    }
+
+    if (gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on data file on %s,"
+               " gfid local = %s, gfid node = %s ",
+               local->loc.path, subvol->name, gfid, uuid_utoa(stbuf->ia_gfid));
+        goto err;
+    }
+
+    if ((stbuf->ia_nlink == 1) && (conf && conf->unhashed_sticky_bit)) {
+        stbuf->ia_prot.sticky = 1;
+    }
+
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+               "Failed to set layout for subvolume %s,"
+               "gfid = %s",
+               prev->name, gfid);
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
 
-        /* This is required for handling stale linkfile deletion,
-         * or any more call which happens from this 'loc'.
-         */
-        if (!op_ret && uuid_is_null (local->gfid))
-                memcpy (local->gfid, stbuf->ia_gfid, 16);
-
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-                gf_log (this->name, GF_LOG_TRACE, "Entry %s missing on subvol"
-                        " %s", loc->path, prev->this->name);
-                if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_ON) {
-                        local->op_errno = ENOENT;
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
-                if ((conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) &&
-                    (loc->parent)) {
-                        ret = inode_ctx_get (loc->parent, this, &tmp_layout);
-                        parent_layout = (dht_layout_t *)(long)tmp_layout;
-                        if (parent_layout->search_unhashed) {
-                                local->op_errno = ENOENT;
-                                dht_lookup_everywhere (frame, this, loc);
-                                return 0;
-                        }
-                }
-        }
+unwind:
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
 
-        if (op_ret == 0) {
-                is_dir      = check_is_dir (inode, stbuf, xattr);
-                if (is_dir) {
-                        local->inode = inode_ref (inode);
-                        local->xattr = dict_ref (xattr);
+    return 0;
+
+err:
+    dht_lookup_everywhere(frame, this, loc);
+out:
+    return 0;
+}
+
+/* Code to get hashed subvol based on inode and loc
+   First it check if loc->parent and loc->path exist then it get
+   hashed subvol based on loc.
+*/
+
+static gf_boolean_t
+dht_should_lookup_everywhere(xlator_t *this, dht_conf_t *conf, loc_t *loc)
+{
+    dht_layout_t *parent_layout = NULL;
+    int ret = 0;
+    gf_boolean_t lookup_everywhere = _gf_true;
+
+    /* lookup-optimize supersedes lookup-unhashed settings.
+     * If it is set, do not process search_unhashed
+     * If lookup-optimize if enabled, lookup everywhere if:
+     *  - this is the rebalance daemon.
+     *  - loc->parent is unavailable.
+     *  - parent_layout is unavailable
+     *  - parent_layout->commit_hash != conf->vol_commit_hash
+     */
+
+    if (conf->lookup_optimize) {
+        if (!conf->defrag && loc->parent) {
+            ret = dht_inode_ctx_layout_get(loc->parent, this, &parent_layout);
+            if (!ret && parent_layout &&
+                (parent_layout->commit_hash == conf->vol_commit_hash)) {
+                lookup_everywhere = _gf_false;
+            }
+        }
+        goto out;
+    } else {
+        if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
+            if (loc->parent) {
+                ret = dht_inode_ctx_layout_get(loc->parent, this,
+                                               &parent_layout);
+                if (ret || !parent_layout ||
+                    (!parent_layout->search_unhashed)) {
+                    lookup_everywhere = _gf_false;
                 }
-        }
+            } else {
+                lookup_everywhere = _gf_false;
+            }
 
-        if (is_dir || (op_ret == -1 && op_errno == ENOTCONN)) {
-                dht_lookup_directory (frame, this, &local->loc);
-                return 0;
+            goto out;
         }
+    }
+out:
+    return lookup_everywhere;
+}
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "Lookup of %s for subvolume"
-                       " %s failed with error %s", loc->path, prev->this->name,
-                       strerror (op_errno));
+int
+dht_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+               struct iatt *postparent)
+{
+    char is_linkfile = 0;
+    char is_dir = 0;
+    xlator_t *subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    xlator_t *prev = NULL;
+    int ret = 0;
+    uint32_t vol_commit_hash = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+
+    conf = this->private;
+
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
+
+    gf_msg_debug(this->name, op_errno,
+                 "%s: fresh_lookup on %s returned with op_ret %d", loc->path,
+                 prev->name, op_ret);
+
+    if (op_ret == -1) {
+        if (ENTRY_MISSING(op_ret, op_errno)) {
+            if (1 == conf->subvolume_cnt) {
+                /* No need to lookup again */
                 goto out;
-        }
-
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+            }
 
-        if (!is_linkfile) {
-                /* non-directory and not a linkfile */
+            gf_msg_debug(this->name, 0, "Entry %s missing on subvol %s",
+                         loc->path, prev->name);
 
-                ret = dht_layout_preset (this, prev->this, inode);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not set pre-set layout for subvolume %s",
-                                prev->this->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto out;
-                }
-                goto out;
-        }
+            if (dht_should_lookup_everywhere(this, conf, loc)) {
+                local->op_errno = ENOENT;
+                dht_lookup_everywhere(frame, this, loc);
+                return 0;
+            }
 
-        subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "linkfile not having link subvolume. path=%s",
-                        loc->path);
-                dht_lookup_everywhere (frame, this, loc);
+        } else {
+            /* posix returns ENODATA if the gfid is not set but the client and
+             * server protocol layers do not send the stbuf. We need to
+             * heal this so check if this is a directory on the other subvols.
+             */
+            if ((op_errno == ENOTCONN) || (op_errno == ENODATA)) {
+                dht_lookup_directory(frame, this, &local->loc);
                 return 0;
+            }
+        }
+        gf_msg_debug(this->name, op_errno, "%s: Lookup on subvolume %s failed",
+                     loc->path, prev->name);
+        goto out;
+    }
+
+    /* Lookup succeeded - op_ret = 0 */
+
+    /* This is required for handling stale linkfile deletion,
+     * or any more call which happens from this 'loc'.
+     */
+    if (gf_uuid_is_null(local->gfid)) {
+        /*This is set from the first successful response*/
+        memcpy(local->gfid, stbuf->ia_gfid, 16);
+    }
+
+    if (!conf->vch_forced) {
+        /* Update the commit hash in conf if it is found */
+        ret = dict_get_uint32(xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
         }
+    }
 
-        STACK_WIND (frame, dht_lookup_linkfile_cbk,
-                    subvol, subvol->fops->lookup,
-                    &local->loc, local->xattr_req);
-
+    is_dir = check_is_dir(inode, stbuf, xattr);
+    if (is_dir) {
+        /* A directory is present on all subvols, send the lookup to
+         * all subvols now */
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
+        dht_lookup_directory(frame, this, &local->loc);
         return 0;
+    }
 
-out:
-        /*
-         * FIXME: postparent->ia_size and postparent->st_blocks do not have
-         * correct values. since, postparent corresponds to a directory these
-         * two members should have values equal to sum of corresponding values
-         * from each of the subvolume. See dht_iatt_merge for reference.
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+
+    if (!is_linkfile) {
+        /* non-directory and not a linkto file. This is a data file
+         * Update the layout to point to the cached subvol
          */
 
-        WIPE (postparent);
+        ret = dht_layout_preset(this, prev, inode);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+                   "%s: could not set pre-set layout for subvolume %s",
+                   loc->path, prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
+        }
+        goto out;
+    }
+
+    /* This is a linkto file. Get the value of the target subvol from the
+     * linkto xattr and lookup there to see if the file exists
+     */
+    subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "%s: No link subvol for linkto", loc->path);
+        dht_lookup_everywhere(frame, this, loc);
+        return 0;
+    }
+
+    gf_msg_debug(this->name, 0, "%s: Calling lookup on linkto target %s",
+                 loc->path, subvol->name);
+
+    STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                      subvol->fops->lookup, &local->loc, local->xattr_req);
+
+    return 0;
 
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
-                          postparent);
+out:
+    /*
+     * FIXME: postparent->ia_size and postparent->st_blocks do not have
+     * correct values. since, postparent corresponds to a directory these
+     * two members should have values equal to sum of corresponding values
+     * from each of the subvolume. See dht_iatt_merge for reference.
+     */
+
+    if (!op_ret && local && local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
 err:
-        return 0;
+    return 0;
 }
 
-/* For directories, check if acl xattrs have been requested (by the acl xlator),
- * if not, request for them. These xattrs are needed for dht dir self-heal to
- * perform proper self-healing of dirs
+/* For directories, check if acl xattrs have been requested (by the acl
+ * xlator), if not, request for them. These xattrs are needed for dht dir
+ * self-heal to perform proper self-healing of dirs
  */
-void
-dht_check_and_set_acl_xattr_req (inode_t *inode, dict_t *xattr_req)
+static void
+dht_check_and_set_acl_xattr_req(xlator_t *this, dict_t *xattr_req)
 {
-        int     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (inode);
-        GF_ASSERT (xattr_req);
+    GF_ASSERT(xattr_req);
 
-        if (inode->ia_type != IA_IFDIR)
-                return;
+    if (!dict_get(xattr_req, POSIX_ACL_ACCESS_XATTR)) {
+        ret = dict_set_int8(xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value:key = %s",
+                   POSIX_ACL_ACCESS_XATTR);
+    }
 
-        if (!dict_get (xattr_req, POSIX_ACL_ACCESS_XATTR)) {
-                ret = dict_set_int8 (xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
-                if (ret)
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "failed to set key %s",
-                                POSIX_ACL_ACCESS_XATTR);
-        }
+    if (!dict_get(xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
+        ret = dict_set_int8(xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value:key = %s",
+                   POSIX_ACL_DEFAULT_XATTR);
+    }
 
-        if (!dict_get (xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
-                ret = dict_set_int8 (xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
-                if (ret)
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "failed to set key %s",
-                                POSIX_ACL_DEFAULT_XATTR);
-        }
+    return;
+}
 
-        return;
+/* for directories, we need the following info:
+ * the layout : trusted.glusterfs.dht
+ * the mds information : trusted.glusterfs.dht.mds
+ * the acl info: See above
+ */
+static int
+dht_set_dir_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    int ret = -EINVAL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    if (!conf) {
+        goto err;
+    }
+
+    if (!xattr_req) {
+        goto err;
+    }
+
+    /* Xattr to get the layout for a directory
+     */
+    ret = dict_set_uint32(xattr_req, conf->xattr_name, 4 * 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->xattr_name, loc->path);
+        goto err;
+    }
+
+    /*Non-fatal failure */
+    ret = dict_set_uint32(xattr_req, conf->mds_xattr_key, 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->mds_xattr_key, loc->path);
+    }
+
+    dht_check_and_set_acl_xattr_req(this, xattr_req);
+    ret = 0;
+err:
+    return ret;
 }
 
-int
-dht_lookup (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, dict_t *xattr_req)
-{
-        xlator_t     *subvol = NULL;
-        xlator_t     *hashed_subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-        loc_t         new_loc = {0,};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        conf = this->private;
-        if (!conf)
-                goto err;
+/* If the hashed subvol is present, send the lookup to only that subvol first.
+ * If no hashed subvol, send a lookup to all subvols and proceed based on the
+ * responses.
+ */
+static int
+dht_do_fresh_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int call_cnt = 0;
+    int i = 0;
+
+    conf = this->private;
+    if (!conf) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Since we don't know whether this is a file or a directory,
+     * request all xattrs*/
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    /* Fuse sets a random value in gfid-req. If the gfid is missing
+     * on one or more subvols, posix will set the gfid to this value,
+     * causing GFID mismatches for directories. Remove the value fuse
+     * has sent before sending the lookup.
+     */
+    ret = dict_get_gfuuid(local->xattr_req, "gfid-req", &local->gfid_req);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "%s: No gfid-req available", loc->path);
+    } else {
+        dict_del(local->xattr_req, "gfid-req");
+    }
+    /* This should have been set in dht_lookup */
+    hashed_subvol = local->hashed_subvol;
+
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "%s: no subvolume in layout for path, "
+                     "checking on all the subvols to see if "
+                     "it is a directory",
+                     loc->path);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-        ret = dht_filter_loc_subvol_key (this, loc, &new_loc,
-                                         &hashed_subvol);
-        if (ret) {
-                loc_wipe (&local->loc);
-                ret = loc_dup (&new_loc, &local->loc);
-
-                /* we no more need 'new_loc' entries */
-                loc_wipe (&new_loc);
-
-                /* check if loc_dup() is successful */
-                if (ret == -1) {
-                        op_errno = errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "copying location failed for path=%s",
-                                loc->path);
-                        goto err;
-                }
+        /* Allocate a layout. This will be populated and saved in
+         * the dht inode_ctx on successful lookup
+         */
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
+        if (!local->layout) {
+            op_errno = ENOMEM;
+            goto err;
         }
 
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
-        } else {
-                local->xattr_req = dict_new ();
-        }
+        gf_msg_debug(this->name, 0,
+                     "%s: Found null hashed subvol. Calling lookup"
+                     " on all nodes.",
+                     loc->path);
 
-        if (uuid_is_null (loc->pargfid) && !uuid_is_null (loc->gfid) &&
-            !__is_root_gfid (loc->inode->gfid)) {
-                local->cached_subvol = NULL;
-                dht_discover (frame, this, loc);
-                return 0;
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
         }
+        return 0;
+    }
 
-        if (!hashed_subvol) {
-                hashed_subvol = dht_subvol_get_hashed (this, loc);
-                if (!hashed_subvol) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get hashed subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-        }
-        local->hashed_subvol = hashed_subvol;
+    /* if the hashed_subvol is non-null, send the lookup there first so
+     * as to see whether we have a file or a directory */
+    gf_msg_debug(this->name, 0, "%s: Calling fresh lookup on %s", loc->path,
+                 hashed_subvol->name);
 
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "revalidate without cache. path=%s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, hashed_subvol, hashed_subvol,
+                      hashed_subvol->fops->lookup, loc, local->xattr_req);
+    return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "incomplete layout failure for path=%s",
-                                loc->path);
+static int
+dht_do_revalidate(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+    int gen = 0;
+
+    conf = this->private;
+    if (!conf) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0,
+                     "path = %s. No layout found in the inode ctx.", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Generation number has changed. This layout may be stale. */
+    if (layout->gen && (layout->gen < conf->gen)) {
+        gen = layout->gen;
+        dht_layout_unref(this, local->layout);
+        local->layout = NULL;
+        local->cached_subvol = NULL;
+
+        gf_msg_debug(this->name, 0,
+                     "path = %s. In memory layout may be stale."
+                     "(layout->gen (%d) is less than "
+                     "conf->gen (%d)). Calling fresh lookup.",
+                     loc->path, gen, conf->gen);
+
+        dht_do_fresh_lookup(frame, this, loc);
+        return 0;
+    }
+
+    local->inode = inode_ref(loc->inode);
+
+    /* Since we don't know whether this has changed,
+     * request all xattrs*/
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    if (IA_ISDIR(local->inode->ia_type)) {
+        ret = dht_inode_ctx_mdsvol_get(local->inode, this, &mds_subvol);
+        if (ret || !mds_subvol) {
+            gf_msg_debug(this->name, 0, "path = %s. No mds subvol in inode ctx",
+                         local->loc.path);
+        }
+        local->mds_subvol = mds_subvol;
+        local->call_cnt = conf->subvolume_cnt;
 
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-                        local->cached_subvol = NULL;
-                        goto do_fresh_lookup;
-                }
+        /* local->call_cnt will change as responses are processed. Always use a
+         * local copy to loop through the STACK_WIND calls
+         */
 
-                local->inode = inode_ref (loc->inode);
+        call_cnt = local->call_cnt;
 
-                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
-                 *       revalidates directly go to the cached-subvolume.
-                 */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
-
-                if (IA_ISDIR (local->inode->ia_type)) {
-                        local->call_cnt = call_cnt = conf->subvolume_cnt;
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND (frame, dht_revalidate_cbk,
-                                            conf->subvolumes[i],
-                                            conf->subvolumes[i]->fops->lookup,
-                                            loc, local->xattr_req);
-                        }
-                        return 0;
-                }
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, loc,
+                              local->xattr_req);
+        }
+        return 0;
+    }
 
-                call_cnt = local->call_cnt = layout->cnt;
+    /* If not a dir, this should be 1 */
+    local->call_cnt = layout->cnt;
+    call_cnt = local->call_cnt;
 
-                /* need it for self-healing linkfiles which is
-                   'in-migration' state */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       GLUSTERFS_OPEN_FD_COUNT, 4);
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
 
-                /* need it for dir self-heal */
-                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+        gf_msg_debug(this->name, 0,
+                     "path = %s. Calling "
+                     "revalidate lookup on %s",
+                     loc->path, subvol->name);
 
-		for (i = 0; i < call_cnt; i++) {
-			subvol = layout->list[i].xlator;
+        STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
+    return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-			STACK_WIND (frame, dht_revalidate_cbk,
-				    subvol, subvol->fops->lookup,
-				    &local->loc, local->xattr_req);
+/* Depending on the input, decide if this is a:
+ * fresh-lookup: loc->name is provided but no dht inode ctx
+ * revalidation: loc->name is provided, dht inode ctx is present
+ * discover: gfid based nameless lookup.
+ */
 
-		}
-        } else {
-        do_fresh_lookup:
-                /* TODO: remove the hard-coding */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
-
-                ret = dict_set_uint32 (local->xattr_req,
-                                       DHT_LINKFILE_KEY, 256);
-
-                /* need it for self-healing linkfiles which is
-                   'in-migration' state */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       GLUSTERFS_OPEN_FD_COUNT, 4);
-
-                /* need it for dir self-heal */
-                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
-
-                if (!hashed_subvol) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no subvolume in layout for path=%s, "
-                                "checking on all the subvols to see if "
-                                "it is a directory", loc->path);
-                        call_cnt        = conf->subvolume_cnt;
-                        local->call_cnt = call_cnt;
-
-                        local->layout = dht_layout_new (this,
-                                                        conf->subvolume_cnt);
-                        if (!local->layout) {
-                                op_errno = ENOMEM;
-                                goto err;
-                        }
+int
+dht_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    loc_t new_loc = {
+        0,
+    };
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    conf = this->private;
+    if (!conf)
+        goto err;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_filter_loc_subvol_key(this, loc, &new_loc, &hashed_subvol);
+    if (ret) {
+        loc_wipe(&local->loc);
+        ret = loc_dup(&new_loc, &local->loc);
+
+        /* we no longer need 'new_loc' entries */
+        loc_wipe(&new_loc);
+
+        /* check if loc_dup() is successful */
+        if (ret == -1) {
+            op_errno = errno;
+            gf_msg_debug(this->name, errno,
+                         "copying location failed for path=%s", loc->path);
+            goto err;
+        }
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    /* Nameless lookup */
+
+    /* This is usually sent by NFS. Lookups are done based on the gfid and
+     * no name information is available. Without the name, dht cannot calculate
+     * the hash and has to send a lookup to all subvols.
+     */
+    if (gf_uuid_is_null(loc->pargfid) && !gf_uuid_is_null(loc->gfid) &&
+        !__is_root_gfid(loc->inode->gfid)) {
+        local->cached_subvol = NULL;
+        dht_do_discover(frame, this, loc);
+        return 0;
+    }
 
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND (frame, dht_lookup_dir_cbk,
-                                            conf->subvolumes[i],
-                                            conf->subvolumes[i]->fops->lookup,
-                                            &local->loc, local->xattr_req);
-                        }
-                        return 0;
-                }
+    if (loc_is_root(loc)) {
+        /* Request the DHT commit hash xattr (trusted.glusterfs.dht.commithash)
+         * set on the brick root.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->commithash_xattr_name,
+                              sizeof(uint32_t));
+    }
 
-                STACK_WIND (frame, dht_lookup_cbk,
-                            hashed_subvol, hashed_subvol->fops->lookup,
-                            loc, local->xattr_req);
-        }
+    if (!hashed_subvol)
+        hashed_subvol = dht_subvol_get_hashed(this, loc);
+    local->hashed_subvol = hashed_subvol;
 
+    if (is_revalidate(loc)) {
+        /* The entry has been looked up before and has a dht inode_ctx
+         */
+        dht_do_revalidate(frame, this, loc);
         return 0;
+    } else {
+        /* Entry has not been looked up before
+         */
+        dht_do_fresh_lookup(frame, this, loc);
+        return 0;
+    }
 
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
-
-int
-dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+static int
+dht_unlink_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev  = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
 
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_ret   = -1;
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
+    local = frame->local;
+    prev = cookie;
 
-                local->op_ret = 0;
+    LOCK(&frame->lock);
+    {
+        if ((op_ret == -1) &&
+            !((op_errno == ENOENT) || (op_errno == ENOTCONN))) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno,
+                         "Unlink link: subvolume %s returned -1", prev->name);
+            goto post_unlock;
+        }
 
-                local->postparent = *postparent;
-                local->preparent = *preparent;
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, xdata);
+
+    return 0;
+}
 
-                WIPE (&local->postparent);
-                WIPE (&local->preparent);
+static int
+dht_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *preparent, struct iatt *postparent,
+               dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *hashed_subvol = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno != ENOENT) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+            } else {
+                local->op_ret = 0;
+            }
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno,
+                         "Unlink: subvolume %s returned -1", prev->name);
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
+        local->op_ret = 0;
 
-        return 0;
-}
+        local->postparent = *postparent;
+        local->preparent = *preparent;
 
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->preparent, 0);
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
+        }
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!local->op_ret) {
+        hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+        if (hashed_subvol && hashed_subvol != local->cached_subvol) {
+            /*
+             * If hashed and cached are different, then we need
+             * to unlink linkfile from hashed subvol if data
+             * file is deleted successfully
+             */
+            STACK_WIND_COOKIE(frame, dht_unlink_linkfile_cbk, hashed_subvol,
+                              hashed_subvol, hashed_subvol->fops->unlink,
+                              &local->loc, local->flags, xdata);
+            return 0;
+        }
+    }
+
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, xdata);
+
+    return 0;
+}
 
-int
-dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
+static int
+dht_common_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
+    DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        xlator_t *cached_subvol = NULL;
+static int
+dht_fix_layout_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
 
+    if (op_ret == 0) {
+        /* update the layout in the inode ctx */
         local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
+        layout = local->selfheal.layout;
 
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
+        dht_layout_set(this, local->loc.inode, layout);
+    }
 
-        if (op_ret == -1)
-                goto err;
+    DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
-        if (!cached_subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s",
-                        local->loc.path);
-                local->op_errno = EINVAL;
-                goto err;
+static int
+dht_err_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+            int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
         }
 
-        STACK_WIND (frame, dht_unlink_cbk,
-                    cached_subvol, cached_subvol->fops->unlink,
-                    &local->loc, local->flags, NULL);
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if ((local->fop == GF_FOP_SETXATTR) ||
+            (local->fop == GF_FOP_FSETXATTR)) {
+            DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                             NULL);
+            /* 'local' itself may not be valid after this */
+            goto out;
+        }
+        if ((local->fop == GF_FOP_REMOVEXATTR) ||
+            (local->fop == GF_FOP_FREMOVEXATTR)) {
+            DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                             NULL);
+        }
+    }
 
-        return 0;
+out:
+    return 0;
+}
 
-err:
-        DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
-                          NULL, NULL, NULL);
-        return 0;
+/* Set the value[] of key into dict after convert from
+   host byte order to network byte order
+*/
+int32_t
+dht_dict_set_array(dict_t *dict, char *key, int32_t value[], int32_t size)
+{
+    int ret = -1;
+    int32_t *ptr = NULL;
+    int32_t vindex;
+
+    if (value == NULL) {
+        return -EINVAL;
+    }
+
+    ptr = GF_MALLOC(sizeof(int32_t) * size, gf_common_mt_char);
+    if (ptr == NULL) {
+        return -ENOMEM;
+    }
+    for (vindex = 0; vindex < size; vindex++) {
+        ptr[vindex] = hton32(value[vindex]);
+    }
+    ret = dict_set_bin(dict, key, ptr, sizeof(int32_t) * size);
+    if (ret)
+        GF_FREE(ptr);
+    return ret;
 }
 
 static int
-dht_ufo_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, dict_t *xdata)
+dht_common_mds_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    call_frame_t *prev = cookie;
 
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
+    local = frame->local;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                  local->op_errno, NULL);
-        }
+    if (op_ret)
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
 
-        return 0;
-}
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, 0, op_errno, local->xdata);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
 
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, 0, op_errno, local->xdata);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
 
-int
-dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int op_ret, int op_errno, dict_t *xdata)
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, 0, op_errno, NULL);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, 0, op_errno, NULL);
+    }
+
+out:
+    return 0;
+}
+
+/* Code to wind a xattrop call to add 1 on current mds internal xattr
+   value
+*/
+static int
+dht_setxattr_non_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int ret = 0;
+    dict_t *xattrop = NULL;
+    int32_t addone[1] = {1};
+    call_frame_t *prev = NULL;
+    dht_conf_t *conf = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret && !local->op_ret) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->this->name);
+            goto post_unlock;
+        }
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        if (!local->op_ret) {
+            xattrop = dict_new();
+            if (!xattrop) {
+                gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, 0,
+                       "dictionary creation failed");
+                ret = -1;
+                goto out;
+            }
+            ret = dht_dict_set_array(xattrop, conf->mds_xattr_key, addone, 1);
+            if (ret != 0) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                       "dictionary set array failed ");
+                ret = -1;
+                goto out;
+            }
+            if ((local->fop == GF_FOP_SETXATTR) ||
+                (local->fop == GF_FOP_REMOVEXATTR)) {
+                STACK_WIND(frame, dht_common_mds_xattrop_cbk, local->mds_subvol,
+                           local->mds_subvol->fops->xattrop, &local->loc,
+                           GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+            } else {
+                STACK_WIND(frame, dht_common_mds_xattrop_cbk, local->mds_subvol,
+                           local->mds_subvol->fops->fxattrop, local->fd,
+                           GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+            }
+        } else {
+            if (local->fop == GF_FOP_SETXATTR) {
+                DHT_STACK_UNWIND(setxattr, frame, 0, 0, local->xdata);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
+
+            if (local->fop == GF_FOP_FSETXATTR) {
+                DHT_STACK_UNWIND(fsetxattr, frame, 0, 0, local->xdata);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
+
+            if (local->fop == GF_FOP_REMOVEXATTR) {
+                DHT_STACK_UNWIND(removexattr, frame, 0, 0, NULL);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
+
+            if (local->fop == GF_FOP_FREMOVEXATTR) {
+                DHT_STACK_UNWIND(fremovexattr, frame, 0, 0, NULL);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
+        }
+    }
+out:
+    if (ret) {
+        if (local->fop == GF_FOP_SETXATTR) {
+            DHT_STACK_UNWIND(setxattr, frame, 0, 0, local->xdata);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
+        }
 
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
+        if (local->fop == GF_FOP_FSETXATTR) {
+            DHT_STACK_UNWIND(fsetxattr, frame, 0, 0, local->xdata);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
+        }
 
-                local->op_ret = 0;
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+            DHT_STACK_UNWIND(removexattr, frame, 0, 0, NULL);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                  local->op_errno, NULL);
+        if (local->fop == GF_FOP_FREMOVEXATTR) {
+            DHT_STACK_UNWIND(fremovexattr, frame, 0, 0, NULL);
         }
+    }
+just_return:
+    if (xattrop)
+        dict_unref(xattrop);
+    return 0;
+}
 
-        return 0;
+static int
+dht_setxattr_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    call_frame_t *prev = NULL;
+    xlator_t *mds_subvol = NULL;
+    int i = 0;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+    mds_subvol = local->mds_subvol;
+
+    if (op_ret == -1) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
+        goto out;
+    }
+
+    local->op_ret = 0;
+    local->call_cnt = conf->subvolume_cnt - 1;
+    local->xdata = dict_ref(xdata);
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol && (mds_subvol == conf->subvolumes[i]))
+            continue;
+        if (local->fop == GF_FOP_SETXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->setxattr, &local->loc,
+                       local->xattr, local->flags, local->xattr_req);
+        }
+
+        if (local->fop == GF_FOP_FSETXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->fsetxattr, local->fd,
+                       local->xattr, local->flags, local->xattr_req);
+        }
+
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->removexattr, &local->loc,
+                       local->key, local->xattr_req);
+        }
+
+        if (local->fop == GF_FOP_FREMOVEXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->fremovexattr, local->fd,
+                       local->key, local->xattr_req);
+        }
+    }
+
+    return 0;
+out:
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+    }
+
+just_return:
+    return 0;
+}
+
+static int
+dht_xattrop_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    call_frame_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        local->op_ret = op_ret;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
+        goto out;
+    }
+
+    if (local->fop == GF_FOP_SETXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->setxattr, &local->loc, local->xattr,
+                   local->flags, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->fsetxattr, local->fd, local->xattr,
+                   local->flags, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->removexattr, &local->loc,
+                   local->key, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->fremovexattr, local->fd, local->key,
+                   local->xattr_req);
+    }
+
+    return 0;
+out:
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+    }
+
+just_return:
+    return 0;
 }
 
 static void
-fill_layout_info (dht_layout_t *layout, char *buf)
+fill_layout_info(dht_layout_t *layout, char *buf)
 {
-        int i = 0;
-        char tmp_buf[128] = {0,};
+    int i = 0;
+    char tmp_buf[128] = {
+        0,
+    };
+
+    for (i = 0; i < layout->cnt; i++) {
+        snprintf(tmp_buf, sizeof(tmp_buf), "(%s %u %u)",
+                 layout->list[i].xlator->name, layout->list[i].start,
+                 layout->list[i].stop);
+        if (i)
+            strcat(buf, " ");
+        strcat(buf, tmp_buf);
+    }
+}
 
-        for (i = 0; i < layout->cnt; i++) {
-                snprintf (tmp_buf, 128, "(%s %u %u)",
-                          layout->list[i].xlator->name,
-                          layout->list[i].start,
-                          layout->list[i].stop);
-                if (i)
-                        strcat (buf, " ");
-                strcat (buf, tmp_buf);
-        }
+static void
+dht_fill_pathinfo_xattr(xlator_t *this, dht_local_t *local, char *xattr_buf,
+                        int32_t alloc_len, int flag, char *layout_buf)
+{
+    if (flag) {
+        if (local->xattr_val) {
+            snprintf(xattr_buf, alloc_len,
+                     "((<" DHT_PATHINFO_HEADER "%s> %s) (%s-layout %s))",
+                     this->name, local->xattr_val, this->name, layout_buf);
+        } else {
+            snprintf(xattr_buf, alloc_len, "(%s-layout %s)", this->name,
+                     layout_buf);
+        }
+    } else if (local->xattr_val) {
+        snprintf(xattr_buf, alloc_len, "(<" DHT_PATHINFO_HEADER "%s> %s)",
+                 this->name, local->xattr_val);
+    } else {
+        xattr_buf[0] = '\0';
+    }
 }
 
-void
-dht_fill_pathinfo_xattr (xlator_t *this, dht_local_t *local,
-                         char *xattr_buf, int32_t alloc_len,
-                         int flag, char *layout_buf)
-{
-        if (flag && local->xattr_val)
-                snprintf (xattr_buf, alloc_len,
-                          "((<"DHT_PATHINFO_HEADER"%s> %s) (%s-layout %s))",
-                          this->name, local->xattr_val, this->name,
-                          layout_buf);
-        else if (local->xattr_val)
-                snprintf (xattr_buf, alloc_len,
-                          "(<"DHT_PATHINFO_HEADER"%s> %s)",
-                          this->name, local->xattr_val);
-        else if (flag)
-                snprintf (xattr_buf, alloc_len, "(%s-layout %s)",
-                          this->name, layout_buf);
+static int
+dht_vgetxattr_alloc_and_fill(dht_local_t *local, dict_t *xattr, xlator_t *this,
+                             int op_errno)
+{
+    int ret = -1;
+    char *value = NULL;
+
+    ret = dict_get_str(xattr, local->xsel, &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_GET_XATTR_FAILED,
+               "Subvolume %s returned -1", this->name);
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto out;
+    }
+
+    local->alloc_len += strlen(value);
+
+    if (!local->xattr_val) {
+        local->alloc_len += (SLEN(DHT_PATHINFO_HEADER) + 10);
+        local->xattr_val = GF_MALLOC(local->alloc_len, gf_common_mt_char);
+        if (!local->xattr_val) {
+            ret = -1;
+            goto out;
+        }
+        local->xattr_val[0] = '\0';
+    }
+
+    int plen = strlen(local->xattr_val);
+    if (plen) {
+        /* extra byte(s) for \0 to be safe */
+        local->alloc_len += (plen + 2);
+        local->xattr_val = GF_REALLOC(local->xattr_val, local->alloc_len);
+        if (!local->xattr_val) {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    (void)strcat(local->xattr_val, value);
+    (void)strcat(local->xattr_val, " ");
+    local->op_ret = 0;
+
+    ret = 0;
+
+out:
+    return ret;
 }
 
-int
-dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
-{
-        dht_local_t *local         = NULL;
-        int          ret           = 0;
-        int          flag          = 0;
-        int          this_call_cnt = 0;
-        char        *value_got     = NULL;
-        char  layout_buf[8192]     = {0,};
-        char        *xattr_buf     = NULL;
-        dict_t      *dict          = NULL;
-        int32_t      alloc_len     = 0;
-        int32_t      plen          = 0;
-        call_frame_t *prev         = NULL;
+static int
+dht_vgetxattr_fill_and_set(dht_local_t *local, dict_t **dict, xlator_t *this,
+                           gf_boolean_t flag)
+{
+    int ret = -1;
+    char *xattr_buf = NULL;
+    char layout_buf[8192] = {
+        0,
+    };
+
+    if (flag)
+        fill_layout_info(local->layout, layout_buf);
+
+    *dict = dict_new();
+    if (!*dict)
+        goto out;
+
+    local->xattr_val[strlen(local->xattr_val) - 1] = '\0';
+
+    /* we would need max this many bytes to create xattr string
+     * extra 40 bytes is just an estimated amount of additional
+     * space required as we include translator name and some
+     * spaces, brackets etc. when forming the pathinfo string.
+     *
+     * For node-uuid we just don't have all the pretty formatting,
+     * but since this is a generic routine for pathinfo & node-uuid
+     * we don't have conditional space allocation and try to be
+     * generic
+     */
+    local->alloc_len += (2 * strlen(this->name)) + strlen(layout_buf) + 40;
+    xattr_buf = GF_MALLOC(local->alloc_len, gf_common_mt_char);
+    if (!xattr_buf)
+        goto out;
+
+    if (XATTR_IS_PATHINFO(local->xsel)) {
+        (void)dht_fill_pathinfo_xattr(this, local, xattr_buf, local->alloc_len,
+                                      flag, layout_buf);
+    } else if ((XATTR_IS_NODE_UUID(local->xsel)) ||
+               (XATTR_IS_NODE_UUID_LIST(local->xsel))) {
+        (void)snprintf(xattr_buf, local->alloc_len, "%s", local->xattr_val);
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GET_XATTR_FAILED,
+               "Unknown local->xsel (%s)", local->xsel);
+        GF_FREE(xattr_buf);
+        goto out;
+    }
+
+    ret = dict_set_dynstr(*dict, local->xsel, xattr_buf);
+    if (ret)
+        GF_FREE(xattr_buf);
+    GF_FREE(local->xattr_val);
 
-        local = frame->local;
-        prev = cookie;
-
-        if (op_ret >= 0) {
-                ret = dict_get_str (xattr, local->xsel, &value_got);
-                if (!ret) {
-                        alloc_len = strlen (value_got);
-
-                        /**
-                         * allocate the buffer:- we allocate 10 bytes extra in
-                         * case we need to append ' Link: ' in the buffer for
-                         * another STACK_WIND
-                         */
-                        if (!local->xattr_val) {
-                                alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
-                                local->xattr_val =
-                                        GF_CALLOC (alloc_len,
-                                                   sizeof (char),
-                                                   gf_common_mt_char);
-                        }
+out:
+    return ret;
+}
 
-                        if (local->xattr_val) {
-                                plen = strlen (local->xattr_val);
-                                if (plen) {
-                                        /* extra byte(s) for \0 to be safe */
-                                        alloc_len += (plen + 2);
-                                        local->xattr_val =
-                                                GF_REALLOC (local->xattr_val,
-                                                            alloc_len);
-                                        if (!local->xattr_val)
-                                                goto out;
-                                }
-
-                                strcat (local->xattr_val, value_got);
-                        }
-                        local->op_ret = 0;
-                }
-        } else {
+static int
+dht_find_local_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+    int ret = 0;
+    char *uuid_str = NULL;
+    char *uuid_list = NULL;
+    char *next_uuid_str = NULL;
+    char *saveptr = NULL;
+    uuid_t node_uuid = {
+        0,
+    };
+    char *uuid_list_copy = NULL;
+    int count = 0;
+    int i = 0;
+    int index = 0;
+    int found = 0;
+    nodeuuid_info_t *tmp_ptr = NULL;
+
+    VALIDATE_OR_GOTO(frame, out);
+    VALIDATE_OR_GOTO(frame->local, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    VALIDATE_OR_GOTO(conf->defrag, out);
+
+    gf_msg_debug(this->name, 0, "subvol %s returned", prev->name);
+
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            if (op_errno == ENODATA)
+                gf_msg_debug(this->name, 0, "failed to get node-uuid");
+            else
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_GET_XATTR_FAILED, "failed to get node-uuid");
+            goto post_unlock;
+        }
+
+        ret = dict_get_str(xattr, local->xsel, &uuid_list);
+
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_GET_FAILED,
+                   "Failed to get %s", local->xsel);
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
+            goto unlock;
+        }
+
+        /* As DHT will not know details of its child xlators
+         * we need to parse this twice to get the count first
+         * and allocate memory later.
+         */
+        count = 0;
+        index = conf->local_subvols_cnt;
+
+        uuid_list_copy = gf_strdup(uuid_list);
+        if (!uuid_list_copy)
+            goto unlock;
+
+        for (uuid_str = strtok_r(uuid_list, " ", &saveptr); uuid_str;
+             uuid_str = next_uuid_str) {
+            next_uuid_str = strtok_r(NULL, " ", &saveptr);
+            if (gf_uuid_parse(uuid_str, node_uuid)) {
                 local->op_ret = -1;
-                local->op_errno = op_errno;
-                gf_log (this->name, GF_LOG_ERROR, "Subvolume %s returned -1 "
-                        "(%s)", prev->this->name, strerror (op_errno));
+                local->op_errno = EINVAL;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UUID_PARSE_ERROR,
+                       "Failed to parse uuid for %s", prev->name);
+                goto post_unlock;
+            }
+
+            count++;
+            if (gf_uuid_compare(node_uuid, conf->defrag->node_uuid)) {
+                gf_msg_debug(this->name, 0,
+                             "subvol %s does not"
+                             "belong to this node",
+                             prev->name);
+            } else {
+                /* handle multiple bricks of the same replica
+                 * on the same node */
+                if (found)
+                    continue;
+                conf->local_subvols[(conf->local_subvols_cnt)++] = prev;
+                found = 1;
+                gf_msg_debug(this->name, 0,
+                             "subvol %s belongs to"
+                             " this node",
+                             prev->name);
+            }
+        }
+
+        if (!found) {
+            local->op_ret = 0;
+            goto unlock;
+        }
+
+        conf->local_nodeuuids[index].count = count;
+        conf->local_nodeuuids[index].elements = GF_CALLOC(
+            count, sizeof(nodeuuid_info_t), 1);
+
+        /* The node-uuids are guaranteed to be returned in the same
+         * order as the bricks
+         * A null node-uuid is returned for a brick that is down.
+         */
+
+        saveptr = NULL;
+        i = 0;
+
+        for (uuid_str = strtok_r(uuid_list_copy, " ", &saveptr); uuid_str;
+             uuid_str = next_uuid_str) {
+            next_uuid_str = strtok_r(NULL, " ", &saveptr);
+            tmp_ptr = &(conf->local_nodeuuids[index].elements[i]);
+            gf_uuid_parse(uuid_str, tmp_ptr->uuid);
+
+            if (!gf_uuid_compare(tmp_ptr->uuid, conf->defrag->node_uuid)) {
+                tmp_ptr->info = REBAL_NODEUUID_MINE;
+            }
+            i++;
+            tmp_ptr = NULL;
         }
+    }
 
- out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
+    local->op_ret = 0;
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!is_last_call(this_call_cnt))
+        goto out;
 
-                if (local->op_ret == -1) {
-                        goto unwind;
-                }
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-                if (local->layout->cnt > 1) {
-                        /* Set it for directory */
-                        fill_layout_info (local->layout, layout_buf);
-                        flag = 1;
-                }
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, xattr, xdata);
+    goto out;
+
+unwind:
 
-                dict = dict_new ();
+    GF_FREE(conf->local_nodeuuids[index].elements);
+    conf->local_nodeuuids[index].elements = NULL;
 
-                /* we would need max this many bytes to create xattr string */
-                alloc_len += (2 * strlen (this->name))
-                          + strlen (layout_buf)
-                          + 40;
-                xattr_buf = GF_CALLOC (alloc_len,
-                                       sizeof (char), gf_common_mt_char);
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, xdata);
+out:
+    GF_FREE(uuid_list_copy);
+    return 0;
+}
 
-                if (XATTR_IS_PATHINFO (local->xsel)) {
-                        (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
-                                                        alloc_len, flag,
-                                                        layout_buf);
-                } else if (XATTR_IS_NODE_UUID (local->xsel)) {
-                        (void) snprintf (xattr_buf, alloc_len, "%s",
-                                         local->xattr_val);
-                } else
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unknown local->xsel (%s)", local->xsel);
+static int
+dht_vgetxattr_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    int ret = 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    dict_t *dict = NULL;
 
-                ret = dict_set_dynstr (dict, local->xsel, xattr_buf);
+    VALIDATE_OR_GOTO(frame, out);
+    VALIDATE_OR_GOTO(frame->local, out);
 
-                GF_FREE (local->xattr_val);
+    local = frame->local;
 
-                DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
-                                  xdata);
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
+        if (op_ret < 0) {
+            if (op_errno != ENOTCONN) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_GET_XATTR_FAILED, "getxattr err for dir");
+                goto post_unlock;
+            }
 
-                if (dict)
-                        dict_unref (dict);
+            goto unlock;
+        }
 
-                return 0;
+        ret = dht_vgetxattr_alloc_and_fill(local, xattr, this, op_errno);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_DICT_SET_FAILED,
+                   "alloc or fill failure");
+            goto post_unlock;
         }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!is_last_call(this_call_cnt))
+        goto out;
+
+    /* -- last call: do patch ups -- */
+
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
+
+    ret = dht_vgetxattr_fill_and_set(local, &dict, this, _gf_true);
+    if (ret)
+        goto unwind;
+
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, dict, xdata);
+    goto cleanup;
 
 unwind:
-        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, NULL);
+cleanup:
+    if (dict)
+        dict_unref(dict);
+out:
+    return 0;
 }
 
-int
-dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr,
-                           dict_t *xdata)
+static int
+dht_vgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int   ret   = 0;
-        char *value = NULL;
+    dht_local_t *local = NULL;
+    int ret = 0;
+    dict_t *dict = NULL;
+    xlator_t *prev = NULL;
+    gf_boolean_t flag = _gf_true;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_GET_XATTR_FAILED,
+               "vgetxattr: Subvolume %s returned -1", prev->name);
+        goto unwind;
+    }
+
+    ret = dht_vgetxattr_alloc_and_fill(local, xattr, this, op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+               "Allocation or fill failure");
+        goto unwind;
+    }
+
+    flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
+
+    ret = dht_vgetxattr_fill_and_set(local, &dict, this, flag);
+    if (ret)
+        goto unwind;
+
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, dict, xdata);
+    goto cleanup;
 
-        if (op_ret != -1) {
-                ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value);
-                if (!ret) {
-                        ret = dict_set_str (xattr, GF_XATTR_LINKINFO_KEY, value);
-                        if (!ret)
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "failed to set linkinfo");
-                }
+unwind:
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, NULL);
+cleanup:
+    if (dict)
+        dict_unref(dict);
+
+    return 0;
+}
+
+static int
+dht_linkinfo_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
+{
+    int ret = 0;
+    char *value = NULL;
+
+    if (op_ret != -1) {
+        ret = dict_get_str(xattr, GF_XATTR_PATHINFO_KEY, &value);
+        if (!ret) {
+            ret = dict_set_str(xattr, GF_XATTR_LINKINFO_KEY, value);
+            if (!ret)
+                gf_msg_trace(this->name, 0, "failed to set linkinfo");
         }
+    }
 
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
 
-        return 0;
+    return 0;
+}
+
+static int
+dht_mds_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(frame->local, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+    local = frame->local;
+
+    if (!xattr || (op_ret == -1)) {
+        local->op_ret = op_ret;
+        goto out;
+    }
+    dict_del(xattr, conf->xattr_name);
+    local->op_ret = 0;
+
+    if (!local->xattr) {
+        local->xattr = dict_copy_with_ref(xattr, NULL);
+    }
+
+out:
+    DHT_STACK_UNWIND(getxattr, frame, local->op_ret, op_errno, local->xattr,
+                     xdata);
+    return 0;
+err:
+    DHT_STACK_UNWIND(getxattr, frame, -1, EINVAL, NULL, NULL);
+    return 0;
 }
 
 int
-dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(frame->local, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-        local = frame->local;
+    conf = this->private;
+    local = frame->local;
 
-        this_call_cnt = dht_frame_return (frame);
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto err;
+        return 0;
+    }
 
-        if (!xattr || (op_ret == -1))
-                goto out;
+    LOCK(&frame->lock);
+    {
+        if (!xattr || (op_ret == -1)) {
+            local->op_ret = op_ret;
+            goto unlock;
+        }
+
+        dict_del(xattr, conf->xattr_name);
+        dict_del(xattr, conf->mds_xattr_key);
 
-        if (dict_get (xattr, "trusted.glusterfs.dht")) {
-                dict_del (xattr, "trusted.glusterfs.dht");
+        dict_del(xattr, conf->commithash_xattr_name);
+
+        if (frame->root->pid >= 0) {
+            GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+            GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
         }
+
         local->op_ret = 0;
 
         if (!local->xattr) {
-                local->xattr = dict_copy_with_ref (xattr, NULL);
+            local->xattr = dict_copy_with_ref(xattr, NULL);
         } else {
-                /* first aggregate everything into xattr and then copy into
-                 * local->xattr. This is required as we want to have
-                 * 'local->xattr' as the proper final dictionary passed above
-                 * distribute xlator.
-                 */
-                dht_aggregate_xattr (xattr, local->xattr);
-                local->xattr = dict_copy (xattr, local->xattr);
+            dht_aggregate_xattr(local->xattr, xattr);
         }
-out:
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
-                                  local->xattr, NULL);
+
+        if (!local->xdata) {
+            local->xdata = dict_ref(xdata);
+        } else if ((local->inode && IA_ISDIR(local->inode->ia_type)) ||
+                   (local->fd && IA_ISDIR(local->fd->inode->ia_type))) {
+            dht_aggregate_xattr(local->xdata, xdata);
         }
-        return 0;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /* If we have a valid xattr received from any one of the
+         * subvolume, let's return it */
+        if (local->xattr) {
+            local->op_ret = 0;
+        }
+
+        DHT_STACK_UNWIND(getxattr, frame, local->op_ret, op_errno, local->xattr,
+                         local->xdata);
+    }
+    return 0;
+err:
+    DHT_STACK_UNWIND(getxattr, frame, -1, EINVAL, NULL, NULL);
+    return 0;
 }
 
-int32_t
-dht_getxattr_unwind (call_frame_t *frame,
-                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+static int32_t
+dht_getxattr_unwind(call_frame_t *frame, int op_ret, int op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
+static int
+dht_getxattr_get_real_filename_cbk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int op_ret, int op_errno,
+                                   dict_t *xattr, dict_t *xdata)
+{
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
 
-int
-dht_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *key, dict_t *xdata)
-{
-        xlator_t     *subvol        = NULL;
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
-        dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        dht_layout_t *layout        = NULL;
-        xlator_t     **sub_volumes  = NULL;
-        int           op_errno      = -1;
-        int           i             = 0;
-        int           cnt           = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf   = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_GETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
+    local = frame->local;
 
-                goto err;
+    LOCK(&frame->lock);
+    {
+        if (local->op_errno == EOPNOTSUPP) {
+            /* Nothing to do here, we have already found
+             * a subvol which does not have the get_real_filename
+             * optimization. If condition is for simple logic.
+             */
+            goto unlock;
         }
 
-        layout = local->layout;
-        if (!layout) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "layout is NULL");
-                op_errno = ENOENT;
-                goto err;
-        }
+        if (op_ret == -1) {
+            if (op_errno == EOPNOTSUPP) {
+                /* This subvol does not have the optimization.
+                 * Better let the user know we don't support it.
+                 * Remove previous results if any.
+                 */
 
-        if (key) {
-                local->key = gf_strdup (key);
-                if (!local->key) {
-                        op_errno = ENOMEM;
-                        goto err;
+                if (local->xattr) {
+                    dict_unref(local->xattr);
+                    local->xattr = NULL;
                 }
-        }
 
-        if (key && ((strcmp (key, GF_XATTR_PATHINFO_KEY) == 0)
-                    || strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)) {
-                cached_subvol = local->cached_subvol;
-                (void) strncpy (local->xsel, key, 256);
+                if (local->xattr_req) {
+                    dict_unref(local->xattr_req);
+                    local->xattr_req = NULL;
+                }
 
-                local->call_cnt = 1;
-                STACK_WIND (frame, dht_vgetxattr_cbk, cached_subvol,
-                            cached_subvol->fops->getxattr, loc, key, NULL);
+                local->op_ret = op_ret;
+                local->op_errno = op_errno;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                       DHT_MSG_UPGRADE_BRICKS,
+                       "At least "
+                       "one of the bricks does not support "
+                       "this operation. Please upgrade all "
+                       "bricks.");
+                goto post_unlock;
+            }
+
+            if (op_errno == ENOATTR) {
+                /* Do nothing, our defaults are set to this.
+                 */
+                goto unlock;
+            }
+
+            /* This is a place holder for every other error
+             * case. I am not sure of how to interpret
+             * ENOTCONN etc. As of now, choosing to ignore
+             * down subvol and return a good result(if any)
+             * from other subvol.
+             */
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   DHT_MSG_GET_XATTR_FAILED, "Failed to get real filename.");
+            goto post_unlock;
+        }
+
+        /* This subvol has the required file.
+         * There could be other subvols which have returned
+         * success already, choosing to return the latest good
+         * result.
+         */
+        if (local->xattr)
+            dict_unref(local->xattr);
+        local->xattr = dict_ref(xattr);
+
+        if (local->xattr_req) {
+            dict_unref(local->xattr_req);
+            local->xattr_req = NULL;
+        }
+        if (xdata)
+            local->xattr_req = dict_ref(xdata);
+
+        local->op_ret = op_ret;
+        local->op_errno = 0;
+        UNLOCK(&frame->lock);
+        gf_msg_debug(this->name, 0, "Found a matching file.");
+        goto post_unlock;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->xattr, local->xattr_req);
+    }
+
+    return 0;
+}
 
-                return 0;
-        }
-        if (key && (strcmp (key, GF_XATTR_LINKINFO_KEY) == 0)) {
-                hashed_subvol = dht_subvol_get_hashed (this, loc);
-                if (!hashed_subvol) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get"
-                                "hashed subvol for %s", loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+static int
+dht_getxattr_get_real_filename(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               const char *key, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_layout_t *layout = NULL;
+    int cnt = 0;
+    xlator_t *subvol = NULL;
 
-                cached_subvol = dht_subvol_get_cached (this, loc->inode);
-                if (!cached_subvol) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get"
-                                "cached subvol for %s", loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    local = frame->local;
+    layout = local->layout;
 
-                if (hashed_subvol == cached_subvol) {
-                        op_errno = ENODATA;
-                        goto err;
-                }
-                if (hashed_subvol) {
-                        STACK_WIND (frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
-                                    hashed_subvol->fops->getxattr, loc,
-                                    GF_XATTR_PATHINFO_KEY, NULL);
-                        return 0;
-                }
-                op_errno = ENODATA;
-                goto err;
-        }
+    cnt = local->call_cnt = layout->cnt;
 
-        if (key && (!strcmp (GF_XATTR_MARKER_KEY, key))
-            && (-1 == frame->root->pid)) {
+    local->op_ret = -1;
+    local->op_errno = ENOATTR;
 
-                if (loc->inode-> ia_type == IA_IFDIR) {
-                        cnt = layout->cnt;
-                } else {
-                        cnt = 1;
-                }
-                sub_volumes = alloca ( cnt * sizeof (xlator_t *));
-                for (i = 0; i < cnt; i++)
-                        *(sub_volumes + i) = layout->list[i].xlator;
-
-                if (cluster_getmarkerattr (frame, this, loc, key,
-                                           local, dht_getxattr_unwind,
-                                           sub_volumes, cnt,
-                                           MARKER_UUID_TYPE, conf->vol_uuid)) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_get_real_filename_cbk, subvol,
+                   subvol->fops->getxattr, loc, key, xdata);
+    }
 
-                return 0;
+    return 0;
+}
+
+static int
+dht_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                         xlator_t **subvols)
+{
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_layout_t *layout = NULL;
+
+    local = frame->local;
+    layout = local->layout;
+
+    for (i = 0; i < layout->cnt; i++)
+        subvols[i] = layout->list[i].xlator;
+
+    return layout->cnt;
+}
+
+static int
+dht_is_debug_xattr_key(const char **array, char *key)
+{
+    int i = 0;
+
+    for (i = 0; array[i]; i++) {
+        if (fnmatch(array[i], key, FNM_NOESCAPE) == 0)
+            return i;
+    }
+
+    return -1;
+}
+
+/* Note we already have frame->local initialised here*/
+
+static int
+dht_handle_debug_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                          const char *key)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENODATA;
+    char *value = NULL;
+    loc_t file_loc = {0};
+    const char *name = NULL;
+
+    local = frame->local;
+
+    if (dht_is_debug_xattr_key(dht_dbg_vxattrs, (char *)key) == -1) {
+        goto out;
+    }
+
+    local->xattr = dict_new();
+    if (!local->xattr) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (strncmp(key, DHT_DBG_HASHED_SUBVOL_KEY,
+                SLEN(DHT_DBG_HASHED_SUBVOL_KEY)) == 0) {
+        name = key + strlen(DHT_DBG_HASHED_SUBVOL_KEY);
+        if (strlen(name) == 0) {
+            op_errno = EINVAL;
+            goto out;
+        }
+
+        ret = dht_build_child_loc(this, &file_loc, loc, (char *)name);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto out;
         }
 
-        if (key && *conf->vol_uuid) {
-                if ((match_uuid_local (key, conf->vol_uuid) == 0) &&
-                    (-1 == frame->root->pid)) {
-                        if (loc->inode-> ia_type == IA_IFDIR) {
-                                cnt = layout->cnt;
-                        } else {
-                                cnt = 1;
-                        }
-                        sub_volumes = alloca ( cnt * sizeof (xlator_t *));
-                        for (i = 0; i < cnt; i++)
-                                sub_volumes[i] = layout->list[i].xlator;
-
-                        if (cluster_getmarkerattr (frame, this, loc, key,
-                                                   local, dht_getxattr_unwind,
-                                                   sub_volumes, cnt,
-                                                   MARKER_XTIME_TYPE,
-                                                   conf->vol_uuid)) {
-                                op_errno = EINVAL;
-                                goto err;
-                        }
+        local->hashed_subvol = dht_subvol_get_hashed(this, &file_loc);
+        if (local->hashed_subvol == NULL) {
+            op_errno = ENODATA;
+            goto out;
+        }
 
-                        return 0;
-                }
+        value = gf_strdup(local->hashed_subvol->name);
+        if (!value) {
+            op_errno = ENOMEM;
+            goto out;
         }
 
-        if (loc->inode-> ia_type == IA_IFDIR) {
-                cnt = local->call_cnt = layout->cnt;
-        } else {
-                cnt = local->call_cnt  = 1;
+        ret = dict_set_dynstr(local->xattr, (char *)key, value);
+        if (ret < 0) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
         }
+        ret = 0;
+        goto out;
+    }
+
+out:
+    loc_wipe(&file_loc);
+    DHT_STACK_UNWIND(getxattr, frame, ret, op_errno, local->xattr, NULL);
+    return 0;
+}
+
+/* Virtual Xattr which returns 1 if all subvols are up,
+   else returns 0. Geo-rep then uses this virtual xattr
+   after a fresh mount and starts the I/O.
+*/
+
+enum dht_vxattr_subvol {
+    DHT_VXATTR_SUBVOLS_UP = 1,
+    DHT_VXATTR_SUBVOLS_DOWN = 0,
+};
+
+int
+dht_vgetxattr_subvol_status(call_frame_t *frame, xlator_t *this,
+                            const char *key)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENODATA;
+    int value = DHT_VXATTR_SUBVOLS_UP;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    local = frame->local;
+
+    if (!key) {
+        op_errno = EINVAL;
+        goto out;
+    }
+    local->xattr = dict_new();
+    if (!local->xattr) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            value = DHT_VXATTR_SUBVOLS_DOWN;
+            gf_msg_debug(this->name, 0, "subvol %s is down ",
+                         conf->subvolumes[i]->name);
+            break;
+        }
+    }
+    ret = dict_set_int8(local->xattr, (char *)key, value);
+    if (ret < 0) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+
+out:
+    DHT_STACK_UNWIND(getxattr, frame, ret, op_errno, local->xattr, NULL);
+    return 0;
+}
 
+int
+dht_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key,
+             dict_t *xdata)
+#define DHT_IS_DIR(layout) (layout->cnt > 1)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int cnt = 0;
+    char *node_uuid_key = NULL;
+    int ret = -1;
+
+    GF_CHECK_XATTR_KEY_AND_GOTO(key, IO_THREADS_QUEUE_SIZE_KEY, op_errno, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_GETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_NULL,
+               "Layout is NULL");
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    /* skip over code which is irrelevant without a valid key */
+    if (!key)
+        goto no_key;
+
+    local->key = gf_strdup(key);
+    if (!local->key) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (strncmp(key, conf->mds_xattr_key, strlen(key)) == 0) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    if (strncmp(key, DHT_SUBVOL_STATUS_KEY, SLEN(DHT_SUBVOL_STATUS_KEY)) == 0) {
+        dht_vgetxattr_subvol_status(frame, this, key);
+        return 0;
+    }
+
+    /* skip over code which is irrelevant if !DHT_IS_DIR(layout) */
+    if (!DHT_IS_DIR(layout))
+        goto no_dht_is_dir;
+
+    if ((strncmp(key, GF_XATTR_GET_REAL_FILENAME_KEY,
+                 SLEN(GF_XATTR_GET_REAL_FILENAME_KEY)) == 0) &&
+        DHT_IS_DIR(layout)) {
+        dht_getxattr_get_real_filename(frame, this, loc, key, xdata);
+        return 0;
+    }
+
+    if (!strcmp(key, GF_REBAL_FIND_LOCAL_SUBVOL)) {
+        ret = gf_asprintf(&node_uuid_key, "%s", GF_XATTR_LIST_NODE_UUIDS_KEY);
+        if (ret == -1 || !node_uuid_key) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+                   "Failed to copy node uuid key");
+            op_errno = ENOMEM;
+            goto err;
+        }
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", node_uuid_key);
+        cnt = local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_find_local_subvol_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              node_uuid_key, xdata);
+        }
+        if (node_uuid_key)
+            GF_FREE(node_uuid_key);
+        return 0;
+    }
+
+    if (!strcmp(key, GF_REBAL_OLD_FIND_LOCAL_SUBVOL)) {
+        ret = gf_asprintf(&node_uuid_key, "%s", GF_XATTR_NODE_UUID_KEY);
+        if (ret == -1 || !node_uuid_key) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+                   "Failed to copy node uuid key");
+            op_errno = ENOMEM;
+            goto err;
+        }
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", node_uuid_key);
+        cnt = local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_find_local_subvol_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              node_uuid_key, xdata);
+        }
+        if (node_uuid_key)
+            GF_FREE(node_uuid_key);
+        return 0;
+    }
+
+    /* for file use cached subvolume (obviously!): see if {}
+     * below
+     * for directory:
+     *  wind to all subvolumes and exclude subvolumes which
+     *  return ENOTCONN (in callback)
+     *
+     * NOTE: Don't trust inode here, as that may not be valid
+     *       (until inode_link() happens)
+     */
+
+    if (XATTR_IS_PATHINFO(key) || (strcmp(key, GF_XATTR_NODE_UUID_KEY) == 0) ||
+        (strcmp(key, GF_XATTR_LIST_NODE_UUIDS_KEY) == 0)) {
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", key);
+        cnt = local->call_cnt = layout->cnt;
         for (i = 0; i < cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_getxattr_cbk,
-                            subvol, subvol->fops->getxattr,
-                            loc, key, NULL);
+            subvol = layout->list[i].xlator;
+            STACK_WIND(frame, dht_vgetxattr_dir_cbk, subvol,
+                       subvol->fops->getxattr, loc, key, xdata);
         }
         return 0;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
+no_dht_is_dir:
+    /* node-uuid or pathinfo for files */
+    if (XATTR_IS_PATHINFO(key) || (strcmp(key, GF_XATTR_NODE_UUID_KEY) == 0)) {
+        cached_subvol = local->cached_subvol;
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", key);
+        local->call_cnt = 1;
+        STACK_WIND_COOKIE(frame, dht_vgetxattr_cbk, cached_subvol,
+                          cached_subvol, cached_subvol->fops->getxattr, loc,
+                          key, xdata);
 
         return 0;
-}
+    }
 
-int
-dht_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *key, dict_t *xdata)
-{
-        xlator_t     *subvol        = NULL;
-        dht_local_t  *local         = NULL;
-        dht_layout_t *layout        = NULL;
-        int           op_errno      = -1;
-        int           i             = 0;
-        int           cnt           = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FGETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-
-                goto err;
+    if (strcmp(key, GF_XATTR_LINKINFO_KEY) == 0) {
+        hashed_subvol = dht_subvol_get_hashed(this, loc);
+        if (!hashed_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        layout = local->layout;
-        if (!layout) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "layout is NULL");
-                op_errno = ENOENT;
-                goto err;
+        cached_subvol = dht_subvol_get_cached(this, loc->inode);
+        if (!cached_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_CACHED_SUBVOL_GET_FAILED,
+                   "Failed to get cached subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (key) {
-                local->key = gf_strdup (key);
-                if (!local->key) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
+        if (hashed_subvol == cached_subvol) {
+            op_errno = ENODATA;
+            goto err;
         }
 
-        if (fd->inode->ia_type == IA_IFDIR) {
-                cnt = local->call_cnt = layout->cnt;
+        STACK_WIND(frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
+                   hashed_subvol->fops->getxattr, loc, GF_XATTR_PATHINFO_KEY,
+                   xdata);
+        return 0;
+    }
+
+    if (dht_is_debug_xattr_key(dht_dbg_vxattrs, (char *)key) >= 0) {
+        dht_handle_debug_getxattr(frame, this, loc, key);
+        return 0;
+    }
+
+no_key:
+    if (cluster_handle_marker_getxattr(frame, loc, key, conf->vol_uuid,
+                                       dht_getxattr_unwind,
+                                       dht_marker_populate_args) == 0)
+        return 0;
+
+    if (DHT_IS_DIR(layout)) {
+        local->call_cnt = conf->subvolume_cnt;
+        cnt = conf->subvolume_cnt;
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+        if (!mds_subvol) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Cannot determine MDS, fetching xattr %s randomly"
+                   " from a subvol for path %s ",
+                   key, loc->path);
         } else {
-                cnt = local->call_cnt  = 1;
+            /* TODO need to handle it, As of now we are
+               choosing availability instead of chossing
+               consistencty, in case of mds_subvol is
+               down winding a getxattr call on other subvol
+               and return xattr
+            */
+            local->mds_subvol = mds_subvol;
+            for (i = 0; i < cnt; i++) {
+                if (conf->subvolumes[i] == mds_subvol) {
+                    if (!conf->subvolume_status[i]) {
+                        gf_msg(this->name, GF_LOG_INFO, 0,
+                               DHT_MSG_HASHED_SUBVOL_DOWN,
+                               "MDS %s is down for path"
+                               " path %s so fetching xattr "
+                               "%s randomly from a subvol ",
+                               local->mds_subvol->name, loc->path, key);
+                        ret = 1;
+                    }
+                }
+            }
         }
 
-        for (i = 0; i < cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_getxattr_cbk,
-                            subvol, subvol->fops->fgetxattr,
-                            fd, key, NULL);
+        if (!ret && key && local->mds_subvol && dht_match_xattr(key)) {
+            STACK_WIND(frame, dht_mds_getxattr_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->getxattr, loc, key, xdata);
+
+            return 0;
         }
-        return 0;
+    } else {
+        cnt = local->call_cnt = 1;
+    }
+
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->getxattr, loc,
+                   key, xdata);
+    }
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(getxattr, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
+#undef DHT_IS_DIR
 
 int
-dht_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *xattr, int flags, dict_t *xdata)
+dht_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+              dict_t *xdata)
 {
-        xlator_t     *subvol   = NULL;
-        dht_local_t  *local    = NULL;
-        int           op_errno = EINVAL;
-        data_pair_t  *trav     = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int cnt = 0;
+    xlator_t *mds_subvol = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FGETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_NULL,
+               "Layout is NULL");
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    if (key) {
+        local->key = gf_strdup(key);
+        if (!local->key) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+    }
+
+    gf_uuid_unparse(fd->inode->gfid, gfid);
+
+    if ((fd->inode->ia_type == IA_IFDIR) && key &&
+        (strncmp(key, GF_XATTR_LOCKINFO_KEY, SLEN(GF_XATTR_LOCKINFO_KEY)) !=
+         0)) {
+        local->call_cnt = conf->subvolume_cnt;
+        cnt = conf->subvolume_cnt;
+        ret = dht_inode_ctx_mdsvol_get(fd->inode, this, &mds_subvol);
+
+        if (!mds_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "cannot determine MDS, fetching xattr %s "
+                   " randomly from a subvol for gfid %s ",
+                   key, gfid);
+        } else {
+            /* TODO need to handle it, As of now we are
+               choosing availability instead of chossing
+               consistencty, in case of hashed_subvol is
+               down winding a getxattr call on other subvol
+               and return xattr
+            */
+            local->mds_subvol = mds_subvol;
+            for (i = 0; i < cnt; i++) {
+                if (conf->subvolumes[i] == mds_subvol) {
+                    if (!conf->subvolume_status[i]) {
+                        gf_msg(this->name, GF_LOG_WARNING, 0,
+                               DHT_MSG_HASHED_SUBVOL_DOWN,
+                               "MDS subvolume %s is down"
+                               " for gfid %s so fetching xattr "
+                               " %s randomly from a subvol ",
+                               local->mds_subvol->name, gfid, key);
+                        ret = 1;
+                    }
+                }
+            }
         }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
+        if (!ret && key && local->mds_subvol && dht_match_xattr(key)) {
+            STACK_WIND(frame, dht_mds_getxattr_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->fgetxattr, fd, key, NULL);
+
+            return 0;
         }
 
-        local->call_cnt = 1;
+    } else {
+        cnt = local->call_cnt = 1;
+    }
 
-        STACK_WIND (frame, dht_err_cbk, subvol, subvol->fops->fsetxattr,
-                    fd, xattr, flags, NULL);
-
-        return 0;
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->fgetxattr, fd,
+                   key, NULL);
+    }
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 static int
-dht_common_setxattr_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *xdata)
+dht_setxattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+
+    if (!frame || !frame->local)
+        goto err;
 
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         local->rebalance.xdata);
         return 0;
+    }
+
+    if (subvol == NULL)
+        goto err;
+
+    local->call_cnt = 2; /* This is the second attempt */
+
+    if (local->fop == GF_FOP_SETXATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->setxattr, &local->loc,
+                          local->rebalance.xattr, local->rebalance.flags,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->fsetxattr, local->fd,
+                          local->rebalance.xattr, local->rebalance.flags,
+                          local->xattr_req);
+    }
+
+    return 0;
+
+err:
+    DHT_STACK_UNWIND(setxattr, frame, (local ? local->op_ret : -1), op_errno,
+                     NULL);
+    return 0;
 }
 
 int
-dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr,
-                           dict_t *xdata)
+dht_file_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata)
 {
-        int           i     = -1;
-        int           ret   = -1;
-        char         *value = NULL;
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf  = NULL;
-        call_frame_t *prev  = NULL;
-        int           this_call_cnt = 0;
-
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    struct iatt *stbuf = NULL;
+    inode_t *inode = NULL;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
 
-        if (op_ret == -1)
-                goto out;
+    local = frame->local;
+    prev = cookie;
 
+    local->op_errno = op_errno;
 
-        ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value);
+    if ((local->fop == GF_FOP_FSETXATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
         if (ret)
-                goto out;
+            goto out;
+        return 0;
+    }
 
-        if (!strcmp (value, local->key)) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->subvolumes[i] == prev->this)
-                                conf->decommissioned_bricks[i] = prev->this;
-                }
-        }
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1.",
+                     prev->name);
+        goto out;
+    }
 
-out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP, NULL);
-        }
-        return 0;
+    if (local->call_cnt != 1)
+        goto out;
 
-}
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
 
-int
-dht_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
-{
-        xlator_t     *subvol   = NULL;
-        dht_local_t  *local    = NULL;
-        dht_conf_t   *conf     = NULL;
-        dht_layout_t *layout   = NULL;
-        int           i        = 0;
-        int           op_errno = EINVAL;
-        int           ret      = -1;
-        data_t       *tmp      = NULL;
-        uint32_t      dir_spread = 0;
-        char          value[4096] = {0,};
-        gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
-        int           call_cnt = 0;
-        data_pair_t  *trav     = NULL;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
-
-        conf   = this->private;
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    if ((!op_ret) && !stbuf) {
+        goto out;
+    }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local->op_ret = op_ret;
+    local->rebalance.target_op_fn = dht_setxattr2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
-        layout = local->layout;
-        if (!layout) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Phase 1 of migration */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        ret = dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            dht_setxattr2(this, subvol2, frame, 0);
+            return 0;
         }
 
-        local->call_cnt = call_cnt = layout->cnt;
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+out:
+
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    } else {
+        DHT_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
+    }
+
+    return 0;
+}
+
+/* Function is call by dict_foreach_fnmatch if key is match with
+   user.* and set boolean flag to true
+*/
+static int
+dht_is_user_xattr(dict_t *this, char *key, data_t *value, void *data)
+{
+    gf_boolean_t *user_xattr_found = data;
+    *user_xattr_found = _gf_true;
+    return 0;
+}
+
+/* Common code to wind a (f)(set|remove)xattr call to set xattr on directory
+ */
+static int
+dht_dir_common_set_remove_xattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                fd_t *fd, dict_t *xattr, int flags,
+                                dict_t *xdata, int *op_errno)
 
-        /*  This key is sent by Unified File and Object storage
-         *  to test xattr support in backend.
+{
+    dict_t *xattrop = NULL;
+    int32_t subone[1] = {-1};
+    gf_boolean_t uxattr_key_found = _gf_false;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *travvol = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int i = 0;
+    int call_cnt = 0;
+    dht_local_t *local = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char **xattrs_to_heal;
+
+    conf = this->private;
+    local = frame->local;
+    call_cnt = conf->subvolume_cnt;
+    local->flags = flags;
+    xattrs_to_heal = get_xattrs_to_heal();
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid_local);
+    }
+
+    if ((local->fop == GF_FOP_SETXATTR) || (local->fop == GF_FOP_FSETXATTR)) {
+        /* Check if any user xattr present in xattr
+         */
+        dict_foreach_fnmatch(xattr, "user*", dht_is_user_xattr,
+                             &uxattr_key_found);
+
+        /* Check if any custom key xattr present in dict xattr
+           and start index from 1 because user xattr already
+           checked in previous line
+        */
+        for (i = 1; xattrs_to_heal[i]; i++)
+            if (dict_get(xattr, xattrs_to_heal[i]))
+                uxattr_key_found = _gf_true;
+    }
+
+    if ((local->fop == GF_FOP_REMOVEXATTR) ||
+        (local->fop == GF_FOP_FREMOVEXATTR)) {
+        /* Check if any custom key xattr present in local->key
          */
-        tmp = dict_get (xattr, "user.ufo-test");
-        if (tmp) {
-                if (IA_ISREG (loc->inode->ia_type)) {
-                        op_errno = ENOTSUP;
-                        goto err;
+        for (i = 0; xattrs_to_heal[i]; i++)
+            if (strstr(local->key, xattrs_to_heal[i]))
+                uxattr_key_found = _gf_true;
+    }
+
+    /* If there is no custom key xattr present or gfid is root
+       or call_cnt is 1 then wind a (f)setxattr call on all subvols
+    */
+    if (!uxattr_key_found || __is_root_gfid(local->gfid) || call_cnt == 1) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            travvol = conf->subvolumes[i];
+            if ((local->fop == GF_FOP_SETXATTR) ||
+                (local->fop == GF_FOP_FSETXATTR)) {
+                if (fd) {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->fsetxattr, fd, xattr,
+                                      flags, xdata);
+                } else {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->setxattr, loc, xattr,
+                                      flags, xdata);
                 }
-                local->op_ret = 0;
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_ufo_xattr_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->setxattr,
-                                    loc, xattr, flags, NULL);
+            }
+
+            if ((local->fop == GF_FOP_REMOVEXATTR) ||
+                (local->fop == GF_FOP_FREMOVEXATTR)) {
+                if (fd) {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->fremovexattr, fd,
+                                      local->key, local->xattr_req);
+                } else {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->removexattr, loc,
+                                      local->key, local->xattr_req);
                 }
-                return 0;
+            }
         }
 
-        tmp = dict_get (xattr, "distribute.migrate-data");
-        if (tmp) {
-                if (IA_ISDIR (loc->inode->ia_type)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
+        return 0;
+    }
+
+    /* Calculate hash subvol based on inode and parent inode
+     */
+    if (fd) {
+        ret = dht_inode_ctx_mdsvol_get(fd->inode, this, &mds_subvol);
+    } else {
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+    }
+    if (ret || !mds_subvol) {
+        if (fd) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get mds subvol for fd %p"
+                   "gfid is %s ",
+                   fd, gfid_local);
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "%s: Failed to get mds subvol. (gfid is %s)", loc->path,
+                   gfid_local);
+        }
+        (*op_errno) = ENOENT;
+        goto err;
+    }
+
+    local->mds_subvol = mds_subvol;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == mds_subvol) {
+            if (!conf->subvolume_status[i]) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_HASHED_SUBVOL_DOWN,
+                       "MDS subvol is down for path "
+                       " %s gfid is %s Unable to set xattr ",
+                       local->loc.path, gfid_local);
+                (*op_errno) = ENOTCONN;
+                goto err;
+            }
+        }
+    }
+
+    if (uxattr_key_found) {
+        xattrop = dict_new();
+        if (!xattrop) {
+            gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, 0,
+                   "dictionary creation failed for path %s "
+                   "for gfid is %s ",
+                   local->loc.path, gfid_local);
+            (*op_errno) = ENOMEM;
+            goto err;
+        }
+        local->xattr = dict_ref(xattr);
+        /* Subtract current MDS xattr value to -1 , value of MDS
+           xattr represents no. of times xattr modification failed
+           on non MDS subvols.
+        */
+        ret = dht_dict_set_array(xattrop, conf->mds_xattr_key, subone, 1);
+        if (ret != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "dictionary set array failed for path %s "
+                   "for gfid is %s ",
+                   local->loc.path, gfid_local);
+            if (xattrop)
+                dict_unref(xattrop);
+            (*op_errno) = ret;
+            goto err;
+        }
+        /* Wind a xattrop call to use ref counting approach
+           update mds xattr to -1 before update xattr on
+           hashed subvol and update mds xattr to +1 after update
+           xattr on all non hashed subvol
+        */
+        if (fd) {
+            STACK_WIND(frame, dht_xattrop_mds_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->fxattrop, fd,
+                       GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+        } else {
+            STACK_WIND(frame, dht_xattrop_mds_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->xattrop, loc,
+                       GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+        }
+        if (xattrop)
+            dict_unref(xattrop);
+    }
 
-                /* TODO: need to interpret the 'value' for more meaning
-                   (ie, 'target' subvolume given there, etc) */
-                memcpy (value, tmp->data, tmp->len);
-                if (strcmp (value, "force") == 0)
-                       forced_rebalance =
-                                GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
-
-                if (conf->decommission_in_progress)
-                        forced_rebalance = GF_DHT_MIGRATE_HARDLINK;
-
-                local->rebalance.target_node = dht_subvol_get_hashed (this, loc);
-                if (!local->rebalance.target_node) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                "hashed subvol for %s", loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    return 0;
+err:
+    return -1;
+}
 
-                local->rebalance.from_subvol = local->cached_subvol;
+int
+dht_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
+              int flags, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    if (!conf->defrag)
+        GF_IF_INTERNAL_XATTR_GOTO(conf->wild_xattr_name, xattr, op_errno, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+    local->call_cnt = call_cnt = layout->cnt;
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, NULL, fd, xattr,
+                                              flags, xdata, &op_errno);
+        if (ret)
+            goto err;
+    } else {
+        local->call_cnt = 1;
+        local->rebalance.xattr = dict_ref(xattr);
+        local->rebalance.flags = flags;
 
-                if (local->rebalance.target_node == local->rebalance.from_subvol) {
-                        op_errno = EEXIST;
-                        goto err;
-                }
-                if (local->rebalance.target_node) {
-                        local->flags = forced_rebalance;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dictionary key %s for fd=%p",
+                         DHT_IATT_IN_XDATA_KEY, fd);
+        }
 
-                        ret = dht_start_rebalance_task (this, frame);
-                        if (!ret)
-                                return 0;
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->fsetxattr, fd, xattr, flags,
+                          local->xattr_req);
+    }
+    return 0;
 
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s: failed to create a new synctask",
-                                loc->path);
-                }
-                op_errno = EINVAL;
-                goto err;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
 
+    return 0;
+}
+
+static int
+dht_checking_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
+{
+    int i = -1;
+    int ret = -1;
+    char *value = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (op_ret == -1)
+        goto out;
+
+    ret = dict_get_str(xattr, GF_XATTR_PATHINFO_KEY, &value);
+    if (ret)
+        goto out;
+
+    if (!strcmp(value, local->key)) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == prev)
+                conf->decommissioned_bricks[i] = prev;
         }
+    }
 
-        tmp = dict_get (xattr, "decommission-brick");
-        if (tmp) {
-                /* This operation should happen only on '/' */
-                if (!__is_root_gfid (loc->inode->gfid)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
+out:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, ENOTSUP, NULL);
+    }
+    return 0;
+}
 
-                memcpy (value, tmp->data, ((tmp->len < 4095) ? tmp->len : 4095));
-                local->key = gf_strdup (value);
-                local->call_cnt = conf->subvolume_cnt;
+static int
+dht_nuke_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, NULL);
+    return 0;
+}
 
-                for (i = 0 ; i < conf->subvolume_cnt; i++) {
-                        /* Get the pathinfo, and then compare */
-                        STACK_WIND (frame, dht_checking_pathinfo_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->getxattr,
-                                    loc, GF_XATTR_PATHINFO_KEY, NULL);
-                }
-                return 0;
+static int
+dht_nuke_dir(call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *tmp)
+{
+    if (!IA_ISDIR(loc->inode->ia_type)) {
+        DHT_STACK_UNWIND(setxattr, frame, -1, ENOTSUP, NULL);
+        return 0;
+    }
+
+    /* Setxattr didn't need the parent, but rmdir does. */
+    loc->parent = inode_parent(loc->inode, NULL, NULL);
+    if (!loc->parent) {
+        DHT_STACK_UNWIND(setxattr, frame, -1, ENOENT, NULL);
+        return 0;
+    }
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
+
+    if (!loc->name && loc->path) {
+        loc->name = strrchr(loc->path, '/');
+        if (loc->name) {
+            ++(loc->name);
+        }
+    }
+
+    /*
+     * We do this instead of calling dht_rmdir_do directly for two reasons.
+     * The first is that we want to reuse all of the initialization that
+     * dht_rmdir does, so if it ever changes we'll just follow along.  The
+     * second (i.e. why we don't use STACK_WIND_TAIL) is so that we don't
+     * obscure the fact that we came in via this path instead of a genuine
+     * rmdir.  That makes debugging just a tiny bit easier.
+     */
+    STACK_WIND(frame, dht_nuke_dir_cbk, this, this->fops->rmdir, loc, 1, NULL);
+
+    return 0;
+}
+
+int
+dht_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr,
+             int flags, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int op_errno = EINVAL;
+    int ret = -1;
+    data_t *tmp = NULL;
+    uint32_t dir_spread = 0;
+    char value[4096] = {
+        0,
+    };
+    gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
+    int call_cnt = 0;
+    uint32_t new_hash = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
+
+    methods = &(conf->methods);
+
+    /* Rebalance daemon is allowed to set internal keys */
+    if (!conf->defrag)
+        GF_IF_INTERNAL_XATTR_GOTO(conf->wild_xattr_name, xattr, op_errno, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local->call_cnt = call_cnt = layout->cnt;
+    tmp = dict_get(xattr, conf->mds_xattr_key);
+    if (tmp) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    tmp = dict_get(xattr, GF_XATTR_FILE_MIGRATE_KEY);
+    if (tmp) {
+        if (IA_ISDIR(loc->inode->ia_type)) {
+            op_errno = ENOTSUP;
+            goto err;
+        }
+
+        /* TODO: need to interpret the 'value' for more meaning
+           (ie, 'target' subvolume given there, etc) */
+        memcpy(value, tmp->data, tmp->len);
+        if (strcmp(value, "force") == 0)
+            forced_rebalance = GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+
+        if (conf->decommission_in_progress)
+            forced_rebalance = GF_DHT_MIGRATE_HARDLINK;
+
+        if (!loc->path) {
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        if (!local->loc.name)
+            local->loc.name = strrchr(local->loc.path, '/') + 1;
+
+        if (!local->loc.parent)
+            local->loc.parent = inode_parent(local->loc.inode, NULL, NULL);
+
+        if ((!local->loc.name) || (!local->loc.parent)) {
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        if (gf_uuid_is_null(local->loc.pargfid))
+            gf_uuid_copy(local->loc.pargfid, local->loc.parent->gfid);
+
+        methods->migration_get_dst_subvol(this, local);
+
+        if (!local->rebalance.target_node) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        local->rebalance.from_subvol = local->cached_subvol;
+
+        if (local->rebalance.target_node == local->rebalance.from_subvol) {
+            op_errno = EEXIST;
+            goto err;
         }
+        if (local->rebalance.target_node) {
+            local->flags = forced_rebalance;
 
-        tmp = dict_get (xattr, GF_XATTR_FIX_LAYOUT_KEY);
-        if (tmp) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "fixing the layout of %s", loc->path);
+            frame->root->pid = GF_CLIENT_PID_DEFRAG;
 
-                dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
-                                          layout);
+            ret = dht_start_rebalance_task(this, frame);
+            if (!ret)
                 return 0;
+
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+                   "%s: failed to create a new rebalance synctask", loc->path);
         }
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        tmp = dict_get (xattr, "distribute.directory-spread-count");
-        if (tmp) {
-                /* Setxattr value is packed as 'binary', not string */
-                memcpy (value, tmp->data, ((tmp->len < 4095)?tmp->len:4095));
-                ret = gf_string2uint32 (value, &dir_spread);
-                if (!ret && ((dir_spread <= conf->subvolume_cnt) &&
-                             (dir_spread > 0))) {
-                        layout->spread_cnt = dir_spread;
+    tmp = dict_get(xattr, "decommission-brick");
+    if (tmp) {
+        /* This operation should happen only on '/' */
+        if (!__is_root_gfid(loc->inode->gfid)) {
+            op_errno = ENOTSUP;
+            goto err;
+        }
 
-                        dht_fix_directory_layout (frame,
-                                                  dht_common_setxattr_cbk,
-                                                  layout);
-                        return 0;
-                }
-                gf_log (this->name, GF_LOG_ERROR,
-                        "wrong 'directory-spread-count' value (%s)", value);
-                op_errno = ENOTSUP;
+        memcpy(value, tmp->data, min(tmp->len, 4095));
+        local->key = gf_strdup(value);
+        local->call_cnt = conf->subvolume_cnt;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            /* Get the pathinfo, and then compare */
+            STACK_WIND_COOKIE(frame, dht_checking_pathinfo_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              GF_XATTR_PATHINFO_KEY, NULL);
+        }
+        return 0;
+    }
+
+    tmp = dict_get(xattr, GF_XATTR_FIX_LAYOUT_KEY);
+    if (tmp) {
+        ret = dict_get_uint32(xattr, "new-commit-hash", &new_hash);
+        if (ret == 0) {
+            gf_msg_debug(this->name, 0,
+                         "updating commit hash for %s from %u to %u",
+                         uuid_utoa(loc->gfid), layout->commit_hash, new_hash);
+            layout->commit_hash = new_hash;
+
+            ret = dht_update_commit_hash_for_layout(frame);
+            if (ret) {
+                op_errno = ENOTCONN;
                 goto err;
+            }
+            return ret;
         }
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_err_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->setxattr,
-                            loc, xattr, flags, xdata);
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_FIX_LAYOUT_INFO,
+               "fixing the layout of %s", loc->path);
+
+        ret = dht_fix_directory_layout(frame, dht_fix_layout_setxattr_cbk,
+                                       layout);
+        if (ret) {
+            op_errno = ENOTCONN;
+            goto err;
         }
+        return ret;
+    }
+
+    tmp = dict_get(xattr, "distribute.directory-spread-count");
+    if (tmp) {
+        /* Setxattr value is packed as 'binary', not string */
+        memcpy(value, tmp->data, min(tmp->len, 4095));
+        ret = gf_string2uint32(value, &dir_spread);
+        if (!ret && ((dir_spread <= conf->subvolume_cnt) && (dir_spread > 0))) {
+            layout->spread_cnt = dir_spread;
+
+            ret = dht_fix_directory_layout(frame, dht_common_setxattr_cbk,
+                                           layout);
+            if (ret) {
+                op_errno = ENOTCONN;
+                goto err;
+            }
+            return ret;
+        }
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_OPERATION_NOT_SUP,
+               "wrong 'directory-spread-count' value (%s)", value);
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    tmp = dict_get(xattr, "glusterfs.dht.nuke");
+    if (tmp) {
+        return dht_nuke_dir(frame, this, loc, tmp);
+    }
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, loc, NULL, xattr,
+                                              flags, xdata, &op_errno);
+        if (ret)
+            goto err;
+    } else {
+        local->rebalance.xattr = dict_ref(xattr);
+        local->rebalance.flags = flags;
+        local->call_cnt = 1;
 
-        return 0;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->setxattr, loc, xattr, flags,
+                          local->xattr_req);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_removexattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
+    if (!frame || !frame->local)
+        goto err;
 
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
-                                  local->op_errno, NULL);
-        }
+    local->call_cnt = 2; /* This is the second attempt */
 
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         local->rebalance.xdata);
         return 0;
-}
+    }
+
+    if (subvol == NULL)
+        goto err;
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->removexattr, &local->loc, local->key,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->fremovexattr, local->fd, local->key,
+                          local->xattr_req);
+    }
 
+    return 0;
+
+err:
+    DHT_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    return 0;
+}
 
 int
-dht_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *key, dict_t *xdata)
+dht_file_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           call_cnt = 0;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    struct iatt *stbuf = NULL;
+    inode_t *inode = NULL;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
 
-        int i;
+    local = frame->local;
+    prev = cookie;
 
-        VALIDATE_OR_GOTO (this, err);
+    local->op_errno = op_errno;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.dht*",
-                                 key, op_errno, err);
+    if ((local->fop == GF_FOP_FREMOVEXATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_REMOVEXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    if (local->call_cnt != 1)
+        goto out;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
 
-        layout = local->layout;
-        if (!local->layout) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    if ((!op_ret) && !stbuf) {
+        goto out;
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
-        local->key = gf_strdup (key);
+    local->op_ret = 0;
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_removexattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->removexattr,
-                            loc, key, NULL);
-        }
+    local->rebalance.target_op_fn = dht_removexattr2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
-        return 0;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+    /* Phase 1 of migration */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
 
-        return 0;
+        ret = dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            dht_removexattr2(this, subvol2, frame, 0);
+            return 0;
+        }
+
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+out:
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
+    } else {
+        DHT_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
+    }
+    return 0;
 }
 
 int
-dht_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *key, dict_t *xdata)
+dht_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           call_cnt = 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    GF_IF_NATIVE_XATTR_GOTO(conf->wild_xattr_name, key, op_errno, err);
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_REMOVEXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!local->layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+
+    local->call_cnt = call_cnt = layout->cnt;
+    local->key = gf_strdup(key);
+
+    if (key && (strncmp(key, conf->mds_xattr_key, strlen(key)) == 0)) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, loc, NULL, NULL, 0,
+                                              local->xattr_req, &op_errno);
+        if (ret)
+            goto err;
 
-        int i;
+    } else {
+        local->call_cnt = 1;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set dictionary key %s for %s",
+                   DHT_IATT_IN_XDATA_KEY, loc->path);
+        }
 
-        VALIDATE_OR_GOTO (this, err);
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->removexattr, loc, key,
+                          local->xattr_req);
+    }
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.dht*",
-                                 key, op_errno, err);
+    return 0;
 
-        VALIDATE_OR_GOTO (frame, err);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
 
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FREMOVEXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    return 0;
+}
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for inode=%s",
-                        uuid_utoa (fd->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
-        }
+int
+dht_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = 0;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    GF_IF_NATIVE_XATTR_GOTO(conf->wild_xattr_name, key, op_errno, err);
+
+    VALIDATE_OR_GOTO(frame, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FREMOVEXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for inode=%s",
+                     uuid_utoa(fd->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!local->layout) {
+        gf_msg_debug(this->name, 0, "no layout for inode=%s",
+                     uuid_utoa(fd->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+
+    local->call_cnt = call_cnt = layout->cnt;
+    local->key = gf_strdup(key);
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, NULL, fd, NULL, 0,
+                                              local->xattr_req, &op_errno);
+        if (ret)
+            goto err;
 
-        layout = local->layout;
-        if (!local->layout) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no layout for inode=%s", uuid_utoa (fd->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
+    } else {
+        local->call_cnt = 1;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set dictionary key %s for fd=%p",
+                   DHT_IATT_IN_XDATA_KEY, fd);
         }
 
-        local->call_cnt = call_cnt = layout->cnt;
-        local->key = gf_strdup (key);
-
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_removexattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->fremovexattr,
-                            fd, key, NULL);
-        }
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->fremovexattr, fd, key,
+                          local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+dht_fd_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+           int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
 
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
+    local = frame->local;
+    prev = cookie;
 
-                local->op_ret = 0;
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                                  local->fd, NULL);
 
-        return 0;
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(open, frame, local->op_ret, local->op_errno, local->fd,
+                         NULL);
+
+    return 0;
 }
 
 /*
  * dht_normalize_stats -
  */
 static void
-dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
-                     unsigned long frsize)
+dht_normalize_stats(struct statvfs *buf, unsigned long bsize,
+                    unsigned long frsize)
 {
-        double factor = 0;
+    double factor = 0;
+
+    if (buf->f_bsize != bsize) {
+        buf->f_bsize = bsize;
+    }
+
+    if (buf->f_frsize != frsize) {
+        factor = ((double)buf->f_frsize) / frsize;
+        buf->f_frsize = frsize;
+        buf->f_blocks = (fsblkcnt_t)(factor * buf->f_blocks);
+        buf->f_bfree = (fsblkcnt_t)(factor * buf->f_bfree);
+        buf->f_bavail = (fsblkcnt_t)(factor * buf->f_bavail);
+    }
+}
+
+static int
+dht_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct statvfs *statvfs, dict_t *xdata)
+{
+    gf_boolean_t event = _gf_false;
+    qdstatfs_action_t action = qdstatfs_action_OFF;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int bsize = 0;
+    int frsize = 0;
+    GF_UNUSED int ret = 0;
+    unsigned long new_usage = 0;
+    unsigned long cur_usage = 0;
+
+    local = frame->local;
+    GF_ASSERT(local);
+
+    if (xdata)
+        ret = dict_get_int8(xdata, "quota-deem-statfs", (int8_t *)&event);
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            goto unlock;
+        }
+        if (!statvfs) {
+            op_errno = EINVAL;
+            local->op_ret = -1;
+            goto unlock;
+        }
+        local->op_ret = 0;
 
-        if (buf->f_bsize != bsize) {
-                buf->f_bsize = bsize;
+        if (local->quota_deem_statfs) {
+            if (event == _gf_true) {
+                action = qdstatfs_action_COMPARE;
+            } else {
+                action = qdstatfs_action_NEGLECT;
+            }
+        } else {
+            if (event == _gf_true) {
+                action = qdstatfs_action_REPLACE;
+                local->quota_deem_statfs = _gf_true;
+            }
         }
 
-        if (buf->f_frsize != frsize) {
-                factor = ((double) buf->f_frsize) / frsize;
-                buf->f_frsize = frsize;
-                buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-                buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-                buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
+        if (local->quota_deem_statfs) {
+            switch (action) {
+                case qdstatfs_action_NEGLECT:
+                    goto unlock;
+
+                case qdstatfs_action_REPLACE:
+                    local->statvfs = *statvfs;
+                    goto unlock;
+
+                case qdstatfs_action_COMPARE:
+                    new_usage = statvfs->f_blocks - statvfs->f_bfree;
+                    cur_usage = local->statvfs.f_blocks -
+                                local->statvfs.f_bfree;
 
+                    /* Take the max of the usage from subvols */
+                    if (new_usage >= cur_usage)
+                        local->statvfs = *statvfs;
+                    goto unlock;
+
+                default:
+                    break;
+            }
         }
+
+        if (local->statvfs.f_bsize != 0) {
+            bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
+            frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
+            dht_normalize_stats(&local->statvfs, bsize, frsize);
+            dht_normalize_stats(statvfs, bsize, frsize);
+        } else {
+            local->statvfs.f_bsize = statvfs->f_bsize;
+            local->statvfs.f_frsize = statvfs->f_frsize;
+        }
+
+        local->statvfs.f_blocks += statvfs->f_blocks;
+        local->statvfs.f_bfree += statvfs->f_bfree;
+        local->statvfs.f_bavail += statvfs->f_bavail;
+        local->statvfs.f_files += statvfs->f_files;
+        local->statvfs.f_ffree += statvfs->f_ffree;
+        local->statvfs.f_favail += statvfs->f_favail;
+        local->statvfs.f_fsid = statvfs->f_fsid;
+        local->statvfs.f_flag = statvfs->f_flag;
+        local->statvfs.f_namemax = statvfs->f_namemax;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(statfs, frame, local->op_ret, local->op_errno,
+                         &local->statvfs, xdata);
+
+    return 0;
 }
 
 int
-dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct statvfs *statvfs, dict_t *xdata)
+dht_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        dht_local_t *local         = NULL;
-        int          this_call_cnt = 0;
-        int          bsize         = 0;
-        int          frsize        = 0;
-
-
-        local = frame->local;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    inode_t *inode = NULL;
+    inode_table_t *itable = NULL;
+    static uuid_t root_gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    loc_t newloc = {
+        0,
+    };
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
-                local->op_ret = 0;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-                if (local->statvfs.f_bsize != 0) {
-                        bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
-                        frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
-                        dht_normalize_stats(&local->statvfs, bsize, frsize);
-                        dht_normalize_stats(statvfs, bsize, frsize);
-                } else {
-                        local->statvfs.f_bsize    = statvfs->f_bsize;
-                        local->statvfs.f_frsize   = statvfs->f_frsize;
-                }
+    conf = this->private;
 
-                local->statvfs.f_blocks  += statvfs->f_blocks;
-                local->statvfs.f_bfree   += statvfs->f_bfree;
-                local->statvfs.f_bavail  += statvfs->f_bavail;
-                local->statvfs.f_files   += statvfs->f_files;
-                local->statvfs.f_ffree   += statvfs->f_ffree;
-                local->statvfs.f_favail  += statvfs->f_favail;
-                local->statvfs.f_fsid     = statvfs->f_fsid;
-                local->statvfs.f_flag     = statvfs->f_flag;
-                local->statvfs.f_namemax  = statvfs->f_namemax;
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_STATFS);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
+    if (loc->inode && !IA_ISDIR(loc->inode->ia_type)) {
+        itable = loc->inode->table;
+        if (!itable) {
+            op_errno = EINVAL;
+            goto err;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
+        loc = &local->loc2;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->statvfs, xdata);
+        inode = inode_find(itable, root_gfid);
+        if (!inode) {
+            op_errno = EINVAL;
+            goto err;
+        }
 
-        return 0;
-}
+        dht_build_root_loc(inode, &newloc);
+        loc = &newloc;
+    }
 
+    local->call_cnt = conf->subvolume_cnt;
 
-int
-dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        xlator_t     *subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, NULL, NULL, GF_FOP_STATFS);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND(frame, dht_statfs_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->statfs, loc, xdata);
+    }
+    return 0;
 
-        if (IA_ISDIR (loc->inode->ia_type)) {
-                local->call_cnt = conf->subvolume_cnt;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        STACK_WIND (frame, dht_statfs_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->statfs, loc,
-                                    xdata);
-                }
-                return 0;
-        }
+    return 0;
+}
 
-        subvol = dht_subvol_get_cached (this, loc->inode);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
+int
+dht_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = 0;
+    gf_boolean_t new_xdata = _gf_false;
+    xlator_t **subvolumes = NULL;
+    int call_count = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_OPENDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    local->first_up_subvol = dht_first_up_subvol(this);
+
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+        new_xdata = _gf_true;
+    }
+
+    ret = dict_set_uint32(xdata, conf->link_xattr_name, 256);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value : key = %s",
+               conf->link_xattr_name);
+
+    /* dht_readdirp will wind to all subvols so open has to be sent to
+     * all subvols whether or not conf->local_subvols is set */
+
+    call_count = local->call_cnt = conf->subvolume_cnt;
+    subvolumes = conf->subvolumes;
+
+    /* In case of parallel-readdir, the readdir-ahead will be loaded
+     * below dht, in this case, if we want to enable or disable SKIP_DIRs
+     * it has to be done in opendir, so that prefetching logic in
+     * readdir-ahead, honors it */
+    for (i = 0; i < call_count; i++) {
+        if (conf->readdir_optimize == _gf_true) {
+            if (subvolumes[i] != local->first_up_subvol) {
+                ret = dict_set_int32(xdata, GF_READDIR_SKIP_DIRS, 1);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary"
+                           " value :key = %s, ret:%d",
+                           GF_READDIR_SKIP_DIRS, ret);
+            }
         }
 
-        local->call_cnt = 1;
+        STACK_WIND_COOKIE(frame, dht_fd_cbk, subvolumes[i], subvolumes[i],
+                          subvolumes[i]->fops->opendir, loc, fd, xdata);
+        dict_del(xdata, GF_READDIR_SKIP_DIRS);
+    }
 
-        STACK_WIND (frame, dht_statfs_cbk,
-                    subvol, subvol->fops->statfs, loc, xdata);
+    if (new_xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(opendir, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+/* dht_readdirp_cbk creates a new dentry and dentry->inode is not assigned.
+   This functions assigns an inode if all of the following conditions are
+   true:
 
-int
-dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-             dict_t *xdata)
+   * DHT has only one child. In this case the entire layout is present on
+   this single child and hence we can set complete layout in inode.
+   * backend has complete layout and there are no anomalies in it and from
+   this information layout can be constructed and set in inode.
+*/
+
+static void
+dht_populate_inode_for_dentry(xlator_t *this, xlator_t *subvol,
+                              gf_dirent_t *entry, gf_dirent_t *orig_entry)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
+    dht_layout_t *layout = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+
+    if (gf_uuid_is_null(orig_entry->d_stat.ia_gfid)) {
+        /* this skips the '..' entry for the root of the volume */
+        return;
+    }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    gf_uuid_copy(loc.gfid, orig_entry->d_stat.ia_gfid);
+    loc.inode = inode_ref(orig_entry->inode);
 
-        conf = this->private;
+    if (is_revalidate(&loc)) {
+        goto out;
+    }
 
-        local = dht_local_init (frame, loc, fd, GF_FOP_OPENDIR);
-        if (!local) {
-                op_errno = ENOMEM;
+    layout = dht_layout_new(this, 1);
+    if (!layout)
+        goto out;
 
-                goto err;
+    ret = dht_layout_merge(this, layout, subvol, 0, 0, orig_entry->dict);
+    if (!ret) {
+        ret = dht_layout_normalize(this, &loc, layout);
+        if (ret == 0) {
+            dht_layout_set(this, orig_entry->inode, layout);
+            entry->inode = inode_ref(orig_entry->inode);
+            layout = NULL;
         }
+    }
 
-        local->call_cnt = conf->subvolume_cnt;
+    if (layout)
+        dht_layout_unref(this, layout);
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_fd_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->opendir,
-                            loc, fd, xdata);
-        }
+out:
+    loc_wipe(&loc);
+    return;
+}
 
-        return 0;
+/* Posix returns op_errno = ENOENT to indicate that there are no more
+ * entries
+ */
+static int
+dht_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *orig_entry = NULL;
+    gf_dirent_t *entry = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *next_subvol = NULL;
+    off_t next_offset = 0;
+    int count = 0;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    xlator_t *subvol = 0;
+    xlator_t *hashed_subvol = 0;
+    int ret = 0;
+    int readdir_optimize = 0;
+    inode_table_t *itable = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t skip_hashed_check = _gf_false;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    prev = cookie;
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO(this->name, local->fd, unwind);
+
+    itable = local->fd->inode->table;
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, unwind);
+
+    methods = &(conf->methods);
+
+    if (op_ret <= 0) {
+        goto done;
+    }
+
+    /* Why aren't we skipping DHT entirely in case of a single subvol?
+     * Because if this was a larger volume earlier and all but one subvol
+     * was removed, there might be stale linkto files on the subvol.
+     */
+    if (conf->subvolume_cnt == 1) {
+        /* return all directory and file entries except
+         * linkto files for a single child DHT
+         */
+        skip_hashed_check = _gf_true;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
+    if (!local->layout)
+        local->layout = dht_layout_get(this, local->fd->inode);
 
-        return 0;
-}
+    layout = local->layout;
 
+    /* This will skip the entries on the subvol without a layout,
+     * hence preventing the crash but rmdir might fail with
+     * "directory not empty" errors*/
 
-int
-dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-                  int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        gf_dirent_t   entries;
-        gf_dirent_t  *orig_entry = NULL;
-        gf_dirent_t  *entry = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *next_subvol = NULL;
-        off_t         next_offset = 0;
-        int           count = 0;
-        dht_layout_t *layout = 0;
-        dht_conf_t   *conf   = NULL;
-        xlator_t     *subvol = 0;
-        int           ret    = 0;
-
-        INIT_LIST_HEAD (&entries.list);
-        prev = cookie;
-        local = frame->local;
-        conf  = this->private;
+    if (layout == NULL)
+        goto done;
 
-        if (op_ret < 0)
-                goto done;
+    if (conf->readdir_optimize == _gf_true)
+        readdir_optimize = 1;
 
-        if (!local->layout)
-                local->layout = dht_layout_get (this, local->fd->inode);
+    gf_msg_debug(this->name, 0, "Processing entries from %s", prev->name);
 
-        layout = local->layout;
+    list_for_each_entry(orig_entry, (&orig_entries->list), list)
+    {
+        next_offset = orig_entry->d_off;
 
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                next_offset = orig_entry->d_off;
-                if ((check_is_dir (NULL, (&orig_entry->d_stat), NULL) &&
-                     (prev->this != dht_first_up_subvol (this))) ||
-                    check_is_linkfile (NULL, (&orig_entry->d_stat),
-                                       orig_entry->dict)) {
-                        continue;
-                }
+        gf_msg_debug(this->name, 0, "%s: entry = %s, type = %d", prev->name,
+                     orig_entry->d_name, orig_entry->d_type);
 
-                entry = gf_dirent_for_name (orig_entry->d_name);
-                if (!entry) {
+        if (IA_ISINVAL(orig_entry->d_stat.ia_type)) {
+            /*stat failed somewhere- display this entry but the data may
+             * be inaccurate.
+             */
+            gf_msg_debug(this->name, EINVAL, "Invalid stat for %s (gfid %s)",
+                         orig_entry->d_name,
+                         uuid_utoa(orig_entry->d_stat.ia_gfid));
+        }
 
-                        goto unwind;
-                }
+        if (check_is_linkfile(NULL, (&orig_entry->d_stat), orig_entry->dict,
+                              conf->link_xattr_name)) {
+            gf_msg_debug(this->name, 0, "%s: %s is a linkto file", prev->name,
+                         orig_entry->d_name);
+            continue;
+        }
 
-                /* Do this if conf->search_unhashed is set to "auto" */
-                if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
-                        subvol = dht_layout_search (this, layout,
-                                                    orig_entry->d_name);
-                        if (!subvol || (subvol != prev->this)) {
-                                /* TODO: Count the number of entries which need
-                                   linkfile to prove its existence in fs */
-                                layout->search_unhashed++;
-                        }
-                }
+        if (skip_hashed_check) {
+            goto list;
+        }
 
-                dht_itransform (this, prev->this, orig_entry->d_off,
-                                &entry->d_off);
-
-                entry->d_stat = orig_entry->d_stat;
-                entry->d_ino  = orig_entry->d_ino;
-                entry->d_type = orig_entry->d_type;
-                entry->d_len  = orig_entry->d_len;
-
-                if (orig_entry->dict)
-                        entry->dict = dict_ref (orig_entry->dict);
-
-                /* making sure we set the inode ctx right with layout,
-                   currently possible only for non-directories, so for
-                   directories don't set entry inodes */
-                if (!IA_ISDIR(entry->d_stat.ia_type)) {
-                        ret = dht_layout_preset (this, prev->this,
-                                                 orig_entry->inode);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to link the layout in inode");
-                        entry->inode = inode_ref (orig_entry->inode);
+        if (check_is_dir(NULL, (&orig_entry->d_stat), NULL)) {
+            /*Directory entries filtering :
+             * a) If rebalance is running, pick from first_up_subvol
+             * b) (rebalance not running)hashed subvolume is NULL or
+             * down then filter in first_up_subvolume. Other wise the
+             * corresponding hashed subvolume will take care of the
+             * directory entry.
+             */
+            if (readdir_optimize) {
+                if (prev == local->first_up_subvol)
+                    goto list;
+                else
+                    continue;
+            }
+
+            hashed_subvol = methods->layout_search(this, layout,
+                                                   orig_entry->d_name);
+
+            if (prev == hashed_subvol)
+                goto list;
+            if ((hashed_subvol && dht_subvol_status(conf, hashed_subvol)) ||
+                (prev != local->first_up_subvol))
+                continue;
+
+            goto list;
+        }
+
+    list:
+        entry = gf_dirent_for_name(orig_entry->d_name);
+        if (!entry) {
+            goto unwind;
+        }
+
+        /* Do this if conf->search_unhashed is set to "auto" */
+        if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
+            subvol = methods->layout_search(this, layout, orig_entry->d_name);
+            if (!subvol || (subvol != prev)) {
+                /* TODO: Count the number of entries which need
+                   linkfile to prove its existence in fs */
+                layout->search_unhashed++;
+            }
+        }
+
+        entry->d_off = orig_entry->d_off;
+        entry->d_stat = orig_entry->d_stat;
+        entry->d_ino = orig_entry->d_ino;
+        entry->d_type = orig_entry->d_type;
+        entry->d_len = orig_entry->d_len;
+
+        if (orig_entry->dict)
+            entry->dict = dict_ref(orig_entry->dict);
+
+        /* making sure we set the inode ctx right with layout,
+           currently possible only for non-directories, so for
+           directories don't set entry inodes */
+        if (IA_ISDIR(entry->d_stat.ia_type)) {
+            entry->d_stat.ia_blocks = DHT_DIR_STAT_BLOCKS;
+            entry->d_stat.ia_size = DHT_DIR_STAT_SIZE;
+            if (orig_entry->inode) {
+                dht_inode_ctx_time_update(orig_entry->inode, this,
+                                          &entry->d_stat, 1);
+
+                if (conf->subvolume_cnt == 1) {
+                    dht_populate_inode_for_dentry(this, prev, entry,
+                                                  orig_entry);
                 }
-
-                list_add_tail (&entry->list, &entries.list);
-                count++;
+            }
+        } else {
+            if (orig_entry->dict &&
+                dict_get(orig_entry->dict, conf->link_xattr_name)) {
+                /* Strip out the S and T flags set by rebalance*/
+                DHT_STRIP_PHASE1_FLAGS(&entry->d_stat);
+            }
+
+            if (orig_entry->inode) {
+                ret = dht_layout_preset(this, prev, orig_entry->inode);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           DHT_MSG_LAYOUT_SET_FAILED,
+                           "failed to link the layout "
+                           "in inode for %s",
+                           orig_entry->d_name);
+
+                entry->inode = inode_ref(orig_entry->inode);
+            } else if (itable) {
+                /*
+                 * orig_entry->inode might be null if any upper
+                 * layer xlators below client set to null, to
+                 * force a lookup on the inode even if the inode
+                 * is present in the inode table. In that case
+                 * we just update the ctx to make sure we didn't
+                 * missed anything.
+                 */
+                inode = inode_find(itable, orig_entry->d_stat.ia_gfid);
+                if (inode) {
+                    ret = dht_layout_preset(this, prev, inode);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_WARNING, 0,
+                               DHT_MSG_LAYOUT_SET_FAILED,
+                               "failed to link the layout"
+                               " in inode for %s",
+                               orig_entry->d_name);
+                    inode_unref(inode);
+                    inode = NULL;
+                }
+            }
         }
-        op_ret = count;
-        /* We need to ensure that only the last subvolume's end-of-directory
-         * notification is respected so that directory reading does not stop
-         * before all subvolumes have been read. That could happen because the
-         * posix for each subvolume sends a ENOENT on end-of-directory but in
-         * distribute we're not concerned only with a posix's view of the
-         * directory but the aggregated namespace' view of the directory.
-         */
-        if (prev->this != dht_last_up_subvol (this))
-                op_errno = 0;
+
+        gf_msg_debug(this->name, 0, "%s: Adding entry = %s", prev->name,
+                     entry->d_name);
+
+        list_add_tail(&entry->list, &entries.list);
+        count++;
+    }
 
 done:
-        if (count == 0) {
-                /* non-zero next_offset means that
-                   EOF is not yet hit on the current subvol
-                */
-                if (next_offset == 0) {
-                        next_subvol = dht_subvol_next (this, prev->this);
-                } else {
-                        next_subvol = prev->this;
-                }
 
-                if (!next_subvol) {
-                        goto unwind;
-                }
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     * Possible values:
+     * op_ret == 0 and op_errno != 0
+     *   if op_errno != ENOENT : Error.Unwind.
+     *   if op_errno == ENOENT : There are no more entries on this subvol.
+     *                           Move to the next one.
+     * op_ret > 0 and count == 0 :
+     *    The subvol returned entries to dht but all were stripped out.
+     *    For example, if they were linkto files or dirs where
+     *    hashed_subvol != prev. Try to get some entries by winding
+     *    to the next subvol. This can be dangerous if parallel readdir
+     *    is enabled as it grows the stack.
+     *
+     * op_ret > 0 and count > 0:
+     *   We found some entries. Unwind even if the buffer is not full.
+     *
+     */
+
+    op_ret = count;
+    if (count == 0) {
+        /* non-zero next_offset means that
+         * EOF is not yet hit on the current subvol
+         */
+        if ((next_offset == 0) || (op_errno == ENOENT)) {
+            next_offset = 0;
+            next_subvol = dht_subvol_next(this, prev);
+        } else {
+            next_subvol = prev;
+        }
 
-                STACK_WIND (frame, dht_readdirp_cbk,
-                            next_subvol, next_subvol->fops->readdirp,
-                            local->fd, local->size, next_offset,
-                            local->xattr);
-                return 0;
+        if (!next_subvol) {
+            goto unwind;
         }
 
+        if (conf->readdir_optimize == _gf_true) {
+            if (next_subvol != local->first_up_subvol) {
+                ret = dict_set_int32(local->xattr, GF_READDIR_SKIP_DIRS, 1);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary value"
+                           ":key = %s",
+                           GF_READDIR_SKIP_DIRS);
+            } else {
+                dict_del(local->xattr, GF_READDIR_SKIP_DIRS);
+            }
+        }
+
+        STACK_WIND_COOKIE(frame, dht_readdirp_cbk, next_subvol, next_subvol,
+                          next_subvol->fops->readdirp, local->fd, local->size,
+                          next_offset, local->xattr);
+        return 0;
+    }
+
 unwind:
-        if (op_ret < 0)
-                op_ret = 0;
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
+    if (op_ret < 0)
+        op_ret = 0;
+
+    if (prev != dht_last_up_subvol(this))
+        op_errno = 0;
+
+    DHT_STACK_UNWIND(readdirp, frame, op_ret, op_errno, &entries, NULL);
+
+    gf_dirent_free(&entries);
+    return 0;
+}
 
-        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries, NULL);
+static int
+dht_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *orig_entry = NULL;
+    gf_dirent_t *entry = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *next_subvol = NULL;
+    off_t next_offset = 0;
+    int count = 0;
+    dht_layout_t *layout = 0;
+    xlator_t *subvol = 0;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    gf_boolean_t skip_hashed_check = _gf_false;
 
-        gf_dirent_free (&entries);
+    INIT_LIST_HEAD(&entries.list);
 
-        return 0;
-}
+    prev = cookie;
+    local = frame->local;
 
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, done);
 
+    methods = &(conf->methods);
 
-int
-dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries,
-                 dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        gf_dirent_t   entries;
-        gf_dirent_t  *orig_entry = NULL;
-        gf_dirent_t  *entry = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *next_subvol = NULL;
-        off_t         next_offset = 0;
-        int           count = 0;
-        dht_layout_t *layout = 0;
-        xlator_t     *subvol = 0;
-
-        INIT_LIST_HEAD (&entries.list);
-        prev = cookie;
-        local = frame->local;
+    if (op_ret <= 0)
+        goto done;
 
-        if (op_ret < 0)
-                goto done;
+    if (!local->layout)
+        local->layout = dht_layout_get(this, local->fd->inode);
 
-        if (!local->layout)
-                local->layout = dht_layout_get (this, local->fd->inode);
+    layout = local->layout;
 
-        layout = local->layout;
+    gf_msg_debug(this->name, 0, "Processing entries from %s", prev->name);
 
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                next_offset = orig_entry->d_off;
+    if (conf->subvolume_cnt == 1) {
+        /*return everything*/
+        skip_hashed_check = _gf_true;
+        count = op_ret;
+        goto done;
+    }
 
-                subvol = dht_layout_search (this, layout, orig_entry->d_name);
+    list_for_each_entry(orig_entry, (&orig_entries->list), list)
+    {
+        next_offset = orig_entry->d_off;
 
-                if (!subvol || (subvol == prev->this)) {
-                        entry = gf_dirent_for_name (orig_entry->d_name);
-                        if (!entry) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "memory allocation failed :(");
-                                goto unwind;
-                        }
+        gf_msg_debug(this->name, 0, "%s: entry = %s, type = %d", prev->name,
+                     orig_entry->d_name, orig_entry->d_type);
 
-                        dht_itransform (this, prev->this, orig_entry->d_off,
-                                        &entry->d_off);
+        subvol = methods->layout_search(this, layout, orig_entry->d_name);
 
-                        entry->d_ino  = orig_entry->d_ino;
-                        entry->d_type = orig_entry->d_type;
-                        entry->d_len  = orig_entry->d_len;
+        if (!subvol || (subvol == prev)) {
+            entry = gf_dirent_for_name(orig_entry->d_name);
+            if (!entry) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                       "Memory allocation failed ");
+                goto unwind;
+            }
 
-                        list_add_tail (&entry->list, &entries.list);
-                        count++;
-                }
-        }
-        op_ret = count;
-        /* We need to ensure that only the last subvolume's end-of-directory
-         * notification is respected so that directory reading does not stop
-         * before all subvolumes have been read. That could happen because the
-         * posix for each subvolume sends a ENOENT on end-of-directory but in
-         * distribute we're not concerned only with a posix's view of the
-         * directory but the aggregated namespace' view of the directory.
-         */
-        if (prev->this != dht_last_up_subvol (this))
-                op_errno = 0;
+            entry->d_off = orig_entry->d_off;
+            entry->d_ino = orig_entry->d_ino;
+            entry->d_type = orig_entry->d_type;
+            entry->d_len = orig_entry->d_len;
 
+            gf_msg_debug(this->name, 0, "%s: Adding = entry %s", prev->name,
+                         entry->d_name);
+
+            list_add_tail(&entry->list, &entries.list);
+            count++;
+        }
+    }
 done:
-        if (count == 0) {
-                /* non-zero next_offset means that
-                   EOF is not yet hit on the current subvol
-                */
-                if (next_offset == 0) {
-                        next_subvol = dht_subvol_next (this, prev->this);
+    op_ret = count;
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
+    if (count == 0) {
+        if ((next_offset == 0) || (op_errno == ENOENT)) {
+            next_offset = 0;
+            next_subvol = dht_subvol_next(this, prev);
+        } else {
+            next_subvol = prev;
+        }
+
+        if (!next_subvol) {
+            goto unwind;
+        }
+
+        STACK_WIND_COOKIE(frame, dht_readdir_cbk, next_subvol, next_subvol,
+                          next_subvol->fops->readdir, local->fd, local->size,
+                          next_offset, NULL);
+        return 0;
+    }
+
+unwind:
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
+
+    if (prev != dht_last_up_subvol(this))
+        op_errno = 0;
+
+    if (!skip_hashed_check) {
+        DHT_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, NULL);
+        gf_dirent_free(&entries);
+
+    } else {
+        DHT_STACK_UNWIND(readdir, frame, op_ret, op_errno, orig_entries, NULL);
+    }
+    return 0;
+}
+
+static int
+dht_do_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t yoff, int whichop, dict_t *dict)
+{
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    xlator_t *xvol = NULL;
+    int ret = 0;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, NULL, NULL, whichop);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->fd = fd_ref(fd);
+    local->size = size;
+    local->xattr_req = (dict) ? dict_ref(dict) : NULL;
+    local->first_up_subvol = dht_first_up_subvol(this);
+    local->op_ret = -1;
+
+    dht_deitransform(this, yoff, &xvol);
+
+    /* TODO: do proper readdir */
+    if (whichop == GF_FOP_READDIRP) {
+        if (dict)
+            local->xattr = dict_ref(dict);
+        else
+            local->xattr = dict_new();
+
+        if (local->xattr) {
+            ret = dict_set_uint32(local->xattr, conf->link_xattr_name, 256);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                       "Failed to set dictionary value"
+                       " : key = %s",
+                       conf->link_xattr_name);
+
+            if (conf->readdir_optimize == _gf_true) {
+                if (xvol != local->first_up_subvol) {
+                    ret = dict_set_int32(local->xattr, GF_READDIR_SKIP_DIRS, 1);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               DHT_MSG_DICT_SET_FAILED,
+                               "Failed to set "
+                               "dictionary value: "
+                               "key = %s",
+                               GF_READDIR_SKIP_DIRS);
                 } else {
-                        next_subvol = prev->this;
+                    dict_del(local->xattr, GF_READDIR_SKIP_DIRS);
                 }
-
-                if (!next_subvol) {
-                        goto unwind;
+            }
+
+            if (conf->subvolume_cnt == 1) {
+                ret = dict_set_uint32(local->xattr, conf->xattr_name, 4 * 4);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+                           DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary "
+                           "value:key = %s ",
+                           conf->xattr_name);
                 }
-
-                STACK_WIND (frame, dht_readdir_cbk,
-                            next_subvol, next_subvol->fops->readdir,
-                            local->fd, local->size, next_offset, NULL);
-                return 0;
+            }
         }
 
-unwind:
-        if (op_ret < 0)
-                op_ret = 0;
+        STACK_WIND_COOKIE(frame, dht_readdirp_cbk, xvol, xvol,
+                          xvol->fops->readdirp, fd, size, yoff, local->xattr);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_readdir_cbk, xvol, xvol,
+                          xvol->fops->readdir, fd, size, yoff, local->xattr);
+    }
 
-        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, NULL);
+    return 0;
 
-        gf_dirent_free (&entries);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t yoff, int whichop, dict_t *dict)
-{
-        dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-        xlator_t     *xvol = NULL;
-        off_t         xoff = 0;
-        int           ret = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, NULL, whichop);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+dht_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t yoff, dict_t *xdata)
+{
+    int op = GF_FOP_READDIR;
+    dht_conf_t *conf = NULL;
+    int i = 0;
 
-        local->fd = fd_ref (fd);
-        local->size = size;
-        local->xattr_req = (dict)? dict_ref (dict) : NULL;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        dht_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            op = GF_FOP_READDIRP;
+            break;
+        }
+    }
 
-        /* TODO: do proper readdir */
-        if (whichop == GF_FOP_READDIRP) {
-                if (dict)
-                        local->xattr = dict_ref (dict);
-                else
-                        local->xattr = dict_new ();
+    if (conf->use_readdirp)
+        op = GF_FOP_READDIRP;
 
-                if (local->xattr) {
-                        ret = dict_set_uint32 (local->xattr,
-                                               "trusted.glusterfs.dht.linkto",
-                                               256);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to set 'glusterfs.dht.linkto'"
-                                        " key");
-                }
+out:
+    dht_do_readdir(frame, this, fd, size, yoff, op, 0);
+    return 0;
+}
 
-                STACK_WIND (frame, dht_readdirp_cbk, xvol, xvol->fops->readdirp,
-                            fd, size, xoff, local->xattr);
-        } else {
-                STACK_WIND (frame, dht_readdir_cbk, xvol, xvol->fops->readdir,
-                            fd, size, xoff, local->xattr);
-        }
+int
+dht_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t yoff, dict_t *dict)
+{
+    dht_do_readdir(frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
+    return 0;
+}
 
-        return 0;
+static int
+dht_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+    local = frame->local;
 
-        return 0;
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1)
+            local->op_errno = op_errno;
+        else if (op_ret == 0)
+            local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(fsyncdir, frame, local->op_ret, local->op_errno,
+                         xdata);
+
+    return 0;
 }
 
-
 int
-dht_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff, dict_t *xdata)
+dht_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+             dict_t *xdata)
 {
-        int          op = GF_FOP_READDIR;
-        dht_conf_t  *conf = NULL;
-        int          i = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->subvolume_status[i]) {
-                        op = GF_FOP_READDIRP;
-                        break;
-                }
-        }
+    conf = this->private;
 
-        if (conf->use_readdirp)
-                op = GF_FOP_READDIRP;
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_FSYNCDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-out:
-        dht_do_readdir (frame, this, fd, size, yoff, op, 0);
-        return 0;
+    local->fd = fd_ref(fd);
+    local->call_cnt = conf->subvolume_cnt;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND(frame, dht_fsyncdir_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->fsyncdir, fd, datasync, xdata);
+    }
+
+    return 0;
+
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
+
+    return 0;
 }
 
 int
-dht_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t yoff, dict_t *dict)
+dht_newfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        dht_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
-        return 0;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    if (op_ret == -1)
+        goto out;
+
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    prev = cookie;
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret < 0) {
+        gf_msg_debug(this->name, EINVAL,
+                     "could not set pre-set layout for subvolume %s",
+                     prev ? prev->name : NULL);
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+    if (local->linked == _gf_true)
+        dht_linkfile_attr_heal(frame, this);
+out:
+    /*
+     * FIXME: ia_size and st_blocks of preparent and postparent do not have
+     * correct values. since, preparent and postparent buffers correspond
+     * to a directory these two members should have values equal to sum of
+     * corresponding values from each of the subvolume.
+     * See dht_iatt_merge for reference.
+     */
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    dht_set_fixed_dir_stat(preparent);
+
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        /* store op_errno for failure case*/
+        local->op_errno = op_errno;
+        local->refresh_layout_unlock(frame, this, op_ret, 1);
+
+        if (op_ret == 0) {
+            DHT_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, stbuf,
+                             preparent, postparent, xdata);
+        }
+    } else {
+        DHT_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, stbuf,
+                         preparent, postparent, xdata);
+    }
+
+    return 0;
 }
 
+static int
+dht_mknod_linkfile_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, inode_t *inode,
+                              struct iatt *stbuf, struct iatt *preparent,
+                              struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
 
+    local = frame->local;
 
-int
-dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, dict_t *xdata)
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    conf = this->private;
+    if (!conf) {
+        local->op_errno = EINVAL;
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    cached_subvol = local->cached_subvol;
+
+    if (local->params) {
+        dict_del(local->params, conf->link_xattr_name);
+        dict_del(local->params, GLUSTERFS_INTERNAL_FOP_KEY);
+    }
+
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)cached_subvol,
+                      cached_subvol, cached_subvol->fops->mknod, &local->loc,
+                      local->mode, local->rdev, local->umask, local->params);
+
+    return 0;
+err:
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    } else {
+        DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                         NULL);
+    }
+    return 0;
+}
+
+static int
+dht_mknod_wind_to_avail_subvol(call_frame_t *frame, xlator_t *this,
+                               xlator_t *subvol, loc_t *loc, dev_t rdev,
+                               mode_t mode, mode_t umask, dict_t *params)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    xlator_t *avail_subvol = NULL;
 
+    local = frame->local;
 
-        local = frame->local;
+    if (!dht_is_subvol_filled(this, subvol)) {
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1)
-                        local->op_errno = op_errno;
+        STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                          subvol->fops->mknod, loc, mode, rdev, umask, params);
+    } else {
+        avail_subvol = dht_free_disk_available_subvol(this, subvol, local);
 
-                if (op_ret == 0)
-                        local->op_ret = 0;
+        if (avail_subvol != subvol) {
+            local->params = dict_ref(params);
+            local->rdev = rdev;
+            local->mode = mode;
+            local->umask = umask;
+            local->cached_subvol = avail_subvol;
+            local->hashed_subvol = subvol;
+
+            gf_msg_debug(this->name, 0, "creating %s on %s (link at %s)",
+                         loc->path, avail_subvol->name, subvol->name);
+
+            dht_linkfile_create(frame, dht_mknod_linkfile_create_cbk, this,
+                                avail_subvol, subvol, loc);
+
+            goto out;
         }
-        UNLOCK (&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                  local->op_errno, xdata);
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
+
+        STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                          subvol->fops->mknod, loc, mode, rdev, umask, params);
+    }
+out:
+    return 0;
+}
+
+static int32_t
+dht_mknod_do(call_frame_t *frame)
+{
+    dht_local_t *local = NULL;
+    dht_layout_t *refreshed = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+
+    local = frame->local;
+
+    this = THIS;
+
+    conf = this->private;
+
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
+
+    methods = &(conf->methods);
+
+    /* We don't need parent_loc anymore */
+    loc_wipe(&local->loc);
+
+    loc_copy(&local->loc, &local->loc2);
+
+    loc_wipe(&local->loc2);
 
+    refreshed = local->selfheal.refreshed_layout;
+
+    subvol = methods->layout_search(this, refreshed, local->loc.name);
+
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in "
+               "layout for path=%s",
+               local->loc.path);
+        local->op_errno = ENOENT;
+        goto err;
+    }
+
+    dht_mknod_wind_to_avail_subvol(frame, this, subvol, &local->loc,
+                                   local->rdev, local->mode, local->umask,
+                                   local->params);
+    return 0;
+err:
+    local->refresh_layout_unlock(frame, this, -1, 1);
+
+    return 0;
+}
+
+static int32_t
+dht_mknod_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
+
+static int32_t
+dht_mknod_finish(call_frame_t *frame, xlator_t *this, int op_ret,
+                 int invoke_cbk)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
+
+    local = frame->local;
+    lock_count = dht_lock_count(local->lock[0].layout.parent_layout.locks,
+                                local->lock[0].layout.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
+
+    lock_local->lock[0]
+        .layout.parent_layout.locks = local->lock[0].layout.parent_layout.locks;
+    lock_local->lock[0].layout.parent_layout.lk_count =
+        local->lock[0].layout.parent_layout.lk_count;
+
+    local->lock[0].layout.parent_layout.locks = NULL;
+    local->lock[0].layout.parent_layout.lk_count = 0;
+
+    dht_unlock_inodelk(lock_frame,
+                       lock_local->lock[0].layout.parent_layout.locks,
+                       lock_local->lock[0].layout.parent_layout.lk_count,
+                       dht_mknod_unlock_cbk);
+    lock_frame = NULL;
+
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
+
+    if (op_ret == 0)
         return 0;
+
+    DHT_STACK_UNWIND(mknod, frame, op_ret, local->op_errno, NULL, NULL, NULL,
+                     NULL, NULL);
+    return 0;
 }
 
+static int32_t
+dht_mknod_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
 
-int
-dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int datasync, dict_t *xdata)
+    local = frame->local;
+
+    if (!local) {
+        goto err;
+    }
+
+    if (op_ret < 0) {
+        gf_msg("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+               "mknod lock failed for file: %s", local->loc2.name);
+
+        local->op_errno = op_errno;
+
+        goto err;
+    }
+
+    local->refresh_layout_unlock = dht_mknod_finish;
+
+    local->refresh_layout_done = dht_mknod_do;
+
+    dht_refresh_layout(frame);
+
+    return 0;
+err:
+    if (local)
+        dht_mknod_finish(frame, this, -1, 0);
+    else
+        DHT_STACK_UNWIND(mknod, frame, -1, EINVAL, NULL, NULL, NULL, NULL,
+                         NULL);
+    return 0;
+}
+
+static int32_t
+dht_mknod_lock(call_frame_t *frame, xlator_t *subvol)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
 
-        conf = this->private;
+    local = frame->local;
 
-        local = dht_local_init (frame, NULL, NULL, GF_FOP_FSYNCDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
 
-        local->fd = fd_ref (fd);
-        local->call_cnt = conf->subvolume_cnt;
+    if (lk_array == NULL)
+        goto err;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_fsyncdir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->fsyncdir,
-                            fd, datasync, xdata);
-        }
+    lk_array[0] = dht_lock_new(frame->this, subvol, &local->loc, F_RDLCK,
+                               DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                               IGNORE_ENOENT_ESTALE);
 
-        return 0;
+    if (lk_array[0] == NULL)
+        goto err;
+
+    local->lock[0].layout.parent_layout.locks = lk_array;
+    local->lock[0].layout.parent_layout.lk_count = count;
 
+    ret = dht_blocking_inodelk(frame, lk_array, count, dht_mknod_lock_cbk);
+
+    if (ret < 0) {
+        local->lock[0].layout.parent_layout.locks = NULL;
+        local->lock[0].layout.parent_layout.lk_count = 0;
+        goto err;
+    }
+
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        return 0;
+    return -1;
 }
 
+static int
+dht_refresh_parent_layout_resume(call_frame_t *frame, xlator_t *this, int ret,
+                                 int invoke_cbk)
+{
+    dht_local_t *local = NULL, *parent_local = NULL;
+    call_stub_t *stub = NULL;
+    call_frame_t *parent_frame = NULL;
 
-int
-dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno,
-                 inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+    local = frame->local;
+
+    stub = local->stub;
+    local->stub = NULL;
+
+    parent_frame = stub->frame;
+    parent_local = parent_frame->local;
+
+    if (ret < 0) {
+        parent_local->op_ret = -1;
+        parent_local->op_errno = local->op_errno ? local->op_errno : EIO;
+    } else {
+        parent_local->op_ret = 0;
+    }
+
+    call_resume(stub);
+
+    DHT_STACK_DESTROY(frame);
+
+    return 0;
+}
+
+static int
+dht_refresh_parent_layout_done(call_frame_t *frame)
 {
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
+    int ret = 0;
 
+    local = frame->local;
 
-        if (op_ret == -1)
-                goto out;
+    if (local->op_ret < 0) {
+        ret = -1;
+        goto resume;
+    }
 
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    dht_layout_set(frame->this, local->loc.inode,
+                   local->selfheal.refreshed_layout);
 
-        prev = cookie;
+resume:
+    dht_refresh_parent_layout_resume(frame, frame->this, ret, 1);
+    return 0;
+}
 
-        if (local->loc.parent) {
-                WIPE (preparent);
-                WIPE (postparent);
-        }
+static int
+dht_handle_parent_layout_change(xlator_t *this, call_stub_t *stub)
+{
+    call_frame_t *refresh_frame = NULL, *frame = NULL;
+    dht_local_t *refresh_local = NULL, *local = NULL;
 
-        ret = dht_layout_preset (this, prev->this, inode);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "could not set pre-set layout for subvolume %s",
-                        prev->this->name);
-                op_ret   = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-out:
-        /*
-         * FIXME: ia_size and st_blocks of preparent and postparent do not have
-         * correct values. since, preparent and postparent buffers correspond
-         * to a directory these two members should have values equal to sum of
-         * corresponding values from each of the subvolume.
-         * See dht_iatt_merge for reference.
-         */
+    frame = stub->frame;
+    local = frame->local;
 
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf,
-                          preparent, postparent, xdata);
-        return 0;
+    refresh_frame = copy_frame(frame);
+    if (!refresh_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "mem allocation failed for refresh_frame");
+        return -1;
+    }
+
+    refresh_local = dht_local_init(refresh_frame, NULL, NULL, stub->fop);
+    if (!refresh_local) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "mem allocation failed for refresh_local");
+        return -1;
+    }
+
+    refresh_local->loc.inode = inode_ref(local->loc.parent);
+    gf_uuid_copy(refresh_local->loc.gfid, local->loc.parent->gfid);
+
+    refresh_local->stub = stub;
+
+    refresh_local->refresh_layout_unlock = dht_refresh_parent_layout_resume;
+    refresh_local->refresh_layout_done = dht_refresh_parent_layout_done;
+
+    dht_refresh_layout(refresh_frame);
+    return 0;
 }
 
-int
-dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this,
-                               int32_t op_ret, int32_t op_errno,
-                               inode_t *inode, struct iatt *stbuf,
-                               struct iatt *preparent, struct iatt *postparent,
-                               dict_t *xdata)
+static int32_t
+dht_call_mkdir_stub(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *cached_subvol = NULL;
+    dht_local_t *local = NULL;
+    call_stub_t *stub = NULL;
 
-        if (op_ret == -1)
-                goto err;
+    local = frame->local;
+    stub = local->stub;
+    local->stub = NULL;
 
-        local = frame->local;
-        cached_subvol = local->cached_subvol;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    } else {
+        local->op_ret = 0;
+    }
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    cached_subvol, cached_subvol->fops->mknod,
-                    &local->loc, local->mode, local->rdev, local->umask,
-                    local->params);
+    call_resume(stub);
 
-        return 0;
+    return 0;
+}
+
+static int32_t
+dht_guard_parent_layout_and_namespace(xlator_t *subvol, call_stub_t *stub)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    loc_t *loc = NULL;
+    xlator_t *hashed_subvol = NULL, *this = NULL;
+    ;
+    call_frame_t *frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t *parent_disk_layout = NULL;
+    dht_layout_t *parent_layout = NULL;
+    dht_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", stub, err);
+
+    frame = stub->frame;
+    this = frame->this;
+
+    conf = this->private;
+
+    local = frame->local;
+
+    local->stub = stub;
+
+    /* TODO: recheck whether we should lock on src or dst if we do similar
+     * stale layout checks for rename.
+     */
+    loc = &stub->args.loc;
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    if (local->params == NULL) {
+        local->params = dict_new();
+        if (local->params == NULL) {
+            local->op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+                   DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "%s (%s/%s) (path: %s): "
+                   "dict allocation failed",
+                   gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+            goto err;
+        }
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        local->op_errno = EINVAL;
+
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "hashed subvolume not found",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_layout = dht_layout_get(this, loc->parent);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        local->op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    memcpy((void *)local->parent_disk_layout, (void *)parent_disk_layout,
+           sizeof(local->parent_disk_layout));
+
+    dht_layout_unref(this, parent_layout);
+    parent_layout = NULL;
+
+    ret = dict_set_str(local->params, GF_PREOP_PARENT_KEY, conf->xattr_name);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path,
+               GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(local->params, conf->xattr_name, parent_disk_layout,
+                       4 * 4);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting parent-layout in params dictionary failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_disk_layout = NULL;
+    local->hashed_subvol = hashed_subvol;
+
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, loc, hashed_subvol, &local->current->ns,
+                                dht_call_mkdir_stub);
+    if (ret < 0)
+        goto err;
+
+    return 0;
 err:
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL);
-        return 0;
+
+    if (parent_disk_layout != NULL)
+        GF_FREE(parent_disk_layout);
+
+    if (parent_layout != NULL)
+        dht_layout_unref(this, parent_layout);
+
+    return -1;
 }
 
 int
-dht_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
+dht_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *params)
 {
-        xlator_t    *subvol = NULL;
-        int          op_errno = -1;
-        xlator_t    *avail_subvol = NULL;
-        dht_local_t *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int ret = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    /* Post remove-brick, the client layout may not be in sync with
+     * disk layout because of lack of lookup. Hence,a mknod call
+     * may fall on the decommissioned brick.  Hence, if the
+     * hashed_subvol is part of decommissioned bricks  list, do a
+     * lookup on parent dir. If a fix-layout is already done by the
+     * remove-brick process, the parent directory layout will be in
+     * sync with that of the disk. If fix-layout is still ending
+     * on the parent directory, we can let the file get created on
+     * the decommissioned brick which will be eventually migrated to
+     * non-decommissioned brick based on the new layout.
+     */
+
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == subvol) {
+                gf_msg_debug(this->name, 0,
+                             "hashed subvol:%s is "
+                             "part of decommission brick list for "
+                             "file: %s",
+                             subvol->name, loc->path);
+
+                /* dht_refresh_layout needs directory info in
+                 * local->loc. Hence, storing the parent_loc in
+                 * local->loc and storing the create context in
+                 * local->loc2. We will restore this information
+                 * in dht_creation do */
+
+                ret = loc_copy(&local->loc2, &local->loc);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                           "loc_copy failed %s", loc->path);
+
+                    goto err;
+                }
 
-        dht_get_du_info (frame, this, loc);
+                local->params = dict_ref(params);
+                local->rdev = rdev;
+                local->mode = mode;
+                local->umask = umask;
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+                loc_wipe(&local->loc);
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume in layout for path=%s",
-                        loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+                ret = dht_build_parent_loc(this, &local->loc, loc, &op_errno);
 
-        if (!dht_is_subvol_filled (this, subvol)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "creating %s on %s", loc->path, subvol->name);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                           "parent loc build failed");
+                    goto err;
+                }
 
-                STACK_WIND (frame, dht_newfile_cbk,
-                            subvol, subvol->fops->mknod,
-                            loc, mode, rdev, umask, params);
-        } else {
-                avail_subvol = dht_free_disk_available_subvol (this, subvol);
-                if (avail_subvol != subvol) {
-                        /* Choose the minimum filled volume, and create the
-                           files there */
-
-                        local->params = dict_ref (params);
-                        local->cached_subvol = avail_subvol;
-                        local->mode = mode;
-                        local->rdev = rdev;
-                        local->umask = umask;
-                        dht_linkfile_create (frame,
-                                             dht_mknod_linkfile_create_cbk,
-                                             avail_subvol, subvol, loc);
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "creating %s on %s", loc->path, subvol->name);
+                ret = dht_mknod_lock(frame, subvol);
 
-                        STACK_WIND (frame, dht_newfile_cbk,
-                                    subvol, subvol->fops->mknod,
-                                    loc, mode, rdev, umask, params);
+                if (ret < 0) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                           "locking parent failed");
+                    goto err;
                 }
+
+                goto done;
+            }
         }
+    }
 
-        return 0;
+    dht_mknod_wind_to_avail_subvol(frame, this, subvol, loc, rdev, mode, umask,
+                                   params);
+
+done:
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+int
+dht_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+            loc_t *loc, mode_t umask, dict_t *params)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SYMLINK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
+
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                      subvol->fops->symlink, linkname, loc, umask, params);
+
+    return 0;
+
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+
+    return 0;
+}
 
 int
-dht_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkname, loc_t *loc, mode_t umask, dict_t *params)
+dht_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
 {
-        xlator_t    *subvol = NULL;
-        int          op_errno = -1;
-        dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_UNLINK);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+
+    cached_subvol = local->cached_subvol;
+    if (!cached_subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local->flags = xflag;
+    STACK_WIND_COOKIE(frame, dht_unlink_cbk, cached_subvol, cached_subvol,
+                      cached_subvol->fops->unlink, loc, xflag, xdata);
+
+    return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    return 0;
+}
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SYMLINK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+static int
+dht_remove_stale_linkto_cbk(int ret, call_frame_t *sync_frame, void *data)
+{
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
+}
+
+static int
+dht_remove_stale_linkto(void *data)
+{
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata_in = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", data, out);
+
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    GF_VALIDATE_OR_GOTO("dht", local->link_subvol, out);
+
+    xdata_in = dict_new();
+    if (!xdata_in)
+        goto out;
+
+    ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(xdata_in);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, 0,
+               "Failed to set keys for stale linkto"
+               "deletion on path %s",
+               local->loc.path);
+        goto out;
+    }
+
+    ret = syncop_unlink(local->link_subvol, &local->loc, xdata_in, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, 0,
+               "Removal of linkto failed"
+               " on path %s at subvol %s",
+               local->loc.path, local->link_subvol->name);
+    }
+out:
+    if (xdata_in)
+        dict_unref(xdata_in);
+    return ret;
+}
 
-        subvol = dht_subvol_get_hashed (this, loc);
+static int
+dht_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, inode_t *inode, struct iatt *stbuf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    gf_boolean_t stbuf_merged = _gf_false;
+    xlator_t *subvol = NULL;
+    call_frame_t *cleanup_frame = NULL;
+    dht_local_t *cleanup_local = NULL;
+
+    local = frame->local;
+
+    if (op_ret == -1) {
+        /* Remove the linkto if exists */
+        if (local->linked) {
+            cleanup_frame = create_frame(this, this->ctx->pool);
+            if (cleanup_frame) {
+                cleanup_local = dht_local_init(cleanup_frame, &local->loc2,
+                                               NULL, 0);
+                if (!cleanup_local || !local->link_subvol) {
+                    DHT_STACK_DESTROY(cleanup_frame);
+                    goto out;
+                }
+                cleanup_local->link_subvol = local->link_subvol;
+                FRAME_SU_DO(cleanup_frame, dht_local_t);
+                ret = synctask_new(this->ctx->env, dht_remove_stale_linkto,
+                                   dht_remove_stale_linkto_cbk, cleanup_frame,
+                                   cleanup_frame);
+            }
+        }
+        /* No continuation on DHT inode missing errors, as we should
+         * then have a good stbuf that states P2 happened. We would
+         * get inode missing if, the file completed migrated between
+         * the lookup and the link call */
+        goto out;
+    }
+
+    /* Update parent on success, even if P1/2 checks are positive.
+     * The second call on success will further update the parent */
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    /* Update linkto attrs, if this is the first call and non-P2,
+     * if we detect P2 then we need to trust the attrs from the
+     * second call, not the first */
+    if (local->linked == _gf_true &&
+        ((local->call_cnt == 1 && !IS_DHT_MIGRATION_PHASE2(stbuf)) ||
+         (local->call_cnt != 1 && IS_DHT_MIGRATION_PHASE2(&local->stbuf)))) {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        stbuf_merged = _gf_true;
+        dht_linkfile_attr_heal(frame, this);
+    }
+
+    /* No further P1/2 checks if we are in the second iteration of
+     * the call */
+    if (local->call_cnt != 1) {
+        goto out;
+    } else {
+        /* Preserve the return values, in case the migration decides
+         * to recreate the link on the same subvol that the current
+         * hased for the link was created on. */
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+        if (!stbuf_merged) {
+            dht_iatt_merge(this, &local->stbuf, stbuf);
+            stbuf_merged = _gf_true;
+        }
+
+        local->inode = inode_ref(inode);
+    }
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    local->rebalance.target_op_fn = dht_link2;
+    dht_set_local_rebalance(this, local, stbuf, preparent, postparent, xdata);
+
+    /* Check if the rebalance phase2 is true */
+    if (IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_inode_ctx_get_mig_info(this, local->loc.inode, NULL, &subvol);
         if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume in layout for path=%s",
-                        loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+            /* Phase 2 of migration */
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            dht_link2(this, subvol, frame, 0);
+            return 0;
+        }
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        ret = dht_inode_ctx_get_mig_info(this, local->loc.inode, NULL, &subvol);
+        if (subvol) {
+            dht_link2(this, subvol, frame, 0);
+            return 0;
+        }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+out:
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "creating %s on %s", loc->path, subvol->name);
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, NULL);
+
+    return 0;
+}
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->symlink,
-                    linkname, loc, umask, params);
+static int
+dht_link2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+{
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    if (!local)
+        goto err;
+
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        dht_set_fixed_dir_stat(&local->preparent);
+        dht_set_fixed_dir_stat(&local->postparent);
+
+        DHT_STACK_UNWIND(link, frame, local->op_ret, op_errno, local->inode,
+                         &local->stbuf, &local->preparent, &local->postparent,
+                         NULL);
+        return 0;
+    }
+
+    if (subvol == NULL) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Second call to create link file could result in EEXIST as the
+     * first call created the linkto in the currently
+     * migrating subvol, which could be the new hashed subvol */
+    if (local->link_subvol == subvol) {
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->preparent);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(link, frame, 0, 0, local->inode, &local->stbuf,
+                         &local->preparent, &local->postparent, NULL);
 
         return 0;
+    }
 
+    local->call_cnt = 2;
+
+    STACK_WIND(frame, dht_link_cbk, subvol, subvol->fops->link, &local->loc,
+               &local->loc2, local->xattr_req);
+
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-            dict_t *xdata)
+static int
+dht_link_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        xlator_t    *cached_subvol = NULL;
-        xlator_t    *hashed_subvol = NULL;
-        int          op_errno = -1;
-        dht_local_t *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        if (dht_filter_loc_subvol_key (this, loc, &local->loc,
-                                       &cached_subvol)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "unlinking %s on %s (given path %s)",
-                        local->loc.path, cached_subvol->name, loc->path);
-                STACK_WIND (frame, dht_unlink_cbk,
-                            cached_subvol, cached_subvol->fops->unlink,
-                            &local->loc, xflag, xdata);
-                goto done;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *srcvol = NULL;
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_UNLINK);
-        if (!local) {
-                op_errno = ENOMEM;
+    if (op_ret == -1)
+        goto err;
 
-                goto err;
-        }
+    local = frame->local;
+    srcvol = local->linkfile.srcvol;
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (!hashed_subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume in layout for path=%s",
-                        loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    STACK_WIND(frame, dht_link_cbk, srcvol, srcvol->fops->link, &local->loc,
+               &local->loc2, local->xattr_req);
 
-        cached_subvol = local->cached_subvol;
-        if (!cached_subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    return 0;
 
-        local->flags = xflag;
-        if (hashed_subvol != cached_subvol) {
-                STACK_WIND (frame, dht_unlink_linkfile_cbk,
-                            hashed_subvol, hashed_subvol->fops->unlink, loc,
-                            xflag, xdata);
-        } else {
-                STACK_WIND (frame, dht_unlink_cbk,
-                            cached_subvol, cached_subvol->fops->unlink, loc,
-                            xflag, xdata);
-        }
-done:
-        return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
+int
+dht_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
+{
+    xlator_t *cached_subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    int op_errno = -1;
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(oldloc, err);
+    VALIDATE_OR_GOTO(newloc, err);
+
+    local = dht_local_init(frame, oldloc, NULL, GF_FOP_LINK);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+    local->call_cnt = 1;
+
+    cached_subvol = local->cached_subvol;
+    if (!cached_subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     oldloc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, newloc);
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     newloc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    ret = loc_copy(&local->loc2, newloc);
+    if (ret == -1) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (hashed_subvol != cached_subvol) {
+        gf_uuid_copy(local->gfid, oldloc->inode->gfid);
+        dht_linkfile_create(frame, dht_link_linkfile_cbk, this, cached_subvol,
+                            hashed_subvol, newloc);
+    } else {
+        STACK_WIND(frame, dht_link_cbk, cached_subvol,
+                   cached_subvol->fops->link, oldloc, newloc, xdata);
+    }
+
+    return 0;
+
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+
+    return 0;
+}
 
 int
-dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
+dht_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        call_frame_t *prev = NULL;
-        dht_layout_t *layout = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    gf_boolean_t parent_layout_changed = _gf_false;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        parent_layout_changed = (xdata &&
+                                 dict_get(xdata, GF_PREOP_CHECK_FAILED))
+                                    ? _gf_true
+                                    : _gf_false;
+
+        if (parent_layout_changed) {
+            if (local && local->lock[0].layout.parent_layout.locks) {
+                /* Returning failure as the layout could not be fixed even under
+                 * the lock */
+                goto out;
+            }
 
-        prev = cookie;
+            gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "create (%s/%s) (path: %s): parent layout "
+                   "changed. Attempting a layout refresh and then a "
+                   "retry",
+                   pgfid, local->loc.name, local->loc.path);
+
+            /*
+              dht_refresh_layout needs directory info in local->loc.Hence,
+              storing the parent_loc in local->loc and storing the create
+              context in local->loc2. We will restore this information in
+              dht_creation_do.
+             */
+
+            loc_wipe(&local->loc2);
+
+            ret = loc_copy(&local->loc2, &local->loc);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                       "loc_copy failed %s", local->loc.path);
 
-        if (op_ret == -1)
                 goto out;
+            }
 
-        layout = dht_layout_for_subvol (this, prev->this);
-        if (!layout) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no pre-set layout for subvolume %s",
-                        prev->this->name);
-                op_ret   = -1;
-                op_errno = EINVAL;
+            loc_wipe(&local->loc);
+
+            ret = dht_build_parent_loc(this, &local->loc, &local->loc2,
+                                       &op_errno);
+
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                       "parent loc build failed");
                 goto out;
+            }
+
+            subvol = dht_subvol_get_hashed(this, &local->loc2);
+
+            ret = dht_create_lock(frame, subvol);
+            if (ret < 0) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                       "locking parent failed");
+                goto out;
+            }
+
+            return 0;
         }
 
-        WIPE (preparent);
-        WIPE (postparent);
+        goto out;
+    }
 
-out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent, NULL);
+    prev = cookie;
 
-        return 0;
-}
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
 
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
 
-int
-dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       inode_t *inode, struct iatt *stbuf,
-                       struct iatt *preparent, struct iatt *postparent,
-                       dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        xlator_t     *srcvol = NULL;
+    ret = dht_fd_ctx_set(this, fd, prev);
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0,
+                     "Possible fd leak. "
+                     "Could not set fd ctx for subvol %s",
+                     prev->name);
+    }
 
-        if (op_ret == -1)
-                goto err;
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0, "could not set preset layout for subvol %s",
+                     prev->name);
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        local = frame->local;
-        srcvol = local->linkfile.srcvol;
+    local->op_errno = op_errno;
 
-        STACK_WIND (frame, dht_link_cbk, srcvol, srcvol->fops->link,
-                    &local->loc, &local->loc2, xdata);
+    if (local->linked == _gf_true) {
+        local->stbuf = *stbuf;
+        dht_linkfile_attr_heal(frame, this);
+    }
+out:
 
-        return 0;
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
 
-err:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent, NULL);
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        /* store op_errno for failure case*/
+        local->op_errno = op_errno;
+        local->refresh_layout_unlock(frame, this, op_ret, 1);
 
-        return 0;
+        if (op_ret == 0) {
+            DHT_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                             preparent, postparent, xdata);
+        }
+    } else {
+        DHT_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                         preparent, postparent, xdata);
+    }
+    return 0;
 }
 
+static int
+dht_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *stbuf,
+                               struct iatt *preparent, struct iatt *postparent,
+                               dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
+
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    conf = this->private;
+    if (!conf) {
+        local->op_errno = EINVAL;
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    cached_subvol = local->cached_subvol;
+
+    if (local->params) {
+        dict_del(local->params, conf->link_xattr_name);
+        dict_del(local->params, GLUSTERFS_INTERNAL_FOP_KEY);
+    }
+
+    STACK_WIND_COOKIE(frame, dht_create_cbk, cached_subvol, cached_subvol,
+                      cached_subvol->fops->create, &local->loc, local->flags,
+                      local->mode, local->umask, local->fd, local->params);
+
+    return 0;
+err:
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    } else {
+        DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                         NULL, NULL);
+    }
+    return 0;
+}
 
-int
-dht_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+static int
+dht_create_wind_to_avail_subvol(call_frame_t *frame, xlator_t *this,
+                                xlator_t *subvol, loc_t *loc, int32_t flags,
+                                mode_t mode, mode_t umask, fd_t *fd,
+                                dict_t *params)
 {
-        xlator_t    *cached_subvol = NULL;
-        xlator_t    *hashed_subvol = NULL;
-        int          op_errno = -1;
-        int          ret = -1;
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *avail_subvol = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (newloc, err);
+    local = frame->local;
 
-        local = dht_local_init (frame, oldloc, NULL, GF_FOP_LINK);
-        if (!local) {
-                op_errno = ENOMEM;
+    if (!dht_is_subvol_filled(this, subvol)) {
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-                goto err;
-        }
+        dht_set_parent_layout_in_dict(loc, this, local);
 
-        cached_subvol = local->cached_subvol;
-        if (!cached_subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s", oldloc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, loc, flags, mode, umask, fd,
+                          params);
 
-        hashed_subvol = dht_subvol_get_hashed (this, newloc);
-        if (!hashed_subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume in layout for path=%s",
-                        newloc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    } else {
+        avail_subvol = dht_free_disk_available_subvol(this, subvol, local);
 
-        ret = loc_copy (&local->loc2, newloc);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+        if (avail_subvol != subvol) {
+            local->cached_subvol = avail_subvol;
+            local->hashed_subvol = subvol;
 
-        if (hashed_subvol != cached_subvol) {
-                uuid_copy (local->gfid, oldloc->inode->gfid);
-                dht_linkfile_create (frame, dht_link_linkfile_cbk,
-                                     cached_subvol, hashed_subvol, newloc);
-        } else {
-                STACK_WIND (frame, dht_link_cbk,
-                            cached_subvol, cached_subvol->fops->link,
-                            oldloc, newloc, xdata);
+            gf_msg_debug(this->name, 0, "creating %s on %s (link at %s)",
+                         loc->path, avail_subvol->name, subvol->name);
+
+            dht_linkfile_create(frame, dht_create_linkfile_create_cbk, this,
+                                avail_subvol, subvol, loc);
+
+            goto out;
         }
 
-        return 0;
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        dht_set_parent_layout_in_dict(loc, this, local);
 
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, loc, flags, mode, umask, fd,
+                          params);
+    }
+out:
+    return 0;
 }
 
-
 int
-dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno,
-                fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+dht_build_parent_loc(xlator_t *this, loc_t *parent, loc_t *child,
+                     int32_t *op_errno)
 {
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        dht_local_t  *local = NULL;
+    inode_table_t *table = NULL;
+    int ret = -1;
 
-        if (op_ret == -1)
-                goto out;
+    if (!parent || !child) {
+        if (op_errno)
+            *op_errno = EINVAL;
+        goto out;
+    }
 
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
+    if (child->parent) {
+        parent->inode = inode_ref(child->parent);
+        if (!parent->inode) {
+            if (op_errno)
+                *op_errno = EINVAL;
+            goto out;
         }
 
-        prev = cookie;
+        gf_uuid_copy(parent->gfid, child->pargfid);
 
-        if (local->loc.parent) {
-                WIPE (preparent);
-                WIPE (postparent);
+        ret = 0;
+
+        goto out;
+    } else {
+        if (gf_uuid_is_null(child->pargfid)) {
+            if (op_errno)
+                *op_errno = EINVAL;
+            goto out;
         }
 
-        ret = dht_layout_preset (this, prev->this, inode);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "could not set preset layout for subvol %s",
-                        prev->this->name);
-                op_ret   = -1;
-                op_errno = EINVAL;
+        table = this->itable;
+
+        if (!table) {
+            if (op_errno) {
+                *op_errno = EINVAL;
+                goto out;
+            }
+        }
+
+        parent->inode = inode_find(table, child->pargfid);
+
+        if (!parent->inode) {
+            if (op_errno) {
+                *op_errno = EINVAL;
                 goto out;
+            }
         }
 
+        gf_uuid_copy(parent->gfid, child->pargfid);
+
+        ret = 0;
+    }
+
 out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, stbuf, preparent,
-                          postparent, NULL);
-        return 0;
+    return ret;
 }
 
-
-int
-dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent,
-                                dict_t *xdata)
+static int32_t
+dht_create_do(call_frame_t *frame)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *cached_subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *refreshed = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        if (op_ret == -1)
-                goto err;
+    local = frame->local;
 
-        local = frame->local;
-        cached_subvol = local->cached_subvol;
+    this = THIS;
 
-        STACK_WIND (frame, dht_create_cbk,
-                    cached_subvol, cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode,
-                    local->umask, local->fd, local->params);
+    conf = this->private;
 
-        return 0;
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
+
+    methods = &(conf->methods);
+
+    /* We don't need parent_loc anymore */
+    loc_wipe(&local->loc);
+
+    loc_copy(&local->loc, &local->loc2);
+
+    loc_wipe(&local->loc2);
+
+    refreshed = local->selfheal.refreshed_layout;
+
+    subvol = methods->layout_search(this, refreshed, local->loc.name);
+
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in "
+               "layout for path=%s",
+               local->loc.path);
+        local->op_errno = ENOENT;
+        goto err;
+    }
+
+    dht_create_wind_to_avail_subvol(frame, this, subvol, &local->loc,
+                                    local->flags, local->mode, local->umask,
+                                    local->fd, local->params);
+    return 0;
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL, NULL);
+    local->refresh_layout_unlock(frame, this, -1, 1);
+
+    return 0;
+}
+
+static int32_t
+dht_create_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
+
+static int32_t
+dht_create_finish(call_frame_t *frame, xlator_t *this, int op_ret,
+                  int invoke_cbk)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
+
+    local = frame->local;
+    lock_count = dht_lock_count(local->lock[0].layout.parent_layout.locks,
+                                local->lock[0].layout.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
+
+    lock_local->lock[0]
+        .layout.parent_layout.locks = local->lock[0].layout.parent_layout.locks;
+    lock_local->lock[0].layout.parent_layout.lk_count =
+        local->lock[0].layout.parent_layout.lk_count;
+
+    local->lock[0].layout.parent_layout.locks = NULL;
+    local->lock[0].layout.parent_layout.lk_count = 0;
+
+    dht_unlock_inodelk(lock_frame,
+                       lock_local->lock[0].layout.parent_layout.locks,
+                       lock_local->lock[0].layout.parent_layout.lk_count,
+                       dht_create_unlock_cbk);
+    lock_frame = NULL;
+
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
+
+    if (op_ret == 0)
         return 0;
+
+    DHT_STACK_UNWIND(create, frame, op_ret, local->op_errno, NULL, NULL, NULL,
+                     NULL, NULL, NULL);
+    return 0;
 }
 
-int
-dht_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *params)
+static int32_t
+dht_create_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int          op_errno = -1;
-        xlator_t    *subvol = NULL;
-        dht_local_t *local = NULL;
-        xlator_t    *avail_subvol = NULL;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    local = frame->local;
 
-        dht_get_du_info (frame, this, loc);
+    if (!local) {
+        goto err;
+    }
 
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    if (op_ret < 0) {
+        gf_msg("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+               "Create lock failed for file: %s", local->loc2.name);
 
-        if (dht_filter_loc_subvol_key (this, loc, &local->loc,
-                                       &subvol)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "creating %s on %s (got create on %s)",
-                        local->loc.path, subvol->name, loc->path);
-                STACK_WIND (frame, dht_create_cbk,
-                            subvol, subvol->fops->create,
-                            &local->loc, flags, mode, umask, fd, params);
-                goto done;
-        }
+        local->op_errno = op_errno;
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume in layout for path=%s",
-                        loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+        goto err;
+    }
 
-        if (!dht_is_subvol_filled (this, subvol)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "creating %s on %s", loc->path, subvol->name);
-                STACK_WIND (frame, dht_create_cbk,
-                            subvol, subvol->fops->create,
-                            loc, flags, mode, umask, fd, params);
-                goto done;
-        }
-        /* Choose the minimum filled volume, and create the
-           files there */
-        avail_subvol = dht_free_disk_available_subvol (this, subvol);
-        if (avail_subvol != subvol) {
-                local->params = dict_ref (params);
-                local->flags = flags;
-                local->mode = mode;
-                local->umask = umask;
-                local->cached_subvol = avail_subvol;
-                local->hashed_subvol = subvol;
-                gf_log (this->name, GF_LOG_TRACE,
-                        "creating %s on %s (link at %s)", loc->path,
-                        avail_subvol->name, subvol->name);
-                dht_linkfile_create (frame,
-                                     dht_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
-                goto done;
-        }
-        gf_log (this->name, GF_LOG_TRACE,
-                "creating %s on %s", loc->path, subvol->name);
-        STACK_WIND (frame, dht_create_cbk,
-                    subvol, subvol->fops->create,
-                    loc, flags, mode, umask, fd, params);
-done:
-        return 0;
+    local->refresh_layout_unlock = dht_create_finish;
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL, NULL);
+    local->refresh_layout_done = dht_create_do;
 
-        return 0;
-}
+    dht_refresh_layout(frame);
 
+    return 0;
+err:
+    if (local)
+        dht_create_finish(frame, this, -1, 0);
+    else
+        DHT_STACK_UNWIND(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL,
+                         NULL, NULL);
+    return 0;
+}
 
-int
-dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int32_t
+dht_create_lock(call_frame_t *frame, xlator_t *subvol)
 {
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
 
-        local = frame->local;
-        layout = local->selfheal.layout;
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
 
-        if (op_ret == 0) {
-                dht_layout_set (this, local->inode, layout);
-                if (local->loc.parent) {
-                        WIPE (&local->preparent);
-                        WIPE (&local->postparent);
-                }
-        }
+    local = frame->local;
 
-        DHT_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-                          local->inode, &local->stbuf, &local->preparent,
-                          &local->postparent, NULL);
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
 
-        return 0;
+    if (lk_array == NULL)
+        goto err;
+
+    lk_array[0] = dht_lock_new(frame->this, subvol, &local->loc, F_RDLCK,
+                               DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                               IGNORE_ENOENT_ESTALE);
+
+    if (lk_array[0] == NULL)
+        goto err;
+
+    local->lock[0].layout.parent_layout.locks = lk_array;
+    local->lock[0].layout.parent_layout.lk_count = count;
+
+    ret = dht_blocking_inodelk(frame, lk_array, count, dht_create_lock_cbk);
+
+    if (ret < 0) {
+        local->lock[0].layout.parent_layout.locks = NULL;
+        local->lock[0].layout.parent_layout.lk_count = 0;
+        goto err;
+    }
+
+    return 0;
+err:
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
+
+    return -1;
 }
 
 int
-dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
-               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+dht_set_parent_layout_in_dict(loc_t *loc, xlator_t *this, dht_local_t *local)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        int           ret = -1;
-        gf_boolean_t subvol_filled = _gf_false;
-        call_frame_t *prev = NULL;
-        dht_layout_t *layout = NULL;
+    dht_conf_t *conf = this->private;
+    dht_layout_t *parent_layout = NULL;
+    int *parent_disk_layout = NULL;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    parent_layout = dht_layout_get(this, loc->parent);
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    ret = dict_set_str_sizen(local->params, GF_PREOP_PARENT_KEY,
+                             conf->xattr_name);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path,
+               GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(local->params, conf->xattr_name, parent_disk_layout,
+                       4 * 4);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting parent-layout in params dictionary failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
 
-        local = frame->local;
-        prev  = cookie;
-        layout = local->layout;
+err:
+    dht_layout_unref(this, parent_layout);
+    return ret;
+}
 
-        subvol_filled = dht_is_subvol_filled (this, prev->this);
+int
+dht_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
+{
+    int op_errno = -1;
+    xlator_t *subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->params = dict_ref(params);
+    local->flags = flags;
+    local->mode = mode;
+    local->umask = umask;
+
+    if (dht_filter_loc_subvol_key(this, loc, &local->loc, &subvol)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "creating %s on %s (got create on %s)", local->loc.path,
+               subvol->name, loc->path);
+
+        /* Since lookup-optimize is enabled by default, we need
+         * to create the linkto file if required.
+         * Note this does not check for decommisioned bricks
+         * and min-free-disk limits as this is a debugging tool
+         * and not expected to be used in production.
+         */
+        hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
 
-        LOCK (&frame->lock);
-        {
-                if (subvol_filled && (op_ret != -1)) {
-                        ret = dht_layout_merge (this, layout, prev->this,
-                                                -1, ENOSPC, NULL);
-                } else {
-                        ret = dht_layout_merge (this, layout, prev->this,
-                                                op_ret, op_errno, NULL);
+        if (hashed_subvol && (hashed_subvol != subvol)) {
+            /* Create the linkto file and then the data file */
+            local->cached_subvol = subvol;
+            local->hashed_subvol = hashed_subvol;
+
+            dht_linkfile_create(frame, dht_create_linkfile_create_cbk, this,
+                                subvol, hashed_subvol, &local->loc);
+            goto done;
+        }
+        /* We either don't have a hashed subvol or the hashed subvol is
+         * the same as the one specified. No need to create the linkto
+         * file as we expect a lookup everywhere if there are problems
+         * with the parent layout
+         */
+
+        dht_set_parent_layout_in_dict(loc, this, local);
+
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, &local->loc, flags, mode, umask,
+                          fd, params);
+        goto done;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in layout for path=%s", loc->path);
+
+        op_errno = EIO;
+        goto err;
+    }
+
+    /* Post remove-brick, the client layout may not be in sync with
+     * disk layout because of lack of lookup. Hence,a create call
+     * may fall on the decommissioned brick.  Hence, if the
+     * hashed_subvol is part of decommissioned bricks  list, do a
+     * lookup on parent dir. If a fix-layout is already done by the
+     * remove-brick process, the parent directory layout will be in
+     * sync with that of the disk. If fix-layout is still ending
+     * on the parent directory, we can let the file get created on
+     * the decommissioned brick which will be eventually migrated to
+     * non-decommissioned brick based on the new layout.
+     */
+
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == subvol) {
+                gf_msg_debug(this->name, 0,
+                             "hashed subvol:%s is "
+                             "part of decommission brick list for "
+                             "file: %s",
+                             subvol->name, loc->path);
+
+                /* dht_refresh_layout needs directory info in
+                 * local->loc. Hence, storing the parent_loc in
+                 * local->loc and storing the create context in
+                 * local->loc2. We will restore this information
+                 * in dht_creation do */
+
+                ret = loc_copy(&local->loc2, &local->loc);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                           "loc_copy failed %s", loc->path);
+
+                    goto err;
                 }
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to merge layouts", local->loc.path);
 
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        goto unlock;
+                loc_wipe(&local->loc);
+
+                ret = dht_build_parent_loc(this, &local->loc, loc, &op_errno);
+
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                           "parent loc build failed");
+                    goto err;
+                }
+
+                ret = dht_create_lock(frame, subvol);
+
+                if (ret < 0) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                           "locking parent failed");
+                    goto err;
                 }
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
-        }
-unlock:
-        UNLOCK (&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_new_directory (frame, dht_mkdir_selfheal_cbk,
-                                            layout);
+                goto done;
+            }
         }
+    }
 
-        return 0;
+    dht_create_wind_to_avail_subvol(frame, this, subvol, loc, flags, mode,
+                                    umask, fd, params);
+done:
+    return 0;
+
+err:
+
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+
+    return 0;
 }
 
-int
-dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int op_ret, int op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           ret = -1;
-        call_frame_t *prev = NULL;
-        dht_layout_t *layout = NULL;
-        dht_conf_t   *conf = NULL;
-        int           i = 0;
-        xlator_t     *hashed_subvol = NULL;
-
-        VALIDATE_OR_GOTO (this->private, err);
+static int
+dht_mkdir_selfheal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
 
-        local = frame->local;
-        prev  = cookie;
-        layout = local->layout;
-        conf = this->private;
-        hashed_subvol = local->hashed_subvol;
+    local = frame->local;
+    layout = local->selfheal.layout;
 
-        if (uuid_is_null (local->loc.gfid) && !op_ret)
-                uuid_copy (local->loc.gfid, stbuf->ia_gfid);
+    FRAME_SU_UNDO(frame, dht_local_t);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        if (dht_is_subvol_filled (this, hashed_subvol))
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        -1, ENOSPC, NULL);
-        else
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        op_ret, op_errno, NULL);
+    if (op_ret == 0) {
+        dht_layout_set(this, local->inode, layout);
 
-        /* TODO: we may have to return from the function
-           if layout merge fails. For now, lets just log an error */
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to merge layouts", local->loc.path);
+        dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->preparent, 0);
 
-        if (op_ret == -1) {
-                local->op_errno = op_errno;
-                goto err;
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
         }
-        local->op_ret = 0;
+    }
 
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-        dht_iatt_merge (this, &local->postparent, postparent, prev->this);
+    DHT_STACK_UNWIND(mkdir, frame, op_ret, op_errno, local->inode,
+                     &local->stbuf, &local->preparent, &local->postparent,
+                     NULL);
 
-        local->call_cnt = conf->subvolume_cnt - 1;
+    return 0;
+}
 
-        if (uuid_is_null (local->loc.gfid))
-                uuid_copy (local->loc.gfid, stbuf->ia_gfid);
-        if (local->call_cnt == 0) {
-                dht_selfheal_directory (frame, dht_mkdir_selfheal_cbk,
-                                        &local->loc, layout);
+static int
+dht_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, inode_t *inode, struct iatt *stbuf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int ret = -1;
+    gf_boolean_t subvol_filled = _gf_false;
+    gf_boolean_t dir_exists = _gf_false;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    layout = local->layout;
+
+    subvol_filled = dht_is_subvol_filled(this, prev);
+
+    LOCK(&frame->lock);
+    {
+        if (subvol_filled && (op_ret != -1)) {
+            ret = dht_layout_merge(this, layout, prev, -1, ENOSPC, NULL);
+        } else {
+            if (op_ret == -1 && op_errno == EEXIST) {
+                /* Very likely just a race between mkdir and
+                   self-heal (from lookup of a concurrent mkdir
+                   attempt).
+                   Ignore error for now. layout setting will
+                   anyways fail if this was a different (old)
+                   pre-existing different directory.
+                */
+                op_ret = 0;
+                dir_exists = _gf_true;
+            }
+            ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, NULL);
         }
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == hashed_subvol)
-                        continue;
-                STACK_WIND (frame, dht_mkdir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->mkdir, &local->loc,
-                            local->mode, local->umask, local->params);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                   "%s: failed to merge layouts for subvol %s", local->loc.path,
+                   prev->name);
+
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            goto unlock;
         }
-        return 0;
-err:
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
-        return 0;
+
+        if (dir_exists)
+            goto unlock;
+
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /*Unlock entrylk and inodelk once mkdir is done on all subvols*/
+        dht_unlock_namespace(frame, &local->lock[0]);
+        FRAME_SU_DO(frame, dht_local_t);
+        dht_selfheal_new_directory(frame, dht_mkdir_selfheal_cbk, layout);
+    }
+
+    return 0;
 }
 
+static int
+dht_mkdir_hashed_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata);
 
- int
-dht_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
+static int
+dht_mkdir_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                 mode_t umask, dict_t *params)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        xlator_t     *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1, ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    int32_t *parent_disk_layout = NULL;
+    dht_layout_t *parent_layout = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    conf = this->private;
+    local = frame->local;
+
+    if (local->op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): refreshing parent layout "
+               "failed.",
+               pgfid, loc->name, loc->path);
+
+        op_errno = local->op_errno;
+        goto err;
+    }
+
+    local->op_ret = -1;
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "mkdir (%s/%s) (path: %s): hashed subvol not "
+                     "found",
+                     pgfid, loc->name, loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    local->hashed_subvol = hashed_subvol;
+
+    parent_layout = dht_layout_get(this, loc->parent);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, EIO, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    if (memcmp(local->parent_disk_layout, parent_disk_layout,
+               sizeof(local->parent_disk_layout)) == 0) {
+        gf_msg(this->name, GF_LOG_WARNING, EIO, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): loop detected. "
+               "parent layout didn't change even though "
+               "previous attempt of mkdir failed because of "
+               "in-memory layout not matching with that on disk.",
+               pgfid, loc->name, loc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    memcpy((void *)local->parent_disk_layout, (void *)parent_disk_layout,
+           sizeof(local->parent_disk_layout));
+
+    dht_layout_unref(this, parent_layout);
+    parent_layout = NULL;
+
+    ret = dict_set_str(params, GF_PREOP_PARENT_KEY, conf->xattr_name);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               pgfid, loc->name, loc->path, GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(params, conf->xattr_name, parent_disk_layout, 4 * 4);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "setting parent-layout in params dictionary failed. "
+               "mkdir (%s/%s) (path: %s)",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_disk_layout = NULL;
+
+    STACK_WIND_COOKIE(frame, dht_mkdir_hashed_cbk, hashed_subvol, hashed_subvol,
+                      hashed_subvol->fops->mkdir, loc, mode, umask, params);
+
+    return 0;
 
+err:
+    dht_unlock_namespace(frame, &local->lock[0]);
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        conf = this->private;
+    if (parent_disk_layout != NULL)
+        GF_FREE(parent_disk_layout);
 
-        dht_get_du_info (frame, this, loc);
+    if (parent_layout != NULL)
+        dht_layout_unref(this, parent_layout);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    return 0;
+}
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (hashed_subvol == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "hashed subvol not found for %s",
-                        loc->path);
-                op_errno = EINVAL;
+static int
+dht_mkdir_hashed_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    gf_boolean_t parent_layout_changed = _gf_false;
+    call_stub_t *stub = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    layout = local->layout;
+    conf = this->private;
+    hashed_subvol = local->hashed_subvol;
+
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    if (gf_uuid_is_null(local->loc.gfid) && !op_ret)
+        gf_uuid_copy(local->loc.gfid, stbuf->ia_gfid);
+
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+
+        parent_layout_changed = (xdata &&
+                                 dict_get(xdata, GF_PREOP_CHECK_FAILED))
+                                    ? 1
+                                    : 0;
+        if (parent_layout_changed) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "mkdir (%s/%s) (path: %s): parent layout "
+                   "changed. Attempting a refresh and then a "
+                   "retry",
+                   pgfid, local->loc.name, local->loc.path);
+
+            stub = fop_mkdir_stub(frame, dht_mkdir_helper, &local->loc,
+                                  local->mode, local->umask, local->params);
+            if (stub == NULL) {
                 goto err;
-        }
+            }
 
-        local->hashed_subvol = hashed_subvol;
-        local->mode = mode;
-        local->umask = umask;
-        local->params = dict_ref (params);
-        local->inode  = inode_ref (loc->inode);
-
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
-        if (!local->layout) {
-                op_errno = ENOMEM;
+            ret = dht_handle_parent_layout_change(this, stub);
+            if (ret) {
                 goto err;
-        }
+            }
+
+            stub = NULL;
+
+            return 0;
+        }
+
+        goto err;
+    }
+
+    dict_del(local->params, GF_PREOP_PARENT_KEY);
+    dict_del(local->params, conf->xattr_name);
+
+    if (dht_is_subvol_filled(this, hashed_subvol))
+        ret = dht_layout_merge(this, layout, prev, -1, ENOSPC, NULL);
+    else
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, NULL);
+
+    /* TODO: we may have to return from the function
+       if layout merge fails. For now, lets just log an error */
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+               "%s: failed to merge layouts for subvol %s", local->loc.path,
+               prev->name);
+
+    local->op_ret = 0;
+
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preparent, preparent);
+    dht_iatt_merge(this, &local->postparent, postparent);
+
+    local->call_cnt = conf->subvolume_cnt - 1;
+    /* Delete internal mds xattr from params dict to avoid store
+      internal mds xattr on other subvols
+    */
+    dict_del(local->params, conf->mds_xattr_key);
+
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, stbuf->ia_gfid);
+
+    /* Set hashed subvol as a mds subvol on inode ctx */
+    /*if (!local->inode)
+            local->inode  = inode_ref (inode);
+    */
+    ret = dht_inode_ctx_mdsvol_set(local->inode, this, hashed_subvol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+               "Failed to set hashed subvol for %s on inode vol is %s",
+               local->loc.path, hashed_subvol->name);
+    }
+
+    if (local->call_cnt == 0) {
+        /*Unlock namespace lock once mkdir is done on all subvols*/
+        dht_unlock_namespace(frame, &local->lock[0]);
+        FRAME_SU_DO(frame, dht_local_t);
+        dht_selfheal_directory(frame, dht_mkdir_selfheal_cbk, &local->loc,
+                               layout);
+        return 0;
+    }
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == hashed_subvol)
+            continue;
+        STACK_WIND_COOKIE(frame, dht_mkdir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i], conf->subvolumes[i]->fops->mkdir,
+                          &local->loc, local->mode, local->umask,
+                          local->params);
+    }
+
+    return 0;
+err:
+    if (local->op_ret != 0) {
+        dht_unlock_namespace(frame, &local->lock[0]);
+    }
 
-        STACK_WIND (frame, dht_mkdir_hashed_cbk,
-                    hashed_subvol,
-                    hashed_subvol->fops->mkdir,
-                    loc, mode, umask, params);
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
+}
 
+static int
+dht_mkdir_guard_parent_layout_cbk(call_frame_t *frame, xlator_t *this,
+                                  loc_t *loc, mode_t mode, mode_t umask,
+                                  dict_t *params)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = 0;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = -1;
+    int32_t zero[1] = {0};
+
+    local = frame->local;
+    conf = this->private;
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "Acquiring lock on parent to guard against "
+               "layout-change failed.",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    local->op_ret = -1;
+    /* Add internal MDS xattr on disk for hashed subvol
+     */
+    ret = dht_dict_set_array(params, conf->mds_xattr_key, zero, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->mds_xattr_key, loc->path);
+    }
+
+    STACK_WIND_COOKIE(frame, dht_mkdir_hashed_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->mkdir,
+                      loc, mode, umask, params);
+
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
+    DHT_STACK_UNWIND(mkdir, frame, -1, local->op_errno, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rmdir_selfheal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *xdata)
+dht_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *params)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = EINVAL, ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    call_stub_t *stub = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    conf = this->private;
+
+    if (!params || !dict_get(params, "gfid-req")) {
+        op_errno = EPERM;
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, op_errno,
+                         DHT_MSG_GFID_NULL,
+                         "mkdir: %s is received "
+                         "without gfid-req %p",
+                         loc->path, params);
+        goto err;
+    }
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        gf_msg_debug(this->name, 0, "hashed subvol not found for %s",
+                     loc->path);
+        local->op_errno = EIO;
+        goto err;
+    }
+
+    local->hashed_subvol = hashed_subvol;
+    local->mode = mode;
+    local->umask = umask;
+    if (params)
+        local->params = dict_ref(params);
+
+    local->inode = inode_ref(loc->inode);
+
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!local->layout) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* set the newly created directory hash to the commit hash
+     * if the configuration option is set. If configuration option
+     * is not set, the older clients may still be connecting to the
+     * volume and hence we need to preserve the 1 in disk[0] part of the
+     * layout xattr */
+    if (conf->lookup_optimize)
+        local->layout->commit_hash = conf->vol_commit_hash;
+    else
+        local->layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+
+    stub = fop_mkdir_stub(frame, dht_mkdir_guard_parent_layout_cbk, loc, mode,
+                          umask, params);
+    if (stub == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "creating stub failed.",
+               pgfid, loc->name, loc->path);
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_guard_parent_layout_and_namespace(this, stub);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s) cannot wind lock request to "
+               "guard parent layout",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    return 0;
 
-        local = frame->local;
-
-        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
+err:
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_rmdir_hashed_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
+static int
+dht_rmdir_selfheal_cbk(call_frame_t *heal_frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        local->op_ret   = -1;
-                        if (op_errno != ENOENT && op_errno != EACCES) {
-                                local->need_selfheal = 1;
-                        }
+    heal_local = heal_frame->local;
+    main_frame = heal_local->main_frame;
+    local = main_frame->local;
 
+    DHT_STACK_DESTROY(heal_frame);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "rmdir on %s for %s failed (%s)",
-                                prev->this->name, local->loc.path,
-                                strerror (op_errno));
-                        goto unlock;
-                }
+    DHT_STACK_UNWIND(rmdir, main_frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
 
-                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+    return 0;
+}
 
+static int
+dht_rmdir_hashed_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int op_ret, int op_errno, struct iatt *preparent,
+                            struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+    dht_conf_t *conf = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            local->op_ret = -1;
+            if (conf->subvolume_cnt != 1) {
+                if (op_errno != ENOENT && op_errno != EACCES &&
+                    op_errno != ESTALE) {
+                    local->need_selfheal = 1;
+                }
+            }
+
+            gf_msg_debug(this->name, op_errno,
+                         "rmdir on %s for %s failed "
+                         "(gfid = %s)",
+                         prev->name, local->loc.path, gfid);
+            goto unlock;
         }
+
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-               if (local->need_selfheal) {
-                        local->layout =
-                                dht_layout_get (this, local->loc.inode);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if (local->need_selfheal) {
+            dht_rmdir_unlock(frame, this);
+            local->layout = dht_layout_get(this, local->loc.inode);
 
-                        /* TODO: neater interface needed below */
-                        local->stbuf.ia_type = local->loc.inode->ia_type;
+            /* TODO: neater interface needed below */
+            local->stbuf.ia_type = local->loc.inode->ia_type;
 
-                        uuid_copy (local->gfid, local->loc.inode->gfid);
-                        dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
-                                              &local->loc, local->layout);
-               } else {
+            gf_uuid_copy(local->gfid, local->loc.inode->gfid);
 
-                        if (local->loc.parent) {
-                                WIPE (&local->preparent);
-                                WIPE (&local->postparent);
-                        }
+            /* Use a different frame or else the rmdir op_ret is
+             * overwritten by that of the selfheal */
+
+            heal_frame = copy_frame(frame);
+
+            if (heal_frame == NULL) {
+                goto err;
+            }
 
-                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                                          local->op_errno, &local->preparent,
-                                          &local->postparent, NULL);
-               }
+            heal_local = dht_local_init(heal_frame, &local->loc, NULL, 0);
+            if (!heal_local) {
+                DHT_STACK_DESTROY(heal_frame);
+                goto err;
+            }
+
+            heal_local->inode = inode_ref(local->loc.inode);
+            heal_local->main_frame = frame;
+            gf_uuid_copy(heal_local->gfid, local->loc.inode->gfid);
+
+            dht_selfheal_restore(heal_frame, dht_rmdir_selfheal_cbk,
+                                 &heal_local->loc, heal_local->layout);
+            return 0;
+        } else {
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->preparent, 0);
+
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
+
+            dht_set_fixed_dir_stat(&local->preparent);
+            dht_set_fixed_dir_stat(&local->postparent);
+
+            dht_rmdir_unlock(frame, this);
+            DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                             &local->preparent, &local->postparent, NULL);
         }
+    }
 
-        return 0;
-}
+    return 0;
 
+err:
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL);
+    return 0;
+}
 
-int
-dht_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
+static int
+dht_rmdir_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-        int           done = 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
-        local = frame->local;
-        prev  = cookie;
+static int
+dht_rmdir_unlock(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        local->op_ret   = -1;
+    local = frame->local;
 
-                        if (op_errno != ENOENT && op_errno != EACCES) {
-                                local->need_selfheal = 1;
-                        }
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[0].ns.directory_ns);
 
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "rmdir on %s for %s failed (%s)",
-                                prev->this->name, local->loc.path,
-                                strerror (op_errno));
-                        goto unlock;
-                }
+    /* Unlock inodelk */
+    lock_count = dht_lock_count(local->lock[0].ns.parent_layout.locks,
+                                local->lock[0].ns.parent_layout.lk_count);
 
-                /* Track if rmdir succeeded on atleast one subvol*/
-                local->fop_succeeded = 1;
-                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
-        }
-unlock:
-        UNLOCK (&frame->lock);
+    if (lock_count == 0)
+        goto done;
 
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL)
+        goto done;
 
-        this_call_cnt = dht_frame_return (frame);
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL)
+        goto done;
 
-        /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
-        if (local->hashed_subvol && (this_call_cnt == 1)) {
-                done = 1;
-        } else if (!local->hashed_subvol && !this_call_cnt) {
-                done = 1;
-        }
+    lock_local->lock[0].ns.parent_layout.locks = local->lock[0]
+                                                     .ns.parent_layout.locks;
+    lock_local->lock[0]
+        .ns.parent_layout.lk_count = local->lock[0].ns.parent_layout.lk_count;
 
+    local->lock[0].ns.parent_layout.locks = NULL;
+    local->lock[0].ns.parent_layout.lk_count = 0;
+    dht_unlock_inodelk(lock_frame, lock_local->lock[0].ns.parent_layout.locks,
+                       lock_local->lock[0].ns.parent_layout.lk_count,
+                       dht_rmdir_unlock_cbk);
+    lock_frame = NULL;
 
-        if (done) {
-                if (local->need_selfheal && local->fop_succeeded) {
-                        local->layout =
-                                dht_layout_get (this, local->loc.inode);
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
-                        /* TODO: neater interface needed below */
-                        local->stbuf.ia_type = local->loc.inode->ia_type;
+    return 0;
+}
 
-                        uuid_copy (local->gfid, local->loc.inode->gfid);
-                        dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
-                                              &local->loc, local->layout);
-                } else if (this_call_cnt) {
-                        /* If non-hashed subvol's have responded, proceed */
+static int
+dht_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int done = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+    int ret = -1;
+
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
 
-                        local->need_selfheal = 0;
-                        STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
-                                    local->hashed_subvol,
-                                    local->hashed_subvol->fops->rmdir,
-                                    &local->loc, local->flags, NULL);
-                } else if (!this_call_cnt) {
-                        /* All subvol's have responded, proceed */
+                if (op_errno != EACCES)
+                    local->need_selfheal = 1;
+            }
 
-                        if (local->loc.parent) {
-                                WIPE (&local->preparent);
-                                WIPE (&local->postparent);
-                        }
+            gf_uuid_unparse(local->loc.gfid, gfid);
 
-                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                                          local->op_errno, &local->preparent,
-                                          &local->postparent, NULL);
-                }
+            gf_msg_debug(this->name, op_errno,
+                         "rmdir on %s for %s failed."
+                         "(gfid = %s)",
+                         prev->name, local->loc.path, gfid);
+            goto unlock;
         }
 
-        return 0;
-}
+        /* Track if rmdir succeeded on at least one subvol*/
+        local->fop_succeeded = 1;
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
+    this_call_cnt = dht_frame_return(frame);
 
-int
-dht_rmdir_do (call_frame_t *frame, xlator_t *this)
-{
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf = NULL;
-        int           i = 0;
-        xlator_t     *hashed_subvol = NULL;
+    /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
+    if (local->hashed_subvol && (this_call_cnt == 1)) {
+        done = 1;
+    } else if (!local->hashed_subvol && !this_call_cnt) {
+        done = 1;
+    }
 
-        VALIDATE_OR_GOTO (this->private, err);
+    if (done) {
+        if (local->need_selfheal && local->fop_succeeded) {
+            dht_rmdir_unlock(frame, this);
+            local->layout = dht_layout_get(this, local->loc.inode);
 
-        conf = this->private;
-        local = frame->local;
+            /* TODO: neater interface needed below */
+            local->stbuf.ia_type = local->loc.inode->ia_type;
 
-        if (local->op_ret == -1)
+            gf_uuid_copy(local->gfid, local->loc.inode->gfid);
+            heal_frame = copy_frame(frame);
+            if (heal_frame == NULL) {
                 goto err;
+            }
 
-        local->call_cnt = conf->subvolume_cnt;
+            heal_local = dht_local_init(heal_frame, &local->loc, NULL, 0);
+            if (!heal_local) {
+                DHT_STACK_DESTROY(heal_frame);
+                goto err;
+            }
+
+            heal_local->inode = inode_ref(local->loc.inode);
+            heal_local->main_frame = frame;
+            gf_uuid_copy(heal_local->gfid, local->loc.inode->gfid);
+            ret = dht_selfheal_restore(heal_frame, dht_rmdir_selfheal_cbk,
+                                       &heal_local->loc, heal_local->layout);
+            if (ret) {
+                DHT_STACK_DESTROY(heal_frame);
+                goto err;
+            }
+
+        } else if (this_call_cnt) {
+            /* If non-hashed subvol's have responded, proceed */
+            if (local->op_ret == 0) {
+                /* Delete the dir from the hashed subvol if:
+                 * The fop succeeded on at least one subvol
+                 *  and did not fail on any
+                 *  or
+                 *  The fop failed with ENOENT/ESTALE on
+                 *  all subvols */
+
+                STACK_WIND_COOKIE(frame, dht_rmdir_hashed_subvol_cbk,
+                                  local->hashed_subvol, local->hashed_subvol,
+                                  local->hashed_subvol->fops->rmdir,
+                                  &local->loc, local->flags, NULL);
+            } else {
+                /* hashed-subvol was non-NULL and rmdir failed on
+                 * all non hashed-subvols. Unwind rmdir with
+                 * local->op_ret and local->op_errno. */
+                dht_rmdir_unlock(frame, this);
+                DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                                 &local->preparent, &local->postparent, NULL);
 
-        /* first remove from non-hashed_subvol */
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
+                return 0;
+            }
+        } else if (!this_call_cnt) {
+            /* All subvol's have responded, proceed */
 
-        if (!hashed_subvol) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to get hashed "
-                        "subvol for %s",local->loc.path);
-        } else {
-                local->hashed_subvol = hashed_subvol;
-        }
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->preparent, 0);
 
-        /* When DHT has only 1 child */
-        if (conf->subvolume_cnt == 1) {
-                STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
-                            conf->subvolumes[0],
-                            conf->subvolumes[0]->fops->rmdir,
-                            &local->loc, local->flags, NULL);
-                return 0;
-        }
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (hashed_subvol &&
-                    (hashed_subvol == conf->subvolumes[i]))
-                        continue;
+            dht_set_fixed_dir_stat(&local->preparent);
+            dht_set_fixed_dir_stat(&local->postparent);
 
-                STACK_WIND (frame, dht_rmdir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->rmdir,
-                            &local->loc, local->flags, NULL);
+            dht_rmdir_unlock(frame, this);
+            DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                             &local->preparent, &local->postparent, NULL);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
-        return 0;
+    DHT_STACK_UNWIND(rmdir, frame, -1, local->op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
+static int
+dht_rmdir_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *hashed_subvol;
+
+    conf = this->private;
+    local = frame->local;
 
-int
-dht_rmdir_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                               int op_ret, int op_errno, struct iatt *preparent,
-                               struct iatt *postparent, dict_t *xdata)
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed rmdir for %s)",
+               local->loc.path);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    hashed_subvol = local->hashed_subvol;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (hashed_subvol && (hashed_subvol == conf->subvolumes[i]))
+            continue;
+
+        STACK_WIND_COOKIE(frame, dht_rmdir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i], conf->subvolumes[i]->fops->rmdir,
+                          &local->loc, local->flags, NULL);
+    }
+
+    return 0;
+
+err:
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
+
+    return 0;
+}
+
+static int
+dht_rmdir_do(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t    *local = NULL;
-        call_frame_t   *prev = NULL;
-        xlator_t       *src = NULL;
-        call_frame_t   *main_frame = NULL;
-        dht_local_t    *main_local = NULL;
-        int             this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local  = frame->local;
-        prev   = cookie;
-        src    = prev->this;
+    VALIDATE_OR_GOTO(frame->local, err);
+    local = frame->local;
+    VALIDATE_OR_GOTO(this->private, out);
+    conf = this->private;
 
-        main_frame = local->main_frame;
-        main_local = main_frame->local;
+    if (local->op_ret == -1)
+        goto out;
 
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "unlinked linkfile %s on %s",
-                        local->loc.path, src->name);
-        } else {
-                main_local->op_ret   = -1;
-                main_local->op_errno = op_errno;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlink of %s on %s failed (%s)",
-                        local->loc.path, src->name, strerror (op_errno));
-        }
+    local->call_cnt = conf->subvolume_cnt;
+
+    /* first remove from non-hashed_subvol */
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
 
-        this_call_cnt = dht_frame_return (main_frame);
-        if (is_last_call (this_call_cnt))
-                dht_rmdir_do (main_frame, this);
+    if (!hashed_subvol) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
 
-        DHT_STACK_DESTROY (frame);
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "Failed to get hashed subvol for %s (gfid = %s)",
+               local->loc.path, gfid);
+    } else {
+        local->hashed_subvol = hashed_subvol;
+    }
+
+    /* When DHT has only 1 child */
+    if (conf->subvolume_cnt == 1) {
+        STACK_WIND_COOKIE(frame, dht_rmdir_hashed_subvol_cbk,
+                          conf->subvolumes[0], conf->subvolumes[0],
+                          conf->subvolumes[0]->fops->rmdir, &local->loc,
+                          local->flags, NULL);
         return 0;
+    }
+
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, &local->loc, local->hashed_subvol,
+                                &local->current->ns, dht_rmdir_lock_cbk);
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = errno ? errno : EINVAL;
+        goto out;
+    }
+
+    return 0;
+
+out:
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
+
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
+    return 0;
+err:
+    DHT_STACK_UNWIND(rmdir, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
 }
 
+static void
+dht_rmdir_readdirp_done(call_frame_t *readdirp_frame, xlator_t *this)
+{
+    call_frame_t *main_frame = NULL;
+    dht_local_t *main_local = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+
+    local = readdirp_frame->local;
+    main_frame = local->main_frame;
+    main_local = main_frame->local;
+
+    /* At least one readdirp failed.
+     * This is a bit hit or miss - if readdirp failed on more than
+     * one subvol, we don't know which error is returned.
+     */
+    if (local->op_ret == -1) {
+        main_local->op_ret = local->op_ret;
+        main_local->op_errno = local->op_errno;
+    }
+
+    this_call_cnt = dht_frame_return(main_frame);
+
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_do(main_frame, this);
+
+    DHT_STACK_DESTROY(readdirp_frame);
+}
 
-int
-dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, inode_t *inode,
-                      struct iatt *stbuf, dict_t *xattr, struct iatt *parent)
+/* Keep sending readdirp on the subvol until it returns no more entries
+ * It is possible that not all entries will fit in a single readdirp in
+ * which case the rmdir will keep failing with ENOTEMPTY
+ */
+
+static int
+dht_rmdir_readdirp_do(call_frame_t *readdirp_frame, xlator_t *this)
 {
-        dht_local_t    *local = NULL;
-        call_frame_t   *prev = NULL;
-        xlator_t       *src = NULL;
-        call_frame_t   *main_frame = NULL;
-        dht_local_t    *main_local = NULL;
-        int             this_call_cnt = 0;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        prev  = cookie;
-        src   = prev->this;
+    local = readdirp_frame->local;
 
-        main_frame = local->main_frame;
-        main_local = main_frame->local;
+    if (local->op_ret == -1) {
+        /* there is no point doing another readdirp on this
+         * subvol . */
+        dht_rmdir_readdirp_done(readdirp_frame, this);
+        return 0;
+    }
 
-        if (op_ret != 0)
-                goto err;
+    STACK_WIND_COOKIE(readdirp_frame, dht_rmdir_readdirp_cbk,
+                      local->hashed_subvol, local->hashed_subvol,
+                      local->hashed_subvol->fops->readdirp, local->fd, 4096, 0,
+                      local->xattr);
 
-        if (check_is_linkfile (inode, stbuf, xattr) == 0) {
-                main_local->op_ret  = -1;
-                main_local->op_errno = ENOTEMPTY;
+    return 0;
+}
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s on %s found to be not a linkfile (type=0%o)",
-                        local->loc.path, src->name, stbuf->ia_type);
-                goto err;
-        }
+static int
+dht_rmdir_linkfile_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int op_ret, int op_errno, struct iatt *preparent,
+                              struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    src = prev;
+
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (op_ret == 0) {
+        gf_msg_trace(this->name, 0, "Unlinked linkfile %s on %s, gfid = %s",
+                     local->loc.path, src->name, gfid);
+    } else {
+        if (op_errno != ENOENT) {
+            readdirp_local->op_ret = -1;
+            readdirp_local->op_errno = op_errno;
+        }
+        gf_msg_debug(this->name, op_errno,
+                     "Unlink of %s on %s failed. (gfid = %s)", local->loc.path,
+                     src->name, gfid);
+    }
+
+    this_call_cnt = dht_frame_return(readdirp_frame);
+
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_readdirp_do(readdirp_frame, this);
+
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
-        STACK_WIND (frame, dht_rmdir_linkfile_unlink_cbk,
-                    src, src->fops->unlink, &local->loc, 0, NULL);
-        return 0;
+static int
+dht_rmdir_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, dict_t *xattr, struct iatt *parent)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    src = prev;
+
+    gf_msg_debug(this->name, 0, "dht_rmdir_lookup_cbk %s", local->loc.path);
+
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
+
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_FILE_LOOKUP_FAILED,
+               "lookup failed for %s on %s", local->loc.path, src->name);
+        goto err;
+    }
+
+    if (!check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = ENOTEMPTY;
+
+        gf_uuid_unparse(local->loc.gfid, gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NOT_LINK_FILE_ERROR,
+               "%s on %s is not a linkfile (type=0%o, gfid = %s)",
+               local->loc.path, src->name, stbuf->ia_type, gfid);
+        goto err;
+    }
+
+    STACK_WIND_COOKIE(frame, dht_rmdir_linkfile_unlink_cbk, src, src,
+                      src->fops->unlink, &local->loc, 0, NULL);
+    return 0;
 err:
 
-        this_call_cnt = dht_frame_return (main_frame);
-        if (is_last_call (this_call_cnt))
-                dht_rmdir_do (main_frame, this);
+    this_call_cnt = dht_frame_return(readdirp_frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_rmdir_readdirp_do(readdirp_frame, this);
+    }
 
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
+static int
+dht_rmdir_cached_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int op_ret, int op_errno, inode_t *inode,
+                            struct iatt *stbuf, dict_t *xattr,
+                            struct iatt *parent)
+{
+    dht_local_t *local = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    dht_conf_t *conf = this->private;
+    dict_t *xattrs = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    src = local->hashed_subvol;
+
+    /* main_frame here is the readdirp_frame */
+
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
+
+    gf_msg_debug(this->name, 0, "returning for %s ", local->loc.path);
+
+    if (op_ret == 0) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = ENOTEMPTY;
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_ERROR,
+               "%s found on cached subvol %s", local->loc.path, src->name);
+        goto err;
+    } else if (op_errno != ENOENT) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = op_errno;
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_SUBVOL_ERROR,
+               "%s not found on cached subvol %s", local->loc.path, src->name);
+        goto err;
+    }
+
+    xattrs = dict_new();
+    if (!xattrs) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dict_new failed");
+        goto err;
+    }
+
+    ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s",
+               conf->link_xattr_name);
+        if (xattrs)
+            dict_unref(xattrs);
+        goto err;
+    }
+    STACK_WIND_COOKIE(frame, dht_rmdir_lookup_cbk, src, src, src->fops->lookup,
+                      &local->loc, xattrs);
+    if (xattrs)
+        dict_unref(xattrs);
+
+    return 0;
+err:
+
+    this_call_cnt = dht_frame_return(readdirp_frame);
 
-int
-dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
-                           gf_dirent_t *entries, xlator_t *src)
+    /* Once all the lookups/unlinks etc have returned, proceed to wind
+     * readdirp on the subvol again until no entries are returned.
+     * This is required if there are more entries than can be returned
+     * in a single readdirp call.
+     */
+
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_readdirp_do(readdirp_frame, this);
+
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
+
+static int
+dht_rmdir_is_subvol_empty(call_frame_t *frame, xlator_t *this,
+                          gf_dirent_t *entries, xlator_t *src)
 {
-        int                 ret = 0;
-        int                 build_ret = 0;
-        gf_dirent_t        *trav = NULL;
-        call_frame_t       *lookup_frame = NULL;
-        dht_local_t        *lookup_local = NULL;
-        dht_local_t        *local = NULL;
-        dict_t             *xattrs = NULL;
+    int ret = 0;
+    int build_ret = 0;
+    gf_dirent_t *trav = NULL;
+    call_frame_t *lookup_frame = NULL;
+    dht_local_t *lookup_local = NULL;
+    dht_local_t *local = NULL;
+    dict_t *xattrs = NULL;
+    dht_conf_t *conf = this->private;
+    xlator_t *subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int count = 0;
+    gf_boolean_t unwind = _gf_false;
+
+    local = frame->local;
+
+    list_for_each_entry(trav, &entries->list, list)
+    {
+        if (strcmp(trav->d_name, ".") == 0)
+            continue;
+        if (strcmp(trav->d_name, "..") == 0)
+            continue;
+        if (check_is_linkfile(NULL, (&trav->d_stat), trav->dict,
+                              conf->link_xattr_name)) {
+            count++;
+            continue;
+        }
+
+        /* this entry is either a directory which is neither "." nor "..",
+           or a non directory which is not a linkfile. the directory is to
+           be treated as non-empty
+        */
+        return 0;
+    }
 
-        local = frame->local;
+    xattrs = dict_new();
+    if (!xattrs) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dict_new failed");
+        return -1;
+    }
 
-        list_for_each_entry (trav, &entries->list, list) {
-                if (strcmp (trav->d_name, ".") == 0)
-                        continue;
-                if (strcmp (trav->d_name, "..") == 0)
-                        continue;
-                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict)) {
-                        ret++;
-                        continue;
-                }
+    ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s",
+               conf->link_xattr_name);
 
-                /* this entry is either a directory which is neither "." nor "..",
-                   or a non directory which is not a linkfile. the directory is to
-                   be treated as non-empty
-                */
-                return 0;
+        if (xattrs)
+            dict_unref(xattrs);
+        return -1;
+    }
+
+    local->call_cnt = count;
+    ret = 0;
+
+    list_for_each_entry(trav, &entries->list, list)
+    {
+        if (strcmp(trav->d_name, ".") == 0)
+            continue;
+        if (strcmp(trav->d_name, "..") == 0)
+            continue;
+
+        lookup_frame = copy_frame(frame);
+
+        if (!lookup_frame) {
+            /* out of memory, let the rmdir fail
+               (as non-empty, unfortunately) */
+            goto err;
         }
 
-        xattrs = dict_new ();
-        if (!xattrs) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                return -1;
+        lookup_local = dht_local_init(lookup_frame, NULL, NULL, GF_FOP_LOOKUP);
+        if (!lookup_local) {
+            goto err;
         }
 
-        ret = dict_set_uint32 (xattrs, DHT_LINKFILE_KEY, 256);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set linkto key"
-                        " in dict");
-                if (xattrs)
-                        dict_unref (xattrs);
-                return -1;
-        }
-
-        list_for_each_entry (trav, &entries->list, list) {
-                if (strcmp (trav->d_name, ".") == 0)
-                        continue;
-                if (strcmp (trav->d_name, "..") == 0)
-                        continue;
-
-                lookup_frame = NULL;
-                lookup_local = NULL;
-
-                lookup_frame = copy_frame (frame);
-                if (!lookup_frame) {
-                        /* out of memory, let the rmdir fail
-                           (as non-empty, unfortunately) */
-                        goto err;
-                }
+        lookup_frame->local = lookup_local;
+        lookup_local->main_frame = frame;
+        lookup_local->hashed_subvol = src;
 
-                lookup_local = mem_get0 (this->local_pool);
-                if (!lookup_local) {
-                        goto err;
-                }
+        build_ret = dht_build_child_loc(this, &lookup_local->loc, &local->loc,
+                                        trav->d_name);
+        if (build_ret != 0)
+            goto err;
 
-                lookup_frame->local = lookup_local;
-                lookup_local->main_frame = frame;
+        gf_uuid_copy(lookup_local->loc.gfid, trav->d_stat.ia_gfid);
 
-                build_ret = dht_build_child_loc (this, &lookup_local->loc,
-                                                 &local->loc, trav->d_name);
-                if (build_ret != 0)
-                        goto err;
+        gf_uuid_unparse(lookup_local->loc.gfid, gfid);
 
-                uuid_copy (lookup_local->loc.gfid, trav->d_stat.ia_gfid);
+        gf_msg_trace(this->name, 0, "looking up %s on subvolume %s, gfid = %s",
+                     lookup_local->loc.path, src->name, gfid);
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "looking up %s on %s",
-                        lookup_local->loc.path, src->name);
+        subvol = dht_linkfile_subvol(this, NULL, &trav->d_stat, trav->dict);
+        if (!subvol || (subvol == src)) {
+            /* we need to delete the linkto file if it does not have a
+             * valid subvol or it points to itself.
+             */
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_INVALID_LINKFILE,
+                   "Linkfile does not have link subvolume. "
+                   "path = %s, gfid = %s",
+                   lookup_local->loc.path, gfid);
 
-                LOCK (&frame->lock);
-                {
-                        local->call_cnt++;
-                }
-                UNLOCK (&frame->lock);
+            gf_msg_debug(this->name, 0, "looking up %s on subvol %s, gfid = %s",
+                         lookup_local->loc.path, src->name, gfid);
 
-                STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
-                            src, src->fops->lookup,
-                            &lookup_local->loc, xattrs);
-                ret++;
+            STACK_WIND_COOKIE(lookup_frame, dht_rmdir_lookup_cbk, src, src,
+                              src->fops->lookup, &lookup_local->loc, xattrs);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "Looking up linkfile target %s on "
+                         " subvol %s, gfid = %s",
+                         lookup_local->loc.path, subvol->name, gfid);
+
+            STACK_WIND(lookup_frame, dht_rmdir_cached_lookup_cbk, subvol,
+                       subvol->fops->lookup, &lookup_local->loc, xattrs);
         }
+        ret++;
 
-        if (xattrs)
-                dict_unref (xattrs);
+        lookup_frame = NULL;
+        lookup_local = NULL;
+    }
 
-        return ret;
+    if (xattrs)
+        dict_unref(xattrs);
+
+    return ret;
 err:
-        if (xattrs)
-                dict_unref (xattrs);
+    if (xattrs)
+        dict_unref(xattrs);
 
-        DHT_STACK_DESTROY (lookup_frame);
-        return 0;
+    if (lookup_frame)
+        DHT_STACK_DESTROY(lookup_frame);
+
+    /* Handle the case where the wound calls have unwound before the
+     * loop processing is done
+     */
+
+    LOCK(&frame->lock);
+    {
+        local->op_ret = -1;
+        local->op_errno = ENOTEMPTY;
+
+        local->call_cnt -= (count - ret);
+        if (!local->call_cnt)
+            unwind = _gf_true;
+    }
+    UNLOCK(&frame->lock);
+
+    if (!unwind) {
+        return ret;
+    }
+    return 0;
 }
 
+/*
+ * No more entries on this subvol. Proceed to the actual rmdir operation.
+ */
 
-int
-dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries,
-                        dict_t *xdata)
+static int
+dht_rmdir_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
-        xlator_t     *src = NULL;
-        int           ret = 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    int ret = 0;
+    char *path = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    src = prev;
+
+    if (op_ret > 2) {
+        /* dht_rmdir_is_subvol_empty() may free the frame,
+         * copy path for logging.
+         */
+        path = gf_strdup(local->loc.path);
 
-        local = frame->local;
-        prev  = cookie;
-        src   = prev->this;
-
-        if (op_ret > 2) {
-                ret = dht_rmdir_is_subvol_empty (frame, this, entries, src);
-
-                switch (ret) {
-                case 0: /* non linkfiles exist */
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "readdir on %s for %s returned %d entries",
-                                prev->this->name, local->loc.path, op_ret);
-                        local->op_ret = -1;
-                        local->op_errno = ENOTEMPTY;
-                        break;
-                default:
-                        /* @ret number of linkfiles are getting unlinked */
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "readdir on %s for %s found %d linkfiles",
-                                prev->this->name, local->loc.path, ret);
-                        break;
-                }
+        ret = dht_rmdir_is_subvol_empty(frame, this, entries, src);
+
+        switch (ret) {
+            case 0: /* non linkfiles exist */
+                gf_msg_trace(this->name, 0,
+                             "readdir on %s for %s returned %d "
+                             "entries",
+                             prev->name, local->loc.path, op_ret);
+                local->op_ret = -1;
+                local->op_errno = ENOTEMPTY;
+                break;
+            default:
+                /* @ret number of linkfiles are getting unlinked */
+                gf_msg_trace(this->name, 0,
+                             "readdir on %s for %s found %d "
+                             "linkfiles",
+                             prev->name, path, ret);
+                break;
         }
+    }
 
-        this_call_cnt = dht_frame_return (frame);
+    /* readdirp failed or no linkto files were found on this subvol */
+    if (!ret)
+        dht_rmdir_readdirp_done(frame, this);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rmdir_do (frame, this);
-        }
+    GF_FREE(path);
+    return 0;
+}
 
+static int
+dht_rmdir_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
+    int ret = 0;
+    dht_conf_t *conf = this->private;
+    dict_t *dict = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *readdirp_local = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    int cnt = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    this_call_cnt = dht_frame_return(frame);
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+
+        gf_msg_debug(this->name, op_errno,
+                     "opendir on %s for %s failed, "
+                     "gfid = %s,",
+                     prev->name, local->loc.path, gfid);
+        if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
+            local->op_ret = -1;
+            local->op_errno = op_errno;
+        }
+        goto err;
+    }
+
+    if (!is_last_call(this_call_cnt))
         return 0;
-}
 
+    if (local->op_ret == -1)
+        goto err;
 
-int
-dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
-        dict_t       *dict = NULL;
-        int           ret = 0;
+    fd_bind(fd);
 
-        local = frame->local;
-        prev  = cookie;
+    dict = dict_new();
+    if (!dict) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "opendir on %s for %s failed (%s)",
-                        prev->this->name, local->loc.path,
-                        strerror (op_errno));
-                if (op_errno != ENOENT) {
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                }
-                goto err;
+    ret = dict_set_uint32(dict, conf->link_xattr_name, 256);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: Failed to set dictionary value:key = %s", local->loc.path,
+               conf->link_xattr_name);
+
+    cnt = local->call_cnt = conf->subvolume_cnt;
+
+    /* Create a separate frame per subvol as we might need
+     * to resend readdirp multiple times to get all the
+     * entries.
+     */
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        readdirp_frame = copy_frame(frame);
+
+        if (!readdirp_frame) {
+            cnt--;
+            /* Reduce the local->call_cnt as well */
+            (void)dht_frame_return(frame);
+            continue;
         }
 
-        dict = dict_new ();
-        if (!dict) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto err;
+        readdirp_local = dht_local_init(readdirp_frame, &local->loc, local->fd,
+                                        0);
+
+        if (!readdirp_local) {
+            DHT_STACK_DESTROY(readdirp_frame);
+            cnt--;
+            /* Reduce the local->call_cnt as well */
+            dht_frame_return(frame);
+            continue;
         }
+        readdirp_local->main_frame = frame;
+        readdirp_local->op_ret = 0;
+        readdirp_local->xattr = dict_ref(dict);
+        /* overload this field to save the subvol info */
+        readdirp_local->hashed_subvol = conf->subvolumes[i];
 
-        ret = dict_set_uint32 (dict,
-                               "trusted.glusterfs.dht.linkto", 256);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht.linkto' key",
-                        local->loc.path);
+        STACK_WIND_COOKIE(readdirp_frame, dht_rmdir_readdirp_cbk,
+                          conf->subvolumes[i], conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->readdirp,
+                          readdirp_local->fd, 4096, 0, readdirp_local->xattr);
+    }
 
-        STACK_WIND (frame, dht_rmdir_readdirp_cbk,
-                    prev->this, prev->this->fops->readdirp,
-                    local->fd, 4096, 0, dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (dict)
-                dict_unref (dict);
+    /* Could not wind readdirp to any subvol */
 
-        return 0;
+    if (!cnt)
+        goto err;
 
-err:
-        this_call_cnt = dht_frame_return (frame);
+    return 0;
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rmdir_do (frame, this);
-        }
+err:
+    if (is_last_call(this_call_cnt)) {
+        dht_rmdir_do(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-           dict_t *xdata)
+dht_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = -1;
+    dict_t *xattr_req = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_RMDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = conf->subvolume_cnt;
+    local->op_ret = 0;
+    local->fop_succeeded = 0;
+
+    local->flags = flags;
+
+    local->fd = fd_create(local->loc.inode, frame->root->pid);
+    if (!local->fd) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (flags) {
+        return dht_rmdir_do(frame, this);
+    }
+    if (xdata) {
+        xattr_req = dict_ref(xdata);
+    } else {
+        xattr_req = dict_new();
+    }
+    if (xattr_req) {
+        ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+        /* If parallel-readdir is enabled, this is required
+         * to handle stale linkto files in the directory
+         * being deleted. If this fails, log an error but
+         * do not prevent the operation.
+         */
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "%s: failed to set key %s",
+                   loc->path, conf->link_xattr_name);
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "%s: failed to set key %s",
+               loc->path, conf->link_xattr_name);
+    }
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_rmdir_opendir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->opendir, loc, local->fd,
+                          xattr_req);
+    }
+
+    if (xattr_req) {
+        dict_unref(xattr_req);
+    }
+    return 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
 
-        conf = this->private;
+    return 0;
+}
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_RMDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+static int
+dht_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-        local->call_cnt = conf->subvolume_cnt;
-        local->op_ret   = 0;
-        local->fop_succeeded = 0;
+{
+    DHT_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        local->flags = flags;
+/* TODO
+ * Sending entrylk to cached subvol can result in stale lock
+ * as described in the bug 1311002.
+ */
+int
+dht_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local->fd = fd_create (local->loc.inode, frame->root->pid);
-        if (!local->fd) {
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
 
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = dht_local_init(frame, loc, NULL, GF_FOP_ENTRYLK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_rmdir_opendir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->opendir,
-                            loc, local->fd, NULL);
-        }
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_uuid_unparse(loc->gfid, gfid);
 
-        return 0;
+        gf_msg_debug(this->name, 0,
+                     "no cached subvolume for path=%s, "
+                     "gfid = %s",
+                     loc->path, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local->call_cnt = 1;
+
+    STACK_WIND(frame, dht_entrylk_cbk, subvol, subvol->fops->entrylk, volume,
+               loc, basename, cmd, type, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rmdir, frame, -1, op_errno,
-                          NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(entrylk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+dht_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    DHT_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
-
 int
-dht_entrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, const char *basename,
-             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
-{
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_ENTRYLK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+dht_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
 
-        local->call_cnt = 1;
+    gf_uuid_unparse(fd->inode->gfid, gfid);
 
-        STACK_WIND (frame, dht_entrylk_cbk,
-                    subvol, subvol->fops->entrylk,
-                    volume, loc, basename, cmd, type, xdata);
+    subvol = dht_subvol_get_cached(this, fd->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0,
+                     "No cached subvolume for fd=%p,"
+                     " gfid = %s",
+                     fd, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        return 0;
+    STACK_WIND(frame, dht_fentrylk_cbk, subvol, subvol->fops->fentrylk, volume,
+               fd, basename, cmd, type, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fentrylk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
+static int32_t
+dht_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
-int
-dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
-{
-        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, NULL);
-        return 0;
-}
+    local = frame->local;
 
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0 && op_errno != ENOTCONN) {
+            local->op_errno = op_errno;
+            goto unlock;
+        }
+        local->op_ret = 0;
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-int
-dht_fentrylk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, const char *basename,
-              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(ipc, frame, local->op_ret, local->op_errno, NULL);
+    }
+
+out:
+    return 0;
+}
+
+int32_t
+dht_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int i = 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
 
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    if (op != GF_IPC_TARGET_UPCALL)
+        goto wind_default;
 
-        STACK_WIND (frame, dht_fentrylk_cbk,
-                    subvol, subvol->fops->fentrylk,
-                    volume, fd, basename, cmd, type, xdata);
+    VALIDATE_OR_GOTO(this->private, err);
+    conf = this->private;
 
-        return 0;
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_IPC);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
+
+    if (xdata) {
+        if (dict_set_int8(xdata, conf->xattr_name, 0) < 0)
+            goto err;
+    }
+
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND(frame, dht_ipc_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->ipc, op, xdata);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+    DHT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
 
-        return 0;
-}
+    return 0;
 
+wind_default:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
+}
 
 int
-dht_forget (xlator_t *this, inode_t *inode)
+dht_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t      tmp_layout = 0;
-        dht_layout_t *layout = NULL;
+    uint64_t ctx_int = 0;
+    dht_inode_ctx_t *ctx = NULL;
+    dht_layout_t *layout = NULL;
 
-        inode_ctx_del (inode, this, &tmp_layout);
-
-        if (!tmp_layout)
-                return 0;
-
-        layout = (dht_layout_t *)(long)tmp_layout;
-        dht_layout_unref (this, layout);
+    inode_ctx_del(inode, this, &ctx_int);
 
+    if (!ctx_int)
         return 0;
-}
 
+    ctx = (dht_inode_ctx_t *)(long)ctx_int;
 
-int
-dht_notify (xlator_t *this, int event, void *data, ...)
-{
-        xlator_t                *subvol = NULL;
-        int                      cnt    = -1;
-        int                      i      = -1;
-        dht_conf_t              *conf   = NULL;
-        int                      ret    = -1;
-        int                      propagate = 0;
-
-        int                      had_heard_from_all = 0;
-        int                      have_heard_from_all = 0;
-        struct timeval           time = {0,};
-        gf_defrag_info_t        *defrag = NULL;
-        dict_t                  *dict   = NULL;
-        gf_defrag_type           cmd    = 0;
-        dict_t                  *output = NULL;
-        va_list                  ap;
-
-
-        conf = this->private;
-        if (!conf)
-                return ret;
+    layout = ctx->layout;
+    ctx->layout = NULL;
+    dht_layout_unref(this, layout);
+    GF_FREE(ctx);
 
-        /* had all subvolumes reported status once till now? */
-        had_heard_from_all = 1;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->last_event[i]) {
-                        had_heard_from_all = 0;
-                }
-        }
+    return 0;
+}
 
-        switch (event) {
+int
+dht_notify(xlator_t *this, int event, void *data, ...)
+{
+    xlator_t *subvol = NULL;
+    int cnt = -1;
+    int i = -1;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int propagate = 0;
+
+    int had_heard_from_all = 0;
+    int have_heard_from_all = 0;
+    gf_defrag_info_t *defrag = NULL;
+    dict_t *dict = NULL;
+    gf_defrag_type cmd = 0;
+    dict_t *output = NULL;
+    va_list ap;
+    struct gf_upcall *up_data = NULL;
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    /* had all subvolumes reported status once till now? */
+    had_heard_from_all = 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->last_event[i]) {
+            had_heard_from_all = 0;
+        }
+    }
+
+    switch (event) {
         case GF_EVENT_CHILD_UP:
-                subvol = data;
+            subvol = data;
 
-                conf->gen++;
+            conf->gen++;
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
+            }
 
-                if (cnt == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "got GF_EVENT_CHILD_UP bad subvolume %s",
-                                subvol->name);
-                        break;
-                }
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_UP bad "
+                             "subvolume %s",
+                             subvol->name);
+                break;
+            }
 
-                gettimeofday (&time, NULL);
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->subvolume_status[cnt] = 1;
-                        conf->last_event[cnt] = event;
-                        conf->subvol_up_time[cnt] = time.tv_sec;
-                }
-                UNLOCK (&conf->subvolume_lock);
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->subvolume_status[cnt] = 1;
+                conf->last_event[cnt] = event;
+                conf->subvol_up_time[cnt] = gf_time();
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                /* one of the node came back up, do a stat update */
-                dht_get_du_info_for_subvol (this, cnt);
+            /* one of the node came back up, do a stat update */
+            dht_get_du_info_for_subvol(this, cnt);
 
-                break;
+            break;
 
-        case GF_EVENT_CHILD_MODIFIED:
-                subvol = data;
+        case GF_EVENT_SOME_DESCENDENT_UP:
+            subvol = data;
+            conf->gen++;
+            propagate = 1;
 
-                conf->gen++;
-                propagate = 1;
+            break;
 
-                break;
+        case GF_EVENT_SOME_DESCENDENT_DOWN:
+            subvol = data;
+            propagate = 1;
 
-        case GF_EVENT_CHILD_DOWN:
-                subvol = data;
-
-                if (conf->assert_no_child_down) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Received CHILD_DOWN. Exiting");
-                        if (conf->defrag) {
-                                gf_defrag_stop (conf->defrag, NULL);
-                        } else {
-                                kill (getpid(), SIGTERM);
-                        }
-                }
+            break;
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
-                }
+        case GF_EVENT_CHILD_DOWN:
+            subvol = data;
 
-                if (cnt == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "got GF_EVENT_CHILD_DOWN bad subvolume %s",
-                                subvol->name);
-                        break;
+            if (conf->assert_no_child_down) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_CHILD_DOWN,
+                       "Received CHILD_DOWN. Exiting");
+                if (conf->defrag) {
+                    gf_defrag_stop(conf, GF_DEFRAG_STATUS_FAILED, NULL);
+                } else {
+                    kill(getpid(), SIGTERM);
                 }
+            }
 
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->subvolume_status[cnt] = 0;
-                        conf->last_event[cnt] = event;
-                        conf->subvol_up_time[cnt] = 0;
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
-                UNLOCK (&conf->subvolume_lock);
+            }
 
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_DOWN bad "
+                             "subvolume %s",
+                             subvol->name);
                 break;
+            }
 
-        case GF_EVENT_CHILD_CONNECTING:
-                subvol = data;
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->subvolume_status[cnt] = 0;
+                conf->last_event[cnt] = event;
+                conf->subvol_up_time[cnt] = 0;
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
-                }
+            for (i = 0; i < conf->subvolume_cnt; i++)
+                if (conf->last_event[i] != event)
+                    event = GF_EVENT_SOME_DESCENDENT_DOWN;
+            break;
 
-                if (cnt == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "got GF_EVENT_CHILD_CONNECTING bad subvolume %s",
-                                subvol->name);
-                        break;
-                }
+        case GF_EVENT_CHILD_CONNECTING:
+            subvol = data;
 
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->last_event[cnt] = event;
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
-                UNLOCK (&conf->subvolume_lock);
+            }
 
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_CONNECTING"
+                             " bad subvolume %s",
+                             subvol->name);
                 break;
-        case GF_EVENT_VOLUME_DEFRAG:
-        {
-                if (!conf->defrag) {
-                        return ret;
-                }
-                defrag = conf->defrag;
+            }
 
-                dict = data;
-                va_start (ap, data);
-                output = va_arg (ap, dict_t*);
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->last_event[cnt] = event;
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                ret = dict_get_int32 (dict, "rebalance-command",
-                                      (int32_t*)&cmd);
-                if (ret)
-                        return ret;
-                LOCK (&defrag->lock);
-                {
-                        if (defrag->is_exiting)
-                                goto unlock;
-                        if (cmd == GF_DEFRAG_CMD_STATUS)
-                                gf_defrag_status_get (defrag, output);
-                        else if (cmd == GF_DEFRAG_CMD_STOP)
-                                gf_defrag_stop (defrag, output);
-                }
-unlock:
-                UNLOCK (&defrag->lock);
-                return 0;
-                break;
-        }
+            break;
+        case GF_EVENT_VOLUME_DEFRAG: {
+            if (!conf->defrag) {
+                return ret;
+            }
+            defrag = conf->defrag;
+
+            dict = data;
+            va_start(ap, data);
+            output = va_arg(ap, dict_t *);
 
+            ret = dict_get_int32(dict, "rebalance-command", (int32_t *)&cmd);
+            if (ret) {
+                va_end(ap);
+                return ret;
+            }
+            LOCK(&defrag->lock);
+            {
+                if (defrag->is_exiting)
+                    goto unlock;
+                if ((cmd == GF_DEFRAG_CMD_STATUS) ||
+                    (cmd == GF_DEFRAG_CMD_DETACH_STATUS))
+                    gf_defrag_status_get(conf, output);
+                else if (cmd == GF_DEFRAG_CMD_DETACH_START)
+                    defrag->cmd = GF_DEFRAG_CMD_DETACH_START;
+                else if (cmd == GF_DEFRAG_CMD_STOP ||
+                         cmd == GF_DEFRAG_CMD_DETACH_STOP)
+                    gf_defrag_stop(conf, GF_DEFRAG_STATUS_STOPPED, output);
+            }
+        unlock:
+            UNLOCK(&defrag->lock);
+            va_end(ap);
+            return ret;
+            break;
+        }
+        case GF_EVENT_UPCALL:
+            up_data = (struct gf_upcall *)data;
+            if (up_data->event_type != GF_UPCALL_CACHE_INVALIDATION)
+                break;
+            up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
+
+            /* Since md-cache will be aggressively filtering lookups,
+             * the stale layout issue will be more pronounced. Hence
+             * when a layout xattr is changed by the rebalance process
+             * notify all the md-cache clients to invalidate the existing
+             * stat cache and send the lookup next time*/
+            if (up_ci->dict && dict_get(up_ci->dict, conf->xattr_name))
+                up_ci->flags |= UP_EXPLICIT_LOOKUP;
+
+            /* TODO: Instead of invalidating iatt, update the new
+             * hashed/cached subvolume in dht inode_ctx */
+            if (IS_DHT_LINKFILE_MODE(&up_ci->stat))
+                up_ci->flags |= UP_EXPLICIT_LOOKUP;
+
+            propagate = 1;
+            break;
         default:
-                propagate = 1;
+            propagate = 1;
+            break;
+    }
+
+    /* have all subvolumes reported status once by now? */
+    have_heard_from_all = 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->last_event[i])
+            have_heard_from_all = 0;
+    }
+
+    /* if all subvols have reported status, no need to hide anything
+       or wait for anything else. Just propagate blindly */
+    if (have_heard_from_all) {
+        propagate = 1;
+    }
+
+    if (!had_heard_from_all && have_heard_from_all) {
+        static int run_defrag = 0;
+        /* This is the first event which completes aggregation
+           of events from all subvolumes. If at least one subvol
+           had come up, propagate CHILD_UP, but only this time
+        */
+        event = GF_EVENT_CHILD_DOWN;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->last_event[i] == GF_EVENT_CHILD_UP) {
+                event = GF_EVENT_CHILD_UP;
                 break;
+            }
+
+            if (conf->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
+                event = GF_EVENT_CHILD_CONNECTING;
+                /* continue to check other events for CHILD_UP */
+            }
         }
 
+        /* Rebalance is started with assert_no_child_down. So we do
+         * not need to handle CHILD_DOWN event here.
+         *
+         * If there is a graph switch, we should not restart the
+         * rebalance daemon. Use 'run_defrag' to indicate if the
+         * thread has already started.
+         */
+        if (conf->defrag && !run_defrag) {
+            run_defrag = 1;
+            ret = gf_thread_create(&conf->defrag->th, NULL, gf_defrag_start,
+                                   this, "dhtdg");
+            if (ret) {
+                GF_FREE(conf->defrag);
+                conf->defrag = NULL;
+                kill(getpid(), SIGTERM);
+            }
+        }
+    }
+
+    ret = 0;
+    if (propagate)
+        ret = default_notify(this, event, data);
+out:
+    return ret;
+}
+
+int
+dht_inode_ctx_layout_get(inode_t *inode, xlator_t *this, dht_layout_t **layout)
+{
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        /* have all subvolumes reported status once by now? */
-        have_heard_from_all = 1;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->last_event[i])
-                        have_heard_from_all = 0;
-        }
+    ret = dht_inode_ctx_get(inode, this, &ctx);
 
-        /* if all subvols have reported status, no need to hide anything
-           or wait for anything else. Just propagate blindly */
-        if (have_heard_from_all) {
-                propagate = 1;
-                if (conf->defrag) {
-                        ret = pthread_create (&conf->defrag->th, NULL,
-                                              gf_defrag_start, this);
-                        if (ret) {
-                                conf->defrag = NULL;
-                                GF_FREE (conf->defrag);
-                                kill (getpid(), SIGTERM);
-                        }
-                }
+    if (!ret && ctx) {
+        if (ctx->layout) {
+            if (layout)
+                *layout = ctx->layout;
+            ret = 0;
+        } else {
+            ret = -1;
         }
+    }
 
+    return ret;
+}
 
-        if (!had_heard_from_all && have_heard_from_all) {
-                /* This is the first event which completes aggregation
-                   of events from all subvolumes. If at least one subvol
-                   had come up, propagate CHILD_UP, but only this time
-                */
-                event = GF_EVENT_CHILD_DOWN;
+void
+dht_log_new_layout_for_dir_selfheal(xlator_t *this, loc_t *loc,
+                                    dht_layout_t *layout)
+{
+    char string[2048] = {0};
+    char *output_string = NULL;
+    int len = 0;
+    int off = 0;
+    int i = 0;
+    gf_loglevel_t log_level = gf_log_get_loglevel();
+    int ret = 0;
+
+    if (log_level < GF_LOG_INFO)
+        return;
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->last_event[i] == GF_EVENT_CHILD_UP) {
-                                event = GF_EVENT_CHILD_UP;
-                                break;
-                        }
+    if (!layout)
+        return;
 
-                        if (conf->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
-                                event = GF_EVENT_CHILD_CONNECTING;
-                                /* continue to check other events for CHILD_UP */
-                        }
-                }
-        }
+    if (!layout->cnt)
+        return;
+
+    if (!loc)
+        return;
+
+    if (!loc->path)
+        return;
+
+    ret = snprintf(string, sizeof(string), "Setting layout of %s with ",
+                   loc->path);
+
+    if (ret < 0)
+        return;
+
+    len += ret;
+
+    /* Calculation  of total length of the string required to calloc
+     * output_string. Log includes subvolume-name, start-range, end-range
+     * and err value.
+     *
+     * This log will help to debug cases where:
+     * a) Different processes set different layout of a directory.
+     * b) Error captured in lookup, which will be filled in layout->err
+     * (like ENOENT, ESTALE etc)
+     */
+
+    for (i = 0; i < layout->cnt; i++) {
+        ret = snprintf(string, sizeof(string),
+                       "[Subvol_name: %s, Err: %d , Start: "
+                       "0x%x, Stop: 0x%x, Hash: 0x%x], ",
+                       layout->list[i].xlator->name, layout->list[i].err,
+                       layout->list[i].start, layout->list[i].stop,
+                       layout->list[i].commit_hash);
+
+        if (ret < 0)
+            return;
+
+        len += ret;
+    }
+
+    len++;
+
+    output_string = GF_MALLOC(len + 1, gf_common_mt_char);
+
+    if (!output_string)
+        return;
+
+    ret = snprintf(output_string, len + 1, "Setting layout of %s with ",
+                   loc->path);
+
+    if (ret < 0)
+        goto err;
+
+    off += ret;
+
+    for (i = 0; i < layout->cnt; i++) {
+        ret = snprintf(output_string + off, len - off,
+                       "[Subvol_name: %s, Err: %d , Start: "
+                       "0x%x, Stop: 0x%x, Hash: 0x%x], ",
+                       layout->list[i].xlator->name, layout->list[i].err,
+                       layout->list[i].start, layout->list[i].stop,
+                       layout->list[i].commit_hash);
+
+        if (ret < 0)
+            goto err;
+
+        off += ret;
+    }
+
+    gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_LOG_FIXED_LAYOUT, "%s",
+           output_string);
+
+err:
+    GF_FREE(output_string);
+}
+
+int32_t
+dht_migration_get_dst_subvol(xlator_t *this, dht_local_t *local)
+{
+    int ret = -1;
 
+    if (!local)
+        goto out;
+
+    local->rebalance.target_node = dht_subvol_get_hashed(this, &local->loc);
+
+    if (local->rebalance.target_node)
         ret = 0;
-        if (propagate)
-                ret = default_notify (this, event, data);
 
-        return ret;
+out:
+    return ret;
+}
+
+/*
+This function should not be called more then once during a FOP
+handling path. It is valid only for for ops on files
+*/
+int32_t
+dht_set_local_rebalance(xlator_t *this, dht_local_t *local, struct iatt *stbuf,
+                        struct iatt *prebuf, struct iatt *postbuf,
+                        dict_t *xdata)
+{
+    if (!local)
+        return -1;
+
+    if (local->rebalance.set) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_REBAL_STRUCT_SET,
+               "local->rebalance already set");
+    }
+
+    if (stbuf)
+        memcpy(&local->rebalance.stbuf, stbuf, sizeof(struct iatt));
+
+    if (prebuf)
+        memcpy(&local->rebalance.prebuf, prebuf, sizeof(struct iatt));
+
+    if (postbuf)
+        memcpy(&local->rebalance.postbuf, postbuf, sizeof(struct iatt));
+
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
+
+    local->rebalance.set = 1;
+
+    return 0;
+}
+
+int32_t
+dht_release(xlator_t *this, fd_t *fd)
+{
+    return dht_fd_ctx_destroy(this, fd);
+}
+
+static int
+dht_pt_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *stbuf,
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (!op_ret) {
+        dht_layout_set(this, inode, local->layout);
+    }
+
+    DHT_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, NULL);
+
+    return 0;
+}
+
+int32_t
+dht_pt_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
+{
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    bool free_xdata = false;
+    int ret = 0;
+    int op_errno = 0;
+    int32_t *disk_layout_p = NULL;
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!layout)
+        goto wind;
+
+    local->layout = layout;
+
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata)
+            goto wind;
+        free_xdata = true;
+    }
+
+    /*Set the xlator or the following will crash*/
+    layout->list[0].xlator = conf->subvolumes[0];
+
+    dht_selfheal_layout_new_directory(frame, loc, layout);
+
+    dht_disk_layout_extract(this, layout, 0, &disk_layout_p);
+
+    ret = dict_set_bin(xdata, conf->xattr_name, disk_layout_p, 4 * 4);
+    if (ret) {
+        gf_msg("dht", GF_LOG_DEBUG, EINVAL, DHT_MSG_DICT_SET_FAILED,
+               "dht layout dict set failed");
+    }
+wind:
+    STACK_WIND(frame, dht_pt_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    if (free_xdata)
+        dict_unref(xdata);
+    return 0;
+
+err:
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+
+    return 0;
+}
+
+static int
+dht_pt_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    dict_del(xattr, conf->xattr_name);
+    dict_del(xattr, conf->mds_xattr_key);
+    dict_del(xattr, conf->commithash_xattr_name);
+
+    if (frame->root->pid >= 0) {
+        GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+    }
+
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
+}
+
+int
+dht_pt_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata)
+{
+    STACK_WIND(frame, dht_pt_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, key, xdata);
+    return 0;
+}
+
+static int
+dht_pt_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    dict_del(xattr, conf->xattr_name);
+
+    if (frame->root->pid >= 0) {
+        GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+    }
+
+    DHT_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
+}
+
+int
+dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata)
+{
+    STACK_WIND(frame, dht_pt_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, key, xdata);
+    return 0;
+}
+
+/* The job of this function is to check if all the xlators have updated
+ * error in the layout. */
+int
+dht_dir_layout_error_check(xlator_t *this, inode_t *inode)
+{
+    dht_layout_t *layout = NULL;
+    int i = 0;
+
+    layout = dht_layout_get(this, inode);
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == 0) {
+            return 0;
+        }
+    }
+
+    /* Returning the first xlator error as all xlators have errors */
+    return layout->list[0].err;
 }
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index d3ccacdb8c6..fe0dc3db34a 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -8,659 +8,1377 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <regex.h>
 
 #include "dht-mem-types.h"
+#include "dht-messages.h"
+#include <glusterfs/call-stub.h>
 #include "libxlator.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/refcount.h>
+#include <glusterfs/timer.h>
+#include "protocol-common.h"
+#include <glusterfs/glusterfs-acl.h>
 
 #ifndef _DHT_H
 #define _DHT_H
 
-#define GF_XATTR_FIX_LAYOUT_KEY     "distribute.fix.layout"
-#define GF_DHT_LOOKUP_UNHASHED_ON   1
+#define GF_XATTR_FIX_LAYOUT_KEY "distribute.fix.layout"
+#define GF_XATTR_FILE_MIGRATE_KEY "trusted.distribute.migrate-data"
+#define DHT_MDS_STR "mds"
+#define GF_DHT_LOOKUP_UNHASHED_OFF 0
+#define GF_DHT_LOOKUP_UNHASHED_ON 1
 #define GF_DHT_LOOKUP_UNHASHED_AUTO 2
-#define DHT_PATHINFO_HEADER         "DISTRIBUTE:"
+#define DHT_PATHINFO_HEADER "DISTRIBUTE:"
+#define DHT_FILE_MIGRATE_DOMAIN "dht.file.migrate"
+/* Layout synchronization */
+#define DHT_LAYOUT_HEAL_DOMAIN "dht.layout.heal"
+/* Namespace synchronization */
+#define DHT_ENTRY_SYNC_DOMAIN "dht.entry.sync"
+#define DHT_LAYOUT_HASH_INVALID 1
+#define MAX_REBAL_THREADS sysconf(_SC_NPROCESSORS_ONLN)
 
-#include <fnmatch.h>
+#define DHT_DIR_STAT_BLOCKS 8
+#define DHT_DIR_STAT_SIZE 4096
 
-typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie,
-                                       xlator_t     *this,
-                                       int32_t       op_ret, int32_t op_errno,
-                                       dict_t *xdata);
-typedef int (*dht_defrag_cbk_fn_t) (xlator_t        *this, call_frame_t *frame,
-                                    int              ret);
+/* Virtual xattr for subvols status */
 
+#define DHT_SUBVOL_STATUS_KEY "dht.subvol.status"
 
-struct dht_layout {
-        int                spread_cnt;  /* layout spread count per directory,
-                                           is controlled by 'setxattr()' with
-                                           special key */
-        int                cnt;
-        int                preset;
-        int                gen;
-        int                type;
-        int                ref; /* use with dht_conf_t->layout_lock */
-        int                search_unhashed;
-        struct {
-                int        err;   /* 0 = normal
-                                     -1 = dir exists and no xattr
-                                     >0 = dir lookup failed with errno
-                                  */
-                uint32_t   start;
-                uint32_t   stop;
-                xlator_t  *xlator;
-        } list[0];
-};
-typedef struct dht_layout  dht_layout_t;
+/* Virtual xattrs for debugging */
 
+#define DHT_DBG_HASHED_SUBVOL_PATTERN "dht.file.hashed-subvol.*"
+#define DHT_DBG_HASHED_SUBVOL_KEY "dht.file.hashed-subvol."
 
-typedef enum {
-        DHT_HASH_TYPE_DM,
-        DHT_HASH_TYPE_DM_USER,
-} dht_hashfn_type_t;
+/* Rebalance nodeuuid flags */
+#define REBAL_NODEUUID_MINE 0x01
 
-/* rebalance related */
-struct dht_rebalance_ {
-        xlator_t            *from_subvol;
-        xlator_t            *target_node;
-        off_t                offset;
-        size_t               size;
-        int32_t              flags;
-        int                  count;
-        struct iobref       *iobref;
-        struct iovec        *vector;
-        struct iatt          stbuf;
-        dht_defrag_cbk_fn_t  target_op_fn;
-        dict_t              *xdata;
-};
+typedef int (*dht_selfheal_dir_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, dict_t *xdata);
+typedef int (*dht_defrag_cbk_fn_t)(xlator_t *this, xlator_t *dst_node,
+                                   call_frame_t *frame, int ret);
 
-struct dht_local {
-        int                      call_cnt;
-        loc_t                    loc;
-        loc_t                    loc2;
-        int                      op_ret;
-        int                      op_errno;
-        int                      layout_mismatch;
-        /* Use stbuf as the postbuf, when we require both
-         * pre and post attrs */
-        struct iatt              stbuf;
-        struct iatt              prebuf;
-        struct iatt              preoldparent;
-        struct iatt              postoldparent;
-        struct iatt              preparent;
-        struct iatt              postparent;
-        struct statvfs           statvfs;
-        fd_t                    *fd;
-        inode_t                 *inode;
-        dict_t                  *params;
-        dict_t                  *xattr;
-        dict_t                  *xattr_req;
-        dht_layout_t            *layout;
-        size_t                   size;
-        ino_t                    ia_ino;
-        xlator_t                *src_hashed, *src_cached;
-        xlator_t                *dst_hashed, *dst_cached;
-        xlator_t                *cached_subvol;
-        xlator_t                *hashed_subvol;
-        char                     need_selfheal;
-        int                      file_count;
-        int                      dir_count;
-        call_frame_t            *main_frame;
-        int                      fop_succeeded;
-        struct {
-                fop_mknod_cbk_t  linkfile_cbk;
-                struct iatt      stbuf;
-                loc_t            loc;
-                inode_t         *inode;
-                dict_t          *xattr;
-                xlator_t        *srcvol;
-        } linkfile;
-        struct {
-                uint32_t         hole_cnt;
-                uint32_t         overlaps_cnt;
-                uint32_t         missing;
-                uint32_t         down;
-                uint32_t         misc;
-                dht_selfheal_dir_cbk_t   dir_cbk;
-                dht_layout_t    *layout;
-        } selfheal;
-        uint32_t                 uid;
-        uint32_t                 gid;
+typedef int (*dht_refresh_layout_unlock)(call_frame_t *frame, xlator_t *this,
+                                         int op_ret, int invoke_cbk);
 
-        /* needed by nufa */
-        int32_t flags;
-        mode_t  mode;
-        dev_t   rdev;
-        mode_t  umask;
+typedef int (*dht_refresh_layout_done_handle)(call_frame_t *frame);
 
-        /* need for file-info */
-        char   *xattr_val;
-        char   *key;
+struct dht_layout {
+    int spread_cnt; /* layout spread count per directory,
+                       is controlled by 'setxattr()' with
+                       special key */
+    int cnt;
+    int preset;
+    /*
+     * The last *configuration* state for which this directory was known
+     * to be in balance.  The corresponding vol_commit_hash changes
+     * whenever bricks are added or removed.  This value changes when a
+     * (full) rebalance is complete.  If they match, it's safe to assume
+     * that every file is where it should be and there's no need to do
+     * lookups for files elsewhere.  If they don't, then we have to do a
+     * global lookup to be sure.
+     */
+    uint32_t commit_hash;
+    /*
+     * The *runtime* state of the volume, changes when connections to
+     * bricks are made or lost.
+     */
+    int gen;
+    int type;
+    gf_atomic_t ref; /* use with dht_conf_t->layout_lock */
+    uint32_t search_unhashed;
+    struct {
+        int err; /* 0 = normal
+                    -1 = dir exists and no xattr
+                    >0 = dir lookup failed with errno
+                 */
+        uint32_t start;
+        uint32_t stop;
+        uint32_t commit_hash;
+        xlator_t *xlator;
+    } list[];
+};
+typedef struct dht_layout dht_layout_t;
+
+struct dht_stat_time {
+    uint32_t atime;
+    uint32_t atime_nsec;
+    uint32_t ctime;
+    uint32_t ctime_nsec;
+    uint32_t mtime;
+    uint32_t mtime_nsec;
+};
+
+typedef struct dht_stat_time dht_stat_time_t;
+
+struct dht_inode_ctx {
+    dht_layout_t *layout;
+    dht_stat_time_t time;
+    xlator_t *lock_subvol;
+    xlator_t *mds_subvol; /* This is only used for directories */
+};
 
-        /* which xattr request? */
-        char xsel[256];
+typedef struct dht_inode_ctx dht_inode_ctx_t;
 
-        char   *newpath;
+typedef enum {
+    DHT_HASH_TYPE_DM,
+    DHT_HASH_TYPE_DM_USER,
+} dht_hashfn_type_t;
 
-        /* gfid related */
-        uuid_t  gfid;
+typedef enum {
+    DHT_INODELK,
+    DHT_ENTRYLK,
+} dht_lock_type_t;
 
-        /*Marker Related*/
-        struct marker_str    marker;
+/* rebalance related */
+struct dht_rebalance_ {
+    xlator_t *from_subvol;
+    xlator_t *target_node;
+    off_t offset;
+    size_t size;
+    int32_t flags;
+    int count;
+    struct iobref *iobref;
+    struct iovec *vector;
+    struct iatt stbuf;
+    struct iatt prebuf;
+    struct iatt postbuf;
+    dht_defrag_cbk_fn_t target_op_fn;
+    dict_t *xdata;
+    dict_t *xattr;
+    dict_t *dict;
+    struct gf_flock flock;
+    int32_t set;
+    int lock_cmd;
+};
 
-        /* flag used to make sure we need to return estale in
-           {lookup,revalidate}_cbk */
-        char return_estale;
-        char need_lookup_everywhere;
+/**
+ * Enum to store decided action based on the qdstatfs (quota-deem-statfs)
+ * events
+ **/
+typedef enum {
+    qdstatfs_action_OFF = 0,
+    qdstatfs_action_REPLACE,
+    qdstatfs_action_NEGLECT,
+    qdstatfs_action_COMPARE,
+} qdstatfs_action_t;
 
-        glusterfs_fop_t      fop;
+typedef enum {
+    REACTION_INVALID,
+    FAIL_ON_ANY_ERROR,
+    IGNORE_ENOENT_ESTALE,
+    IGNORE_ENOENT_ESTALE_EIO,
+} dht_reaction_type_t;
+
+struct dht_skip_linkto_unlink {
+    xlator_t *hash_links_to;
+    uuid_t cached_gfid;
+    uuid_t hashed_gfid;
+    int opend_fd_count;
+    gf_boolean_t handle_valid_link;
+};
 
-        struct dht_rebalance_ rebalance;
+typedef struct {
+    xlator_t *xl;
+    loc_t loc;      /* contains/points to inode to lock on. */
+    char *domain;   /* Only locks within a single domain
+                     * contend with each other
+                     */
+    char *basename; /* Required for entrylk */
+    gf_boolean_t locked;
+    dht_reaction_type_t do_on_failure;
+    short type; /* read/write lock.                     */
+    gf_lkowner_t lk_owner;
+} dht_lock_t;
+
+/* The lock structure represents inodelk. */
+typedef struct {
+    fop_inodelk_cbk_t inodelk_cbk;
+    dht_lock_t **locks;
+    int lk_count;
+    dht_reaction_type_t reaction;
+
+    /* whether locking failed on _any_ of the "locks" above */
+    int op_ret;
+    int op_errno;
+} dht_ilock_wrap_t;
+
+/* The lock structure represents entrylk. */
+typedef struct {
+    fop_entrylk_cbk_t entrylk_cbk;
+    dht_lock_t **locks;
+    int lk_count;
+    dht_reaction_type_t reaction;
+
+    /* whether locking failed on _any_ of the "locks" above */
+    int op_ret;
+    int op_errno;
+} dht_elock_wrap_t;
+
+/* The first member of dht_dir_transaction_t should be of type dht_ilock_wrap_t.
+ * Otherwise it can result in subtle memory corruption issues as in most of the
+ * places we use lock[0].layout.my_layout or lock[0].layout.parent_layout and
+ * lock[0].ns.parent_layout (like in dht_local_wipe).
+ */
+typedef union {
+    union {
+        dht_ilock_wrap_t my_layout;
+        dht_ilock_wrap_t parent_layout;
+    } layout;
+    struct dht_namespace {
+        dht_ilock_wrap_t parent_layout;
+        dht_elock_wrap_t directory_ns;
+        fop_entrylk_cbk_t ns_cbk;
+    } ns;
+} dht_dir_transaction_t;
+
+typedef int (*dht_selfheal_layout_t)(call_frame_t *frame, loc_t *loc,
+                                     dht_layout_t *layout);
+
+typedef gf_boolean_t (*dht_need_heal_t)(call_frame_t *frame,
+                                        dht_layout_t **inmem,
+                                        dht_layout_t **ondisk);
 
+struct dht_local {
+    loc_t loc;
+    loc_t loc2;
+    int call_cnt;
+    int op_ret;
+    int op_errno;
+    int layout_mismatch;
+    /* Use stbuf as the postbuf, when we require both
+     * pre and post attrs */
+    struct iatt stbuf;
+    struct iatt mds_stbuf;
+    struct iatt prebuf;
+    struct iatt preoldparent;
+    struct iatt postoldparent;
+    struct iatt preparent;
+    struct iatt postparent;
+    struct statvfs statvfs;
+    fd_t *fd;
+    inode_t *inode;
+    dict_t *params;
+    dict_t *xattr;
+    dict_t *mds_xattr;
+    dict_t *xdata; /* dict used to save xdata response by xattr fop */
+    dict_t *xattr_req;
+    dht_layout_t *layout;
+    size_t size;
+    ino_t ia_ino;
+    xlator_t *src_hashed, *src_cached;
+    xlator_t *dst_hashed, *dst_cached;
+    xlator_t *cached_subvol;
+    xlator_t *hashed_subvol;
+    xlator_t *mds_subvol; /* This is use for dir only */
+    int file_count;
+    int dir_count;
+    call_frame_t *main_frame;
+    int fop_succeeded;
+    struct {
+        fop_mknod_cbk_t linkfile_cbk;
+        struct iatt stbuf;
+        loc_t loc;
+        inode_t *inode;
+        dict_t *xattr;
+        xlator_t *srcvol;
+    } linkfile;
+    struct {
+        uint32_t hole_cnt;
+        uint32_t overlaps_cnt;
+        uint32_t down;
+        uint32_t misc;
+        dht_selfheal_dir_cbk_t dir_cbk;
+        dht_selfheal_layout_t healer;
+        dht_need_heal_t should_heal;
+        dht_layout_t *layout, *refreshed_layout;
+        uint32_t missing_cnt;
+        gf_boolean_t force_mkdir;
+    } selfheal;
+
+    dht_refresh_layout_unlock refresh_layout_unlock;
+    dht_refresh_layout_done_handle refresh_layout_done;
+
+    uint32_t uid;
+    uint32_t gid;
+    pid_t pid;
+
+    glusterfs_fop_t fop;
+
+    /* need for file-info */
+    char *xattr_val;
+    char *key;
+
+    /* needed by nufa */
+    int32_t flags;
+    mode_t mode;
+    dev_t rdev;
+    mode_t umask;
+
+    /* which xattr request? */
+    char xsel[256];
+    int32_t alloc_len;
+
+    /* gfid related */
+    uuid_t gfid;
+    uuid_t gfid_req;
+
+    xlator_t *link_subvol;
+
+    struct dht_rebalance_ rebalance;
+    xlator_t *first_up_subvol;
+
+    struct dht_skip_linkto_unlink skip_unlink;
+
+    dht_dir_transaction_t lock[2], *current;
+
+    /* inodelks during filerename for backward compatibility */
+    dht_lock_t **rename_inodelk_backward_compatible;
+
+    call_stub_t *stub;
+    int32_t parent_disk_layout[4];
+
+    /* rename rollback */
+    int *ret_cache;
+
+    loc_t loc2_copy;
+
+    int rename_inodelk_bc_count;
+    /* This is use only for directory operation */
+    int32_t valid;
+    int32_t mds_heal_fresh_lookup;
+    short lock_type;
+    char need_selfheal;
+    char need_xattr_heal;
+    char need_attrheal;
+    /* flag used to make sure we need to return estale in
+       {lookup,revalidate}_cbk */
+    char return_estale;
+    char need_lookup_everywhere;
+    /* fd open check */
+    gf_boolean_t fd_checked;
+    gf_boolean_t linked;
+    gf_boolean_t added_link;
+    gf_boolean_t is_linkfile;
+    gf_boolean_t quota_deem_statfs;
+    gf_boolean_t heal_layout;
+    gf_boolean_t locked;
+    gf_boolean_t dont_create_linkto;
+    gf_boolean_t gfid_missing;
 };
 typedef struct dht_local dht_local_t;
 
 /* du - disk-usage */
 struct dht_du {
-        double   avail_percent;
-	double   avail_inodes;
-        uint64_t avail_space;
-        uint32_t log;
+    double avail_percent;
+    double avail_inodes;
+    uint64_t avail_space;
+    uint32_t log;
+    uint32_t chunks;
+    uint32_t total_blocks;
+    uint32_t avail_blocks;
+    uint32_t frsize; /*fragment size*/
 };
 typedef struct dht_du dht_du_t;
 
 enum gf_defrag_type {
-        GF_DEFRAG_CMD_START = 1,
-        GF_DEFRAG_CMD_STOP = 1 + 1,
-        GF_DEFRAG_CMD_STATUS = 1 + 2,
-        GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
-        GF_DEFRAG_CMD_START_FORCE = 1 + 4,
+    GF_DEFRAG_CMD_NONE = 0,
+    GF_DEFRAG_CMD_START = 1,
+    GF_DEFRAG_CMD_STOP = 1 + 1,
+    GF_DEFRAG_CMD_STATUS = 1 + 2,
+    GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
+    GF_DEFRAG_CMD_START_FORCE = 1 + 4,
+    GF_DEFRAG_CMD_DETACH_STATUS = 1 + 11,
+    GF_DEFRAG_CMD_DETACH_START = 1 + 13,
+    GF_DEFRAG_CMD_DETACH_COMMIT = 1 + 14,
+    GF_DEFRAG_CMD_DETACH_COMMIT_FORCE = 1 + 15,
+    GF_DEFRAG_CMD_DETACH_STOP = 1 + 16,
+    /* new labels are used so it will help
+     * while removing old labels by easily differentiating.
+     * A few labels are added so that the count remains same
+     * between this enum and the ones on the xdr file.
+     * different values for the same enum cause errors and
+     * confusion.
+     */
 };
 typedef enum gf_defrag_type gf_defrag_type;
 
 enum gf_defrag_status_t {
-        GF_DEFRAG_STATUS_NOT_STARTED,
-        GF_DEFRAG_STATUS_STARTED,
-        GF_DEFRAG_STATUS_STOPPED,
-        GF_DEFRAG_STATUS_COMPLETE,
-        GF_DEFRAG_STATUS_FAILED,
+    GF_DEFRAG_STATUS_NOT_STARTED,
+    GF_DEFRAG_STATUS_STARTED,
+    GF_DEFRAG_STATUS_STOPPED,
+    GF_DEFRAG_STATUS_COMPLETE,
+    GF_DEFRAG_STATUS_FAILED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
 };
 typedef enum gf_defrag_status_t gf_defrag_status_t;
 
+typedef struct gf_defrag_pattern_list gf_defrag_pattern_list_t;
 
-struct gf_defrag_info_ {
-        uint64_t                     total_files;
-        uint64_t                     total_data;
-        uint64_t                     num_files_lookedup;
-        uint64_t                     total_failures;
-        gf_lock_t                    lock;
-        int                          cmd;
-        pthread_t                    th;
-        gf_defrag_status_t           defrag_status;
-        struct rpc_clnt             *rpc;
-        uint32_t                     connected;
-        uint32_t                     is_exiting;
-        pid_t                        pid;
-        inode_t                     *root_inode;
-        uuid_t                       node_uuid;
-        struct timeval               start_time;
-        gf_boolean_t                 stats;
+struct gf_defrag_pattern_list {
+    char path_pattern[256];
+    uint64_t size;
+    gf_defrag_pattern_list_t *next;
+};
 
+struct dht_container {
+    union {
+        struct list_head list;
+        struct {
+            struct _gf_dirent_t *next;
+            struct _gf_dirent_t *prev;
+        };
+    };
+    gf_dirent_t *df_entry;
+    xlator_t *this;
+    loc_t *parent_loc;
+    dict_t *migrate_data;
+    int local_subvol_index;
+};
+
+typedef struct nodeuuid_info {
+    char info;   /* Set to 1 is this is my node's uuid*/
+    uuid_t uuid; /* Store the nodeuuid as well for debugging*/
+} nodeuuid_info_t;
+
+typedef struct subvol_nodeuuids_info {
+    nodeuuid_info_t *elements;
+    int count;
+} subvol_nodeuuids_info_t;
+
+struct gf_defrag_info_ {
+    uint64_t total_files;
+    uint64_t total_data;
+    uint64_t num_files_lookedup;
+    uint64_t total_failures;
+    uint64_t skipped;
+    uint64_t num_dirs_processed;
+    uint64_t size_processed;
+    gf_lock_t lock;
+    pthread_t th;
+    struct rpc_clnt *rpc;
+    uint32_t connected;
+    uint32_t is_exiting;
+    pid_t pid;
+    int cmd;
+    inode_t *root_inode;
+    uuid_t node_uuid;
+    time_t start_time;
+    uint32_t new_commit_hash;
+    gf_defrag_status_t defrag_status;
+    gf_defrag_pattern_list_t *defrag_pattern;
+
+    pthread_cond_t parallel_migration_cond;
+    pthread_mutex_t dfq_mutex;
+    pthread_cond_t rebalance_crawler_alarm;
+    int32_t q_entry_count;
+    int32_t global_error;
+    struct dht_container *queue;
+    int32_t crawl_done;
+    int32_t abort;
+    int32_t wakeup_crawler;
+
+    /*Throttle params*/
+    /*stands for reconfigured thread count*/
+    int32_t recon_thread_count;
+    pthread_cond_t df_wakeup_thread;
+
+    /* backpointer to make it easier to write functions for rebalance */
+    xlator_t *this;
+
+    pthread_cond_t fc_wakeup_cond;
+    pthread_mutex_t fc_mutex;
+
+    /*stands for current running thread count*/
+    int32_t current_thread_count;
+
+    gf_boolean_t stats;
+    /* lock migration flag */
+    gf_boolean_t lock_migration_enabled;
 };
 
 typedef struct gf_defrag_info_ gf_defrag_info_t;
 
+struct dht_methods_s {
+    int32_t (*migration_get_dst_subvol)(xlator_t *this, dht_local_t *local);
+    int32_t (*migration_other)(xlator_t *this, gf_defrag_info_t *defrag);
+    xlator_t *(*layout_search)(xlator_t *this, dht_layout_t *layout,
+                               const char *name);
+};
+
+typedef struct dht_methods_s dht_methods_t;
+
 struct dht_conf {
-        gf_lock_t      subvolume_lock;
-        int            subvolume_cnt;
-        xlator_t     **subvolumes;
-        char          *subvolume_status;
-        int           *last_event;
-        dht_layout_t **file_layouts;
-        dht_layout_t **dir_layouts;
-        gf_boolean_t   search_unhashed;
-        int            gen;
-        dht_du_t      *du_stats;
-        uint64_t       min_free_disk;
-	uint32_t       min_free_inodes;
-        char           disk_unit;
-        int32_t        refresh_interval;
-        gf_boolean_t   unhashed_sticky_bit;
-        struct timeval last_stat_fetch;
-        gf_lock_t      layout_lock;
-        void          *private;     /* Can be used by wrapper xlators over
-                                       dht */
-        gf_boolean_t   use_readdirp;
-        char           vol_uuid[UUID_SIZE + 1];
-        gf_boolean_t   assert_no_child_down;
-        time_t        *subvol_up_time;
-
-        /* This is the count used as the distribute layout for a directory */
-        /* Will be a global flag to control the layout spread count */
-        uint32_t       dir_spread_cnt;
-
-        /* to keep track of nodes which are decomissioned */
-        xlator_t     **decommissioned_bricks;
-        int            decommission_in_progress;
-
-        /* defrag related */
-        gf_defrag_info_t *defrag;
+    xlator_t **subvolumes;
+    char *subvolume_status;
+    int *last_event;
+    dht_layout_t **file_layouts;
+    dht_layout_t **dir_layouts;
+    unsigned int search_unhashed;
+    int gen;
+    dht_du_t *du_stats;
+    double min_free_disk;
+    double min_free_inodes;
+    int subvolume_cnt;
+    int32_t refresh_interval;
+    gf_lock_t subvolume_lock;
+    time_t last_stat_fetch;
+    gf_lock_t layout_lock;
+    dict_t *leaf_to_subvol;
+    void *private; /* Can be used by wrapper xlators over
+                      dht */
+    time_t *subvol_up_time;
+
+    /* to keep track of nodes which are decommissioned */
+    xlator_t **decommissioned_bricks;
+    int decommission_in_progress;
+    int decommission_subvols_cnt;
+
+    /* defrag related */
+    gf_defrag_info_t *defrag;
+
+    /* Support regex-based name reinterpretation. */
+    regex_t rsync_regex;
+    regex_t extra_regex;
+
+    /* Support variable xattr names. */
+    char *xattr_name;
+    char *mds_xattr_key;
+    char *link_xattr_name;
+    char *commithash_xattr_name;
+    char *wild_xattr_name;
+
+    dht_methods_t methods;
+
+    struct mem_pool *lock_pool;
+
+    /*local subvol storage for rebalance*/
+    xlator_t **local_subvols;
+    subvol_nodeuuids_info_t *local_nodeuuids;
+    int32_t local_subvols_cnt;
+
+    int dthrottle;
+
+    /* Hard link handle requirement for migration triggered from client*/
+    synclock_t link_lock;
+
+    /* lock migration */
+    gf_lock_t lock;
+
+    /* This is the count used as the distribute layout for a directory */
+    /* Will be a global flag to control the layout spread count */
+    uint32_t dir_spread_cnt;
+
+    /*
+     * "Commit hash" for this volume topology.  Changed whenever bricks
+     * are added or removed.
+     */
+    uint32_t vol_commit_hash;
+
+    char vol_uuid[UUID_SIZE + 1];
+
+    char disk_unit;
+
+    gf_boolean_t lock_migration_enabled;
+
+    gf_boolean_t vch_forced;
+
+    gf_boolean_t use_fallocate;
+
+    gf_boolean_t force_migration;
+
+    gf_boolean_t lookup_optimize;
+
+    gf_boolean_t unhashed_sticky_bit;
+
+    gf_boolean_t assert_no_child_down;
+
+    gf_boolean_t use_readdirp;
+
+    /* Request to filter directory entries in readdir request */
+    gf_boolean_t readdir_optimize;
+
+    gf_boolean_t rsync_regex_valid;
+
+    gf_boolean_t extra_regex_valid;
+
+    /* Support size-weighted rebalancing (heterogeneous bricks). */
+    gf_boolean_t do_weighting;
+
+    gf_boolean_t randomize_by_gfid;
 };
 typedef struct dht_conf dht_conf_t;
 
+struct dht_dfoffset_ctx {
+    xlator_t *this;
+    off_t offset;
+    int32_t readdir_done;
+};
+typedef struct dht_dfoffset_ctx dht_dfoffset_ctx_t;
 
 struct dht_disk_layout {
-        uint32_t           cnt;
-        uint32_t           type;
-        struct {
-                uint32_t   start;
-                uint32_t   stop;
-        } list[1];
+    uint32_t cnt;
+    uint32_t type;
+    struct {
+        uint32_t start;
+        uint32_t stop;
+    } list[1];
 };
 typedef struct dht_disk_layout dht_disk_layout_t;
 
 typedef enum {
-        GF_DHT_MIGRATE_DATA,
-        GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS,
-        GF_DHT_MIGRATE_HARDLINK,
-        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS
+    GF_DHT_MIGRATE_DATA,
+    GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS,
+    GF_DHT_MIGRATE_HARDLINK,
+    GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS
 } gf_dht_migrate_data_type_t;
 
+typedef enum {
+    GF_DHT_EQUAL_DISTRIBUTION,
+    GF_DHT_WEIGHTED_DISTRIBUTION
+} dht_distribution_type_t;
+
+struct dir_dfmeta {
+    gf_dirent_t *equeue;
+    dht_dfoffset_ctx_t *offset_var;
+    struct list_head **head;
+    struct list_head **iterator;
+    int *fetch_entries;
+    /* fds corresponding to local subvols only */
+    fd_t **lfd;
+};
+
+typedef struct dht_migrate_info {
+    xlator_t *src_subvol;
+    xlator_t *dst_subvol;
+    GF_REF_DECL;
+} dht_migrate_info_t;
+
+typedef struct dht_fd_ctx {
+    uint64_t opened_on_dst;
+    GF_REF_DECL;
+} dht_fd_ctx_t;
+
 #define ENTRY_MISSING(op_ret, op_errno) (op_ret == -1 && op_errno == ENOENT)
 
-#define is_revalidate(loc) (inode_ctx_get (loc->inode, this, NULL) == 0)
+#define is_revalidate(loc)                                                     \
+    (dht_inode_ctx_layout_get((loc)->inode, this, NULL) == 0)
 
 #define is_last_call(cnt) (cnt == 0)
 
 #define DHT_MIGRATION_IN_PROGRESS 1
-#define DHT_MIGRATION_COMPLETED   2
+#define DHT_MIGRATION_COMPLETED 2
 
-#define DHT_LINKFILE_KEY         "trusted.glusterfs.dht.linkto"
-#define DHT_LINKFILE_MODE        (S_ISVTX)
+#define check_is_linkfile(i, s, x, n)                                          \
+    (IS_DHT_LINKFILE_MODE(s) && dict_get(x, n))
 
-#define check_is_linkfile(i,s,x) (                                      \
-                ((st_mode_from_ia ((s)->ia_prot, (s)->ia_type) & ~S_IFMT) \
-                 == DHT_LINKFILE_MODE) &&                               \
-                dict_get (x, DHT_LINKFILE_KEY))
+#define IS_DHT_MIGRATION_PHASE2(buf)                                           \
+    (IA_ISREG((buf)->ia_type) &&                                               \
+     ((st_mode_from_ia((buf)->ia_prot, (buf)->ia_type) & ~S_IFMT) ==           \
+      DHT_LINKFILE_MODE))
 
-#define IS_DHT_MIGRATION_PHASE2(buf)  (                                 \
-                IA_ISREG ((buf)->ia_type) &&                            \
-                ((st_mode_from_ia ((buf)->ia_prot, (buf)->ia_type) &    \
-                  ~S_IFMT) == DHT_LINKFILE_MODE))
+#define IS_DHT_MIGRATION_PHASE1(buf)                                           \
+    (IA_ISREG((buf)->ia_type) && ((buf)->ia_prot.sticky == 1) &&               \
+     ((buf)->ia_prot.sgid == 1))
 
-#define IS_DHT_MIGRATION_PHASE1(buf)  (                                 \
-                IA_ISREG ((buf)->ia_type) &&                            \
-                ((buf)->ia_prot.sticky == 1) &&                         \
-                ((buf)->ia_prot.sgid == 1))
+#define DHT_STRIP_PHASE1_FLAGS(buf)                                            \
+    do {                                                                       \
+        if ((buf) && IS_DHT_MIGRATION_PHASE1(buf)) {                           \
+            (buf)->ia_prot.sticky = 0;                                         \
+            (buf)->ia_prot.sgid = 0;                                           \
+        }                                                                      \
+    } while (0)
 
-#define DHT_STRIP_PHASE1_FLAGS(buf)  do {                       \
-                if ((buf) && IS_DHT_MIGRATION_PHASE1(buf)) {    \
-                        (buf)->ia_prot.sticky = 0;              \
-                        (buf)->ia_prot.sgid = 0;                \
-                }                                               \
-        } while (0)
+#define dht_inode_missing(op_errno) (op_errno == ENOENT || op_errno == ESTALE)
 
-#define check_is_dir(i,s,x) (IA_ISDIR(s->ia_type))
+#define check_is_dir(i, s, x) (IA_ISDIR(s->ia_type))
 
 #define layout_is_sane(layout) ((layout) && (layout->cnt > 0))
 
-#define DHT_STACK_UNWIND(fop, frame, params ...) do {           \
-                dht_local_t *__local = NULL;                    \
-                xlator_t    *__xl    = NULL;                    \
-                if (frame) {                                    \
-                        __xl         = frame->this;             \
-                        __local      = frame->local;            \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                dht_local_wipe (__xl, __local);                 \
-        } while (0)
-
-#define DHT_STACK_DESTROY(frame) do {                   \
-                dht_local_t *__local = NULL;            \
-                xlator_t    *__xl    = NULL;            \
-                __xl                 = frame->this;     \
-                __local              = frame->local;    \
-                frame->local         = NULL;            \
-                STACK_DESTROY (frame->root);            \
-                dht_local_wipe (__xl, __local);         \
-        } while (0)
-
-dht_layout_t                            *dht_layout_new (xlator_t *this, int cnt);
-dht_layout_t                            *dht_layout_get (xlator_t *this, inode_t *inode);
-dht_layout_t                            *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_layout_search (xlator_t   *this, dht_layout_t *layout,
-                             const char *name);
-int                                      dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout);
-int dht_layout_anomalies (xlator_t      *this, loc_t *loc, dht_layout_t *layout,
-                          uint32_t      *holes_p, uint32_t *overlaps_p,
-                          uint32_t      *missing_p, uint32_t *down_p,
-                          uint32_t      *misc_p);
-int dht_layout_dir_mismatch (xlator_t   *this, dht_layout_t *layout,
-                             xlator_t   *subvol, loc_t *loc, dict_t *xattr);
-
-xlator_t *dht_linkfile_subvol (xlator_t *this, inode_t *inode,
-                               struct iatt *buf, dict_t *xattr);
-int dht_linkfile_unlink (call_frame_t      *frame, xlator_t *this,
-                         xlator_t          *subvol, loc_t *loc);
-
-int dht_layouts_init (xlator_t *this, dht_conf_t *conf);
-int dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                      int       op_ret, int op_errno, dict_t *xattr);
-
-int dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-                             int       pos, int32_t **disk_layout_p);
-int dht_disk_layout_merge (xlator_t   *this, dht_layout_t *layout,
-                           int         pos, void *disk_layout_raw, int disk_layout_len);
-
-
-int dht_frame_return (call_frame_t *frame);
-
-int                             dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y);
-int dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol,
-                      uint64_t *x);
-
-void dht_local_wipe (xlator_t *this, dht_local_t *local);
-dht_local_t *dht_local_init (call_frame_t    *frame, loc_t *loc, fd_t *fd,
-                             glusterfs_fop_t  fop);
-int dht_iatt_merge (xlator_t                 *this, struct iatt *to, struct iatt *from,
-                    xlator_t                 *subvol);
-
-xlator_t *dht_subvol_get_hashed (xlator_t *this, loc_t *loc);
-xlator_t *dht_subvol_get_cached (xlator_t *this, inode_t *inode);
-xlator_t *dht_subvol_next (xlator_t *this, xlator_t *prev);
-int       dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
-
-int dht_hash_compute (int type, const char *name, uint32_t *hash_p);
-
-int dht_linkfile_create (call_frame_t    *frame, fop_mknod_cbk_t linkfile_cbk,
-                         xlator_t        *tovol, xlator_t *fromvol, loc_t *loc);
-int                                       dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc);
-int                                       dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc);
-int
-dht_selfheal_directory (call_frame_t     *frame, dht_selfheal_dir_cbk_t cbk,
-                        loc_t            *loc, dht_layout_t *layout);
-int
-dht_selfheal_new_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
-                            dht_layout_t *layout);
-int
-dht_selfheal_restore (call_frame_t       *frame, dht_selfheal_dir_cbk_t cbk,
-                      loc_t              *loc, dht_layout_t *layout);
-int
-dht_layout_sort_volname (dht_layout_t *layout);
-
-int dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
-
-gf_boolean_t dht_is_subvol_filled (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol);
-int       dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx);
-
-int dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode);
-int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);;
-void          dht_layout_unref (xlator_t *this, dht_layout_t *layout);
-dht_layout_t *dht_layout_ref (xlator_t *this, dht_layout_t *layout);
-xlator_t     *dht_first_up_subvol (xlator_t *this);
-xlator_t     *dht_last_up_subvol (xlator_t *this);
-
-int dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
-
-int dht_filter_loc_subvol_key (xlator_t  *this, loc_t *loc, loc_t *new_loc,
-                               xlator_t **subvol);
-
-int                                     dht_rename_cleanup (call_frame_t *frame);
-int dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t           op_ret, int32_t op_errno,
-                          inode_t          *inode, struct iatt *stbuf,
-                          struct iatt      *preparent, struct iatt *postparent,
-                          dict_t *xdata);
-
-int dht_fix_directory_layout (call_frame_t *frame,
-                              dht_selfheal_dir_cbk_t  dir_cbk,
-                              dht_layout_t           *layout);
-
-int dht_init_subvolumes (xlator_t *this, dht_conf_t *conf);
+#define we_are_not_migrating(x) ((x) == 1)
+
+#define DHT_STACK_UNWIND(fop, frame, params...)                                \
+    do {                                                                       \
+        dht_local_t *__local = NULL;                                           \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __xl = frame->this;                                                \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        dht_local_wipe(__xl, __local);                                         \
+    } while (0)
+
+#define DHT_STACK_DESTROY(frame)                                               \
+    do {                                                                       \
+        dht_local_t *__local = NULL;                                           \
+        xlator_t *__xl = NULL;                                                 \
+        __xl = frame->this;                                                    \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        dht_local_wipe(__xl, __local);                                         \
+    } while (0)
+
+#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, post)            \
+    do {                                                                       \
+        if (ctx_sec == new_sec)                                                \
+            new_nsec = max(new_nsec, ctx_nsec);                                \
+        else if (ctx_sec > new_sec) {                                          \
+            new_sec = ctx_sec;                                                 \
+            new_nsec = ctx_nsec;                                               \
+        }                                                                      \
+        if (post) {                                                            \
+            ctx_sec = new_sec;                                                 \
+            ctx_nsec = new_nsec;                                               \
+        }                                                                      \
+    } while (0)
+
+#define is_greater_time(a, an, b, bn)                                          \
+    (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
+
+#define DHT_MARK_FOP_INTERNAL(xattr)                                           \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr)                                                        \
+                break;                                                         \
+        }                                                                      \
+        tmp = dict_set_str(xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes");          \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   GLUSTERFS_INTERNAL_FOP_KEY, local->loc.path);               \
+        }                                                                      \
+    } while (0)
+
+dht_layout_t *
+dht_layout_new(xlator_t *this, int cnt);
+dht_layout_t *
+dht_layout_get(xlator_t *this, inode_t *inode);
+dht_layout_t *
+dht_layout_for_subvol(xlator_t *this, xlator_t *subvol);
+xlator_t *
+dht_layout_search(xlator_t *this, dht_layout_t *layout, const char *name);
+int32_t
+dht_migration_get_dst_subvol(xlator_t *this, dht_local_t *local);
+int32_t
+dht_migration_needed(xlator_t *this);
+int
+dht_layout_normalize(xlator_t *this, loc_t *loc, dht_layout_t *layout);
+void
+dht_layout_anomalies(xlator_t *this, loc_t *loc, dht_layout_t *layout,
+                     uint32_t *holes_p, uint32_t *overlaps_p,
+                     uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                     uint32_t *no_space_p);
+int
+dht_layout_dir_mismatch(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                        loc_t *loc, dict_t *xattr);
+xlator_t *
+dht_linkfile_subvol(xlator_t *this, inode_t *inode, struct iatt *buf,
+                    dict_t *xattr);
+int
+dht_linkfile_unlink(call_frame_t *frame, xlator_t *this, xlator_t *subvol,
+                    loc_t *loc);
+
+int
+dht_layouts_init(xlator_t *this, dht_conf_t *conf);
+int
+dht_layout_merge(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                 int op_ret, int op_errno, dict_t *xattr);
+
+int
+dht_disk_layout_extract(xlator_t *this, dht_layout_t *layout, int pos,
+                        int32_t **disk_layout_p);
+int
+dht_disk_layout_extract_for_subvol(xlator_t *this, dht_layout_t *layout,
+                                   xlator_t *subvol, int32_t **disk_layout_p);
+
+int
+dht_frame_return(call_frame_t *frame);
+
+int
+dht_deitransform(xlator_t *this, uint64_t y, xlator_t **subvol);
+
+void
+dht_local_wipe(xlator_t *this, dht_local_t *local);
+dht_local_t *
+dht_local_init(call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop);
+int
+dht_iatt_merge(xlator_t *this, struct iatt *to, struct iatt *from);
+
+xlator_t *
+dht_subvol_get_hashed(xlator_t *this, loc_t *loc);
+xlator_t *
+dht_subvol_get_cached(xlator_t *this, inode_t *inode);
+xlator_t *
+dht_subvol_next(xlator_t *this, xlator_t *prev);
+xlator_t *
+dht_subvol_next_available(xlator_t *this, xlator_t *prev);
+int
+dht_subvol_cnt(xlator_t *this, xlator_t *subvol);
+
+int
+dht_hash_compute(xlator_t *this, int type, const char *name, uint32_t *hash_p);
+
+int
+dht_linkfile_create(call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
+                    xlator_t *this, xlator_t *tovol, xlator_t *fromvol,
+                    loc_t *loc);
+int
+dht_lookup_everywhere(call_frame_t *frame, xlator_t *this, loc_t *loc);
+int
+dht_selfheal_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                       loc_t *loc, dht_layout_t *layout);
+int
+dht_selfheal_new_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                           dht_layout_t *layout);
+int
+dht_selfheal_restore(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                     loc_t *loc, dht_layout_t *layout);
+void
+dht_layout_sort_volname(dht_layout_t *layout);
+
+int
+dht_get_du_info(call_frame_t *frame, xlator_t *this, loc_t *loc);
+
+gf_boolean_t
+dht_is_subvol_filled(xlator_t *this, xlator_t *subvol);
+xlator_t *
+dht_free_disk_available_subvol(xlator_t *this, xlator_t *subvol,
+                               dht_local_t *layout);
+int
+dht_get_du_info_for_subvol(xlator_t *this, int subvol_idx);
+
+int
+dht_layout_preset(xlator_t *this, xlator_t *subvol, inode_t *inode);
+int
+dht_layout_set(xlator_t *this, inode_t *inode, dht_layout_t *layout);
+;
+void
+dht_layout_unref(xlator_t *this, dht_layout_t *layout);
+dht_layout_t *
+dht_layout_ref(xlator_t *this, dht_layout_t *layout);
+int
+dht_layout_index_for_subvol(dht_layout_t *layout, xlator_t *subvol);
+xlator_t *
+dht_first_up_subvol(xlator_t *this);
+xlator_t *
+dht_last_up_subvol(xlator_t *this);
+
+int
+dht_build_child_loc(xlator_t *this, loc_t *child, loc_t *parent, char *name);
+
+int
+dht_filter_loc_subvol_key(xlator_t *this, loc_t *loc, loc_t *new_loc,
+                          xlator_t **subvol);
+
+int
+dht_rename_cleanup(call_frame_t *frame);
+int
+dht_rename_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata);
+
+int
+dht_update_commit_hash_for_layout(call_frame_t *frame);
+int
+dht_fix_directory_layout(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                         dht_layout_t *layout);
+
+int
+dht_init_subvolumes(xlator_t *this, dht_conf_t *conf);
 
 /* migration/rebalance */
-int dht_start_rebalance_task (xlator_t *this, call_frame_t *frame);
+int
+dht_start_rebalance_task(xlator_t *this, call_frame_t *frame);
 
-int dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame);
-int dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame);
+int
+dht_rebalance_in_progress_check(xlator_t *this, call_frame_t *frame);
+int
+dht_rebalance_complete_check(xlator_t *this, call_frame_t *frame);
 
+int
+dht_init_local_subvolumes(xlator_t *this, dht_conf_t *conf);
 
 /* FOPS */
-int32_t dht_lookup (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc,
-                    dict_t   *xattr_req);
-
-int32_t dht_stat (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *loc, dict_t *xdata);
-
-int32_t dht_fstat (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd, dict_t *xdata);
-
-int32_t dht_truncate (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      off_t     offset, dict_t *xdata);
-
-int32_t dht_ftruncate (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t     *fd,
-                       off_t     offset, dict_t *xdata);
-
-int32_t dht_access (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc,
-                    int32_t   mask, dict_t *xdata);
-
-int32_t dht_readlink (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      size_t    size, dict_t *xdata);
-
-int32_t dht_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
-
-int32_t dht_mkdir (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
-
-int32_t dht_unlink (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc, int xflag, dict_t *xdata);
-
-int32_t dht_rmdir (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, int flags, dict_t *xdata);
-
-int32_t dht_symlink (call_frame_t   *frame, xlator_t *this,
-                     const char *linkpath, loc_t *loc, mode_t umask,
-                     dict_t *xdata);
-
-int32_t dht_rename (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *oldloc,
-                    loc_t    *newloc, dict_t *xdata);
-
-int32_t dht_link (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *oldloc,
-                  loc_t    *newloc, dict_t *xdata);
-
-int32_t dht_create (call_frame_t *frame, xlator_t *this,
-                    loc_t    *loc, int32_t flags, mode_t mode,
-                    mode_t umask, fd_t     *fd, dict_t *params);
-
-int32_t dht_open (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *loc,
-                  int32_t   flags, fd_t *fd, dict_t *xdata);
-
-int32_t dht_readv (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd,
-                   size_t    size,
-                   off_t     offset, uint32_t flags, dict_t *xdata);
-
-int32_t dht_writev (call_frame_t      *frame,
-                    xlator_t      *this,
-                    fd_t          *fd,
-                    struct iovec  *vector,
-                    int32_t        count,
-                    off_t          offset,
-                    uint32_t       flags,
-                    struct iobref *iobref, dict_t *xdata);
-
-int32_t dht_flush (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd, dict_t *xdata);
-
-int32_t dht_fsync (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd,
-                   int32_t   datasync, dict_t *xdata);
-
-int32_t dht_opendir (call_frame_t *frame,
-                     xlator_t *this,
-                     loc_t    *loc, fd_t *fd, dict_t *xdata);
-
-int32_t dht_fsyncdir (call_frame_t *frame,
-                      xlator_t *this,
-                      fd_t     *fd,
-                      int32_t   datasync, dict_t *xdata);
-
-int32_t dht_statfs (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc, dict_t *xdata);
-
-int32_t dht_setxattr (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      dict_t   *dict,
-                      int32_t   flags, dict_t *xdata);
-
-int32_t dht_getxattr (call_frame_t   *frame,
-                      xlator_t   *this,
-                      loc_t      *loc,
-                      const char *name, dict_t *xdata);
-
-int32_t dht_fsetxattr (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t     *fd,
-                       dict_t   *dict,
-                       int32_t   flags, dict_t *xdata);
-
-int32_t dht_fgetxattr (call_frame_t   *frame,
-                       xlator_t   *this,
-                       fd_t       *fd,
-                       const char *name, dict_t *xdata);
-
-int32_t dht_removexattr (call_frame_t   *frame,
-                         xlator_t   *this,
-                         loc_t      *loc,
-                         const char *name, dict_t *xdata);
-int32_t dht_fremovexattr (call_frame_t   *frame,
-                          xlator_t   *this,
-                          fd_t      *fd,
-                          const char *name, dict_t *xdata);
-
-int32_t dht_lk (call_frame_t        *frame,
-                xlator_t        *this,
-                fd_t            *fd,
-                int32_t          cmd,
-                struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_inodelk (call_frame_t *frame, xlator_t *this,
-                     const char      *volume, loc_t *loc, int32_t cmd,
-                     struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_finodelk (call_frame_t        *frame, xlator_t *this,
-                      const char      *volume, fd_t *fd, int32_t cmd,
-                      struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_entrylk (call_frame_t    *frame, xlator_t *this,
-                     const char  *volume, loc_t *loc, const char *basename,
-                     entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
-
-int32_t dht_fentrylk (call_frame_t    *frame, xlator_t *this,
-                      const char  *volume, fd_t *fd, const char *basename,
-                      entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
-
-int32_t dht_readdir (call_frame_t  *frame,
-                     xlator_t *this,
-                     fd_t     *fd,
-                     size_t    size, off_t off, dict_t *xdata);
-
-int32_t dht_readdirp (call_frame_t *frame,
-                      xlator_t *this,
-                      fd_t     *fd,
-                      size_t    size, off_t off, dict_t *dict);
-
-int32_t dht_xattrop (call_frame_t           *frame,
-                     xlator_t           *this,
-                     loc_t              *loc,
-                     gf_xattrop_flags_t  flags,
-                     dict_t             *dict, dict_t *xdata);
-
-int32_t dht_fxattrop (call_frame_t *frame,
-                      xlator_t           *this,
-                      fd_t               *fd,
-                      gf_xattrop_flags_t  flags,
-                      dict_t             *dict, dict_t *xdata);
-
-int32_t dht_forget (xlator_t *this, inode_t *inode);
-int32_t dht_setattr (call_frame_t  *frame, xlator_t *this, loc_t *loc,
-                     struct iatt   *stbuf, int32_t valid, dict_t *xdata);
-int32_t dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      struct iatt  *stbuf, int32_t valid, dict_t *xdata);
-
-int32_t dht_notify (xlator_t *this, int32_t event, void *data, ...);
+int32_t
+dht_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req);
+
+int32_t
+dht_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int32_t
+dht_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
+
+int32_t
+dht_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata);
+
+int32_t
+dht_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata);
+
+int32_t
+dht_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata);
+
+int32_t
+dht_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata);
+
+int32_t
+dht_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata);
+
+int32_t
+dht_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata);
+
+int32_t
+dht_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
+
+int32_t
+dht_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
+
+int32_t
+dht_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata);
+
+int32_t
+dht_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata);
+
+int32_t
+dht_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata);
+
+int32_t
+dht_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *params);
+
+int32_t
+dht_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata);
+
+int32_t
+dht_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata);
+
+int32_t
+dht_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata);
+
+int32_t
+dht_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
+
+int32_t
+dht_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata);
+
+int32_t
+dht_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata);
+
+int32_t
+dht_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+             dict_t *xdata);
+
+int32_t
+dht_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int32_t
+dht_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata);
+
+int32_t
+dht_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata);
+
+int32_t
+dht_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata);
+
+int32_t
+dht_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata);
+
+int32_t
+dht_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata);
+int32_t
+dht_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata);
+
+int32_t
+dht_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata);
+
+int32_t
+dht_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata);
+
+int32_t
+dht_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata);
+
+int32_t
+dht_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata);
+
+int32_t
+dht_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t off, dict_t *dict);
+
+int32_t
+dht_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+dht_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+dht_forget(xlator_t *this, inode_t *inode);
+int32_t
+dht_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata);
+int32_t
+dht_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int32_t valid, dict_t *xdata);
+int32_t
+dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata);
+int32_t
+dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata);
+int32_t
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata);
+int32_t
+dht_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata);
+
+int
+dht_set_subvol_range(xlator_t *this);
+int32_t
+dht_init(xlator_t *this);
+void
+dht_fini(xlator_t *this);
+int
+dht_reconfigure(xlator_t *this, dict_t *options);
+int32_t
+dht_notify(xlator_t *this, int32_t event, void *data, ...);
 
 /* definitions for nufa/switch */
-int dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, inode_t *inode,
-                        struct iatt *stbuf, dict_t *xattr,
-                        struct iatt *postparent);
-int dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+int
+dht_revalidate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent);
+int
+dht_lookup_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent);
+int
+dht_lookup_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                         int op_ret, int op_errno, inode_t *inode,
                         struct iatt *stbuf, dict_t *xattr,
                         struct iatt *postparent);
-int dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int op_ret, int op_errno,
-                             inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                             struct iatt *postparent);
-int dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent);
-int dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                    struct iatt *preparent, struct iatt *postparent,
-                    dict_t *xdata);
-int dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno,
-                     inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata);
+int
+dht_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+               struct iatt *postparent);
+int
+dht_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata);
+int
+dht_newfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata);
+
+int
+dht_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int
+dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xattr, dict_t *xdata);
+
+int
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata);
+int
+gf_defrag_status_get(dht_conf_t *conf, dict_t *dict);
+
+int
+gf_defrag_stop(dht_conf_t *conf, gf_defrag_status_t status, dict_t *output);
+
+void *
+gf_defrag_start(void *this);
+
+int32_t
+gf_defrag_handle_hardlink(xlator_t *this, loc_t *loc, int *fop_errno);
+int
+dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                 int flag, int *fop_errno);
+int
+dht_inode_ctx_layout_get(inode_t *inode, xlator_t *this,
+                         dht_layout_t **layout_int);
+int
+dht_inode_ctx_layout_set(inode_t *inode, xlator_t *this,
+                         dht_layout_t *layout_int);
+int
+dht_inode_ctx_time_update(inode_t *inode, xlator_t *this, struct iatt *stat,
+                          int32_t update_ctx);
+void
+dht_inode_ctx_time_set(inode_t *inode, xlator_t *this, struct iatt *stat);
+
+int
+dht_inode_ctx_get(inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
+int
+dht_inode_ctx_set(inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
+int
+dht_dir_attr_heal(void *data);
+int
+dht_dir_attr_heal_done(int ret, call_frame_t *sync_frame, void *data);
+xlator_t *
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  xlator_t *ignore, dht_layout_t *layout,
+                                  uint64_t filesize);
+xlator_t *
+dht_subvol_maxspace_nonzeroinode(xlator_t *this, xlator_t *subvol,
+                                 dht_layout_t *layout);
+int
+dht_dir_has_layout(dict_t *xattr, char *name);
+int
+dht_linkfile_attr_heal(call_frame_t *frame, xlator_t *this);
+
+int32_t
+dht_priv_dump(xlator_t *this);
+int32_t
+dht_inodectx_dump(xlator_t *this, inode_t *inode);
+
+gf_boolean_t
+dht_is_subvol_in_layout(dht_layout_t *layout, xlator_t *xlator);
+
+int
+dht_inode_ctx_get_mig_info(xlator_t *this, inode_t *inode,
+                           xlator_t **src_subvol, xlator_t **dst_subvol);
+gf_boolean_t
+dht_mig_info_is_invalid(xlator_t *current, xlator_t *src_subvol,
+                        xlator_t *dst_subvol);
+
+int
+dht_subvol_status(dht_conf_t *conf, xlator_t *subvol);
+
+void
+dht_log_new_layout_for_dir_selfheal(xlator_t *this, loc_t *loc,
+                                    dht_layout_t *layout);
 
 int
-gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict);
+dht_layout_sort(dht_layout_t *layout);
 
 int
-gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output);
+dht_heal_full_path(void *data);
 
-void*
-gf_defrag_start (void *this);
+int
+dht_heal_full_path_done(int op_ret, call_frame_t *frame, void *data);
+
+int
+dht_layout_missing_dirs(dht_layout_t *layout);
+
+int
+dht_refresh_layout(call_frame_t *frame);
+
+int
+dht_build_parent_loc(xlator_t *this, loc_t *parent, loc_t *child,
+                     int32_t *op_errno);
+
+int32_t
+dht_set_local_rebalance(xlator_t *this, dht_local_t *local, struct iatt *stbuf,
+                        struct iatt *prebuf, struct iatt *postbuf,
+                        dict_t *xdata);
+void
+dht_build_root_loc(inode_t *inode, loc_t *loc);
+
+gf_boolean_t
+dht_fd_open_on_dst(xlator_t *this, fd_t *fd, xlator_t *dst);
+
+int32_t
+dht_fd_ctx_destroy(xlator_t *this, fd_t *fd);
+
+int32_t
+dht_release(xlator_t *this, fd_t *fd);
+
+int32_t
+dht_set_fixed_dir_stat(struct iatt *stat);
+
+xlator_t *
+dht_get_lock_subvolume(xlator_t *this, struct gf_flock *lock,
+                       dht_local_t *local);
+
+int
+dht_lk_inode_unref(call_frame_t *frame, int32_t op_ret);
+
+int
+dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *subvol);
+
+int
+dht_check_and_open_fd_on_subvol(xlator_t *this, call_frame_t *frame);
+
+/* FD fop callbacks */
+
+int
+dht_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata);
+
+int
+dht_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, dict_t *xdata);
+
+int
+dht_file_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata);
+
+int
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata);
+
+int
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                dict_t *xdata);
+
+int
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                  dict_t *xdata);
+
+int
+dht_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata);
+
+int
+dht_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata);
+
+int
+dht_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iovec *vector, int count, struct iatt *stbuf,
+              struct iobref *iobref, dict_t *xdata);
+
+int
+dht_file_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *stbuf, dict_t *xdata);
+
+int
+dht_file_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata);
+
+int
+dht_file_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata);
+
+/* All custom xattr heal functions */
+int
+dht_dir_heal_xattrs(void *data);
+
+int
+dht_dir_heal_xattrs_done(int ret, call_frame_t *sync_frame, void *data);
+
+int32_t
+dht_dict_set_array(dict_t *dict, char *key, int32_t value[], int32_t size);
+
+int
+dht_set_user_xattr(dict_t *dict, char *k, data_t *v, void *data);
+
+void
+dht_dir_set_heal_xattr(xlator_t *this, dht_local_t *local, dict_t *dst,
+                       dict_t *src, int *uret, int *uflag);
+
+int
+dht_dir_xattr_heal(xlator_t *this, dht_local_t *local, int *op_errno);
+
+int
+dht_common_mark_mdsxattr(call_frame_t *frame, int *errst, int flag);
+
+int
+dht_inode_ctx_mdsvol_get(inode_t *inode, xlator_t *this, xlator_t **mdsvol);
+
+int
+dht_selfheal_dir_setattr(call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
+                         int32_t valid, dht_layout_t *layout);
+
+/* Abstract out the DHT-IATT-IN-DICT */
+
+void
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *new_layout);
+
+int
+dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata);
+
+int
+dht_pt_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata);
 
 int32_t
-gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
-                           struct iatt *stbuf);
+dht_pt_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata);
+
+int
+dht_pt_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata);
+
+int32_t
+dht_check_remote_fd_failed_error(dht_local_t *local, int op_ret, int op_errno);
+
+int
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata);
+
+int32_t
+dht_create_lock(call_frame_t *frame, xlator_t *subvol);
+
+int
+dht_set_parent_layout_in_dict(loc_t *loc, xlator_t *this, dht_local_t *local);
+
+int
+dht_dir_layout_error_check(xlator_t *this, inode_t *inode);
+
 int
-dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
-                 int flag);
-#endif/* _DHT_H */
+dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol);
+#endif /* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c
index 52ea3a32aca..c0588828fdb 100644
--- a/xlators/cluster/dht/src/dht-diskusage.c
+++ b/xlators/cluster/dht/src/dht-diskusage.c
@@ -8,290 +8,480 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 /* TODO: add NS locking */
 
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "defaults.h"
 
 #include <sys/time.h>
-
+#include <glusterfs/events.h>
 
 int
-dht_du_info_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, struct statvfs *statvfs,
-                 dict_t *xdata)
+dht_du_info_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
-	dht_conf_t    *conf         = NULL;
-	call_frame_t  *prev          = NULL;
-	int            this_call_cnt = 0;
-	int            i = 0;
-	double         percent = 0;
-	double         percent_inodes = 0;
-	uint64_t       bytes = 0;
-
-	conf = this->private;
-	prev = cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"failed to get disk info from %s", prev->this->name);
-		goto out;
-	}
-
-	if (statvfs && statvfs->f_blocks) {
-		percent = (statvfs->f_bavail * 100) / statvfs->f_blocks;
-		bytes = (statvfs->f_bavail * statvfs->f_frsize);
-	}
-
-	if (statvfs && statvfs->f_files) {
-		percent_inodes = (statvfs->f_ffree * 100) / statvfs->f_files;
-	} else {
-		/* set percent inodes to 100 for dynamically allocated inode filesystems
-		   this logic holds good so that, distribute has nothing to worry about
-		   total inodes rather let the 'create()' to be scheduled on the hashed
-		   subvol regardless of the total inodes. since we have no awareness on
-		   loosing inodes this logic fits well
-		*/
-		percent_inodes = 100;
-	}
-
-	LOCK (&conf->subvolume_lock);
-	{
-		for (i = 0; i < conf->subvolume_cnt; i++)
-			if (prev->this == conf->subvolumes[i]) {
-				conf->du_stats[i].avail_percent = percent;
-				conf->du_stats[i].avail_space   = bytes;
-				conf->du_stats[i].avail_inodes  = percent_inodes;
-				gf_log (this->name, GF_LOG_DEBUG,
-					"on subvolume '%s': avail_percent is: "
-					"%.2f and avail_space is: %"PRIu64" "
-					"and avail_inodes is: %.2f",
-					prev->this->name,
-					conf->du_stats[i].avail_percent,
-					conf->du_stats[i].avail_space,
-					conf->du_stats[i].avail_inodes);
-			}
-	}
-	UNLOCK (&conf->subvolume_lock);
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+    int i = 0;
+    double percent = 0;
+    double percent_inodes = 0;
+    uint64_t bytes = 0;
+    uint32_t bpc; /* blocks per chunk */
+    uint32_t chunks = 0;
+
+    conf = this->private;
+    prev = cookie;
+
+    if (op_ret == -1 || !statvfs) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno,
+               DHT_MSG_GET_DISK_INFO_ERROR, "failed to get disk info from %s",
+               prev->name);
+        goto out;
+    }
+
+    if (statvfs->f_blocks) {
+        percent = (statvfs->f_bavail * 100) / statvfs->f_blocks;
+        bytes = (statvfs->f_bavail * statvfs->f_frsize);
+        /*
+         * A 32-bit count of 1MB chunks allows a maximum brick size of
+         * ~4PB.  It's possible that we could see a single local FS
+         * bigger than that some day, but this code is likely to be
+         * irrelevant by then.  Meanwhile, it's more important to keep
+         * the chunk size small so the layout-calculation code that
+         * uses this value can be tested on normal machines.
+         */
+        bpc = (1 << 20) / statvfs->f_bsize;
+        chunks = (statvfs->f_blocks + bpc - 1) / bpc;
+    }
+
+    if (statvfs->f_files) {
+        percent_inodes = (statvfs->f_ffree * 100) / statvfs->f_files;
+    } else {
+        /*
+         * Set percent inodes to 100 for dynamically allocated inode
+         * filesystems. The rationale is that distribute need not
+         * worry about total inodes; rather, let the 'create()' be
+         * scheduled on the hashed subvol regardless of the total
+         * inodes.
+         */
+        percent_inodes = 100;
+    }
+
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++)
+            if (prev == conf->subvolumes[i]) {
+                conf->du_stats[i].avail_percent = percent;
+                conf->du_stats[i].avail_space = bytes;
+                conf->du_stats[i].avail_inodes = percent_inodes;
+                conf->du_stats[i].chunks = chunks;
+                conf->du_stats[i].total_blocks = statvfs->f_blocks;
+                conf->du_stats[i].avail_blocks = statvfs->f_bavail;
+                conf->du_stats[i].frsize = statvfs->f_frsize;
+
+                gf_msg_debug(this->name, 0,
+                             "subvolume '%s': avail_percent "
+                             "is: %.2f and avail_space "
+                             "is: %" PRIu64
+                             " and avail_inodes"
+                             " is: %.2f",
+                             prev->name, conf->du_stats[i].avail_percent,
+                             conf->du_stats[i].avail_space,
+                             conf->du_stats[i].avail_inodes);
+                break; /* no point in looping further */
+            }
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_DESTROY (frame);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_DESTROY(frame);
 
-	return 0;
+    return 0;
 }
 
 int
-dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx)
+dht_get_du_info_for_subvol(xlator_t *this, int subvol_idx)
 {
-	dht_conf_t    *conf         = NULL;
-	call_frame_t  *statfs_frame = NULL;
-	dht_local_t   *statfs_local = NULL;
-	call_pool_t   *pool         = NULL;
-	loc_t          tmp_loc      = {0,};
-
-	conf = this->private;
-	pool = this->ctx->pool;
-
-	statfs_frame = create_frame (this, pool);
-	if (!statfs_frame) {
-		goto err;
-	}
-
-	/* local->fop value is not used in this case */
-	statfs_local = dht_local_init (statfs_frame, NULL, NULL,
-				       GF_FOP_MAXVALUE);
-	if (!statfs_local) {
-		goto err;
-	}
-
-        /* make it root gfid, should be enough to get the proper info back */
-        tmp_loc.gfid[15] = 1;
-
-	statfs_local->call_cnt = 1;
-	STACK_WIND (statfs_frame, dht_du_info_cbk,
-		    conf->subvolumes[subvol_idx],
-		    conf->subvolumes[subvol_idx]->fops->statfs,
-		    &tmp_loc, NULL);
-
-	return 0;
+    dht_conf_t *conf = NULL;
+    call_frame_t *statfs_frame = NULL;
+    dht_local_t *statfs_local = NULL;
+    call_pool_t *pool = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+
+    conf = this->private;
+    pool = this->ctx->pool;
+
+    statfs_frame = create_frame(this, pool);
+    if (!statfs_frame) {
+        goto err;
+    }
+
+    /* local->fop value is not used in this case */
+    statfs_local = dht_local_init(statfs_frame, NULL, NULL, GF_FOP_MAXVALUE);
+    if (!statfs_local) {
+        goto err;
+    }
+
+    /* make it root gfid, should be enough to get the proper info back */
+    tmp_loc.gfid[15] = 1;
+
+    statfs_local->call_cnt = 1;
+    STACK_WIND_COOKIE(
+        statfs_frame, dht_du_info_cbk, conf->subvolumes[subvol_idx],
+        conf->subvolumes[subvol_idx],
+        conf->subvolumes[subvol_idx]->fops->statfs, &tmp_loc, NULL);
+
+    return 0;
 err:
-	if (statfs_frame)
-		DHT_STACK_DESTROY (statfs_frame);
+    if (statfs_frame)
+        DHT_STACK_DESTROY(statfs_frame);
 
-	return -1;
+    return -1;
 }
 
 int
-dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_get_du_info(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-	int            i            = 0;
-	dht_conf_t    *conf         = NULL;
-	call_frame_t  *statfs_frame = NULL;
-	dht_local_t   *statfs_local = NULL;
-	struct timeval tv           = {0,};
-        loc_t          tmp_loc      = {0,};
-
-	conf  = this->private;
-
-	gettimeofday (&tv, NULL);
-
-        /* make it root gfid, should be enough to get the proper
-           info back */
-        tmp_loc.gfid[15] = 1;
-
-	if (tv.tv_sec > (conf->refresh_interval
-			 + conf->last_stat_fetch.tv_sec)) {
-
-		statfs_frame = copy_frame (frame);
-		if (!statfs_frame) {
-			goto err;
-		}
-
-		/* In this case, 'local->fop' is not used */
-		statfs_local = dht_local_init (statfs_frame, loc, NULL,
-					       GF_FOP_MAXVALUE);
-		if (!statfs_local) {
-			goto err;
-		}
-
-		statfs_local->call_cnt = conf->subvolume_cnt;
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			STACK_WIND (statfs_frame, dht_du_info_cbk,
-				    conf->subvolumes[i],
-				    conf->subvolumes[i]->fops->statfs,
-				    &tmp_loc, NULL);
-		}
-
-		conf->last_stat_fetch.tv_sec = tv.tv_sec;
-	}
-	return 0;
+    int i = 0;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    call_frame_t *statfs_frame = NULL;
+    dht_local_t *statfs_local = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    time_t now;
+
+    conf = this->private;
+    now = gf_time();
+    /* make it root gfid, should be enough to get the proper
+       info back */
+    tmp_loc.gfid[15] = 1;
+
+    if (now > (conf->refresh_interval + conf->last_stat_fetch)) {
+        statfs_frame = copy_frame(frame);
+        if (!statfs_frame) {
+            goto err;
+        }
+
+        /* In this case, 'local->fop' is not used */
+        statfs_local = dht_local_init(statfs_frame, loc, NULL, GF_FOP_MAXVALUE);
+        if (!statfs_local) {
+            goto err;
+        }
+
+        statfs_local->params = dict_new();
+        if (!statfs_local->params)
+            goto err;
+
+        ret = dict_set_int8(statfs_local->params,
+                            GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set " GF_INTERNAL_IGNORE_DEEM_STATFS " in dict");
+            goto err;
+        }
+
+        statfs_local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            STACK_WIND_COOKIE(statfs_frame, dht_du_info_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->statfs, &tmp_loc,
+                              statfs_local->params);
+        }
+
+        conf->last_stat_fetch = now;
+    }
+    return 0;
 err:
-	if (statfs_frame)
-		DHT_STACK_DESTROY (statfs_frame);
+    if (statfs_frame)
+        DHT_STACK_DESTROY(statfs_frame);
 
-	return -1;
+    return -1;
 }
 
-
 gf_boolean_t
-dht_is_subvol_filled (xlator_t *this, xlator_t *subvol)
+dht_is_subvol_filled(xlator_t *this, xlator_t *subvol)
+{
+    int i = 0;
+    char vol_name[256];
+    dht_conf_t *conf = NULL;
+    gf_boolean_t subvol_filled_inodes = _gf_false;
+    gf_boolean_t subvol_filled_space = _gf_false;
+    gf_boolean_t is_subvol_filled = _gf_false;
+    double usage = 0;
+
+    conf = this->private;
+
+    /* Check for values above specified percent or free disk */
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (subvol == conf->subvolumes[i]) {
+                if (conf->disk_unit == 'p') {
+                    if (conf->du_stats[i].avail_percent < conf->min_free_disk) {
+                        subvol_filled_space = _gf_true;
+                        break;
+                    }
+
+                } else {
+                    if (conf->du_stats[i].avail_space < conf->min_free_disk) {
+                        subvol_filled_space = _gf_true;
+                        break;
+                    }
+                }
+                if (conf->du_stats[i].avail_inodes < conf->min_free_inodes) {
+                    subvol_filled_inodes = _gf_true;
+                    break;
+                }
+            }
+        }
+    }
+    UNLOCK(&conf->subvolume_lock);
+
+    if (subvol_filled_space && conf->subvolume_status[i]) {
+        if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+            usage = 100 - conf->du_stats[i].avail_percent;
+
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+                   "disk space on subvolume '%s' is getting "
+                   "full (%.2f %%), consider adding more bricks",
+                   subvol->name, usage);
+
+            (void)snprintf(vol_name, sizeof(vol_name), "%s", this->name);
+            vol_name[(strlen(this->name) - 4)] = '\0';
+
+            gf_event(EVENT_DHT_DISK_USAGE, "volume=%s;subvol=%s;usage=%.2f %%",
+                     vol_name, subvol->name, usage);
+        }
+    }
+
+    if (subvol_filled_inodes && conf->subvolume_status[i]) {
+        if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+            usage = 100 - conf->du_stats[i].avail_inodes;
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_SUBVOL_INSUFF_INODES,
+                   "inodes on subvolume '%s' are at "
+                   "(%.2f %%), consider adding more bricks",
+                   subvol->name, usage);
+
+            (void)snprintf(vol_name, sizeof(vol_name), "%s", this->name);
+            vol_name[(strlen(this->name) - 4)] = '\0';
+
+            gf_event(EVENT_DHT_INODES_USAGE,
+                     "volume=%s;subvol=%s;usage=%.2f %%", vol_name,
+                     subvol->name, usage);
+        }
+    }
+
+    is_subvol_filled = (subvol_filled_space || subvol_filled_inodes);
+
+    return is_subvol_filled;
+}
+
+/*Get the best subvolume to create the file in*/
+xlator_t *
+dht_free_disk_available_subvol(xlator_t *this, xlator_t *subvol,
+                               dht_local_t *local)
+{
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    loc_t *loc = NULL;
+
+    conf = this->private;
+    if (!local)
+        goto out;
+    loc = &local->loc;
+    if (!local->layout) {
+        layout = dht_layout_get(this, loc->parent);
+
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "Missing layout. path=%s,"
+                         " parent gfid = %s",
+                         loc->path, uuid_utoa(loc->parent->gfid));
+            goto out;
+        }
+    } else {
+        layout = dht_layout_ref(this, local->layout);
+    }
+
+    LOCK(&conf->subvolume_lock);
+    {
+        avail_subvol = dht_subvol_with_free_space_inodes(this, subvol, NULL,
+                                                         layout, 0);
+        if (!avail_subvol) {
+            avail_subvol = dht_subvol_maxspace_nonzeroinode(this, subvol,
+                                                            layout);
+        }
+    }
+    UNLOCK(&conf->subvolume_lock);
+out:
+    if (!avail_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "No subvolume has enough free space \
+                              and/or inodes to create");
+        avail_subvol = subvol;
+    }
+
+    if (layout)
+        dht_layout_unref(this, layout);
+    return avail_subvol;
+}
+
+static inline int32_t
+dht_subvol_has_err(dht_conf_t *conf, xlator_t *this, xlator_t *ignore,
+                   dht_layout_t *layout)
+{
+    int ret = -1;
+    int i = 0;
+
+    if (!this || !layout)
+        goto out;
+
+    /* this check is meant for rebalance process. The source of the file
+     * should be ignored for space check */
+    if (this == ignore) {
+        goto out;
+    }
+
+    /* check if subvol has layout errors, before selecting it */
+    for (i = 0; i < layout->cnt; i++) {
+        if (!strcmp(layout->list[i].xlator->name, this->name) &&
+            (layout->list[i].err != 0)) {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    /* discard decommissioned subvol */
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == this) {
+                ret = -1;
+                goto out;
+            }
+        }
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+/*Get subvolume which has both space and inodes more than the min criteria*/
+xlator_t *
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  xlator_t *ignore, dht_layout_t *layout,
+                                  uint64_t filesize)
 {
-	int         i = 0;
-	dht_conf_t *conf = NULL;
-	gf_boolean_t subvol_filled_inodes = _gf_false;
-	gf_boolean_t subvol_filled_space = _gf_false;
-	gf_boolean_t is_subvol_filled = _gf_false;
-
-	conf = this->private;
-
-	/* Check for values above specified percent or free disk */
-	LOCK (&conf->subvolume_lock);
-	{
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (subvol == conf->subvolumes[i]) {
-				if (conf->disk_unit == 'p') {
-					if (conf->du_stats[i].avail_percent <
-					    conf->min_free_disk) {
-						subvol_filled_space = _gf_true;
-						break;
-					}
-
-				} else {
-					if (conf->du_stats[i].avail_space <
-					    conf->min_free_disk) {
-						subvol_filled_space = _gf_true;
-						break;
-					}
-				}
-				if (conf->du_stats[i].avail_inodes <
-				    conf->min_free_inodes) {
-					subvol_filled_inodes = _gf_true;
-					break;
-				}
-			}
-		}
-	}
-	UNLOCK (&conf->subvolume_lock);
-
-	if (subvol_filled_space && conf->subvolume_status[i]) {
-		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"disk space on subvolume '%s' is getting "
-				"full (%.2f %%), consider adding more nodes",
-				subvol->name,
-				(100 - conf->du_stats[i].avail_percent));
-		}
-	}
-
-	if (subvol_filled_inodes && conf->subvolume_status[i]) {
-		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
-			gf_log (this->name, GF_LOG_CRITICAL,
-				"inodes on subvolume '%s' are at "
-				"(%.2f %%), consider adding more nodes",
-				subvol->name,
-				(100 - conf->du_stats[i].avail_inodes));
-		}
-	}
-
-	is_subvol_filled = (subvol_filled_space || subvol_filled_inodes);
-
-	return is_subvol_filled;
+    int i = 0;
+    double max = 0;
+    double max_inodes = 0;
+    int ignore_subvol = 0;
+    uint64_t total_blocks = 0;
+    uint64_t avail_blocks = 0;
+    uint64_t frsize = 0;
+    double post_availspace = 0;
+    double post_percent = 0;
+
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        /* check if subvol has layout errors and also it is not a
+         * decommissioned brick, before selecting it */
+        ignore_subvol = dht_subvol_has_err(conf, conf->subvolumes[i], ignore,
+                                           layout);
+        if (ignore_subvol)
+            continue;
+
+        if ((conf->disk_unit == 'p') &&
+            (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
+            (conf->du_stats[i].avail_inodes > conf->min_free_inodes)) {
+            if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                (conf->du_stats[i].avail_percent > max)) {
+                max = conf->du_stats[i].avail_percent;
+                max_inodes = conf->du_stats[i].avail_inodes;
+                avail_subvol = conf->subvolumes[i];
+                total_blocks = conf->du_stats[i].total_blocks;
+                avail_blocks = conf->du_stats[i].avail_blocks;
+                frsize = conf->du_stats[i].frsize;
+            }
+        }
+
+        if ((conf->disk_unit != 'p') &&
+            (conf->du_stats[i].avail_space > conf->min_free_disk) &&
+            (conf->du_stats[i].avail_inodes > conf->min_free_inodes)) {
+            if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                (conf->du_stats[i].avail_space > max)) {
+                max = conf->du_stats[i].avail_space;
+                max_inodes = conf->du_stats[i].avail_inodes;
+                avail_subvol = conf->subvolumes[i];
+            }
+        }
+    }
+
+    if (avail_subvol) {
+        if (conf->disk_unit == 'p') {
+            post_availspace = (avail_blocks * frsize) - filesize;
+            post_percent = (post_availspace * 100) / (total_blocks * frsize);
+            if (post_percent < conf->min_free_disk)
+                avail_subvol = NULL;
+        }
+        if (conf->disk_unit != 'p') {
+            if ((max - filesize) < conf->min_free_disk)
+                avail_subvol = NULL;
+        }
+    }
+
+    return avail_subvol;
 }
 
+/* Get subvol which has at least one inode and maximum space */
 xlator_t *
-dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol)
+dht_subvol_maxspace_nonzeroinode(xlator_t *this, xlator_t *subvol,
+                                 dht_layout_t *layout)
 {
-	int         i = 0;
-	double      max = 0;
-	double      max_inodes = 0;
-	xlator_t   *avail_subvol = NULL;
-	dht_conf_t *conf = NULL;
-
-	conf = this->private;
-
-	LOCK (&conf->subvolume_lock);
-	{
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (conf->disk_unit == 'p') {
-				if ((conf->du_stats[i].avail_percent > max)
-				    && (conf->du_stats[i].avail_inodes > max_inodes)) {
-					max = conf->du_stats[i].avail_percent;
-					max_inodes = conf->du_stats[i].avail_inodes;
-					avail_subvol = conf->subvolumes[i];
-				}
-			} else {
-				if ((conf->du_stats[i].avail_space > max)
-				    && (conf->du_stats[i].avail_inodes > max_inodes)) {
-					max = conf->du_stats[i].avail_space;
-					max_inodes = conf->du_stats[i].avail_inodes;
-					avail_subvol = conf->subvolumes[i];
-				}
-
-			}
-		}
-	}
-	UNLOCK (&conf->subvolume_lock);
-
-	if (!avail_subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume has enough free space and inodes to create");
-	}
-
-	if ((max < conf->min_free_disk) && (max_inodes < conf->min_free_inodes))
-		avail_subvol = subvol;
-
-	if (!avail_subvol)
-		avail_subvol = subvol;
-
-	return avail_subvol;
+    int i = 0;
+    double max = 0;
+    int ignore_subvol = 0;
+
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        /* check if subvol has layout errors and also it is not a
+         * decommissioned brick, before selecting it*/
+
+        ignore_subvol = dht_subvol_has_err(conf, conf->subvolumes[i], NULL,
+                                           layout);
+        if (ignore_subvol)
+            continue;
+
+        if (conf->disk_unit == 'p') {
+            if ((conf->du_stats[i].avail_percent > max) &&
+                (conf->du_stats[i].avail_inodes > 0)) {
+                max = conf->du_stats[i].avail_percent;
+                avail_subvol = conf->subvolumes[i];
+            }
+        } else {
+            if ((conf->du_stats[i].avail_space > max) &&
+                (conf->du_stats[i].avail_inodes > 0)) {
+                max = conf->du_stats[i].avail_space;
+                avail_subvol = conf->subvolumes[i];
+            }
+        }
+    }
+
+    return avail_subvol;
 }
diff --git a/xlators/cluster/dht/src/dht-hashfn.c b/xlators/cluster/dht/src/dht-hashfn.c
index 97eb1f05fa8..acda67c312a 100644
--- a/xlators/cluster/dht/src/dht-hashfn.c
+++ b/xlators/cluster/dht/src/dht-hashfn.c
@@ -8,66 +8,103 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "hashfn.h"
+#include <glusterfs/hashfn.h>
 
-
-int
-dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
+static int
+dht_hash_compute_internal(int type, const char *name, const int len,
+                          uint32_t *hash_p)
 {
-        int      ret = 0;
-        uint32_t hash = 0;
+    int ret = 0;
+    uint32_t hash = 0;
 
-        switch (type) {
+    switch (type) {
         case DHT_HASH_TYPE_DM:
         case DHT_HASH_TYPE_DM_USER:
-                hash = gf_dm_hashfn (name, strlen (name));
-                break;
+            hash = gf_dm_hashfn(name, len);
+            break;
         default:
-                ret = -1;
-                break;
-        }
+            ret = -1;
+            break;
+    }
 
-        if (ret == 0) {
-                *hash_p = hash;
-        }
+    if (ret == 0) {
+        *hash_p = hash;
+    }
 
-        return ret;
+    return ret;
 }
 
+/* The function returns:
+ * 0  : in case no munge took place
+ * >0 : the length (inc. terminating NULL!) of the newly modified string,
+ *      if it was munged.
+ */
+static int
+dht_munge_name(const char *original, char *modified, size_t len, regex_t *re)
+{
+    regmatch_t matches[2] = {
+        {0},
+    };
+    size_t new_len = 0;
+    int ret = 0;
+
+    ret = regexec(re, original, 2, matches, 0);
 
-#define MAKE_RSYNC_FRIENDLY_NAME(rsync_frndly_name, name) do {          \
-                rsync_frndly_name = (char *) name;                      \
-                if (name[0] == '.') {                                   \
-                        char *dot   = 0;                                \
-                        int namelen = 0;                                \
-                                                                        \
-                        dot = strrchr (name, '.');                      \
-                        if (dot && dot > (name + 1) && *(dot + 1)) {    \
-                                namelen = (dot - name);                 \
-                                rsync_frndly_name = alloca (namelen);   \
-                                strncpy (rsync_frndly_name, name + 1,   \
-                                         namelen);                      \
-                                rsync_frndly_name[namelen - 1] = 0;     \
-                        }                                               \
-                }                                                       \
-        } while (0);
+    if (ret != REG_NOMATCH) {
+        if (matches[1].rm_so != -1) {
+            new_len = matches[1].rm_eo - matches[1].rm_so;
+            /* Equal would fail due to the NUL at the end. */
+            if (new_len < len) {
+                memcpy(modified, original + matches[1].rm_so, new_len);
+                modified[new_len] = '\0';
+                return new_len + 1; /* +1 for the terminating NULL */
+            }
+        }
+    }
 
+    /* This is guaranteed safe because of how the dest was allocated. */
+    strcpy(modified, original);
+    return 0;
+}
 
 int
-dht_hash_compute (int type, const char *name, uint32_t *hash_p)
+dht_hash_compute(xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
-        char     *rsync_friendly_name = NULL;
+    char *rsync_friendly_name = NULL;
+    dht_conf_t *priv = NULL;
+    size_t len = 0;
+    int munged = 0;
+
+    priv = this->private;
+
+    if (name == NULL)
+        return -1;
 
-        MAKE_RSYNC_FRIENDLY_NAME (rsync_friendly_name, name);
+    len = strlen(name) + 1;
+    rsync_friendly_name = alloca(len);
+
+    LOCK(&priv->lock);
+    {
+        if (priv->extra_regex_valid) {
+            munged = dht_munge_name(name, rsync_friendly_name, len,
+                                    &priv->extra_regex);
+        }
+
+        if (!munged && priv->rsync_regex_valid) {
+            gf_msg_trace(this->name, 0, "trying regex for %s", name);
+            munged = dht_munge_name(name, rsync_friendly_name, len,
+                                    &priv->rsync_regex);
+        }
+    }
+    UNLOCK(&priv->lock);
+    if (munged) {
+        gf_msg_debug(this->name, 0, "munged down to %s", rsync_friendly_name);
+        len = munged;
+    } else {
+        rsync_friendly_name = (char *)name;
+    }
 
-        return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
+    return dht_hash_compute_internal(type, rsync_friendly_name, len - 1,
+                                     hash_p);
 }
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index f0e74a15423..3f2fe43d5f3 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -8,895 +8,2297 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include "dht-common.h"
+#include "dht-lock.h"
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
 
+static void
+dht_free_fd_ctx(dht_fd_ctx_t *fd_ctx)
+{
+    GF_FREE(fd_ctx);
+}
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "dht-common.h"
+int32_t
+dht_fd_ctx_destroy(xlator_t *this, fd_t *fd)
+{
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    ret = fd_ctx_del(fd, this, &value);
+    if (ret) {
+        goto out;
+    }
+
+    fd_ctx = (dht_fd_ctx_t *)(uintptr_t)value;
+    if (fd_ctx) {
+        GF_REF_PUT(fd_ctx);
+    }
+out:
+    return ret;
+}
+
+static int
+__dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *dst)
+{
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
+    fd_ctx = GF_CALLOC(1, sizeof(*fd_ctx), gf_dht_mt_fd_ctx_t);
+
+    if (!fd_ctx) {
+        goto out;
+    }
+
+    fd_ctx->opened_on_dst = (uint64_t)(uintptr_t)dst;
+    GF_REF_INIT(fd_ctx, dht_free_fd_ctx);
+
+    value = (uint64_t)(uintptr_t)fd_ctx;
+
+    ret = __fd_ctx_set(fd, this, value);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FD_CTX_SET_FAILED,
+                "fd=0x%p", fd, NULL);
+        GF_REF_PUT(fd_ctx);
+    }
+out:
+    return ret;
+}
 
 int
-dht_frame_return (call_frame_t *frame)
+dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *dst)
+{
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, this, &value);
+        if (ret && value) {
+            fd_ctx = (dht_fd_ctx_t *)(uintptr_t)value;
+            if (fd_ctx->opened_on_dst == (uint64_t)(uintptr_t)dst) {
+                /* This could happen due to racing
+                 * check_progress tasks*/
+                goto unlock;
+            } else {
+                /* This would be a big problem*/
+                /* Overwrite and hope for the best*/
+                fd_ctx->opened_on_dst = (uint64_t)(uintptr_t)dst;
+                UNLOCK(&fd->lock);
+                gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_VALUE,
+                        NULL);
+
+                goto out;
+            }
+        }
+        ret = __dht_fd_ctx_set(this, fd, dst);
+    }
+unlock:
+    UNLOCK(&fd->lock);
+out:
+    return ret;
+}
+
+static dht_fd_ctx_t *
+dht_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        dht_local_t *local = NULL;
-        int          this_call_cnt = -1;
+    dht_fd_ctx_t *fd_ctx = NULL;
+    int ret = -1;
+    uint64_t tmp_val = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, this, &tmp_val);
+        if ((ret < 0) || (tmp_val == 0)) {
+            goto unlock;
+        }
 
-        if (!frame)
-                return -1;
+        fd_ctx = (dht_fd_ctx_t *)(uintptr_t)tmp_val;
+        GF_REF_GET(fd_ctx);
+    }
+unlock:
+    UNLOCK(&fd->lock);
 
-        local = frame->local;
+out:
+    return fd_ctx;
+}
 
-        LOCK (&frame->lock);
-        {
-                this_call_cnt = --local->call_cnt;
+gf_boolean_t
+dht_fd_open_on_dst(xlator_t *this, fd_t *fd, xlator_t *dst)
+{
+    dht_fd_ctx_t *fd_ctx = NULL;
+    gf_boolean_t opened = _gf_false;
+
+    fd_ctx = dht_fd_ctx_get(this, fd);
+
+    if (fd_ctx) {
+        if (fd_ctx->opened_on_dst == (uint64_t)(uintptr_t)dst) {
+            opened = _gf_true;
         }
-        UNLOCK (&frame->lock);
+        GF_REF_PUT(fd_ctx);
+    }
 
-        return this_call_cnt;
+    return opened;
 }
 
+void
+dht_free_mig_info(void *data)
+{
+    dht_migrate_info_t *miginfo = NULL;
 
-int
-dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
+    miginfo = data;
+    GF_FREE(miginfo);
+
+    return;
+}
+
+static int
+dht_inode_ctx_set_mig_info(xlator_t *this, inode_t *inode, xlator_t *src_subvol,
+                           xlator_t *dst_subvol)
 {
-        dht_conf_t *conf = NULL;
-        int         cnt = 0;
-        int         max = 0;
-        uint64_t    y = 0;
+    dht_migrate_info_t *miginfo = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        if (x == ((uint64_t) -1)) {
-                y = (uint64_t) -1;
-                goto out;
+    miginfo = GF_CALLOC(1, sizeof(*miginfo), gf_dht_mt_miginfo_t);
+    if (miginfo == NULL)
+        goto out;
+
+    miginfo->src_subvol = src_subvol;
+    miginfo->dst_subvol = dst_subvol;
+    GF_REF_INIT(miginfo, dht_free_mig_info);
+
+    value = (uint64_t)(uintptr_t)miginfo;
+
+    ret = inode_ctx_set1(inode, this, &value);
+    if (ret < 0) {
+        GF_REF_PUT(miginfo);
+    }
+
+out:
+    return ret;
+}
+
+int
+dht_inode_ctx_get_mig_info(xlator_t *this, inode_t *inode,
+                           xlator_t **src_subvol, xlator_t **dst_subvol)
+{
+    int ret = -1;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get1(inode, this, &tmp_miginfo);
+        if ((ret < 0) || (tmp_miginfo == 0)) {
+            UNLOCK(&inode->lock);
+            goto out;
         }
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+        miginfo = (dht_migrate_info_t *)(uintptr_t)tmp_miginfo;
+        GF_REF_GET(miginfo);
+    }
+    UNLOCK(&inode->lock);
+
+    if (src_subvol)
+        *src_subvol = miginfo->src_subvol;
 
-        max = conf->subvolume_cnt;
-        cnt = dht_subvol_cnt (this, subvol);
+    if (dst_subvol)
+        *dst_subvol = miginfo->dst_subvol;
 
-        y = ((x * max) + cnt);
+    GF_REF_PUT(miginfo);
 
 out:
-        if (y_p)
-                *y_p = y;
+    return ret;
+}
 
-        return 0;
+gf_boolean_t
+dht_mig_info_is_invalid(xlator_t *current, xlator_t *src_subvol,
+                        xlator_t *dst_subvol)
+{
+    /* Not set
+     */
+    if (!src_subvol || !dst_subvol)
+        return _gf_true;
+
+    /* Invalid scenarios:
+     * The src_subvol does not match the subvol on which the current op was sent
+     * so the cached subvol has changed between the last mig_info_set and now.
+     * src_subvol == dst_subvol. The file was migrated without any FOP detecting
+     * a P2 so the old dst is now the current subvol.
+     *
+     * There is still one scenario where the info could be outdated - if
+     * file has undergone multiple migrations and ends up on the same src_subvol
+     * on which the mig_info was first set.
+     */
+    if ((current == dst_subvol) || (current != src_subvol))
+        return _gf_true;
+
+    return _gf_false;
 }
 
+/* Used to check if fd fops have the fd opened on the cached subvol
+ * This is required when:
+ * 1. an fd is opened on FILE1 on subvol1
+ * 2. the file is migrated to subvol2
+ * 3. a lookup updates the cached subvol in the inode_ctx to subvol2
+ * 4. a write comes on the fd
+ * The write is sent to subvol2 on an fd which has been opened only on fd1
+ * Since the migration phase checks don't kick in, the fop fails with EBADF
+ *
+ */
+
 int
-dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
-                           xlator_t **subvol)
+dht_check_and_open_fd_on_subvol_complete(int ret, call_frame_t *frame,
+                                         void *data)
 {
-        char          *new_name  = NULL;
-        char          *new_path  = NULL;
-        xlator_list_t *trav      = NULL;
-        char           key[1024] = {0,};
-        int            ret       = 0; /* not found */
+    glusterfs_fop_t fop = 0;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    fd_t *fd = NULL;
+    int op_errno = -1;
+
+    local = frame->local;
+    this = frame->this;
+    fop = local->fop;
+    subvol = local->cached_subvol;
+    fd = local->fd;
+
+    if (ret) {
+        op_errno = local->op_errno;
+        goto handle_err;
+    }
+
+    switch (fop) {
+        case GF_FOP_WRITE:
+            STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                              subvol->fops->writev, fd, local->rebalance.vector,
+                              local->rebalance.count, local->rebalance.offset,
+                              local->rebalance.flags, local->rebalance.iobref,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_FLUSH:
+            STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, fd,
+                       local->xattr_req);
+            break;
+
+        case GF_FOP_FSETATTR:
+            STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                              subvol->fops->fsetattr, fd,
+                              &local->rebalance.stbuf, local->rebalance.flags,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_ZEROFILL:
+            STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                              subvol->fops->zerofill, fd,
+                              local->rebalance.offset, local->rebalance.size,
+                              local->xattr_req);
+
+            break;
+
+        case GF_FOP_DISCARD:
+            STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                              subvol->fops->discard, local->fd,
+                              local->rebalance.offset, local->rebalance.size,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_FALLOCATE:
+            STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                              subvol->fops->fallocate, fd,
+                              local->rebalance.flags, local->rebalance.offset,
+                              local->rebalance.size, local->xattr_req);
+            break;
+
+        case GF_FOP_FTRUNCATE:
+            STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                              subvol->fops->ftruncate, fd,
+                              local->rebalance.offset, local->xattr_req);
+            break;
+
+        case GF_FOP_FSYNC:
+            STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol,
+                              subvol->fops->fsync, local->fd,
+                              local->rebalance.flags, local->xattr_req);
+            break;
+
+        case GF_FOP_READ:
+            STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv,
+                       local->fd, local->rebalance.size,
+                       local->rebalance.offset, local->rebalance.flags,
+                       local->xattr_req);
+            break;
+
+        case GF_FOP_FSTAT:
+            STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                              subvol->fops->fstat, fd, local->xattr_req);
+            break;
+
+        case GF_FOP_FSETXATTR:
+            STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                              subvol->fops->fsetxattr, local->fd,
+                              local->rebalance.xattr, local->rebalance.flags,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_FREMOVEXATTR:
+            STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                              subvol->fops->fremovexattr, local->fd, local->key,
+                              local->xattr_req);
+
+            break;
+
+        case GF_FOP_FXATTROP:
+            STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                       subvol->fops->fxattrop, local->fd,
+                       local->rebalance.flags, local->rebalance.xattr,
+                       local->xattr_req);
+            break;
+
+        case GF_FOP_FGETXATTR:
+            STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->fgetxattr,
+                       local->fd, local->key, NULL);
+            break;
+
+        case GF_FOP_FINODELK:
+            STACK_WIND(frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk,
+                       local->key, local->fd, local->rebalance.lock_cmd,
+                       &local->rebalance.flock, local->xattr_req);
+            break;
+        default:
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, "fd=%p",
+                    fd, "gfid=%s", uuid_utoa(fd->inode->gfid), "name=%s",
+                    subvol->name, NULL);
+            break;
+    }
+
+    goto out;
+
+    /* Could not open the fd on the dst. Unwind */
+
+handle_err:
+
+    switch (fop) {
+        case GF_FOP_WRITE:
+            DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_FLUSH:
+            DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+            break;
+
+        case GF_FOP_FSETATTR:
+            DHT_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_ZEROFILL:
+            DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_DISCARD:
+            DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_FALLOCATE:
+            DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_FTRUNCATE:
+            DHT_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_FSYNC:
+            DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_READ:
+            DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
+                             NULL);
+            break;
+
+        case GF_FOP_FSTAT:
+            DHT_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
+            break;
+
+        case GF_FOP_FSETXATTR:
+            DHT_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+            break;
+
+        case GF_FOP_FREMOVEXATTR:
+            DHT_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
+            break;
+
+        case GF_FOP_FXATTROP:
+            DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+            break;
+
+        case GF_FOP_FGETXATTR:
+            DHT_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
+            break;
+
+        case GF_FOP_FINODELK:
+            DHT_STACK_UNWIND(finodelk, frame, -1, op_errno, NULL);
+            break;
+
+        default:
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, "fd=%p",
+                    fd, "gfid=%s", uuid_utoa(fd->inode->gfid), "name=%s",
+                    subvol->name, NULL);
+            break;
+    }
 
-        /* Why do other tasks if first required 'char' itself is not there */
-        if (!new_loc || !loc || !loc->name || !strchr (loc->name, '@'))
-                goto out;
+out:
 
-        trav = this->children;
-        while (trav) {
-                snprintf (key, 1024, "*@%s:%s", this->name, trav->xlator->name);
-                if (fnmatch (key, loc->name, FNM_NOESCAPE) == 0) {
-                        new_name = GF_CALLOC(strlen (loc->name),
-                                             sizeof (char),
-                                             gf_common_mt_char);
-                        if (!new_name)
-                                goto out;
-                        if (fnmatch (key, loc->path, FNM_NOESCAPE) == 0) {
-                                new_path = GF_CALLOC(strlen (loc->path),
-                                                     sizeof (char),
-                                                     gf_common_mt_char);
-                                if (!new_path)
-                                        goto out;
-                                strncpy (new_path, loc->path, (strlen (loc->path) -
-                                                               strlen (key) + 1));
-                        }
-                        strncpy (new_name, loc->name, (strlen (loc->name) -
-                                                       strlen (key) + 1));
-
-                        if (new_loc) {
-                                new_loc->path   = ((new_path) ? new_path:
-                                                   gf_strdup (loc->path));
-                                new_loc->name   = new_name;
-                                new_loc->inode  = inode_ref (loc->inode);
-                                new_loc->parent = inode_ref (loc->parent);
-                        }
-                        *subvol         = trav->xlator;
-                        ret = 1;  /* success */
-                        goto out;
-                }
-                trav = trav->next;
+    return 0;
+}
+
+/* Check once again if the fd has been opened on the cached subvol.
+ * If not, open and update the fd_ctx.
+ */
+
+int
+dht_check_and_open_fd_on_subvol_task(void *data)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    fd_t *fd = NULL;
+    xlator_t *this = NULL;
+    xlator_t *subvol = NULL;
+
+    frame = data;
+    local = frame->local;
+    this = THIS;
+    fd = local->fd;
+    subvol = local->cached_subvol;
+
+    local->fd_checked = _gf_true;
+
+    if (fd_is_anonymous(fd) || dht_fd_open_on_dst(this, fd, subvol)) {
+        ret = 0;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "Opening fd (%p, flags=0%o) on file %s @ %s",
+                 fd, fd->flags, uuid_utoa(fd->inode->gfid), subvol->name);
+
+    loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(loc.gfid, fd->inode->gfid);
+
+    /* Open this on the dst subvol */
+
+    SYNCTASK_SETID(0, 0);
+
+    ret = syncop_open(subvol, &loc, (fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                      fd, NULL, NULL);
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_OPEN_FD_ON_DST_FAILED,
+                "fd=%p", fd, "flags=0%o", fd->flags, "gfid=%s",
+                uuid_utoa(fd->inode->gfid), "name=%s", subvol->name, NULL);
+        /* This can happen if the cached subvol was updated in the
+         * inode_ctx and the fd was opened on the new cached suvol
+         * after this fop was wound on the old cached subvol.
+         * As we do not close the fd on the old subvol (a leak)
+         * don't treat ENOENT as an error and allow the phase1/phase2
+         * checks to handle it.
+         */
+
+        if ((-ret != ENOENT) && (-ret != ESTALE)) {
+            local->op_errno = -ret;
+            ret = -1;
+        } else {
+            ret = 0;
         }
+
+        local->op_errno = -ret;
+        ret = -1;
+
+    } else {
+        dht_fd_ctx_set(this, fd, subvol);
+    }
+
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
 out:
-        if (!ret) {
-                /* !success */
-                GF_FREE (new_path);
-                GF_FREE (new_name);
-        }
-        return ret;
+    loc_wipe(&loc);
+
+    return ret;
+}
+
+int
+dht_check_and_open_fd_on_subvol(xlator_t *this, call_frame_t *frame)
+{
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    /*
+            if (dht_fd_open_on_dst (this, fd, subvol))
+                    goto out;
+    */
+    local = frame->local;
+
+    ret = synctask_new(this->ctx->env, dht_check_and_open_fd_on_subvol_task,
+                       dht_check_and_open_fd_on_subvol_complete, frame, frame);
+
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SYNCTASK_CREATE_FAILED,
+                "to-check-and-open fd=%p", local->fd, NULL);
+    }
+
+    return ret;
 }
 
 int
-dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p,
-                  uint64_t *x_p)
+dht_frame_return(call_frame_t *frame)
 {
-        dht_conf_t *conf = NULL;
-        int         cnt = 0;
-        int         max = 0;
-        uint64_t    x = 0;
-        xlator_t   *subvol = 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+
+    if (!frame)
+        return -1;
+
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
+    }
+    UNLOCK(&frame->lock);
 
-        if (!this->private)
+    return this_call_cnt;
+}
+
+/*
+ * Use this function to specify which subvol you want the file created
+ * on - this need not be the hashed subvol.
+ * Format: <filename>@<this->name>:<subvol-name>
+ * Eg: file-1@vol1-dht:vol1-client-0
+ *     where vol1 is a pure distribute volume
+ *     will create file-1 on vol1-client-0
+ */
+
+int
+dht_filter_loc_subvol_key(xlator_t *this, loc_t *loc, loc_t *new_loc,
+                          xlator_t **subvol)
+{
+    char *new_name = NULL;
+    char *new_path = NULL;
+    xlator_list_t *trav = NULL;
+    char key[1024] = {
+        0,
+    };
+    int ret = 0; /* not found */
+    int keylen = 0;
+    int name_len = 0;
+    int path_len = 0;
+
+    /* Why do other tasks if first required 'char' itself is not there */
+    if (!new_loc || !loc || !loc->name || !strchr(loc->name, '@')) {
+        /* Skip the GF_FREE checks here */
+        return ret;
+    }
+
+    trav = this->children;
+    while (trav) {
+        keylen = snprintf(key, sizeof(key), "*@%s:%s", this->name,
+                          trav->xlator->name);
+        /* Ignore '*' */
+        keylen = keylen - 1;
+        if (fnmatch(key, loc->name, FNM_NOESCAPE) == 0) {
+            name_len = strlen(loc->name) - keylen;
+            new_name = GF_MALLOC(name_len + 1, gf_common_mt_char);
+            if (!new_name)
                 goto out;
+            if (fnmatch(key, loc->path, FNM_NOESCAPE) == 0) {
+                path_len = strlen(loc->path) - keylen;
+                new_path = GF_MALLOC(path_len + 1, gf_common_mt_char);
+                if (!new_path)
+                    goto out;
+                snprintf(new_path, path_len + 1, "%s", loc->path);
+            }
+            snprintf(new_name, name_len + 1, "%s", loc->name);
+
+            if (new_loc) {
+                new_loc->path = ((new_path) ? new_path : gf_strdup(loc->path));
+                new_loc->name = new_name;
+                new_loc->inode = inode_ref(loc->inode);
+                new_loc->parent = inode_ref(loc->parent);
+            }
+            *subvol = trav->xlator;
+            ret = 1; /* success */
+            goto out;
+        }
+        trav = trav->next;
+    }
+out:
+    if (!ret) {
+        /* !success */
+        GF_FREE(new_path);
+        GF_FREE(new_name);
+    }
+    return ret;
+}
 
-        conf = this->private;
-        max = conf->subvolume_cnt;
+static xlator_t *
+dht_get_subvol_from_id(xlator_t *this, int client_id)
+{
+    xlator_t *xl = NULL;
+    dht_conf_t *conf = NULL;
+    char *sid = NULL;
+    int32_t ret = -1;
 
-        cnt = y % max;
-        x   = y / max;
+    conf = this->private;
 
-        subvol = conf->subvolumes[cnt];
+    ret = gf_asprintf(&sid, "%d", client_id);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_ASPRINTF_FAILED, NULL);
+        goto out;
+    }
 
-        if (subvol_p)
-                *subvol_p = subvol;
+    if (dict_get_ptr(conf->leaf_to_subvol, sid, (void **)&xl))
+        xl = NULL;
 
-        if (x_p)
-                *x_p = x;
+    GF_FREE(sid);
 
 out:
-        return 0;
+    return xl;
 }
 
+int
+dht_deitransform(xlator_t *this, uint64_t y, xlator_t **subvol_p)
+{
+    int client_id = 0;
+    xlator_t *subvol = 0;
+    dht_conf_t *conf = NULL;
+
+    if (!this->private)
+        return -1;
+
+    conf = this->private;
+
+    client_id = gf_deitransform(this, y);
+
+    subvol = dht_get_subvol_from_id(this, client_id);
+
+    if (!subvol)
+        subvol = conf->subvolumes[0];
+
+    if (subvol_p)
+        *subvol_p = subvol;
+
+    return 0;
+}
 
 void
-dht_local_wipe (xlator_t *this, dht_local_t *local)
+dht_local_wipe(xlator_t *this, dht_local_t *local)
 {
-        if (!local)
-                return;
+    int i = 0;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->loc2);
+    if (!local)
+        return;
 
-        if (local->xattr)
-                dict_unref (local->xattr);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->loc2);
+    loc_wipe(&local->loc2_copy);
 
-        if (local->inode)
-                inode_unref (local->inode);
+    if (local->xattr)
+        dict_unref(local->xattr);
 
-        if (local->layout) {
-                dht_layout_unref (this, local->layout);
-                local->layout = NULL;
-        }
+    if (local->inode)
+        inode_unref(local->inode);
 
-        loc_wipe (&local->linkfile.loc);
+    if (local->layout) {
+        dht_layout_unref(this, local->layout);
+        local->layout = NULL;
+    }
 
-        if (local->linkfile.xattr)
-                dict_unref (local->linkfile.xattr);
+    loc_wipe(&local->linkfile.loc);
 
-        if (local->linkfile.inode)
-                inode_unref (local->linkfile.inode);
+    if (local->linkfile.xattr)
+        dict_unref(local->linkfile.xattr);
 
-        if (local->fd) {
-                fd_unref (local->fd);
-                local->fd = NULL;
-        }
+    if (local->linkfile.inode)
+        inode_unref(local->linkfile.inode);
 
-        if (local->params) {
-                dict_unref (local->params);
-                local->params = NULL;
-        }
+    if (local->fd) {
+        fd_unref(local->fd);
+        local->fd = NULL;
+    }
 
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
+    if (local->params) {
+        dict_unref(local->params);
+        local->params = NULL;
+    }
 
-        if (local->selfheal.layout) {
-                dht_layout_unref (this, local->selfheal.layout);
-                local->selfheal.layout = NULL;
-        }
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
+    if (local->mds_xattr)
+        dict_unref(local->mds_xattr);
+    if (local->xdata)
+        dict_unref(local->xdata);
 
-        GF_FREE (local->newpath);
+    if (local->selfheal.layout) {
+        dht_layout_unref(this, local->selfheal.layout);
+        local->selfheal.layout = NULL;
+    }
 
-        GF_FREE (local->key);
+    if (local->selfheal.refreshed_layout) {
+        dht_layout_unref(this, local->selfheal.refreshed_layout);
+        local->selfheal.refreshed_layout = NULL;
+    }
 
-        GF_FREE (local->rebalance.vector);
+    for (i = 0; i < 2; i++) {
+        dht_lock_array_free(local->lock[i].ns.parent_layout.locks,
+                            local->lock[i].ns.parent_layout.lk_count);
 
-        if (local->rebalance.iobref)
-                iobref_unref (local->rebalance.iobref);
+        GF_FREE(local->lock[i].ns.parent_layout.locks);
 
-        mem_put (local);
-}
+        dht_lock_array_free(local->lock[i].ns.directory_ns.locks,
+                            local->lock[i].ns.directory_ns.lk_count);
+        GF_FREE(local->lock[i].ns.directory_ns.locks);
+    }
 
+    GF_FREE(local->key);
 
-dht_local_t *
-dht_local_init (call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
-{
-        dht_local_t *local = NULL;
-        inode_t     *inode = NULL;
-        int          ret   = 0;
+    if (local->rebalance.xdata)
+        dict_unref(local->rebalance.xdata);
 
-        local = mem_get0 (THIS->local_pool);
-        if (!local)
-                goto out;
+    if (local->rebalance.xattr)
+        dict_unref(local->rebalance.xattr);
 
-        if (loc) {
-                ret = loc_copy (&local->loc, loc);
-                if (ret)
-                        goto out;
-
-                inode = loc->inode;
-        }
+    if (local->rebalance.dict)
+        dict_unref(local->rebalance.dict);
 
-        if (fd) {
-                local->fd = fd_ref (fd);
-                if (!inode)
-                        inode = fd->inode;
-        }
+    GF_FREE(local->rebalance.vector);
 
-        local->op_ret   = -1;
-        local->op_errno = EUCLEAN;
-        local->fop      = fop;
+    if (local->rebalance.iobref)
+        iobref_unref(local->rebalance.iobref);
 
-        if (inode) {
-                local->layout   = dht_layout_get (frame->this, inode);
-                local->cached_subvol = dht_subvol_get_cached (frame->this,
-                                                              inode);
-        }
+    if (local->stub) {
+        call_stub_destroy(local->stub);
+        local->stub = NULL;
+    }
 
-        frame->local = local;
+    if (local->ret_cache)
+        GF_FREE(local->ret_cache);
 
-out:
-        if (ret) {
-                if (local)
-                        mem_put (local);
-                local = NULL;
-        }
-        return local;
+    mem_put(local);
 }
 
-
-char *
-basestr (const char *str)
+dht_local_t *
+dht_local_init(call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
 {
-        char *basestr = NULL;
+    dht_local_t *local = NULL;
+    inode_t *inode = NULL;
+    int ret = 0;
 
-        basestr = strrchr (str, '/');
-        if (basestr)
-                basestr ++;
+    local = mem_get0(THIS->local_pool);
+    if (!local)
+        goto out;
 
-        return basestr;
-}
+    if (loc) {
+        ret = loc_copy(&local->loc, loc);
+        if (ret)
+            goto out;
+
+        inode = loc->inode;
+    }
+
+    if (fd) {
+        local->fd = fd_ref(fd);
+        if (!inode)
+            inode = fd->inode;
+    }
+
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop = fop;
 
+    if (inode) {
+        local->layout = dht_layout_get(frame->this, inode);
+        local->cached_subvol = dht_subvol_get_cached(frame->this, inode);
+    }
+
+    frame->local = local;
+
+out:
+    if (ret) {
+        if (local)
+            mem_put(local);
+        local = NULL;
+    }
+    return local;
+}
 
 xlator_t *
-dht_first_up_subvol (xlator_t *this)
+dht_first_up_subvol(xlator_t *this)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *child = NULL;
-        int         i = 0;
-        time_t      time = 0;
+    dht_conf_t *conf = NULL;
+    xlator_t *child = NULL;
+    int i = 0;
+    time_t time = 0;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        LOCK (&conf->subvolume_lock);
-        {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->subvol_up_time[i]) {
-                                if (!time) {
-                                        time = conf->subvol_up_time[i];
-                                        child = conf->subvolumes[i];
-                                } else if (time > conf->subvol_up_time[i]) {
-                                        time  = conf->subvol_up_time[i];
-                                        child = conf->subvolumes[i];
-                                }
-                        }
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvol_up_time[i]) {
+                if (!time) {
+                    time = conf->subvol_up_time[i];
+                    child = conf->subvolumes[i];
+                } else if (time > conf->subvol_up_time[i]) {
+                    time = conf->subvol_up_time[i];
+                    child = conf->subvolumes[i];
                 }
+            }
         }
-        UNLOCK (&conf->subvolume_lock);
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-        return child;
+    return child;
 }
 
 xlator_t *
-dht_last_up_subvol (xlator_t *this)
+dht_last_up_subvol(xlator_t *this)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *child = NULL;
-        int         i = 0;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        LOCK (&conf->subvolume_lock);
-        {
-                for (i = conf->subvolume_cnt-1; i >= 0; i--) {
-                        if (conf->subvolume_status[i]) {
-                                child = conf->subvolumes[i];
-                                break;
-                        }
-                }
+    dht_conf_t *conf = NULL;
+    xlator_t *child = NULL;
+    int i = 0;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = conf->subvolume_cnt - 1; i >= 0; i--) {
+            if (conf->subvolume_status[i]) {
+                child = conf->subvolumes[i];
+                break;
+            }
         }
-        UNLOCK (&conf->subvolume_lock);
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-        return child;
+    return child;
 }
 
 xlator_t *
-dht_subvol_get_hashed (xlator_t *this, loc_t *loc)
+dht_subvol_get_hashed(xlator_t *this, loc_t *loc)
 {
-        dht_layout_t *layout = NULL;
-        xlator_t     *subvol = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
 
-        if (__is_root_gfid (loc->gfid)) {
-                subvol = dht_first_up_subvol (this);
-                goto out;
-        }
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        GF_VALIDATE_OR_GOTO (this->name, loc->parent, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
+    methods = &(conf->methods);
 
-        layout = dht_layout_get (this, loc->parent);
+    if (__is_root_gfid(loc->gfid)) {
+        subvol = dht_first_up_subvol(this);
+        goto out;
+    }
 
-        if (!layout) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "layout missing path=%s parent=%s",
-                        loc->path, uuid_utoa (loc->parent->gfid));
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO(this->name, loc->parent, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->name, out);
 
-        subvol = dht_layout_search (this, layout, loc->name);
+    layout = dht_layout_get(this, loc->parent);
 
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "could not find subvolume for path=%s",
-                        loc->path);
-                goto out;
-        }
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "Missing layout. path=%s, parent gfid =%s",
+                     loc->path, uuid_utoa(loc->parent->gfid));
+        goto out;
+    }
+
+    subvol = methods->layout_search(this, layout, loc->name);
+
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "No hashed subvolume for path=%s",
+                     loc->path);
+        goto out;
+    }
 
 out:
-        if (layout) {
-                dht_layout_unref (this, layout);
-        }
+    if (layout) {
+        dht_layout_unref(this, layout);
+    }
 
-        return subvol;
+    return subvol;
 }
 
-
 xlator_t *
-dht_subvol_get_cached (xlator_t *this, inode_t *inode)
+dht_subvol_get_cached(xlator_t *this, inode_t *inode)
 {
-        dht_layout_t *layout = NULL;
-        xlator_t     *subvol = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *subvol = NULL;
 
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        layout = dht_layout_get (this, inode);
+    layout = dht_layout_get(this, inode);
 
-        if (!layout) {
-                goto out;
-        }
+    if (!layout) {
+        goto out;
+    }
 
-        subvol = layout->list[0].xlator;
+    subvol = layout->list[0].xlator;
 
 out:
-        if (layout) {
-                dht_layout_unref (this, layout);
-        }
+    if (layout) {
+        dht_layout_unref(this, layout);
+    }
 
-        return subvol;
+    return subvol;
 }
 
-
 xlator_t *
-dht_subvol_next (xlator_t *this, xlator_t *prev)
+dht_subvol_next(xlator_t *this, xlator_t *prev)
 {
-        dht_conf_t *conf = NULL;
-        int         i = 0;
-        xlator_t   *next = NULL;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == prev) {
-                        if ((i + 1) < conf->subvolume_cnt)
-                                next = conf->subvolumes[i + 1];
-                        break;
-                }
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *next = NULL;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == prev) {
+            if ((i + 1) < conf->subvolume_cnt)
+                next = conf->subvolumes[i + 1];
+            break;
         }
+    }
 
 out:
-        return next;
+    return next;
 }
 
+/* This func wraps around, if prev is actually the last subvol.
+ */
+xlator_t *
+dht_subvol_next_available(xlator_t *this, xlator_t *prev)
+{
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *next = NULL;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == prev) {
+            /* if prev is last in conf->subvolumes, then wrap
+             * around.
+             */
+            if ((i + 1) < conf->subvolume_cnt) {
+                next = conf->subvolumes[i + 1];
+            } else {
+                next = conf->subvolumes[0];
+            }
+            break;
+        }
+    }
 
+out:
+    return next;
+}
 int
-dht_subvol_cnt (xlator_t *this, xlator_t *subvol)
+dht_subvol_cnt(xlator_t *this, xlator_t *subvol)
 {
-        int i = 0;
-        int ret = -1;
-        dht_conf_t *conf = NULL;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (subvol == conf->subvolumes[i]) {
-                        ret = i;
-                        break;
-                }
+    int i = 0;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (subvol == conf->subvolumes[i]) {
+            ret = i;
+            break;
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
+#define set_if_greater(a, b)                                                   \
+    do {                                                                       \
+        if ((a) < (b))                                                         \
+            (a) = (b);                                                         \
+    } while (0)
 
-#define set_if_greater(a, b) do {               \
-                if ((a) < (b))                  \
-                        (a) = (b);              \
-        } while (0)
-
+#define set_if_greater_time(a, an, b, bn)                                      \
+    do {                                                                       \
+        if (((a) < (b)) || (((a) == (b)) && ((an) < (bn)))) {                  \
+            (a) = (b);                                                         \
+            (an) = (bn);                                                       \
+        }                                                                      \
+    } while (0)
 
-#define set_if_greater_time(a, an, b, bn) do {                          \
-                if (((a) < (b)) || (((a) == (b)) && ((an) < (bn)))){    \
-                        (a) = (b);                                      \
-                        (an) = (bn);                                    \
-                }                                                       \
-        } while (0)                                                     \
+int
+dht_iatt_merge(xlator_t *this, struct iatt *to, struct iatt *from)
+{
+    if (!from || !to)
+        return 0;
 
+    to->ia_dev = from->ia_dev;
+
+    gf_uuid_copy(to->ia_gfid, from->ia_gfid);
+
+    to->ia_ino = from->ia_ino;
+    to->ia_prot = from->ia_prot;
+    to->ia_type = from->ia_type;
+    to->ia_nlink = from->ia_nlink;
+    to->ia_rdev = from->ia_rdev;
+    to->ia_size += from->ia_size;
+    to->ia_blksize = from->ia_blksize;
+    to->ia_blocks += from->ia_blocks;
+
+    if (IA_ISDIR(from->ia_type)) {
+        to->ia_blocks = DHT_DIR_STAT_BLOCKS;
+        to->ia_size = DHT_DIR_STAT_SIZE;
+    }
+    set_if_greater(to->ia_uid, from->ia_uid);
+    set_if_greater(to->ia_gid, from->ia_gid);
+
+    set_if_greater_time(to->ia_atime, to->ia_atime_nsec, from->ia_atime,
+                        from->ia_atime_nsec);
+    set_if_greater_time(to->ia_mtime, to->ia_mtime_nsec, from->ia_mtime,
+                        from->ia_mtime_nsec);
+    set_if_greater_time(to->ia_ctime, to->ia_ctime_nsec, from->ia_ctime,
+                        from->ia_ctime_nsec);
+
+    return 0;
+}
 
 int
-dht_iatt_merge (xlator_t *this, struct iatt *to,
-                struct iatt *from, xlator_t *subvol)
+dht_build_child_loc(xlator_t *this, loc_t *child, loc_t *parent, char *name)
 {
-        if (!from || !to)
-                return 0;
+    if (!child) {
+        goto err;
+    }
 
-        to->ia_dev      = from->ia_dev;
+    if (strcmp(parent->path, "/") == 0)
+        gf_asprintf((char **)&child->path, "/%s", name);
+    else
+        gf_asprintf((char **)&child->path, "%s/%s", parent->path, name);
 
-        uuid_copy (to->ia_gfid, from->ia_gfid);
+    if (!child->path) {
+        goto err;
+    }
 
-        to->ia_ino      = from->ia_ino;
-        to->ia_prot     = from->ia_prot;
-        to->ia_type     = from->ia_type;
-        to->ia_nlink    = from->ia_nlink;
-        to->ia_rdev     = from->ia_rdev;
-        to->ia_size    += from->ia_size;
-        to->ia_blksize  = from->ia_blksize;
-        to->ia_blocks  += from->ia_blocks;
+    child->name = strrchr(child->path, '/');
+    if (child->name)
+        child->name++;
 
-        set_if_greater (to->ia_uid, from->ia_uid);
-        set_if_greater (to->ia_gid, from->ia_gid);
+    child->parent = inode_ref(parent->inode);
+    child->inode = inode_new(parent->inode->table);
 
-        set_if_greater_time(to->ia_atime, to->ia_atime_nsec,
-                            from->ia_atime, from->ia_atime_nsec);
-        set_if_greater_time (to->ia_mtime, to->ia_mtime_nsec,
-                             from->ia_mtime, from->ia_mtime_nsec);
-        set_if_greater_time (to->ia_ctime, to->ia_ctime_nsec,
-                             from->ia_ctime, from->ia_ctime_nsec);
+    if (!child->inode) {
+        goto err;
+    }
 
-        return 0;
+    return 0;
+err:
+    if (child) {
+        loc_wipe(child);
+    }
+    return -1;
 }
 
 int
-dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
+dht_init_local_subvolumes(xlator_t *this, dht_conf_t *conf)
 {
-        if (!child) {
-                goto err;
-        }
-
-        if (strcmp (parent->path, "/") == 0)
-                gf_asprintf ((char **)&child->path, "/%s", name);
-        else
-                gf_asprintf ((char **)&child->path, "%s/%s", parent->path, name);
+    xlator_list_t *subvols = NULL;
+    int cnt = 0;
 
-        if (!child->path) {
-                goto err;
-        }
+    if (!conf)
+        return -1;
 
-        child->name = strrchr (child->path, '/');
-        if (child->name)
-                child->name++;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        cnt++;
 
-        child->parent = inode_ref (parent->inode);
-        child->inode = inode_new (parent->inode->table);
+    conf->local_subvols = GF_CALLOC(cnt, sizeof(xlator_t *),
+                                    gf_dht_mt_xlator_t);
 
-        if (!child->inode) {
-                goto err;
-        }
+    /* FIX FIX : do this dynamically*/
+    conf->local_nodeuuids = GF_CALLOC(cnt, sizeof(subvol_nodeuuids_info_t),
+                                      gf_dht_nodeuuids_t);
 
-        return 0;
-err:
-        loc_wipe (child);
+    if (!conf->local_subvols || !conf->local_nodeuuids) {
         return -1;
-}
+    }
 
+    conf->local_subvols_cnt = 0;
 
+    return 0;
+}
 
 int
-dht_init_subvolumes (xlator_t *this, dht_conf_t *conf)
+dht_init_subvolumes(xlator_t *this, dht_conf_t *conf)
 {
-        xlator_list_t *subvols = NULL;
-        int            cnt = 0;
+    xlator_list_t *subvols = NULL;
+    int cnt = 0;
 
-        if (!conf)
-                return -1;
+    if (!conf)
+        return -1;
 
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                cnt++;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        cnt++;
 
-        conf->subvolumes = GF_CALLOC (cnt, sizeof (xlator_t *),
-                                      gf_dht_mt_xlator_t);
-        if (!conf->subvolumes) {
-                return -1;
-        }
-        conf->subvolume_cnt = cnt;
+    conf->subvolumes = GF_CALLOC(cnt, sizeof(xlator_t *), gf_dht_mt_xlator_t);
+    if (!conf->subvolumes) {
+        return -1;
+    }
+    conf->subvolume_cnt = cnt;
+    /* Doesn't make sense to do any dht layer tasks
+       if the subvol count is 1. Set it as pass_through */
+    if (cnt == 1)
+        this->pass_through = _gf_true;
 
-        cnt = 0;
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                conf->subvolumes[cnt++] = subvols->xlator;
+    conf->local_subvols_cnt = 0;
 
-        conf->subvolume_status = GF_CALLOC (cnt, sizeof (char),
-                                            gf_dht_mt_char);
-        if (!conf->subvolume_status) {
-                return -1;
-        }
+    dht_set_subvol_range(this);
 
-        conf->last_event = GF_CALLOC (cnt, sizeof (int),
-                                      gf_dht_mt_char);
-        if (!conf->last_event) {
-                return -1;
-        }
+    cnt = 0;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        conf->subvolumes[cnt++] = subvols->xlator;
 
-        conf->subvol_up_time = GF_CALLOC (cnt, sizeof (time_t),
-                                          gf_dht_mt_subvol_time);
-        if (!conf->subvol_up_time) {
-                return -1;
-        }
+    conf->subvolume_status = GF_CALLOC(cnt, sizeof(char), gf_dht_mt_char);
+    if (!conf->subvolume_status) {
+        return -1;
+    }
 
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_dht_mt_dht_du_t);
-        if (!conf->du_stats) {
-                return -1;
-        }
+    conf->last_event = GF_CALLOC(cnt, sizeof(int), gf_dht_mt_char);
+    if (!conf->last_event) {
+        return -1;
+    }
 
-        conf->decommissioned_bricks = GF_CALLOC (cnt, sizeof (xlator_t *),
-                                                 gf_dht_mt_xlator_t);
-        if (!conf->decommissioned_bricks) {
-                return -1;
-        }
+    conf->subvol_up_time = GF_CALLOC(cnt, sizeof(time_t),
+                                     gf_dht_mt_subvol_time);
+    if (!conf->subvol_up_time) {
+        return -1;
+    }
 
-        return 0;
-}
+    conf->du_stats = GF_CALLOC(conf->subvolume_cnt, sizeof(dht_du_t),
+                               gf_dht_mt_dht_du_t);
+    if (!conf->du_stats) {
+        return -1;
+    }
 
+    conf->decommissioned_bricks = GF_CALLOC(cnt, sizeof(xlator_t *),
+                                            gf_dht_mt_xlator_t);
+    if (!conf->decommissioned_bricks) {
+        return -1;
+    }
 
+    return 0;
+}
 
+/*
+ op_ret values :
+  0 : Success.
+ -1 : Failure.
+  1 : File is being migrated but not by this DHT layer.
+*/
 
 static int
-dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data)
+dht_migration_complete_check_done(int op_ret, call_frame_t *frame, void *data)
 {
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->rebalance.target_op_fn (THIS, frame, op_ret);
+    if (op_ret != 0)
+        goto out;
 
-        return 0;
-}
+    if (local->cached_subvol == NULL) {
+        local->op_errno = EINVAL;
+        goto out;
+    }
 
+    subvol = local->cached_subvol;
 
-int
-dht_migration_complete_check_task (void *data)
-{
-        int           ret      = -1;
-        xlator_t     *src_node = NULL;
-        xlator_t     *dst_node = NULL;
-        dht_local_t  *local    = NULL;
-        dict_t       *dict     = NULL;
-        dht_layout_t *layout   = NULL;
-        struct iatt   stbuf    = {0,};
-        xlator_t     *this     = NULL;
-        call_frame_t *frame    = NULL;
-        loc_t         tmp_loc  = {0,};
-        char         *path     = NULL;
-
-        this  = THIS;
-        frame = data;
-        local = frame->local;
-
-        src_node = local->cached_subvol;
-
-        /* getxattr on cached_subvol for 'linkto' value */
-        if (!local->loc.inode)
-                ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        DHT_LINKFILE_KEY);
-        else
-                ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       DHT_LINKFILE_KEY);
-
-        if (!ret)
-                dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
+out:
+    local->rebalance.target_op_fn(THIS, subvol, frame, op_ret);
 
-        if (ret) {
-                if ((errno != ENOENT) || (!local->loc.inode)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s: failed to get the 'linkto' xattr %s",
-                                local->loc.path, strerror (errno));
-                        goto out;
-                }
-                /* Need to do lookup on hashed subvol, then get the file */
-                ret = syncop_lookup (this, &local->loc, NULL, &stbuf, NULL,
-                                     NULL);
-                if (ret)
-                        goto out;
-                dst_node = dht_subvol_get_cached (this, local->loc.inode);
-        }
+    return 0;
+}
 
-        if (!dst_node) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to get the destination node",
-                        local->loc.path);
-                ret = -1;
-                goto out;
+int
+dht_migration_complete_check_task(void *data)
+{
+    int ret = -1;
+    xlator_t *src_node = NULL;
+    xlator_t *dst_node = NULL, *linkto_target = NULL;
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    char *path = NULL;
+    dht_conf_t *conf = NULL;
+    inode_t *inode = NULL;
+    fd_t *iter_fd = NULL;
+    fd_t *tmp = NULL;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
+    gf_boolean_t skip_open = _gf_false;
+    int open_failed = 0;
+
+    this = THIS;
+    frame = data;
+    local = frame->local;
+    conf = this->private;
+
+    src_node = local->cached_subvol;
+
+    if (!local->loc.inode && !local->fd) {
+        local->op_errno = EINVAL;
+        goto out;
+    }
+
+    inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+    /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+     * as root:root. If a fd is already open, access check won't be done*/
+
+    if (!local->loc.inode) {
+        ret = syncop_fgetxattr(src_node, local->fd, &dict,
+                               conf->link_xattr_name, NULL, NULL);
+    } else {
+        SYNCTASK_SETID(0, 0);
+        ret = syncop_getxattr(src_node, &local->loc, &dict,
+                              conf->link_xattr_name, NULL, NULL);
+        SYNCTASK_SETID(frame->root->uid, frame->root->gid);
+    }
+
+    /*
+     * Each DHT xlator layer has its own name for the linkto xattr.
+     * If the file mode bits indicate the the file is being migrated but
+     * this layer's linkto xattr is not set, it means that another
+     * DHT layer is migrating the file. In this case, return 1 so
+     * the mode bits can be passed on to the higher layer for appropriate
+     * action.
+     */
+    if (-ret == ENODATA) {
+        /* This DHT translator is not migrating this file */
+
+        ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+        if (tmp_miginfo) {
+            /* This can be a problem if the file was
+             * migrated by two different layers. Raise
+             * a warning here.
+             */
+            gf_smsg(
+                this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), NULL);
+
+            miginfo = (void *)(uintptr_t)tmp_miginfo;
+            GF_REF_PUT(miginfo);
+        }
+        ret = 1;
+        goto out;
+    }
+
+    if (!ret)
+        linkto_target = dht_linkfile_subvol(this, NULL, NULL, dict);
+
+    if (local->loc.inode) {
+        loc_copy(&tmp_loc, &local->loc);
+    } else {
+        tmp_loc.inode = inode_ref(inode);
+        gf_uuid_copy(tmp_loc.gfid, inode->gfid);
+    }
+
+    ret = syncop_lookup(this, &tmp_loc, &stbuf, 0, 0, 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_FILE_LOOKUP_FAILED,
+                "tmp=%s", tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", this->name, NULL);
+        local->op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    dst_node = dht_subvol_get_cached(this, tmp_loc.inode);
+    if (linkto_target && dst_node != linkto_target) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_LINKFILE,
+                "linkto_target_name=%s", linkto_target->name, "dst_name=%s",
+                dst_node->name, NULL);
+    }
+
+    if (gf_uuid_compare(stbuf.ia_gfid, tmp_loc.inode->gfid)) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "dst_name=%s", dst_node->name, NULL);
+        ret = -1;
+        local->op_errno = EIO;
+        goto out;
+    }
+
+    /* update local. A layout is set in inode-ctx in lookup already */
+
+    dht_layout_unref(this, local->layout);
+
+    local->layout = dht_layout_get(frame->this, inode);
+    local->cached_subvol = dst_node;
+
+    ret = 0;
+
+    /* once we detect the migration complete, the inode-ctx2 is no more
+       required.. delete the ctx and also, it means, open() already
+       done on all the fd of inode */
+    ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+    if (tmp_miginfo) {
+        miginfo = (void *)(uintptr_t)tmp_miginfo;
+        GF_REF_PUT(miginfo);
+        goto out;
+    }
+
+    /* perform 'open()' on all the fd's present on the inode */
+    if (tmp_loc.path == NULL) {
+        inode_path(inode, NULL, &path);
+        if (path)
+            tmp_loc.path = path;
+    }
+
+    LOCK(&inode->lock);
+
+    if (list_empty(&inode->fd_list))
+        goto unlock;
+
+    /* perform open as root:root. There is window between linkfile
+     * creation(root:root) and setattr with the correct uid/gid
+     */
+    SYNCTASK_SETID(0, 0);
+
+    /* It's possible that we are the last user of iter_fd after each
+     * iteration. In this case the fd_unref() of iter_fd at the end of
+     * the loop will cause the destruction of the fd. So we need to
+     * iterate the list safely because iter_fd cannot be trusted.
+     */
+    iter_fd = list_entry((&inode->fd_list)->next, typeof(*iter_fd), inode_list);
+    while (&iter_fd->inode_list != (&inode->fd_list)) {
+        if (fd_is_anonymous(iter_fd) ||
+            (dht_fd_open_on_dst(this, iter_fd, dst_node))) {
+            if (!tmp) {
+                iter_fd = list_entry(iter_fd->inode_list.next, typeof(*iter_fd),
+                                     inode_list);
+                continue;
+            }
+            skip_open = _gf_true;
+        }
+        /* We need to release the inode->lock before calling
+         * syncop_open() to avoid possible deadlocks. However this
+         * can cause the iter_fd to be released by other threads.
+         * To avoid this, we take a reference before releasing the
+         * lock.
+         */
+        fd_ref(iter_fd);
+
+        UNLOCK(&inode->lock);
+
+        if (tmp) {
+            fd_unref(tmp);
+            tmp = NULL;
+        }
+        if (skip_open)
+            goto next;
+
+        /* flags for open are stripped down to allow following the
+         * new location of the file, otherwise we can get EEXIST or
+         * truncate the file again as rebalance is moving the data */
+        ret = syncop_open(dst_node, &tmp_loc,
+                          (iter_fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                          iter_fd, NULL, NULL);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_OPEN_FD_ON_DST_FAILED, "id=%p", iter_fd,
+                    "flags=0%o", iter_fd->flags, "path=%s", path, "name=%s",
+                    dst_node->name, NULL);
+
+            open_failed = 1;
+            local->op_errno = -ret;
+            ret = -1;
+        } else {
+            dht_fd_ctx_set(this, iter_fd, dst_node);
         }
 
-        /* lookup on dst */
-        if (local->loc.inode) {
-                ret = syncop_lookup (dst_node, &local->loc, NULL, &stbuf, NULL, NULL);
+    next:
+        LOCK(&inode->lock);
+        skip_open = _gf_false;
+        tmp = iter_fd;
+        iter_fd = list_entry(tmp->inode_list.next, typeof(*tmp), inode_list);
+    }
 
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s: failed to lookup the file on %s",
-                                local->loc.path, dst_node->name);
-                        goto out;
-                }
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
 
-                if (uuid_compare (stbuf.ia_gfid, local->loc.inode->gfid)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s: gfid different on the target file on %s",
-                                local->loc.path, dst_node->name);
-                        ret = -1;
-                        goto out;
-                }
-        }
+    if (open_failed) {
+        ret = -1;
+        goto unlock;
+    }
+    ret = 0;
 
-        /* update inode ctx (the layout) */
-        dht_layout_unref (this, local->layout);
-
-        if (!local->loc.inode)
-                ret = dht_layout_preset (this, dst_node, local->fd->inode);
-        else
-                ret = dht_layout_preset (this, dst_node, local->loc.inode);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%s: could not set preset layout for subvol %s",
-                        local->loc.path, dst_node->name);
-                ret   = -1;
-                goto out;
-        }
+unlock:
+    UNLOCK(&inode->lock);
+    if (tmp) {
+        fd_unref(tmp);
+        tmp = NULL;
+    }
 
-        layout = dht_layout_for_subvol (this, dst_node);
-        if (!layout) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: no pre-set layout for subvolume %s",
-                        local->loc.path, dst_node ? dst_node->name : "<nil>");
-                ret = -1;
-                goto out;
-        }
+out:
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        if (!local->loc.inode)
-                ret = dht_layout_set (this, local->fd->inode, layout);
-        else
-                ret = dht_layout_set (this, local->loc.inode, layout);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set the new layout",
-                        local->loc.path);
-                goto out;
+    loc_wipe(&tmp_loc);
+
+    return ret;
+}
+
+int
+dht_rebalance_complete_check(xlator_t *this, call_frame_t *frame)
+{
+    int ret = -1;
+
+    ret = synctask_new(this->ctx->env, dht_migration_complete_check_task,
+                       dht_migration_complete_check_done, frame, frame);
+    return ret;
+}
+
+/* During 'in-progress' state, both nodes should have the file */
+/*
+ op_ret values :
+  0 : Success
+ -1 : Failure.
+  1 : File is being migrated but not by this DHT layer.
+*/
+static int
+dht_inprogress_check_done(int op_ret, call_frame_t *frame, void *data)
+{
+    dht_local_t *local = NULL;
+    xlator_t *dst_subvol = NULL, *src_subvol = NULL;
+    inode_t *inode = NULL;
+
+    local = frame->local;
+
+    if (op_ret != 0)
+        goto out;
+
+    inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+
+    dht_inode_ctx_get_mig_info(THIS, inode, &src_subvol, &dst_subvol);
+    if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol, dst_subvol)) {
+        dst_subvol = dht_subvol_get_cached(THIS, inode);
+        if (!dst_subvol) {
+            local->op_errno = EINVAL;
+            goto out;
         }
+    }
 
-        local->cached_subvol = dst_node;
-        ret = 0;
+out:
+    local->rebalance.target_op_fn(THIS, dst_subvol, frame, op_ret);
 
-        if (!local->fd)
-                goto out;
+    return 0;
+}
 
-        /* once we detect the migration complete, the fd-ctx is no more
-           required.. delete the ctx, and do one extra 'fd_unref' for open fd */
-        ret = fd_ctx_del (local->fd, this, NULL);
-        if (!ret) {
-                fd_unref (local->fd);
-                ret = 0;
-                goto out;
+static int
+dht_rebalance_inprogress_task(void *data)
+{
+    int ret = -1;
+    xlator_t *src_node = NULL;
+    xlator_t *dst_node = NULL;
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    char *path = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    loc_t tmp_loc = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+    inode_t *inode = NULL;
+    fd_t *iter_fd = NULL;
+    fd_t *tmp = NULL;
+    int open_failed = 0;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
+    gf_boolean_t skip_open = _gf_false;
+
+    this = THIS;
+    frame = data;
+    local = frame->local;
+    conf = this->private;
+
+    src_node = local->cached_subvol;
+
+    if (!local->loc.inode && !local->fd)
+        goto out;
+
+    inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+    /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+     * as root:root. If a fd is already open, access check won't be done*/
+    if (local->loc.inode) {
+        SYNCTASK_SETID(0, 0);
+        ret = syncop_getxattr(src_node, &local->loc, &dict,
+                              conf->link_xattr_name, NULL, NULL);
+        SYNCTASK_SETID(frame->root->uid, frame->root->gid);
+    } else {
+        ret = syncop_fgetxattr(src_node, local->fd, &dict,
+                               conf->link_xattr_name, NULL, NULL);
+    }
+
+    /*
+     * Each DHT xlator layer has its own name for the linkto xattr.
+     * If the file mode bits indicate the the file is being migrated but
+     * this layer's linkto xattr is not present, it means that another
+     * DHT layer is migrating the file. In this case, return 1 so
+     * the mode bits can be passed on to the higher layer for appropriate
+     * action.
+     */
+
+    if (-ret == ENODATA) {
+        /* This DHT layer is not migrating this file */
+        ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+        if (tmp_miginfo) {
+            /* This can be a problem if the file was
+             * migrated by two different layers. Raise
+             * a warning here.
+             */
+            gf_smsg(
+                this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), NULL);
+            miginfo = (void *)(uintptr_t)tmp_miginfo;
+            GF_REF_PUT(miginfo);
+        }
+        ret = 1;
+        goto out;
+    }
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_GET_XATTR_FAILED,
+                "path=%s", local->loc.path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    dst_node = dht_linkfile_subvol(this, NULL, NULL, dict);
+    if (!dst_node) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GET_XATTR_FAILED,
+                "path=%s", local->loc.path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    local->rebalance.target_node = dst_node;
+
+    if (local->loc.inode) {
+        loc_copy(&tmp_loc, &local->loc);
+    } else {
+        tmp_loc.inode = inode_ref(inode);
+        gf_uuid_copy(tmp_loc.gfid, inode->gfid);
+    }
+
+    /* lookup on dst */
+    ret = syncop_lookup(dst_node, &tmp_loc, &stbuf, NULL, NULL, NULL);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_FILE_LOOKUP_FAILED,
+                "tmp=%s", tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", dst_node->name, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (gf_uuid_compare(stbuf.ia_gfid, tmp_loc.inode->gfid)) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", dst_node->name, NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+
+    if (tmp_loc.path == NULL) {
+        inode_path(inode, NULL, &path);
+        if (path)
+            tmp_loc.path = path;
+    }
+
+    LOCK(&inode->lock);
+
+    if (list_empty(&inode->fd_list))
+        goto unlock;
+
+    /* perform open as root:root. There is window between linkfile
+     * creation(root:root) and setattr with the correct uid/gid
+     */
+    SYNCTASK_SETID(0, 0);
+
+    /* It's possible that we are the last user of iter_fd after each
+     * iteration. In this case the fd_unref() of iter_fd at the end of
+     * the loop will cause the destruction of the fd. So we need to
+     * iterate the list safely because iter_fd cannot be trusted.
+     */
+    iter_fd = list_entry((&inode->fd_list)->next, typeof(*iter_fd), inode_list);
+    while (&iter_fd->inode_list != (&inode->fd_list)) {
+        /* We need to release the inode->lock before calling
+         * syncop_open() to avoid possible deadlocks. However this
+         * can cause the iter_fd to be released by other threads.
+         * To avoid this, we take a reference before releasing the
+         * lock.
+         */
+
+        if (fd_is_anonymous(iter_fd) ||
+            (dht_fd_open_on_dst(this, iter_fd, dst_node))) {
+            if (!tmp) {
+                iter_fd = list_entry(iter_fd->inode_list.next, typeof(*iter_fd),
+                                     inode_list);
+                continue;
+            }
+            skip_open = _gf_true;
         }
 
-        /* if 'local->fd' (ie, fd based operation), send a 'open()' on
-           destination if not already done */
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
-        } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
-                ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                GF_FREE (path);
+        /* Yes, this is ugly but there isn't a cleaner way to do this
+         * the fd_ref is an atomic increment so not too bad. We want to
+         * reduce the number of inode locks and unlocks.
+         */
+
+        fd_ref(iter_fd);
+        UNLOCK(&inode->lock);
 
+        if (tmp) {
+            fd_unref(tmp);
+            tmp = NULL;
         }
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
-                goto out;
+        if (skip_open)
+            goto next;
+
+        /* flags for open are stripped down to allow following the
+         * new location of the file, otherwise we can get EEXIST or
+         * truncate the file again as rebalance is moving the data */
+        ret = syncop_open(dst_node, &tmp_loc,
+                          (iter_fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                          iter_fd, NULL, NULL);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_OPEN_FD_ON_DST_FAILED, "fd=%p", iter_fd,
+                    "flags=0%o", iter_fd->flags, "path=%s", path, "name=%s",
+                    dst_node->name, NULL);
+            ret = -1;
+            open_failed = 1;
+        } else {
+            /* Potential fd leak if this fails here as it will be
+               reopened at the next Phase1/2 check */
+            dht_fd_ctx_set(this, iter_fd, dst_node);
         }
 
-        /* need this unref for the fd on src_node */
-        fd_unref (local->fd);
-        ret = 0;
+    next:
+        LOCK(&inode->lock);
+        skip_open = _gf_false;
+        tmp = iter_fd;
+        iter_fd = list_entry(tmp->inode_list.next, typeof(*tmp), inode_list);
+    }
+
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
+
+unlock:
+    UNLOCK(&inode->lock);
+
+    if (tmp) {
+        fd_unref(tmp);
+        tmp = NULL;
+    }
+    if (open_failed) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dht_inode_ctx_set_mig_info(this, inode, src_node, dst_node);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                "path=%s", local->loc.path, "name=%s", dst_node->name, NULL);
+        goto out;
+    }
+
+    ret = 0;
 out:
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        return ret;
+    loc_wipe(&tmp_loc);
+    return ret;
 }
 
 int
-dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
+dht_rebalance_in_progress_check(xlator_t *this, call_frame_t *frame)
 {
-        int         ret     = -1;
+    int ret = -1;
 
-        ret = synctask_new (this->ctx->env, dht_migration_complete_check_task,
-                            dht_migration_complete_check_done,
-                            frame, frame);
-        return ret;
+    ret = synctask_new(this->ctx->env, dht_rebalance_inprogress_task,
+                       dht_inprogress_check_done, frame, frame);
+    return ret;
 }
 
-/* During 'in-progress' state, both nodes should have the file */
-static int
-dht_inprogress_check_done (int op_ret, call_frame_t *sync_frame, void *data)
+int
+dht_inode_ctx_layout_set(inode_t *inode, xlator_t *this,
+                         dht_layout_t *layout_int)
 {
-        dht_local_t *local = NULL;
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+
+    ret = dht_inode_ctx_get(inode, this, &ctx);
+    if (!ret && ctx) {
+        ctx->layout = layout_int;
+    } else {
+        ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+        if (!ctx)
+            return ret;
+        ctx->layout = layout_int;
+    }
+
+    ret = dht_inode_ctx_set(inode, this, ctx);
+
+    return ret;
+}
 
-        local = sync_frame->local;
+void
+dht_inode_ctx_time_set(inode_t *inode, xlator_t *this, struct iatt *stat)
+{
+    dht_inode_ctx_t *ctx = NULL;
+    dht_stat_time_t *time = 0;
+    int ret = -1;
 
-        local->rebalance.target_op_fn (THIS, sync_frame, op_ret);
+    ret = dht_inode_ctx_get(inode, this, &ctx);
 
-        return 0;
+    if (ret)
+        return;
+
+    time = &ctx->time;
+
+    time->mtime = stat->ia_mtime;
+    time->mtime_nsec = stat->ia_mtime_nsec;
+
+    time->ctime = stat->ia_ctime;
+    time->ctime_nsec = stat->ia_ctime_nsec;
+
+    time->atime = stat->ia_atime;
+    time->atime_nsec = stat->ia_atime_nsec;
+
+    return;
 }
 
-static int
-dht_rebalance_inprogress_task (void *data)
-{
-        int           ret      = -1;
-        xlator_t     *src_node = NULL;
-        xlator_t     *dst_node = NULL;
-        dht_local_t  *local    = NULL;
-        dict_t       *dict     = NULL;
-        call_frame_t *frame    = NULL;
-        xlator_t     *this     = NULL;
-        char         *path     = NULL;
-        struct iatt   stbuf    = {0,};
-        loc_t         tmp_loc  = {0,};
-
-        this  = THIS;
-        frame = data;
-        local = frame->local;
-
-        src_node = local->cached_subvol;
-
-        /* getxattr on cached_subvol for 'linkto' value */
-        if (local->loc.inode)
-                ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       DHT_LINKFILE_KEY);
-        else
-                ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        DHT_LINKFILE_KEY);
+int
+dht_inode_ctx_time_update(inode_t *inode, xlator_t *this, struct iatt *stat,
+                          int32_t post)
+{
+    dht_inode_ctx_t *ctx = NULL;
+    dht_stat_time_t *time = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(this->name, stat, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    ret = dht_inode_ctx_get(inode, this, &ctx);
+
+    if (ret) {
+        ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+        if (!ctx)
+            return -1;
+    }
+
+    time = &ctx->time;
+
+    LOCK(&inode->lock);
+    {
+        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec, stat->ia_mtime,
+                        stat->ia_mtime_nsec, post);
+        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec, stat->ia_ctime,
+                        stat->ia_ctime_nsec, post);
+        DHT_UPDATE_TIME(time->atime, time->atime_nsec, stat->ia_atime,
+                        stat->ia_atime_nsec, post);
+    }
+    UNLOCK(&inode->lock);
+
+    ret = dht_inode_ctx_set(inode, this, ctx);
+out:
+    return 0;
+}
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to get the 'linkto' xattr %s",
-                        local->loc.path, strerror (errno));
-                        goto out;
+int
+dht_inode_ctx_get(inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
+{
+    int ret = -1;
+    uint64_t ctx_int = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    ret = inode_ctx_get(inode, this, &ctx_int);
+
+    if (ret)
+        return ret;
+
+    if (ctx)
+        *ctx = (dht_inode_ctx_t *)(uintptr_t)ctx_int;
+out:
+    return ret;
+}
+
+int
+dht_inode_ctx_set(inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
+{
+    int ret = -1;
+    uint64_t ctx_int = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, ctx, out);
+
+    ctx_int = (long)ctx;
+    ret = inode_ctx_set(inode, this, &ctx_int);
+out:
+    return ret;
+}
+
+int
+dht_subvol_status(dht_conf_t *conf, xlator_t *subvol)
+{
+    int i;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == subvol) {
+            return conf->subvolume_status[i];
         }
+    }
+    return 0;
+}
 
-        dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
-        if (!dst_node) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to get the 'linkto' xattr from dict",
-                        local->loc.path);
-                ret = -1;
+inode_t *
+dht_heal_path(xlator_t *this, char *path, inode_table_t *itable)
+{
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    char *bname = NULL;
+    char *save_ptr = NULL;
+    static uuid_t gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    char *tmp_path = NULL;
+
+    tmp_path = gf_strdup(path);
+    if (!tmp_path) {
+        goto out;
+    }
+
+    gf_uuid_copy(loc.pargfid, gfid);
+    loc.parent = inode_ref(itable->root);
+
+    bname = strtok_r(tmp_path, "/", &save_ptr);
+
+    /* sending a lookup on parent directory,
+     * Eg:  if  path is like /a/b/c/d/e/f/g/
+     * then we will send a lookup on a first and then b,c,d,etc
+     */
+
+    while (bname) {
+        linked_inode = NULL;
+        loc.inode = inode_grep(itable, loc.parent, bname);
+        if (loc.inode == NULL) {
+            loc.inode = inode_new(itable);
+            if (loc.inode == NULL) {
+                ret = -ENOMEM;
                 goto out;
+            }
+        } else {
+            /*
+             * Inode is already populated in the inode table.
+             * Which means we already looked up the inode and
+             * linked with a dentry. So that we will skip
+             * lookup on this entry, and proceed to next.
+             */
+            linked_inode = loc.inode;
+            bname = strtok_r(NULL, "/", &save_ptr);
+            if (!bname) {
+                goto out;
+            }
+            inode_unref(loc.parent);
+            loc.parent = loc.inode;
+            gf_uuid_copy(loc.pargfid, loc.inode->gfid);
+            loc.inode = NULL;
+            continue;
         }
 
-        local->rebalance.target_node = dst_node;
-
-        if (local->loc.inode) {
-                /* lookup on dst */
-                ret = syncop_lookup (dst_node, &local->loc, NULL,
-                                     &stbuf, NULL, NULL);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s: failed to lookup the file on %s",
-                                local->loc.path, dst_node->name);
-                        goto out;
-                }
+        loc.name = bname;
+        ret = loc_path(&loc, bname);
 
-                if (uuid_compare (stbuf.ia_gfid, local->loc.inode->gfid)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s: gfid different on the target file on %s",
-                                local->loc.path, dst_node->name);
-                        ret = -1;
-                        goto out;
-                }
+        ret = syncop_lookup(this, &loc, &iatt, NULL, NULL, NULL);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_SELFHEAL_FAILED,
+                    "path=%s", path, "subvolume=%s", this->name, "bname=%s",
+                    bname, NULL);
+            goto out;
         }
 
-        ret = 0;
+        linked_inode = inode_link(loc.inode, loc.parent, bname, &iatt);
+        if (!linked_inode)
+            goto out;
 
-        if (!local->fd)
-                goto out;
+        loc_wipe(&loc);
+        gf_uuid_copy(loc.pargfid, linked_inode->gfid);
+        loc.inode = NULL;
+
+        bname = strtok_r(NULL, "/", &save_ptr);
+        if (bname)
+            loc.parent = linked_inode;
+    }
+out:
+    inode_ref(linked_inode);
+    loc_wipe(&loc);
+    GF_FREE(tmp_path);
+
+    return linked_inode;
+}
 
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
+int
+dht_heal_full_path(void *data)
+{
+    call_frame_t *heal_frame = data;
+    dht_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *path = NULL;
+    int ret = -1;
+    xlator_t *source = NULL;
+    xlator_t *this = NULL;
+    inode_table_t *itable = NULL;
+    inode_t *inode = NULL;
+    inode_t *tmp_inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("DHT", heal_frame, out);
+
+    local = heal_frame->local;
+    this = heal_frame->this;
+    source = heal_frame->cookie;
+    heal_frame->cookie = NULL;
+    gf_uuid_copy(loc.gfid, local->gfid);
+
+    if (local->loc.inode)
+        loc.inode = inode_ref(local->loc.inode);
+    else
+        goto out;
+
+    itable = loc.inode->table;
+    ret = syncop_getxattr(source, &loc, &dict, GET_ANCESTRY_PATH_KEY, NULL,
+                          NULL);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_HEAL_ABORT,
+                "subvol=%s", source->name, NULL);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, GET_ANCESTRY_PATH_KEY, &path);
+    if (path) {
+        inode = dht_heal_path(this, path, itable);
+        if (inode && inode != local->inode) {
+            /*
+             * if inode returned by heal function is different
+             * from what we passed, which means a racing thread
+             * already linked a different inode for dentry.
+             * So we will update our local->inode, so that we can
+             * retrurn proper inode.
+             */
+            tmp_inode = local->inode;
+            local->inode = inode;
+            inode_unref(tmp_inode);
+            tmp_inode = NULL;
         } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
-                ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                GF_FREE (path);
+            inode_unref(inode);
         }
+    }
 
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
-                goto out;
-        }
+out:
+    loc_wipe(&loc);
+    if (dict)
+        dict_unref(dict);
+    return 0;
+}
 
-        ret = fd_ctx_set (local->fd, this, (uint64_t)(long)dst_node);
+int
+dht_heal_full_path_done(int op_ret, call_frame_t *heal_frame, void *data)
+{
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    int op_errno = 0;
+
+    local = heal_frame->local;
+    main_frame = local->main_frame;
+    local->main_frame = NULL;
+    this = heal_frame->this;
+
+    dht_set_fixed_dir_stat(&local->postparent);
+    if (local->need_xattr_heal) {
+        local->need_xattr_heal = 0;
+        ret = dht_dir_xattr_heal(this, local, &op_errno);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set fd-ctx target file at %s",
-                        local->loc.path, dst_node->name);
-                goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    NULL);
         }
+    }
 
-        ret = 0;
-out:
-        return ret;
+    DHT_STACK_UNWIND(lookup, main_frame, 0, 0, local->inode, &local->stbuf,
+                     local->xattr, &local->postparent);
+
+    DHT_STACK_DESTROY(heal_frame);
+    return 0;
 }
 
+/* This function must be called inside an inode lock */
 int
-dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame)
+__dht_lock_subvol_set(inode_t *inode, xlator_t *this, xlator_t *lock_subvol)
 {
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+    uint64_t value = 0;
 
-        int         ret     = -1;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        ret = synctask_new (this->ctx->env, dht_rebalance_inprogress_task,
-                            dht_inprogress_check_done,
-                            frame, frame);
-        return ret;
+    ret = __inode_ctx_get0(inode, this, &value);
+    if (ret || !value) {
+        return -1;
+    }
+
+    ctx = (dht_inode_ctx_t *)(uintptr_t)value;
+    ctx->lock_subvol = lock_subvol;
+out:
+    return ret;
+}
+
+xlator_t *
+dht_get_lock_subvolume(xlator_t *this, struct gf_flock *lock,
+                       dht_local_t *local)
+{
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    int32_t ret = -1;
+    uint64_t value = 0;
+    xlator_t *cached_subvol = NULL;
+    dht_inode_ctx_t *ctx = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO(this->name, lock, out);
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
+
+    cached_subvol = local->cached_subvol;
+
+    if (local->loc.inode || local->fd) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+    }
+
+    if (!inode)
+        goto out;
+
+    if (!(IA_ISDIR(inode->ia_type) || IA_ISINVAL(inode->ia_type))) {
+        /*
+         * We may get non-linked inode for directories as part
+         * of the selfheal code path. So checking  for IA_INVAL
+         * type also. This will only happen for directory.
+         */
+        subvol = local->cached_subvol;
+        goto out;
+    }
+
+    if (lock->l_type != F_UNLCK) {
+        /*
+         * inode purging might happen on NFS between a lk
+         * and unlk. Due to this lk and unlk might be sent
+         * to different subvols.
+         * So during a lock request, taking a ref on inode
+         * to prevent inode purging. inode unref will happen
+         * in unlock cbk code path.
+         */
+        inode_ref(inode);
+    }
+
+    LOCK(&inode->lock);
+    ret = __inode_ctx_get0(inode, this, &value);
+    if (!ret && value) {
+        ctx = (dht_inode_ctx_t *)(uintptr_t)value;
+        subvol = ctx->lock_subvol;
+    }
+    if (!subvol && lock->l_type != F_UNLCK && cached_subvol) {
+        ret = __dht_lock_subvol_set(inode, this, cached_subvol);
+        if (ret) {
+            gf_uuid_unparse(inode->gfid, gfid);
+            UNLOCK(&inode->lock);
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                    "lock_subvol gfid=%s", gfid, NULL);
+            goto post_unlock;
+        }
+        subvol = cached_subvol;
+    }
+    UNLOCK(&inode->lock);
+post_unlock:
+    if (!subvol && inode && lock->l_type != F_UNLCK) {
+        inode_unref(inode);
+    }
+out:
+    return subvol;
+}
+
+int
+dht_lk_inode_unref(call_frame_t *frame, int32_t op_ret)
+{
+    int ret = -1;
+    dht_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    this = frame->this;
+
+    if (local->loc.inode || local->fd) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+    }
+    if (!inode) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LOCK_INODE_UNREF_FAILED,
+                NULL);
+        goto out;
+    }
+
+    if (!(IA_ISDIR(inode->ia_type) || IA_ISINVAL(inode->ia_type))) {
+        ret = 0;
+        goto out;
+    }
+
+    switch (local->lock_type) {
+        case F_RDLCK:
+        case F_WRLCK:
+            if (op_ret) {
+                gf_uuid_unparse(inode->gfid, gfid);
+                gf_msg_debug(this->name, 0, "lock request failed for gfid %s",
+                             gfid);
+                inode_unref(inode);
+                goto out;
+            }
+            break;
+
+        case F_UNLCK:
+            if (!op_ret) {
+                inode_unref(inode);
+            } else {
+                gf_uuid_unparse(inode->gfid, gfid);
+                gf_smsg(this->name, GF_LOG_WARNING, 0,
+                        DHT_MSG_LOCK_INODE_UNREF_FAILED, "gfid=%s", gfid, NULL);
+                goto out;
+            }
+        default:
+            break;
+    }
+    ret = 0;
+out:
+    return ret;
+}
+
+/* Code to update custom extended attributes from src dict to dst dict
+ */
+void
+dht_dir_set_heal_xattr(xlator_t *this, dht_local_t *local, dict_t *dst,
+                       dict_t *src, int *uret, int *uflag)
+{
+    int ret = -1;
+    data_t *keyval = NULL;
+    int luret = -1;
+    int luflag = -1;
+    int i = 0;
+    char **xattrs_to_heal;
+
+    if (!src || !dst) {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_DST_NULL_SET_FAILED,
+                "path=%s", local->loc.path, NULL);
+        return;
+    }
+    /* Check if any user xattr present in src dict and set
+       it to dst dict
+    */
+    luret = dict_foreach_fnmatch(src, "user.*", dht_set_user_xattr, dst);
+    /* Check if any other custom xattr present in src dict
+       and set it to dst dict, here index start from 1 because
+       user xattr already checked in previous statement
+    */
+
+    xattrs_to_heal = get_xattrs_to_heal();
+
+    for (i = 1; xattrs_to_heal[i]; i++) {
+        keyval = dict_get(src, xattrs_to_heal[i]);
+        if (keyval) {
+            luflag = 1;
+            ret = dict_set(dst, xattrs_to_heal[i], keyval);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_WARNING, ENOMEM,
+                        DHT_MSG_DICT_SET_FAILED, "key=%s", xattrs_to_heal[i],
+                        "path=%s", local->loc.path, NULL);
+            keyval = NULL;
+        }
+    }
+    if (uret)
+        (*uret) = luret;
+    if (uflag)
+        (*uflag) = luflag;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index 7e29a7a8ce6..dbb8070b0da 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -8,1108 +8,1651 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "dht-common.h"
 
-int dht_access2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_readv2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_attr2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_open2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_flush2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_lk2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_fsync2 (xlator_t *this, call_frame_t *frame, int ret);
-
-int
-dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+static int
+dht_access2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_readv2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_attr2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_open2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_flush2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_lk2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_fsync2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_common_xattrop2(xlator_t *this, xlator_t *subvol, call_frame_t *frame,
+                    int ret);
+
+static int
+dht_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = 0;
-
-        local = frame->local;
-        prev = cookie;
-
-        local->op_errno = op_errno;
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "subvolume %s returned -1 (%s)",
-                        prev->this->name, strerror (op_errno));
-                goto out;
-        }
-
-        if (!op_ret || (local->call_cnt != 1))
-                goto out;
-
-        /* rebalance would have happened */
-        local->rebalance.target_op_fn = dht_open2;
-        ret = dht_rebalance_complete_check (this, frame);
-        if (!ret)
-                return 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    local->op_errno = op_errno;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    /* Update ctx if the fd has been opened on the target*/
+    if (!op_ret && (local->call_cnt == 1)) {
+        dht_fd_ctx_set(this, fd, prev);
+        goto out;
+    }
+
+    if (!op_ret || (local->call_cnt != 1))
+        goto out;
+
+    /* rebalance would have happened */
+    local->rebalance.target_op_fn = dht_open2;
+    ret = dht_rebalance_complete_check(this, frame);
+    if (!ret)
+        return 0;
 
 out:
-        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd, xdata);
+    DHT_STACK_UNWIND(open, frame, op_ret, op_errno, local->fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_open2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_open2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        xlator_t    *subvol = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    if (!frame || !frame->local)
+        goto out;
 
-        op_errno = ENOENT;
-        if (op_ret)
-                goto out;
-
-        local->call_cnt = 2;
-        subvol = local->cached_subvol;
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
-                    &local->loc, local->rebalance.flags, local->fd,
-                    NULL);
+    if (we_are_not_migrating(ret)) {
+        /* This DHT layer is not migrating the file */
+        DHT_STACK_UNWIND(open, frame, -1, local->op_errno, NULL,
+                         local->rebalance.xdata);
         return 0;
+    }
+
+    if (subvol == NULL)
+        goto out;
+
+    local->call_cnt = 2;
+
+    STACK_WIND_COOKIE(frame, dht_open_cbk, subvol, subvol, subvol->fops->open,
+                      &local->loc, local->rebalance.flags, local->fd,
+                      local->xattr_req);
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 int
-dht_open (call_frame_t *frame, xlator_t *this,
-          loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
+dht_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd,
+         dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_OPEN);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->rebalance.flags = flags;
-        local->call_cnt = 1;
+    local = dht_local_init(frame, loc, fd, GF_FOP_OPEN);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
-                    loc, flags, fd, xdata);
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        return 0;
+    local->rebalance.flags = flags;
+    local->call_cnt = 1;
+
+    STACK_WIND_COOKIE(frame, dht_open_cbk, subvol, subvol, subvol->fops->open,
+                      loc, flags, fd, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+dht_file_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
-        uint64_t      tmp_subvol = 0;
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
-                local->op_errno = op_errno;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "subvolume %s returned -1 (%s)",
-                        prev->this->name, strerror (op_errno));
-                goto out;
-        }
+    xlator_t *subvol1 = 0;
+    xlator_t *subvol2 = 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    if ((local->fop == GF_FOP_FSTAT) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        if (local->call_cnt != 1)
-                goto out;
-
-        /* Check if the rebalance phase2 is true */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                if (local->fd)
-                        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-                if (ret) {
-                        /* Phase 2 of migration */
-                        local->rebalance.target_op_fn = dht_attr2;
-                        ret = dht_rebalance_complete_check (this, frame);
-                } else {
-                        /* value is already set in fd_ctx, that means no need
-                           to check for whether its complete or not. */
-                        dht_attr2 (this, frame, 0);
-                }
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1)
+        goto out;
+
+    local->op_errno = op_errno;
+    local->op_ret = op_ret;
+
+    /* Check if the rebalance phase2 is true */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        local->rebalance.target_op_fn = dht_attr2;
+        dht_set_local_rebalance(this, local, NULL, NULL, stbuf, xdata);
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            /* Phase 2 of migration */
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            /* it is a non-fd op or it is an fd based Fop and
+               opened on the dst.*/
+            if (local->fd && !dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                ret = dht_rebalance_complete_check(this, frame);
                 if (!ret)
-                        return 0;
+                    return 0;
+            } else {
+                dht_attr2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    DHT_STACK_UNWIND(stat, frame, op_ret, op_errno, stbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_attr2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_attr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        xlator_t    *subvol = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(stat, frame, local->op_ret, op_errno,
+                         &local->rebalance.postbuf, local->rebalance.xdata);
+        return 0;
+    }
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        op_errno = local->op_errno;
-        if (op_ret == -1)
-                goto out;
+    local->call_cnt = 2;
 
-        subvol = local->cached_subvol;
-        local->call_cnt = 2;
+    if (local->fop == GF_FOP_FSTAT) {
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, local->fd, local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, &local->loc, local->xattr_req);
+    }
 
-        if (local->fop == GF_FOP_FSTAT) {
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, local->fd, NULL);
-        } else {
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, &local->loc, NULL);
-        }
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
+
+static int
+dht_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, struct iatt *stbuf, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+
+            goto post_unlock;
+        }
+
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(stat, frame, local->op_ret, local->op_errno,
+                         &local->stbuf, xdata);
+    }
+
+    return 0;
 }
 
 int
-dht_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+dht_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_STAT);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(loc->inode->ia_type)) {
+        local->call_cnt = 1;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+        subvol = local->cached_subvol;
 
-        local = frame->local;
-        prev = cookie;
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, loc, xdata);
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
+        return 0;
+    }
 
-                        goto unlock;
-                }
+    local->call_cnt = call_cnt = layout->cnt;
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
+
+        STACK_WIND_COOKIE(frame, dht_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, loc, xdata);
+    }
+
+    return 0;
 
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf, xdata);
-        }
 err:
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
+
+    return 0;
 }
 
 int
-dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+dht_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_STAT);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSTAT);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(fd->inode->ia_type)) {
+        local->call_cnt = 1;
 
-        layout = local->layout;
-        if (!layout) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+        subvol = local->cached_subvol;
 
-        if (IA_ISREG (loc->inode->ia_type)) {
-                local->call_cnt = 1;
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, fd, xdata);
+        return 0;
+    }
+
+    local->call_cnt = call_cnt = layout->cnt;
+
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND_COOKIE(frame, dht_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, fd, xdata);
+    }
 
-                subvol = local->cached_subvol;
+    return 0;
+
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
 
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, loc, xdata);
+    return 0;
+}
 
+int
+dht_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iovec *vector, int count, struct iatt *stbuf,
+              struct iobref *iobref, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = 0;
+    xlator_t *src_subvol = 0;
+    xlator_t *dst_subvol = 0;
+
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /* This is already second try, no need for re-check */
+    if (local->call_cnt != 1)
+        goto out;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
+
+    if ((op_ret == -1) && !dht_inode_missing(op_errno))
+        goto out;
+
+    local->op_errno = op_errno;
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        local->op_ret = op_ret;
+        local->rebalance.target_op_fn = dht_readv2;
+        dht_set_local_rebalance(this, local, NULL, NULL, stbuf, xdata);
+        /* File would be migrated to other node */
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                         &dst_subvol);
+
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
                 return 0;
+        } else {
+            /* value is already set in fd_ctx, that means no need
+               to check for whether its complete or not. */
+            dht_readv2(this, dst_subvol, frame, 0);
+            return 0;
         }
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
+out:
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
 
-        for (i = 0; i < call_cnt; i++) {
-                subvol = layout->list[i].xlator;
+    DHT_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                     iobref, xdata);
 
-                STACK_WIND (frame, dht_attr_cbk,
-                            subvol, subvol->fops->stat,
-                            loc, xdata);
-        }
+    return 0;
+}
 
+static int
+dht_readv2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+{
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(readv, frame, local->op_ret, op_errno, NULL, 0,
+                         &local->rebalance.postbuf, NULL,
+                         local->rebalance.xdata);
         return 0;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+    if (subvol == NULL)
+        goto out;
 
-        return 0;
-}
+    local->call_cnt = 2;
+
+    STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv, local->fd,
+               local->rebalance.size, local->rebalance.offset,
+               local->rebalance.flags, local->xattr_req);
 
+    return 0;
+
+out:
+    DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+    return 0;
+}
 
 int
-dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+dht_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, off_t off,
+          uint32_t flags, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSTAT);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        layout = local->layout;
-        if (!layout) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no layout for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        if (IA_ISREG (fd->inode->ia_type)) {
-                local->call_cnt = 1;
+    local = dht_local_init(frame, NULL, fd, GF_FOP_READ);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-                subvol = local->cached_subvol;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, fd, xdata);
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-                return 0;
-        }
+    local->rebalance.offset = off;
+    local->rebalance.size = size;
+    local->rebalance.flags = flags;
+    local->call_cnt = 1;
 
-        local->call_cnt = call_cnt = layout->cnt;
+    STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv, local->fd,
+               local->rebalance.size, local->rebalance.offset,
+               local->rebalance.flags, local->xattr_req);
 
-        for (i = 0; i < call_cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_attr_cbk,
-                            subvol, subvol->fops->fstat,
-                            fd, xdata);
-        }
-
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno,
-               struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref, dict_t *xdata)
+static int
+dht_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, dict_t *xdata)
 {
-        dht_local_t *local      = NULL;
-        int          ret        = 0;
-
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (!prev)
+        goto out;
+    if (local->call_cnt != 1)
+        goto out;
+    if ((op_ret == -1) &&
+        ((op_errno == ENOTCONN) || dht_inode_missing(op_errno)) &&
+        IA_ISDIR(local->loc.inode->ia_type)) {
+        subvol = dht_subvol_next_available(this, prev);
+        if (!subvol)
+            goto out;
 
-        /* This is already second try, no need for re-check */
-        if (local->call_cnt != 1)
-                goto out;
-
-        if ((op_ret == -1) && (op_errno != ENOENT))
-                goto out;
-
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                /* File would be migrated to other node */
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (ret) {
-                        local->rebalance.target_op_fn = dht_readv2;
-                        ret = dht_rebalance_complete_check (this, frame);
-                } else {
-                        /* value is already set in fd_ctx, that means no need
-                           to check for whether its complete or not. */
-                        dht_readv2 (this, frame, 0);
-                }
-                if (!ret)
-                        return 0;
+        /* check if we are done with visiting every node */
+        if (subvol == local->cached_subvol) {
+            goto out;
         }
 
-out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
-                          iobref, xdata);
-
+        STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                          subvol->fops->access, &local->loc,
+                          local->rebalance.flags, NULL);
         return 0;
+    }
+    if ((op_ret == -1) && dht_inode_missing(op_errno) &&
+        !(IA_ISDIR(local->loc.inode->ia_type))) {
+        /* File would be migrated to other node */
+        local->op_errno = op_errno;
+        local->rebalance.target_op_fn = dht_access2;
+        ret = dht_rebalance_complete_check(frame->this, frame);
+        if (!ret)
+            return 0;
+    }
+
+out:
+    DHT_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-int
-dht_readv2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_access2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        xlator_t    *subvol = NULL;
-        int          op_errno = EINVAL;
-
-        local = frame->local;
-        if (!local)
-                goto out;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        op_errno = local->op_errno;
-        if (op_ret == -1)
-                goto out;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        local->call_cnt = 2;
-        subvol = local->cached_subvol;
+    op_errno = local->op_errno;
 
-        STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
-                    local->fd, local->rebalance.size, local->rebalance.offset,
-                    local->rebalance.flags, NULL);
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
 
+        DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
         return 0;
+    }
+
+    if (subvol == NULL)
+        goto out;
+
+    local->call_cnt = 2;
+
+    STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                      subvol->fops->access, &local->loc, local->rebalance.flags,
+                      local->xattr_req);
+
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
+    return 0;
 }
 
 int
-dht_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
+dht_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_READ);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_ACCESS);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.flags = mask;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                      subvol->fops->access, loc, mask, xdata);
+
+    return 0;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
 
-        local->rebalance.offset = off;
-        local->rebalance.size   = size;
-        local->rebalance.flags  = flags;
-        local->call_cnt = 1;
+    return 0;
+}
 
-        STACK_WIND (frame, dht_readv_cbk,
-                    subvol, subvol->fops->readv,
-                    fd, size, off, flags, xdata);
+int
+dht_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *subvol = 0;
+    int ret = 0;
 
-        return 0;
+    local = frame->local;
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+    local->op_errno = op_errno;
+
+    if (local->call_cnt != 1)
+        goto out;
 
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
         return 0;
-}
+    }
 
-int
-dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, dict_t *xdata)
-{
-        int          ret = -1;
-        dht_local_t *local = NULL;
+    local->rebalance.target_op_fn = dht_flush2;
 
-        local = frame->local;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        if (local->call_cnt != 1)
-                goto out;
+    /* If context is set, then send flush() it to the destination */
+    dht_inode_ctx_get_mig_info(this, local->fd->inode, NULL, &subvol);
+    if (subvol && dht_fd_open_on_dst(this, local->fd, subvol)) {
+        dht_flush2(this, subvol, frame, 0);
+        return 0;
+    }
 
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
-                /* File would be migrated to other node */
-                local->rebalance.target_op_fn = dht_access2;
-                ret = dht_rebalance_complete_check (frame->this, frame);
-                if (!ret)
-                        return 0;
+    if (op_errno == EREMOTE) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret) {
+            return 0;
         }
+    }
 
 out:
-        DHT_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
-        return 0;
+    DHT_STACK_UNWIND(flush, frame, op_ret, op_errno, xdata);
+
+    return 0;
 }
 
-int
-dht_access2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_flush2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        xlator_t    *subvol = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
 
-        op_errno = local->op_errno;
-        if (op_ret == -1)
-                goto out;
+    local = frame->local;
 
-        local->call_cnt = 2;
-        subvol = local->cached_subvol;
+    op_errno = local->op_errno;
 
-        STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    &local->loc, local->rebalance.flags, NULL);
+    if (subvol == NULL)
+        goto out;
 
-        return 0;
+    local->call_cnt = 2; /* This is the second attempt */
+
+    STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, local->fd,
+               local->xattr_req);
+
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
-        return 0;
+    DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-
 int
-dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-            dict_t *xdata)
+dht_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_ACCESS);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        local->rebalance.flags = mask;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    loc, mask, xdata);
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FLUSH);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        return 0;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->call_cnt = 1;
+
+    STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, fd,
+               local->xattr_req);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, dict_t *xdata)
+dht_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+    xlator_t *src_subvol = 0;
+    xlator_t *dst_subvol = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    local->op_errno = op_errno;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
+
+    if (op_ret == -1 && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
+        }
+        goto out;
+    }
 
-        local = frame->local;
+    local->op_ret = op_ret;
+    inode = local->fd->inode;
 
-        local->op_errno = op_errno;
+    local->rebalance.target_op_fn = dht_fsync2;
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-        if (local->call_cnt != 1)
-                goto out;
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        /* If context is set, then send flush() it to the destination */
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (!ret) {
-                dht_flush2 (this, frame, 0);
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
+
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_in_progress_check(this, frame);
+            if (!ret)
                 return 0;
+        } else {
+            dht_fsync2(this, dst_subvol, frame, 0);
+            return 0;
         }
+    }
 
 out:
-        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        return 0;
+    DHT_STACK_UNWIND(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+
+    return 0;
 }
 
-int
-dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_fsync2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
-
-        local = frame->local;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(fsync, frame, local->op_ret, op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+    if (subvol == NULL)
+        goto out;
 
-        if (!subvol)
-                subvol = local->cached_subvol;
+    local->call_cnt = 2; /* This is the second attempt */
 
-        local->call_cnt = 2; /* This is the second attempt */
+    STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol, subvol->fops->fsync,
+                      local->fd, local->rebalance.flags, local->xattr_req);
 
-        STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, local->fd, NULL);
+    return 0;
 
-        return 0;
+out:
+    DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+dht_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+          dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FLUSH);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->call_cnt = 1;
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSYNC);
+    if (!local) {
+        op_errno = ENOMEM;
 
-        STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, fd, xdata);
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        return 0;
+    local->call_cnt = 1;
+    local->rebalance.flags = datasync;
+
+    subvol = local->cached_subvol;
+
+    STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol, subvol->fops->fsync,
+                      local->fd, local->rebalance.flags, local->xattr_req);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
-               dict_t *xdata)
+/* TODO: for 'lk()' call, we need some other special error, may be ESTALE to
+   indicate that lock migration happened on the fd, so we can consider it as
+   phase 2 of migration */
+static int
+dht_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+           int op_errno, struct gf_flock *flock, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
+    dht_local_t *local = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
 
-        local->op_errno = op_errno;
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "subvolume %s returned -1 (%s)",
-                        prev->this->name, strerror (op_errno));
-                goto out;
-        }
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
+    if (local->call_cnt != 1)
+        goto out;
 
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (ret) {
-                local->rebalance.target_op_fn = dht_fsync2;
+    local->rebalance.target_op_fn = dht_lk2;
 
-                /* Check if the rebalance phase1 is true */
-                if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                        dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                        dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-                        ret = dht_rebalance_in_progress_check (this, frame);
-                }
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
-                /* Check if the rebalance phase2 is true */
-                if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                        ret = dht_rebalance_complete_check (this, frame);
-                }
+    if (op_errno == EREMOTE) {
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, NULL, &subvol);
+        if (subvol && dht_fd_open_on_dst(this, local->fd, subvol)) {
+            dht_lk2(this, subvol, frame, 0);
+            return 0;
         } else {
-                dht_fsync2 (this, frame, 0);
-        }
-        if (!ret)
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret) {
                 return 0;
+            }
+        }
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
-        DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(lk, frame, op_ret, op_errno, flock, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_fsync2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_lk2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+    local = frame->local;
 
-        if (!subvol)
-                subvol = local->cached_subvol;
+    op_errno = local->op_errno;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    if (subvol == NULL)
+        goto out;
 
-        STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    local->fd, local->rebalance.flags, NULL);
+    local->call_cnt = 2; /* This is the second attempt */
 
-        return 0;
+    STACK_WIND(frame, dht_lk_cbk, subvol, subvol->fops->lk, local->fd,
+               local->rebalance.lock_cmd, &local->rebalance.flock,
+               local->xattr_req);
+
+    return 0;
+
+out:
+    DHT_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 int
-dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
-           dict_t *xdata)
+dht_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+       struct gf_flock *flock, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
+    xlator_t *lock_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_LK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->lock_type = flock->l_type;
+    lock_subvol = dht_get_lock_subvolume(this, flock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for path=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /*
+            local->cached_subvol = lock_subvol;
+            ret = dht_check_and_open_fd_on_subvol (this, frame);
+            if (ret)
+                    goto err;
+    */
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->rebalance.flock = *flock;
+    local->rebalance.lock_cmd = cmd;
+
+    local->call_cnt = 1;
+
+    STACK_WIND(frame, dht_lk_cbk, lock_subvol, lock_subvol->fops->lk, fd, cmd,
+               flock, xdata);
+
+    return 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
 
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSYNC);
-        if (!local) {
-                op_errno = ENOMEM;
+    return 0;
+}
 
-                goto err;
-        }
+static int
+dht_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct gf_lease *lease, dict_t *xdata)
+{
+    DHT_STACK_UNWIND(lease, frame, op_ret, op_errno, lease, xdata);
 
-        local->call_cnt = 1;
-        local->rebalance.flags = datasync;
+    return 0;
+}
 
-        subvol = local->cached_subvol;
+int
+dht_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
 
-        STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    fd, datasync, xdata);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
 
-        return 0;
+    subvol = dht_subvol_get_cached(this, loc->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* TODO: for rebalance, we need to preserve the fop arguments */
+    STACK_WIND(frame, dht_lease_cbk, subvol, subvol->fops->lease, loc, lease,
+               xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-/* TODO: for 'lk()' call, we need some other special error, may be ESTALE to
-   indicate that lock migration happened on the fd, so we can consider it as
-   phase 2 of migration */
-int
-dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, struct gf_flock *flock, dict_t *xdata)
+/* Symlinks are currently not migrated, so no need for any check here */
+static int
+dht_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, const char *path, struct iatt *stbuf,
+                 dict_t *xdata)
 {
-        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock, xdata);
+    dht_local_t *local = NULL;
 
-        return 0;
-}
+    local = frame->local;
+    if (op_ret == -1)
+        goto err;
+
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
+
+err:
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    DHT_STACK_UNWIND(readlink, frame, op_ret, op_errno, path, stbuf, xdata);
 
+    return 0;
+}
 
 int
-dht_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int cmd, struct gf_flock *flock, dict_t *xdata)
+dht_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_READLINK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    STACK_WIND(frame, dht_readlink_cbk, subvol, subvol->fops->readlink, loc,
+               size, xdata);
+
+    return 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    return 0;
+}
 
-        /* TODO: for rebalance, we need to preserve the fop arguments */
-        STACK_WIND (frame, dht_lk_cbk, subvol, subvol->fops->lk, fd,
-                    cmd, flock, xdata);
+/* Get both DHT_IATT_IN_XDATA_KEY and DHT_MODE_IN_XDATA_KEY
+ * Use DHT_MODE_IN_XDATA_KEY if available, else fall back to
+ * DHT_IATT_IN_XDATA_KEY
+ * This will return a dummy iatt with only the mode and type set
+ */
+static int
+dht_read_iatt_from_xdata(dict_t *xdata, struct iatt *stbuf)
+{
+    int ret = -1;
+    int32_t mode = 0;
 
-        return 0;
+    ret = dict_get_int32(xdata, DHT_MODE_IN_XDATA_KEY, &mode);
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+    if (ret) {
+        ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+    } else {
+        stbuf->ia_prot = ia_prot_from_st_mode(mode);
+        stbuf->ia_type = ia_type_from_st_mode(mode);
+    }
 
-        return 0;
+    return ret;
 }
 
-/* Symlinks are currently not migrated, so no need for any check here */
 int
-dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, const char *path,
-                  struct iatt *stbuf, dict_t *xdata)
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
+    call_frame_t *call_frame = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = -1;
+    inode_t *inode = NULL;
+
+    local = frame->local;
+    call_frame = cookie;
+    prev = call_frame->this;
+
+    local->op_errno = op_errno;
+
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1.",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1)
+        goto out;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
+
+    ret = dht_read_iatt_from_xdata(xdata, &stbuf);
+
+    if ((!op_ret) && (ret)) {
+        /* This is a potential problem and can cause corruption
+         * with sharding.
+         * Oh well. We tried.
+         */
+        goto out;
+    }
+
+    local->op_ret = op_ret;
+    local->rebalance.target_op_fn = dht_common_xattrop2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
-        local = frame->local;
-        if (op_ret == -1)
-                goto err;
+    if (dict)
+        local->rebalance.dict = dict_ref(dict);
 
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(&stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(&stbuf)) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
+
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_in_progress_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            dht_common_xattrop2(this, dst_subvol, frame, 0);
+            return 0;
         }
+    }
 
-err:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf, xdata);
+out:
+    if (local->fop == GF_FOP_XATTROP) {
+        DHT_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    } else {
+        DHT_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-              dict_t *xdata)
+static int
+dht_common_xattrop2(xlator_t *this, xlator_t *subvol, call_frame_t *frame,
+                    int ret)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_READLINK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        if (local->fop == GF_FOP_XATTROP) {
+            DHT_STACK_UNWIND(xattrop, frame, local->op_ret, op_errno,
+                             local->rebalance.dict, local->rebalance.xdata);
+        } else {
+            DHT_STACK_UNWIND(fxattrop, frame, local->op_ret, op_errno,
+                             local->rebalance.dict, local->rebalance.xdata);
         }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+        return 0;
+    }
 
-        STACK_WIND (frame, dht_readlink_cbk,
-                    subvol, subvol->fops->readlink,
-                    loc, size, xdata);
+    if (subvol == NULL)
+        goto out;
 
-        return 0;
+    local->call_cnt = 2; /* This is the second attempt */
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
+    if (local->fop == GF_FOP_XATTROP) {
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol, subvol->fops->xattrop,
+                   &local->loc, local->rebalance.flags, local->rebalance.xattr,
+                   local->xattr_req);
+    } else {
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                   subvol->fops->fxattrop, local->fd, local->rebalance.flags,
+                   local->rebalance.xattr, local->xattr_req);
+    }
 
-        return 0;
-}
+    return 0;
 
-/* Currently no translators on top of 'distribute' will be using
- * below fops, hence not implementing 'migration' related checks
- */
+out:
 
-int
-dht_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+    /* If local is unavailable we could be unwinding the wrong
+     * function here */
+
+    if (local && (local->fop == GF_FOP_XATTROP)) {
+        DHT_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    } else {
+        DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+    }
+    return 0;
+}
+
+static int
+dht_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    DHT_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
+/* Set both DHT_IATT_IN_XDATA_KEY and DHT_MODE_IN_XDATA_KEY
+ * Use DHT_MODE_IN_XDATA_KEY if available. Else fall back to
+ * DHT_IATT_IN_XDATA_KEY
+ */
+static int
+dht_request_iatt_in_xdata(dict_t *xattr_req)
+{
+    int ret = -1;
+
+    ret = dict_set_int8(xattr_req, DHT_MODE_IN_XDATA_KEY, 1);
+    ret = dict_set_int8(xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+
+    /* At least one call succeeded */
+    return ret;
+}
 
 int
-dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+dht_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_XATTROP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_XATTROP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for gfid=%s",
+                     uuid_utoa(loc->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Todo : Handle dirs as well. At the moment the only xlator above dht
+     * that uses xattrop is sharding and that is only for files */
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        STACK_WIND(frame, dht_xattrop_cbk, subvol, subvol->fops->xattrop, loc,
+                   flags, dict, xdata);
+
+    } else {
+        local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+        local->call_cnt = 1;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+        local->rebalance.xattr = dict_ref(dict);
+        local->rebalance.flags = flags;
 
-        local->call_cnt = 1;
+        ret = dht_request_iatt_in_xdata(local->xattr_req);
+
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dictionary key %s file=%s",
+                         DHT_IATT_IN_XDATA_KEY, loc->path);
+        }
 
-        STACK_WIND (frame,
-                    dht_xattrop_cbk,
-                    subvol, subvol->fops->xattrop,
-                    loc, flags, dict, xdata);
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol, subvol->fops->xattrop,
+                   loc, local->rebalance.flags, local->rebalance.xattr,
+                   local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+static int
+dht_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    DHT_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-dht_fxattrop (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+dht_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    subvol = dht_subvol_get_cached(this, fd->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FXATTROP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* Todo : Handle dirs as well. At the moment the only xlator above dht
+     * that uses xattrop is sharding and that is only for files */
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        STACK_WIND(frame, dht_fxattrop_cbk, subvol, subvol->fops->fxattrop, fd,
+                   flags, dict, xdata);
+
+    } else {
+        local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+        local->call_cnt = 1;
+
+        local->rebalance.xattr = dict_ref(dict);
+        local->rebalance.flags = flags;
+
+        ret = dht_request_iatt_in_xdata(local->xattr_req);
+
+        if (ret) {
+            gf_msg_debug(this->name, 0, "Failed to set dictionary key %s fd=%p",
+                         DHT_IATT_IN_XDATA_KEY, fd);
         }
 
-        STACK_WIND (frame,
-                    dht_fxattrop_cbk,
-                    subvol, subvol->fops->fxattrop,
-                    fd, flags, dict, xdata);
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                   subvol->fops->fxattrop, fd, local->rebalance.flags,
+                   local->rebalance.xattr, local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+/* Currently no translators on top of 'distribute' will be using
+ * below fops, hence not implementing 'migration' related checks
+ */
 
-int
-dht_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+dht_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-dht_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+dht_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
+    xlator_t *lock_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_INODELK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = dht_local_init(frame, loc, NULL, GF_FOP_INODELK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local->lock_type = lock->l_type;
+    lock_subvol = dht_get_lock_subvolume(this, lock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        local->call_cnt = 1;
+    local->call_cnt = 1;
 
-        STACK_WIND (frame,
-                    dht_inodelk_cbk,
-                    subvol, subvol->fops->inodelk,
-                    volume, loc, cmd, lock, xdata);
+    STACK_WIND(frame, dht_inodelk_cbk, lock_subvol, lock_subvol->fops->inodelk,
+               volume, loc, cmd, lock, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(inodelk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
+    dht_local_t *local = NULL;
+    int ret = 0;
 
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
-int
-dht_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-              fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
-{
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = frame->local;
 
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        STACK_WIND (frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk,
-                    volume, fd, cmd, lock, xdata);
+out:
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
+}
+
+int
+dht_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    xlator_t *lock_subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_INODELK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = 1;
+    local->lock_type = lock->l_type;
+
+    lock_subvol = dht_get_lock_subvolume(this, lock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /*
+            local->cached_subvol = lock_subvol;
+            ret = dht_check_and_open_fd_on_subvol (this, frame);
+            if (ret)
+                    goto err;
+    */
+    local->rebalance.flock = *lock;
+    local->rebalance.lock_cmd = cmd;
+    local->key = gf_strdup(volume);
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND(frame, dht_finodelk_cbk, lock_subvol,
+               lock_subvol->fops->finodelk, volume, fd, cmd, lock, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(finodelk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index d4a3ecc391c..2f23ce90fbd 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -8,605 +8,1397 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "dht-common.h"
 
-int dht_writev2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_truncate2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_setattr2 (xlator_t *this, call_frame_t *frame, int ret);
+static int
+dht_writev2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_truncate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_setattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_fallocate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_discard2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_zerofill2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
 
 int
-dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+dht_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        int          ret   = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *subvol1 = NULL;
+    xlator_t *subvol2 = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /* writev fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could be a valid bad fd error.
+     */
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        if (op_ret == -1) {
-                goto out;
+    if (op_ret == -1 && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, 0, "subvolume %s returned -1 (%s)", prev->name,
+                     strerror(op_errno));
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        /* preserve the modes of source */
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    local->rebalance.target_op_fn = dht_writev2;
 
-        if (local->call_cnt != 1) {
-                /* preserve the modes of source */
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    /* We might need to pass the stbuf information to the higher DHT
+     * layer for appropriate handling.
+     */
+
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        if (!local->xattr_req) {
+            local->xattr_req = dict_new();
+            if (!local->xattr_req) {
+                gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+                       "insufficient memory");
+                local->op_errno = ENOMEM;
+                local->op_ret = -1;
                 goto out;
+            }
         }
 
-        local->rebalance.target_op_fn = dht_writev2;
-
-        /* Phase 2 of migration */
-        if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+        ret = dict_set_uint32(local->xattr_req, GF_PROTECT_FROM_EXTERNAL_WRITES,
+                              1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_DICT_SET_FAILED, 0,
+                   "Failed to set key %s in dictionary",
+                   GF_PROTECT_FROM_EXTERNAL_WRITES);
+            local->op_errno = ENOMEM;
+            local->op_ret = -1;
+            goto out;
         }
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
 
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
-                        dht_writev2 (this, frame, 0);
-                        return 0;
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &subvol1,
+                                         &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            if (dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                dht_writev2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
+    DHT_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_writev2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local = frame->local;
+    if (subvol == NULL)
+        goto out;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+    local->call_cnt = 2; /* This is the second attempt */
 
-        if (!subvol)
-                subvol = local->cached_subvol;
+    STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                      subvol->fops->writev, local->fd, local->rebalance.vector,
+                      local->rebalance.count, local->rebalance.offset,
+                      local->rebalance.flags, local->rebalance.iobref,
+                      local->xattr_req);
 
-        local->call_cnt = 2; /* This is the second attempt */
+    return 0;
 
-        STACK_WIND (frame, dht_writev_cbk,
-                    subvol, subvol->fops->writev,
-                    local->fd, local->rebalance.vector, local->rebalance.count,
-                    local->rebalance.offset, local->rebalance.flags,
-                    local->rebalance.iobref, NULL);
+out:
+    DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int count, off_t off, uint32_t flags,
-            struct iobref *iobref, dict_t *xdata)
+dht_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int count, off_t off, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_WRITE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->rebalance.vector = iov_dup(vector, count);
+    local->rebalance.offset = off;
+    local->rebalance.count = count;
+    local->rebalance.flags = flags;
+    local->rebalance.iobref = iobref_ref(iobref);
+    local->call_cnt = 1;
+
+    STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                      subvol->fops->writev, fd, local->rebalance.vector,
+                      local->rebalance.count, local->rebalance.offset,
+                      local->rebalance.flags, local->rebalance.iobref,
+                      local->xattr_req);
+
+    return 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        local = dht_local_init (frame, NULL, fd, GF_FOP_WRITE);
-        if (!local) {
+    return 0;
+}
 
-                op_errno = ENOMEM;
-                goto err;
-        }
+int
+dht_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* Needs to be checked only for ftruncate.
+     * ftruncate fails with EBADF/EINVAL if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+
+    if ((local->fop == GF_FOP_FTRUNCATE) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
+    local->rebalance.target_op_fn = dht_truncate2;
 
-        local->rebalance.vector = iov_dup (vector, count);
-        local->rebalance.offset = off;
-        local->rebalance.count = count;
-        local->rebalance.flags = flags;
-        local->rebalance.iobref = iobref_ref (iobref);
-        local->call_cnt = 1;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        STACK_WIND (frame, dht_writev_cbk,
-                    subvol, subvol->fops->writev,
-                    fd, vector, count, off, flags, iobref, xdata);
+    /* We might need to pass the stbuf information to the higher DHT
+     * layer for appropriate handling.
+     */
 
-        return 0;
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if ((!local->fd) ||
+                ((local->fd) &&
+                 dht_fd_open_on_dst(this, local->fd, dst_subvol))) {
+                dht_truncate2(this, dst_subvol, frame, 0);
+                return 0;
+            }
+        }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+out:
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
+    DHT_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
+
+static int
+dht_truncate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+{
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if (!frame || !frame->local)
+        goto out;
 
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    /* This dht xlator is not migrating the file  */
+    if (we_are_not_migrating(ret)) {
+        DHT_STACK_UNWIND(truncate, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
         return 0;
+    }
+
+    if (subvol == NULL)
+        goto out;
+
+    local->call_cnt = 2; /* This is the second attempt */
+
+    if (local->fop == GF_FOP_TRUNCATE) {
+        STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                          subvol->fops->truncate, &local->loc,
+                          local->rebalance.offset, local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                          subvol->fops->ftruncate, local->fd,
+                          local->rebalance.offset, local->xattr_req);
+    }
+
+    return 0;
+
+out:
+    DHT_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
+int
+dht_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_TRUNCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.offset = offset;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for gfid=%s",
+                     uuid_utoa(loc->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                      subvol->fops->truncate, loc, offset, xdata);
+
+    return 0;
 
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
+}
 
 int
-dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
+dht_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FTRUNCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.offset = offset;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                      subvol->fops->ftruncate, fd, local->rebalance.offset,
+                      local->xattr_req);
+    return 0;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        local = frame->local;
-        prev = cookie;
+    return 0;
+}
 
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "subvolume %s returned -1 (%s)",
-                        prev->this->name, strerror (op_errno));
+int
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                  dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* fallocate fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-                goto out;
-        }
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
 
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
-        local->rebalance.target_op_fn = dht_truncate2;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    local->rebalance.target_op_fn = dht_fallocate2;
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
-                        dht_truncate2 (this, frame, 0);
-                        return 0;
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                   &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if (dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+                dht_fallocate2(this, dst_subvol, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
-        DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
+
+    DHT_STACK_UNWIND(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                     xdata);
 err:
-        return 0;
+    return 0;
 }
 
-
-int
-dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_fallocate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if (!frame || !frame->local)
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(fallocate, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local = frame->local;
+    if (subvol == NULL)
+        goto out;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+    local->call_cnt = 2; /* This is the second attempt */
 
-        if (!subvol)
-                subvol = local->cached_subvol;
+    STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                      subvol->fops->fallocate, local->fd,
+                      local->rebalance.flags, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
 
-        local->call_cnt = 2; /* This is the second attempt */
+    return 0;
 
-        if (local->fop == GF_FOP_TRUNCATE) {
-                STACK_WIND (frame, dht_truncate_cbk, subvol,
-                            subvol->fops->truncate, &local->loc,
-                            local->rebalance.offset, NULL);
-        } else {
-                STACK_WIND (frame, dht_truncate_cbk, subvol,
-                            subvol->fops->ftruncate, local->fd,
-                            local->rebalance.offset, NULL);
-        }
+out:
+    DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
 
-        return 0;
+int
+dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FALLOCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.flags = mode;
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
+
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                      subvol->fops->fallocate, fd, local->rebalance.flags,
+                      local->rebalance.offset, local->rebalance.size,
+                      local->xattr_req);
+
+    return 0;
+
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
 }
 
 int
-dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-              dict_t *xdata)
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_TRUNCATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* discard fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->rebalance.offset = offset;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
-        STACK_WIND (frame, dht_truncate_cbk,
-                    subvol, subvol->fops->truncate,
-                    loc, offset, xdata);
+    local->rebalance.target_op_fn = dht_discard2;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        return 0;
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                   &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if (dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+                dht_discard2(this, dst_subvol, frame, 0);
+                return 0;
+            }
+        }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+out:
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
+    DHT_STACK_UNWIND(discard, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
 
+static int
+dht_discard2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+{
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if (!frame || !frame->local)
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(discard, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
         return 0;
+    }
+
+    if (subvol == NULL)
+        goto out;
+
+    local->call_cnt = 2; /* This is the second attempt */
+
+    STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                      subvol->fops->discard, local->fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
+
+    return 0;
+
+out:
+    DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               dict_t *xdata)
+dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FTRUNCATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        local->rebalance.offset = offset;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        STACK_WIND (frame, dht_truncate_cbk,
-                    subvol, subvol->fops->ftruncate,
-                    fd, offset, xdata);
+    local = dht_local_init(frame, NULL, fd, GF_FOP_DISCARD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        return 0;
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
+
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                      subvol->fops->discard, fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-/* handle cases of migration here for 'setattr()' calls */
 int
-dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-
-        local = frame->local;
-        prev = cookie;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* zerofill fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
         local->op_errno = op_errno;
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "subvolume %s returned -1 (%s)",
-                        prev->this->name, strerror (op_errno));
-                goto out;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
-        if (local->call_cnt != 1)
-                goto out;
+    local->rebalance.target_op_fn = dht_zerofill2;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        local->rebalance.target_op_fn = dht_setattr2;
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &subvol1,
+                                         &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            if (dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                dht_zerofill2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
 
-        /* At the end of the migration process, whatever 'attr' we
-           have on source file will be migrated to destination file
-           in one shot, hence we don't need to check for in progress
-           state here (ie, PHASE1) */
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
-        DHT_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        return 0;
+    DHT_STACK_UNWIND(zerofill, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+err:
+    return 0;
 }
 
-int
-dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_zerofill2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
+    if (!frame || !frame->local)
+        goto out;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
-
-        if (!subvol)
-                subvol = local->cached_subvol;
-
-        local->call_cnt = 2; /* This is the second attempt */
-
-        if (local->fop == GF_FOP_SETATTR) {
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->setattr, &local->loc,
-                            &local->rebalance.stbuf, local->rebalance.flags,
-                            NULL);
-        } else {
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->fsetattr, local->fd,
-                            &local->rebalance.stbuf, local->rebalance.flags,
-                            NULL);
-        }
+    local = frame->local;
+
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(zerofill, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
 
         return 0;
-}
+    }
 
+    if (subvol == NULL)
+        goto out;
+
+    local->call_cnt = 2; /* This is the second attempt */
+
+    STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                      subvol->fops->zerofill, local->fd,
+                      local->rebalance.offset, local->rebalance.size,
+                      local->xattr_req);
+
+    return 0;
+
+out:
+
+    DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
 
-/* Keep the existing code same for all the cases other than regular file */
 int
-dht_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, struct iatt *statpre,
-                 struct iatt *statpost, dict_t *xdata)
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-                dht_iatt_merge (this, &local->prebuf, statpre, prev->this);
-                dht_iatt_merge (this, &local->stbuf, statpost, prev->this);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
+    local = dht_local_init(frame, NULL, fd, GF_FOP_ZEROFILL);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (setattr, frame, local->op_ret, local->op_errno,
-                                  &local->prebuf, &local->stbuf, xdata);
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
 
-        return 0;
-}
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                      subvol->fops->zerofill, fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
 
+    return 0;
 
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
+}
+
+/* handle cases of migration here for 'setattr()' calls */
 int
-dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+dht_file_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        dht_layout_t *layout = NULL;
-        dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-        int           call_cnt = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SETATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
 
-        layout = local->layout;
-        if (!layout) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = frame->local;
+    prev = cookie;
 
-        if (!layout_is_sane (layout)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "layout is not sane for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local->op_errno = op_errno;
 
-        if (IA_ISREG (loc->inode->ia_type)) {
-                /* in the regular file _cbk(), we need to check for
-                   migration possibilities */
-                local->rebalance.stbuf = *stbuf;
-                local->rebalance.flags = valid;
-                local->call_cnt = 1;
-                subvol = local->cached_subvol;
+    if ((local->fop == GF_FOP_FSETATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->setattr,
-                            loc, stbuf, valid, xdata);
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
 
-                return 0;
-        }
+    if (local->call_cnt != 1)
+        goto out;
 
-        local->call_cnt = call_cnt = layout->cnt;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_setattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->setattr,
-                            loc, stbuf, valid, xdata);
-        }
+    local->rebalance.target_op_fn = dht_setattr2;
 
-        return 0;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        return 0;
+    /* At the end of the migration process, whatever 'attr' we
+       have on source file will be migrated to destination file
+       in one shot, hence we don't need to check for in progress
+       state here (ie, PHASE1) */
+out:
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
+
+    DHT_STACK_UNWIND(setattr, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+
+    return 0;
 }
 
+static int
+dht_setattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+{
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if (!frame || !frame->local)
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
+
+    if (subvol == NULL)
+        goto out;
+
+    local->call_cnt = 2; /* This is the second attempt */
+
+    if (local->fop == GF_FOP_SETATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->setattr, &local->loc,
+                          &local->rebalance.stbuf, local->rebalance.flags,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->fsetattr, local->fd,
+                          &local->rebalance.stbuf, local->rebalance.flags,
+                          local->xattr_req);
+    }
+
+    return 0;
+
+out:
+    DHT_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
 
+/* Keep the existing code same for all the cases other than regular file */
 int
-dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
-              int32_t valid, dict_t *xdata)
+dht_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *statpre, struct iatt *statpost,
+                dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        dht_layout_t *layout = NULL;
-        dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
 
-        layout = local->layout;
-        if (!layout) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no layout for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = frame->local;
+    prev = cookie;
 
-        if (!layout_is_sane (layout)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "layout is not sane for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
         }
 
-        if (IA_ISREG (fd->inode->ia_type)) {
-                /* in the regular file _cbk(), we need to check for
-                   migration possibilities */
-                local->rebalance.stbuf = *stbuf;
-                local->rebalance.flags = valid;
-                local->call_cnt = 1;
-                subvol = local->cached_subvol;
+        dht_iatt_merge(this, &local->prebuf, statpre);
+        dht_iatt_merge(this, &local->stbuf, statpost);
+
+        local->op_ret = 0;
+        local->op_errno = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if (local->op_ret == 0)
+            dht_inode_ctx_time_set(local->loc.inode, this, &local->stbuf);
+        DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                         &local->prebuf, &local->stbuf, xdata);
+    }
+
+    return 0;
+}
+
+/* Keep the existing code same for all the cases other than regular file */
+int
+dht_non_mds_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *statpre,
+                        struct iatt *statpost, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (op_ret == -1) {
+        gf_msg(this->name, op_errno, 0, 0, "subvolume %s returned -1",
+               prev->name);
+        goto post_unlock;
+    }
+
+    LOCK(&frame->lock);
+    {
+        dht_iatt_merge(this, &local->prebuf, statpre);
+        dht_iatt_merge(this, &local->stbuf, statpost);
+
+        local->op_ret = 0;
+        local->op_errno = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_inode_ctx_time_set(local->loc.inode, this, &local->stbuf);
+        DHT_STACK_UNWIND(setattr, frame, 0, 0, &local->prebuf, &local->stbuf,
+                         xdata);
+    }
+
+    return 0;
+}
 
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->fsetattr,
-                            fd, stbuf, valid, xdata);
+int
+dht_mds_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
 
-                return 0;
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *mds_subvol = NULL;
+    struct iatt loc_stbuf = {
+        0,
+    };
+    int i = 0;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+    mds_subvol = local->mds_subvol;
+
+    if (op_ret == -1) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    local->op_ret = 0;
+    loc_stbuf = local->stbuf;
+    dht_iatt_merge(this, &local->prebuf, statpre);
+    dht_iatt_merge(this, &local->stbuf, statpost);
+
+    local->call_cnt = conf->subvolume_cnt - 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol == conf->subvolumes[i])
+            continue;
+        STACK_WIND_COOKIE(frame, dht_non_mds_setattr_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->setattr, &local->loc,
+                          &loc_stbuf, local->valid, local->xattr_req);
+    }
+
+    return 0;
+out:
+    DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                     &local->prebuf, &local->stbuf, xdata);
+
+    return 0;
+}
+
+int
+dht_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    dht_layout_t *layout = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = -1;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SETATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (!layout_is_sane(layout)) {
+        gf_msg_debug(this->name, 0, "layout is not sane for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(loc->inode->ia_type)) {
+        /* in the regular file _cbk(), we need to check for
+           migration possibilities */
+        local->rebalance.stbuf = *stbuf;
+        local->rebalance.flags = valid;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->setattr, loc, stbuf, valid, xdata);
+
+        return 0;
+    }
+
+    local->call_cnt = call_cnt = layout->cnt;
+
+    if (IA_ISDIR(loc->inode->ia_type) && !__is_root_gfid(loc->inode->gfid) &&
+        call_cnt != 1) {
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+        if (ret || !mds_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get mds subvol for path %s", local->loc.path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        local->call_cnt = call_cnt = layout->cnt;
+        local->mds_subvol = mds_subvol;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == mds_subvol) {
+                if (!conf->subvolume_status[i]) {
+                    gf_msg(this->name, GF_LOG_WARNING, layout->list[i].err,
+                           DHT_MSG_HASHED_SUBVOL_DOWN,
+                           "MDS subvol is down for path "
+                           " %s Unable to set attr ",
+                           local->loc.path);
+                    op_errno = ENOTCONN;
+                    goto err;
+                }
+            }
+        }
+        local->valid = valid;
+        local->stbuf = *stbuf;
 
+        STACK_WIND_COOKIE(frame, dht_mds_setattr_cbk, local->mds_subvol,
+                          local->mds_subvol, local->mds_subvol->fops->setattr,
+                          loc, stbuf, valid, xdata);
+        return 0;
+    } else {
         for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_setattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->fsetattr,
-                            fd, stbuf, valid, xdata);
+            STACK_WIND_COOKIE(frame, dht_setattr_cbk, layout->list[i].xlator,
+                              layout->list[i].xlator,
+                              layout->list[i].xlator->fops->setattr, loc, stbuf,
+                              valid, xdata);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
+}
+
+int
+dht_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int32_t valid, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    dht_layout_t *layout = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSETATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (!layout_is_sane(layout)) {
+        gf_msg_debug(this->name, 0, "layout is not sane for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(fd->inode->ia_type)) {
+        /* in the regular file _cbk(), we need to check for
+           migration possibilities */
+        local->rebalance.stbuf = *stbuf;
+        local->rebalance.flags = valid;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
 
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->fsetattr, fd, &local->rebalance.stbuf,
+                          local->rebalance.flags, local->xattr_req);
         return 0;
+    }
+
+    local->call_cnt = call_cnt = layout->cnt;
+
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_setattr_cbk, layout->list[i].xlator,
+                          layout->list[i].xlator,
+                          layout->list[i].xlator->fops->fsetattr, fd, stbuf,
+                          valid, xdata);
+    }
+
+    return 0;
+
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index 97bbe987fdc..fda904c92c9 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -8,734 +8,801 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
+#include "unittest/unittest.h"
 
-#define layout_base_size (sizeof (dht_layout_t))
+#define layout_base_size (sizeof(dht_layout_t))
 
-#define layout_entry_size (sizeof ((dht_layout_t *)NULL)->list[0])
+#define layout_entry_size (sizeof((dht_layout_t *)NULL)->list[0])
 
 #define layout_size(cnt) (layout_base_size + (cnt * layout_entry_size))
 
-
 dht_layout_t *
-dht_layout_new (xlator_t *this, int cnt)
+dht_layout_new(xlator_t *this, int cnt)
 {
-        dht_layout_t *layout = NULL;
-        dht_conf_t   *conf = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
 
+    REQUIRE(NULL != this);
+    REQUIRE(cnt >= 0);
 
-        conf = this->private;
+    conf = this->private;
 
-        layout = GF_CALLOC (1, layout_size (cnt),
-                            gf_dht_mt_dht_layout_t);
-        if (!layout) {
-                goto out;
-        }
+    layout = GF_CALLOC(1, layout_size(cnt), gf_dht_mt_dht_layout_t);
+    if (!layout) {
+        goto out;
+    }
 
-        layout->type = DHT_HASH_TYPE_DM;
-        layout->cnt = cnt;
+    layout->type = DHT_HASH_TYPE_DM;
+    layout->cnt = cnt;
 
-        if (conf) {
-                layout->spread_cnt = conf->dir_spread_cnt;
-                layout->gen = conf->gen;
-        }
+    if (conf) {
+        layout->spread_cnt = conf->dir_spread_cnt;
+        layout->gen = conf->gen;
+    }
+
+    GF_ATOMIC_INIT(layout->ref, 1);
 
-        layout->ref = 1;
+    ENSURE(NULL != layout);
+    ENSURE(layout->type == DHT_HASH_TYPE_DM);
+    ENSURE(layout->cnt == cnt);
+    ENSURE(GF_ATOMIC_GET(layout->ref) == 1);
 out:
-        return layout;
+    return layout;
 }
 
-
 dht_layout_t *
-dht_layout_get (xlator_t *this, inode_t *inode)
+dht_layout_get(xlator_t *this, inode_t *inode)
 {
-        dht_conf_t   *conf = NULL;
-        uint64_t      layout_int = 0;
-        dht_layout_t *layout = NULL;
-        int           ret    = -1;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        LOCK (&conf->layout_lock);
-        {
-                ret = inode_ctx_get (inode, this, &layout_int);
-                if (ret == 0) {
-                        layout = (dht_layout_t *) (unsigned long) layout_int;
-                        layout->ref++;
-                }
-        }
-        UNLOCK (&conf->layout_lock);
-
-out:
-        return layout;
+    dht_layout_t *layout = NULL;
+    int ret = 0;
+
+    ret = dht_inode_ctx_layout_get(inode, this, &layout);
+    if ((!ret) && layout) {
+        GF_ATOMIC_INC(layout->ref);
+    }
+    return layout;
 }
 
-
 int
-dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
+dht_layout_set(xlator_t *this, inode_t *inode, dht_layout_t *layout)
 {
-        dht_conf_t   *conf = NULL;
-        int           oldret = -1;
-        int           ret = 0;
-        dht_layout_t *old_layout;
-        uint64_t      old_layout_int;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        LOCK (&conf->layout_lock);
-        {
-                oldret = inode_ctx_get (inode, this, &old_layout_int);
-
-                layout->ref++;
-                ret = inode_ctx_put (inode, this, (uint64_t) (unsigned long)
-                                     layout);
-        }
-        UNLOCK (&conf->layout_lock);
-
-        if (oldret == 0) {
-                old_layout = (dht_layout_t *) (unsigned long) old_layout_int;
-                dht_layout_unref (this, old_layout);
-        }
+    dht_conf_t *conf = NULL;
+    int oldret = -1;
+    int ret = -1;
+    dht_layout_t *old_layout;
+
+    conf = this->private;
+    if (!conf || !layout)
+        goto out;
+
+    LOCK(&conf->layout_lock);
+    {
+        oldret = dht_inode_ctx_layout_get(inode, this, &old_layout);
+        if (layout)
+            GF_ATOMIC_INC(layout->ref);
+        ret = dht_inode_ctx_layout_set(inode, this, layout);
+    }
+    UNLOCK(&conf->layout_lock);
+
+    if (!oldret) {
+        dht_layout_unref(this, old_layout);
+    }
+    if (ret)
+        GF_ATOMIC_DEC(layout->ref);
 
 out:
-        return ret;
+    return ret;
 }
 
-
 void
-dht_layout_unref (xlator_t *this, dht_layout_t *layout)
+dht_layout_unref(xlator_t *this, dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
-        int          ref = 0;
+    int ref = 0;
 
-        if (layout->preset || !this->private)
-                return;
+    if (!layout || layout->preset || !this->private)
+        return;
 
-        conf = this->private;
+    ref = GF_ATOMIC_DEC(layout->ref);
 
-        LOCK (&conf->layout_lock);
-        {
-                ref = --layout->ref;
-        }
-        UNLOCK (&conf->layout_lock);
-
-        if (!ref)
-                GF_FREE (layout);
+    if (!ref)
+        GF_FREE(layout);
 }
 
-
 dht_layout_t *
-dht_layout_ref (xlator_t *this, dht_layout_t *layout)
+dht_layout_ref(xlator_t *this, dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
-
-        if (layout->preset || !this->private)
-                return layout;
+    if (layout->preset || !this->private)
+        return layout;
 
-        conf = this->private;
-        LOCK (&conf->layout_lock);
-        {
-                layout->ref++;
-        }
-        UNLOCK (&conf->layout_lock);
+    GF_ATOMIC_INC(layout->ref);
 
-        return layout;
+    return layout;
 }
 
-
 xlator_t *
-dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
+dht_layout_search(xlator_t *this, dht_layout_t *layout, const char *name)
 {
-        uint32_t   hash = 0;
-        xlator_t  *subvol = NULL;
-        int        i = 0;
-        int        ret = 0;
-
-
-        ret = dht_hash_compute (layout->type, name, &hash);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "hash computation failed for type=%d name=%s",
-                        layout->type, name);
-                goto out;
-        }
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].start <= hash
-                    && layout->list[i].stop >= hash) {
-                        subvol = layout->list[i].xlator;
-                        break;
-                }
-        }
-
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "no subvolume for hash (value) = %u", hash);
-        }
+    uint32_t hash = 0;
+    xlator_t *subvol = NULL;
+    int i = 0;
+    int ret = 0;
+
+    ret = dht_hash_compute(this, layout->type, name, &hash);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_COMPUTE_HASH_FAILED,
+                "type=%d", layout->type, "name=%s", name, NULL);
+        goto out;
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].start <= hash && layout->list[i].stop >= hash) {
+            subvol = layout->list[i].xlator;
+            break;
+        }
+    }
+
+    if (!subvol) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                "hash-value=0x%x", hash, NULL);
+    }
 
 out:
-        return subvol;
+    return subvol;
 }
 
-
 dht_layout_t *
-dht_layout_for_subvol (xlator_t *this, xlator_t *subvol)
+dht_layout_for_subvol(xlator_t *this, xlator_t *subvol)
 {
-        dht_conf_t   *conf = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == subvol) {
-                        layout = conf->file_layouts[i];
-                        break;
-                }
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == subvol) {
+            layout = conf->file_layouts[i];
+            break;
         }
+    }
 
 out:
-        return layout;
+    return layout;
 }
 
-
 int
-dht_layouts_init (xlator_t *this, dht_conf_t *conf)
+dht_layouts_init(xlator_t *this, dht_conf_t *conf)
 {
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           ret = -1;
-
-        if (!conf)
-                goto out;
-
-        conf->file_layouts = GF_CALLOC (conf->subvolume_cnt,
-                                        sizeof (dht_layout_t *),
-                                        gf_dht_mt_dht_layout_t);
-        if (!conf->file_layouts) {
-                goto out;
-        }
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                layout = dht_layout_new (this, 1);
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int ret = -1;
 
-                if (!layout) {
-                        goto out;
-                }
+    if (!conf)
+        goto out;
 
-                layout->preset = 1;
+    conf->file_layouts = GF_CALLOC(conf->subvolume_cnt, sizeof(dht_layout_t *),
+                                   gf_dht_mt_dht_layout_t);
+    if (!conf->file_layouts) {
+        goto out;
+    }
 
-                layout->list[0].xlator = conf->subvolumes[i];
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        layout = dht_layout_new(this, 1);
 
-                conf->file_layouts[i] = layout;
+        if (!layout) {
+            goto out;
         }
 
-        ret = 0;
+        layout->preset = 1;
+
+        layout->list[0].xlator = conf->subvolumes[i];
+
+        conf->file_layouts[i] = layout;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-                         int pos, int32_t **disk_layout_p)
+dht_disk_layout_extract(xlator_t *this, dht_layout_t *layout, int pos,
+                        int32_t **disk_layout_p)
 {
-        int      ret = -1;
-        int32_t *disk_layout = NULL;
+    int ret = -1;
+    int32_t *disk_layout = NULL;
 
-        disk_layout = GF_CALLOC (5, sizeof (int),
-                                 gf_dht_mt_int32_t);
-        if (!disk_layout) {
-                goto out;
-        }
+    disk_layout = GF_CALLOC(5, sizeof(int), gf_dht_mt_int32_t);
+    if (!disk_layout) {
+        goto out;
+    }
 
-        disk_layout[0] = hton32 (1);
-        disk_layout[1] = hton32 (layout->type);
-        disk_layout[2] = hton32 (layout->list[pos].start);
-        disk_layout[3] = hton32 (layout->list[pos].stop);
+    disk_layout[0] = hton32(layout->list[pos].commit_hash);
+    disk_layout[1] = hton32(layout->type);
+    disk_layout[2] = hton32(layout->list[pos].start);
+    disk_layout[3] = hton32(layout->list[pos].stop);
 
-        if (disk_layout_p)
-                *disk_layout_p = disk_layout;
-        else
-                GF_FREE (disk_layout);
+    if (disk_layout_p)
+        *disk_layout_p = disk_layout;
+    else
+        GF_FREE(disk_layout);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_disk_layout_merge (xlator_t *this, dht_layout_t *layout,
-		       int pos, void *disk_layout_raw, int disk_layout_len)
+dht_disk_layout_extract_for_subvol(xlator_t *this, dht_layout_t *layout,
+                                   xlator_t *subvol, int32_t **disk_layout_p)
 {
-        int      cnt = 0;
-        int      type = 0;
-        int      start_off = 0;
-        int      stop_off = 0;
-        int      disk_layout[4];
-
-	if (!disk_layout_raw) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-                        "error no layout on disk for merge");
-		return -1;
-	}
-
-	GF_ASSERT (disk_layout_len == sizeof (disk_layout));
-
-        memcpy (disk_layout, disk_layout_raw, disk_layout_len);
-
-        cnt  = ntoh32 (disk_layout[0]);
-        if (cnt != 1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "disk layout has invalid count %d", cnt);
-                return -1;
-        }
+    int i = 0;
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol)
+            break;
+    }
+
+    if (i == layout->cnt)
+        return -1;
+
+    return dht_disk_layout_extract(this, layout, i, disk_layout_p);
+}
+
+static int
+dht_disk_layout_merge(xlator_t *this, dht_layout_t *layout, int pos,
+                      void *disk_layout_raw, int disk_layout_len)
+{
+    int type = 0;
+    int start_off = 0;
+    int stop_off = 0;
+    int commit_hash = 0;
+    int disk_layout[4];
+
+    if (!disk_layout_raw) {
+        gf_smsg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                NULL);
+        return -1;
+    }
 
-        type = ntoh32 (disk_layout[1]);
-	switch (type) {
+    GF_ASSERT(disk_layout_len == sizeof(disk_layout));
+
+    memcpy(disk_layout, disk_layout_raw, disk_layout_len);
+
+    type = ntoh32(disk_layout[1]);
+    switch (type) {
         case DHT_HASH_TYPE_DM_USER:
-                gf_log (this->name, GF_LOG_DEBUG, "found user-set layout");
-                layout->type = type;
-                /* Fall through. */
-	case DHT_HASH_TYPE_DM:
-		break;
+            gf_msg_debug(this->name, 0, "found user-set layout");
+            layout->type = type;
+            /* Fall through. */
+        case DHT_HASH_TYPE_DM:
+            break;
         default:
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"Catastrophic error layout with unknown type found %d",
-			disk_layout[1]);
-		return -1;
-	}
+            gf_smsg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_INVALID_DISK_LAYOUT,
+                    "layout=%d", disk_layout[1], NULL);
+            return -1;
+    }
 
-        start_off = ntoh32 (disk_layout[2]);
-        stop_off  = ntoh32 (disk_layout[3]);
+    commit_hash = ntoh32(disk_layout[0]);
+    start_off = ntoh32(disk_layout[2]);
+    stop_off = ntoh32(disk_layout[3]);
 
-        layout->list[pos].start = start_off;
-        layout->list[pos].stop  = stop_off;
+    layout->list[pos].commit_hash = commit_hash;
+    layout->list[pos].start = start_off;
+    layout->list[pos].stop = stop_off;
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "merged to layout: %u - %u (type %d) from %s",
-                start_off, stop_off, type,
-                layout->list[pos].xlator->name);
+    gf_msg_trace(this->name, 0,
+                 "merged to layout: 0x%x - 0x%x (hash 0x%x, type %d) from %s",
+                 start_off, stop_off, commit_hash, type,
+                 layout->list[pos].xlator->name);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                  int op_ret, int op_errno, dict_t *xattr)
+dht_layout_merge(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                 int op_ret, int op_errno, dict_t *xattr)
 {
-        int      i     = 0;
-        int      ret   = -1;
-        int      err   = -1;
-        void    *disk_layout_raw = NULL;
-        int      disk_layout_len = 0;
-
-        if (op_ret != 0) {
-                err = op_errno;
-        }
+    int i = 0;
+    int ret = -1;
+    int err = -1;
+    void *disk_layout_raw = NULL;
+    int disk_layout_len = 0;
+    dht_conf_t *conf = this->private;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == NULL) {
-                        layout->list[i].err    = err;
-                        layout->list[i].xlator = subvol;
-                        break;
-                }
-        }
+    if (op_ret != 0) {
+        err = op_errno;
+    }
 
-        if (op_ret != 0) {
-                ret = 0;
-                goto out;
-        }
+    if (!layout)
+        goto out;
 
-        if (xattr) {
-                /* during lookup and not mkdir */
-                ret = dict_get_ptr_and_len (xattr, "trusted.glusterfs.dht",
-					    &disk_layout_raw, &disk_layout_len);
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == NULL) {
+            layout->list[i].err = err;
+            layout->list[i].xlator = subvol;
+            break;
         }
+    }
 
-        if (ret != 0) {
-                layout->list[i].err = -1;
-                gf_log (this->name, GF_LOG_TRACE,
-                        "missing disk layout on %s. err = %d",
-                        subvol->name, err);
-                ret = 0;
-                goto out;
-        }
+    if (op_ret != 0) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = dht_disk_layout_merge (this, layout, i, disk_layout_raw,
-				     disk_layout_len);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "layout merge from subvolume %s failed",
-                        subvol->name);
-                goto out;
-        }
+    if (xattr) {
+        /* during lookup and not mkdir */
+        ret = dict_get_ptr_and_len(xattr, conf->xattr_name, &disk_layout_raw,
+                                   &disk_layout_len);
+    }
+
+    if (ret != 0) {
         layout->list[i].err = 0;
+        gf_msg_trace(this->name, 0, "Missing disk layout on %s. err = %d",
+                     subvol->name, err);
+        ret = 0;
+        goto out;
+    }
+
+    ret = dht_disk_layout_merge(this, layout, i, disk_layout_raw,
+                                disk_layout_len);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                "subvolume=%s", subvol->name, NULL);
+        goto out;
+    }
+
+    if (layout->commit_hash == 0) {
+        layout->commit_hash = layout->list[i].commit_hash;
+    } else if (layout->commit_hash != layout->list[i].commit_hash) {
+        layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+    }
+
+    layout->list[i].err = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
+void
+dht_layout_entry_swap(dht_layout_t *layout, int i, int j)
+{
+    uint32_t start_swap = 0;
+    uint32_t stop_swap = 0;
+    uint32_t commit_hash_swap = 0;
+    xlator_t *xlator_swap = 0;
+    int err_swap = 0;
+
+    start_swap = layout->list[i].start;
+    stop_swap = layout->list[i].stop;
+    xlator_swap = layout->list[i].xlator;
+    err_swap = layout->list[i].err;
+    commit_hash_swap = layout->list[i].commit_hash;
+
+    layout->list[i].start = layout->list[j].start;
+    layout->list[i].stop = layout->list[j].stop;
+    layout->list[i].xlator = layout->list[j].xlator;
+    layout->list[i].err = layout->list[j].err;
+    layout->list[i].commit_hash = layout->list[j].commit_hash;
+
+    layout->list[j].start = start_swap;
+    layout->list[j].stop = stop_swap;
+    layout->list[j].xlator = xlator_swap;
+    layout->list[j].err = err_swap;
+    layout->list[j].commit_hash = commit_hash_swap;
+}
 
 void
-dht_layout_entry_swap (dht_layout_t *layout, int i, int j)
+dht_layout_range_swap(dht_layout_t *layout, int i, int j)
+{
+    uint32_t start_swap = 0;
+    uint32_t stop_swap = 0;
+
+    start_swap = layout->list[i].start;
+    stop_swap = layout->list[i].stop;
+
+    layout->list[i].start = layout->list[j].start;
+    layout->list[i].stop = layout->list[j].stop;
+
+    layout->list[j].start = start_swap;
+    layout->list[j].stop = stop_swap;
+}
+static int64_t
+dht_layout_entry_cmp_volname(dht_layout_t *layout, int i, int j)
 {
-        uint32_t  start_swap = 0;
-        uint32_t  stop_swap = 0;
-        xlator_t *xlator_swap = 0;
-        int       err_swap = 0;
-
-        start_swap  = layout->list[i].start;
-        stop_swap   = layout->list[i].stop;
-        xlator_swap = layout->list[i].xlator;
-        err_swap    = layout->list[i].err;
-
-        layout->list[i].start  = layout->list[j].start;
-        layout->list[i].stop   = layout->list[j].stop;
-        layout->list[i].xlator = layout->list[j].xlator;
-        layout->list[i].err    = layout->list[j].err;
-
-        layout->list[j].start  = start_swap;
-        layout->list[j].stop   = stop_swap;
-        layout->list[j].xlator = xlator_swap;
-        layout->list[j].err    = err_swap;
+    return (strcmp(layout->list[i].xlator->name, layout->list[j].xlator->name));
 }
 
-int64_t
-dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
+gf_boolean_t
+dht_is_subvol_in_layout(dht_layout_t *layout, xlator_t *xlator)
 {
-        return (strcmp (layout->list[i].xlator->name,
-                        layout->list[j].xlator->name));
+    int i = 0;
+
+    for (i = 0; i < layout->cnt; i++) {
+        /* Check if xlator is already part of layout, and layout is
+         * non-zero. */
+        if (!strcmp(layout->list[i].xlator->name, xlator->name)) {
+            if (layout->list[i].start != layout->list[i].stop)
+                return _gf_true;
+            break;
+        }
+    }
+    return _gf_false;
 }
 
-int64_t
-dht_layout_entry_cmp (dht_layout_t *layout, int i, int j)
+static int64_t
+dht_layout_entry_cmp(dht_layout_t *layout, int i, int j)
 {
-        int64_t diff = 0;
+    int64_t diff = 0;
 
-        if (layout->list[i].err || layout->list[j].err)
-                diff = layout->list[i].err - layout->list[j].err;
-        else
-                diff = (int64_t) layout->list[i].start
-                        - (int64_t) layout->list[j].start;
+    /* swap zero'ed out layouts to front, if needed */
+    if (!layout->list[j].start && !layout->list[j].stop) {
+        diff = (int64_t)layout->list[i].stop - (int64_t)layout->list[j].stop;
+        goto out;
+    }
+    diff = (int64_t)layout->list[i].start - (int64_t)layout->list[j].start;
 
-        return diff;
+out:
+    return diff;
 }
 
-
 int
-dht_layout_sort (dht_layout_t *layout)
+dht_layout_sort(dht_layout_t *layout)
 {
-        int       i = 0;
-        int       j = 0;
-        int64_t   ret = 0;
+    int i = 0;
+    int j = 0;
+    int64_t ret = 0;
 
-        /* TODO: O(n^2) -- bad bad */
+    /* TODO: O(n^2) -- bad bad */
 
-        for (i = 0; i < layout->cnt - 1; i++) {
-                for (j = i + 1; j < layout->cnt; j++) {
-                        ret = dht_layout_entry_cmp (layout, i, j);
-                        if (ret > 0)
-                                dht_layout_entry_swap (layout, i, j);
-                }
+    for (i = 0; i < layout->cnt - 1; i++) {
+        for (j = i + 1; j < layout->cnt; j++) {
+            ret = dht_layout_entry_cmp(layout, i, j);
+            if (ret > 0)
+                dht_layout_entry_swap(layout, i, j);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_layout_sort_volname (dht_layout_t *layout)
+void
+dht_layout_sort_volname(dht_layout_t *layout)
 {
-        int       i = 0;
-        int       j = 0;
-        int64_t   ret = 0;
+    int i = 0;
+    int j = 0;
+    int64_t ret = 0;
 
-        /* TODO: O(n^2) -- bad bad */
+    /* TODO: O(n^2) -- bad bad */
 
-        for (i = 0; i < layout->cnt - 1; i++) {
-                for (j = i + 1; j < layout->cnt; j++) {
-                        ret = dht_layout_entry_cmp_volname (layout, i, j);
-                        if (ret > 0)
-                                dht_layout_entry_swap (layout, i, j);
-                }
+    for (i = 0; i < layout->cnt - 1; i++) {
+        for (j = i + 1; j < layout->cnt; j++) {
+            ret = dht_layout_entry_cmp_volname(layout, i, j);
+            if (ret > 0)
+                dht_layout_entry_swap(layout, i, j);
         }
-
-        return 0;
+    }
 }
 
-
-int
-dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
-                      uint32_t *holes_p, uint32_t *overlaps_p,
-                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p)
+void
+dht_layout_anomalies(xlator_t *this, loc_t *loc, dht_layout_t *layout,
+                     uint32_t *holes_p, uint32_t *overlaps_p,
+                     uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                     uint32_t *no_space_p)
 {
-        uint32_t    overlaps = 0;
-        uint32_t    missing  = 0;
-        uint32_t    down     = 0;
-        uint32_t    misc     = 0;
-        uint32_t    hole_cnt = 0;
-        uint32_t    overlap_cnt = 0;
-        int         i = 0;
-        int         ret = 0;
-        uint32_t    prev_stop = 0;
-        uint32_t    last_stop = 0;
-        char        is_virgin = 1;
-
-        /* TODO: explain what is happening */
-
-        last_stop = layout->list[0].start - 1;
-        prev_stop = last_stop;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err) {
-                        switch (layout->list[i].err) {
-                        case -1:
-                        case ENOENT:
-                                missing++;
-                                break;
-                        case ENOTCONN:
-                                down++;
-                                break;
-                        case ENOSPC:
-                                down++;
-                                break;
-                        default:
-                                misc++;
-                        }
-                        continue;
+    uint32_t overlaps = 0;
+    uint32_t missing = 0;
+    uint32_t down = 0;
+    uint32_t misc = 0;
+    uint32_t hole_cnt = 0;
+    uint32_t overlap_cnt = 0;
+    int i = 0;
+    uint32_t prev_stop = 0;
+    uint32_t last_stop = 0;
+    char is_virgin = 1;
+    uint32_t no_space = 0;
+
+    /* This function scans through the layout spread of a directory to
+       check if there are any anomalies. Prior to calling this function
+       the layout entries should be sorted in the ascending order.
+
+       If the layout entry has err != 0
+            then increment the corresponding anomaly.
+       else
+            if (start of the current layout entry > stop + 1 of previous
+               non erroneous layout entry)
+                    then it indicates a hole in the layout
+            if (start of the current layout entry < stop + 1 of previous
+                non erroneous layout entry)
+                     then it indicates an overlap in the layout
+    */
+    last_stop = layout->list[0].start - 1;
+    prev_stop = last_stop;
+
+    for (i = 0; i < layout->cnt; i++) {
+        switch (layout->list[i].err) {
+            case -1:
+            case ENOENT:
+            case ESTALE:
+                missing++;
+                continue;
+            case ENOTCONN:
+                down++;
+                continue;
+            case ENOSPC:
+                no_space++;
+                continue;
+            case 0:
+                /* if err == 0 and start == stop, then it is a non misc++;
+                 * participating subvolume(spread-cnt). Then, do not
+                 * check for anomalies. If start != stop, then treat it
+                 * as misc err */
+                if (layout->list[i].start == layout->list[i].stop) {
+                    continue;
                 }
+                break;
+            default:
+                misc++;
+                continue;
+        }
 
-                is_virgin = 0;
+        is_virgin = 0;
 
-                if ((prev_stop + 1) < layout->list[i].start) {
-                        hole_cnt++;
-                }
+        if ((prev_stop + 1) < layout->list[i].start) {
+            hole_cnt++;
+        }
 
-                if ((prev_stop + 1) > layout->list[i].start) {
-                        overlap_cnt++;
-                        overlaps += ((prev_stop + 1) - layout->list[i].start);
-                }
-                prev_stop = layout->list[i].stop;
+        if ((prev_stop + 1) > layout->list[i].start) {
+            overlap_cnt++;
+            overlaps += ((prev_stop + 1) - layout->list[i].start);
         }
+        prev_stop = layout->list[i].stop;
+    }
 
-        if ((last_stop - prev_stop) || is_virgin)
-                hole_cnt++;
+    if ((last_stop - prev_stop) || is_virgin)
+        hole_cnt++;
 
-        if (holes_p)
-                *holes_p = hole_cnt;
+    if (holes_p)
+        *holes_p = hole_cnt;
 
-        if (overlaps_p)
-                *overlaps_p = overlap_cnt;
+    if (overlaps_p)
+        *overlaps_p = overlap_cnt;
 
-        if (missing_p)
-                *missing_p = missing;
+    if (missing_p)
+        *missing_p = missing;
 
-        if (down_p)
-                *down_p = down;
+    if (down_p)
+        *down_p = down;
 
-        if (misc_p)
-                *misc_p = misc;
+    if (misc_p)
+        *misc_p = misc;
 
-        return ret;
+    if (no_space_p)
+        *no_space_p = no_space;
 }
 
-
 int
-dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
+dht_layout_missing_dirs(dht_layout_t *layout)
 {
-        int          ret   = 0;
-        int          i = 0;
-        uint32_t     holes = 0;
-        uint32_t     overlaps = 0;
-        uint32_t     missing = 0;
-        uint32_t     down = 0;
-        uint32_t     misc = 0;
-
-        ret = dht_layout_sort (layout);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "sort failed?! how the ....");
-                goto out;
-        }
+    int i = 0, missing = 0;
 
-        ret = dht_layout_anomalies (this, loc, layout,
-                                    &holes, &overlaps,
-                                    &missing, &down, &misc);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "error while finding anomalies in %s -- not good news",
-                        loc->path);
-                goto out;
-        }
+    if (layout == NULL)
+        goto out;
 
-        if (holes || overlaps) {
-                if (missing == layout->cnt) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "directory %s looked up first time",
-                                loc->path);
-                } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "found anomalies in %s. holes=%d overlaps=%d",
-                                loc->path, holes, overlaps);
-                }
-                ret = -1;
+    for (i = 0; i < layout->cnt; i++) {
+        if ((layout->list[i].err == ENOENT) ||
+            ((layout->list[i].err == -1) && (layout->list[i].start == 0) &&
+             (layout->list[i].stop == 0))) {
+            missing++;
         }
+    }
 
-        for (i = 0; i < layout->cnt; i++) {
-                /* TODO During DHT selfheal rewrite (almost) find a better place
-                 * to detect this - probably in dht_layout_anomalies()
-                 */
-                if (layout->list[i].err > 0) {
-                        gf_log_callingfn (this->name, GF_LOG_DEBUG,
-                                          "path=%s err=%s on subvol=%s",
-                                          loc->path,
-                                          strerror (layout->list[i].err),
-                                          (layout->list[i].xlator ?
-                                           layout->list[i].xlator->name
-                                           : "<>"));
-                        if ((layout->list[i].err == ENOENT) && (ret >= 0)) {
-                                ret++;
-                        }
-                }
-        }
+out:
+    return missing;
+}
 
+int
+dht_layout_normalize(xlator_t *this, loc_t *loc, dht_layout_t *layout)
+{
+    int ret = 0;
+    uint32_t holes = 0;
+    uint32_t overlaps = 0;
+    uint32_t missing = 0;
+    uint32_t down = 0;
+    uint32_t misc = 0, missing_dirs = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    ret = dht_layout_sort(layout);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_SORT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    gf_uuid_unparse(loc->gfid, gfid);
+
+    dht_layout_anomalies(this, loc, layout, &holes, &overlaps, &missing, &down,
+                         &misc, NULL);
+
+    if (holes || overlaps) {
+        if (missing == layout->cnt) {
+            gf_msg_debug(this->name, 0,
+                         "Directory %s looked up first time"
+                         " gfid = %s",
+                         loc->path, gfid);
+        } else {
+            gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_ANOMALIES_INFO,
+                    "path=%s", loc->path, "gfid=%s", gfid, "holes=%d", holes,
+                    "overlaps=%d", overlaps, NULL);
+        }
+        ret = -1;
+    }
+
+    if (ret >= 0) {
+        missing_dirs = dht_layout_missing_dirs(layout);
+        /* TODO During DHT selfheal rewrite (almost) find a better place
+         * to detect this - probably in dht_layout_anomalies()
+         */
+        if (missing_dirs > 0)
+            ret += missing_dirs;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                         loc_t *loc, dict_t *xattr)
+dht_dir_has_layout(dict_t *xattr, char *name)
 {
-        int       idx = 0;
-        int       pos = -1;
-        int       ret = 0;
-        int       err = 0;
-        int       dict_ret = 0;
-        int32_t   disk_layout[4];
-        void     *disk_layout_raw = NULL;
-        int32_t   count = -1;
-        uint32_t  start_off = -1;
-        uint32_t  stop_off = -1;
-
-
-        for (idx = 0; idx < layout->cnt; idx++) {
-                if (layout->list[idx].xlator == subvol) {
-                        pos = idx;
-                        break;
-                }
-        }
+    void *disk_layout_raw = NULL;
 
-        if (pos == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%s - no layout info for subvolume %s",
-                        loc->path, subvol->name);
-                ret = 1;
-                goto out;
-        }
+    return dict_get_ptr(xattr, name, &disk_layout_raw);
+}
 
-        err = layout->list[pos].err;
+int
+dht_layout_dir_mismatch(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                        loc_t *loc, dict_t *xattr)
+{
+    int idx = 0;
+    int pos = -1;
+    int ret = 0;
+    int err = 0;
+    int dict_ret = 0;
+    int32_t disk_layout[4];
+    void *disk_layout_raw = NULL;
+    uint32_t start_off = -1;
+    uint32_t stop_off = -1;
+    uint32_t commit_hash = -1;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    if (loc && loc->inode)
+        gf_uuid_unparse(loc->inode->gfid, gfid);
+
+    for (idx = 0; idx < layout->cnt; idx++) {
+        if (layout->list[idx].xlator == subvol) {
+            pos = idx;
+            break;
+        }
+    }
+
+    if (pos == -1) {
+        if (loc) {
+            gf_msg_debug(this->name, 0, "%s - no layout info for subvolume %s",
+                         loc ? loc->path : "path not found", subvol->name);
+        }
+        ret = 1;
+        goto out;
+    }
+
+    err = layout->list[pos].err;
+
+    if (!xattr) {
+        if (err == 0) {
+            if (loc) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_XATTR_DICT_NULL,
+                        "path=%s", loc->path, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_XATTR_DICT_NULL,
+                        "path not found", NULL);
+            }
+            ret = -1;
+        }
+        goto out;
+    }
+
+    dict_ret = dict_get_ptr(xattr, conf->xattr_name, &disk_layout_raw);
+
+    if (dict_ret < 0) {
+        if (err == 0 && layout->list[pos].stop) {
+            if (loc) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_DISK_LAYOUT_MISSING,
+                        "path=%s", loc->path, "gfid=%s", gfid, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_DISK_LAYOUT_MISSING,
+                        "path not found"
+                        "gfid=%s",
+                        gfid, NULL);
+            }
+            ret = -1;
+        }
+        goto out;
+    }
+
+    memcpy(disk_layout, disk_layout_raw, sizeof(disk_layout));
+
+    start_off = ntoh32(disk_layout[2]);
+    stop_off = ntoh32(disk_layout[3]);
+    commit_hash = ntoh32(disk_layout[0]);
+
+    if ((layout->list[pos].start != start_off) ||
+        (layout->list[pos].stop != stop_off) ||
+        (layout->list[pos].commit_hash != commit_hash)) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_INFO, "subvol=%s",
+                layout->list[pos].xlator->name, "inode-layout:start=0x%x",
+                layout->list[pos].start, "inode-layout:stop=0x%x",
+                layout->list[pos].stop, "layout-commit-hash=0x%x; ",
+                layout->list[pos].commit_hash, "disk-layout:start-off=0x%x",
+                start_off, "disk-layout:top-off=0x%x", stop_off,
+                "commit-hash=0x%x", commit_hash, NULL);
+        ret = 1;
+    } else {
+        ret = 0;
+    }
+out:
+    return ret;
+}
 
-        if (!xattr) {
-                if (err == 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%s - xattr dictionary is NULL",
-                                loc->path);
-                        ret = -1;
-                }
-                goto out;
-        }
+int
+dht_layout_preset(xlator_t *this, xlator_t *subvol, inode_t *inode)
+{
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
 
-        dict_ret = dict_get_ptr (xattr, "trusted.glusterfs.dht",
-                                 &disk_layout_raw);
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        if (dict_ret < 0) {
-                if (err == 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%s - disk layout missing", loc->path);
-                        ret = -1;
-                }
-                goto out;
-        }
+    layout = dht_layout_for_subvol(this, subvol);
+    if (!layout) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_NO_LAYOUT_INFO,
+                "subvolume=%s", subvol ? subvol->name : "<nil>", NULL);
+        ret = -1;
+        goto out;
+    }
 
-        memcpy (disk_layout, disk_layout_raw, sizeof (disk_layout));
+    gf_msg_debug(this->name, 0, "file = %s, subvol = %s",
+                 uuid_utoa(inode->gfid), subvol ? subvol->name : "<nil>");
 
-        count  = ntoh32 (disk_layout[0]);
-        if (count != 1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s - disk layout has invalid count %d",
-                        loc->path, count);
-                ret = -1;
-                goto out;
-        }
+    LOCK(&conf->layout_lock);
+    {
+        dht_inode_ctx_layout_set(inode, this, layout);
+    }
 
-        start_off = ntoh32 (disk_layout[2]);
-        stop_off  = ntoh32 (disk_layout[3]);
-
-        if ((layout->list[pos].start != start_off)
-            || (layout->list[pos].stop != stop_off)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "subvol: %s; inode layout - %"PRIu32" - %"PRIu32"; "
-                        "disk layout - %"PRIu32" - %"PRIu32,
-                        layout->list[pos].xlator->name,
-                        layout->list[pos].start, layout->list[pos].stop,
-                        start_off, stop_off);
-                ret = 1;
-        } else {
-                ret = 0;
-        }
+    UNLOCK(&conf->layout_lock);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode)
+dht_layout_index_for_subvol(dht_layout_t *layout, xlator_t *subvol)
 {
-        dht_layout_t *layout = NULL;
-        int           ret = -1;
-        dht_conf_t   *conf = NULL;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        layout = dht_layout_for_subvol (this, subvol);
-        if (!layout) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no pre-set layout for subvolume %s",
-                        subvol ? subvol->name : "<nil>");
-                ret = -1;
-                goto out;
-        }
+    int i = 0, ret = -1;
 
-        LOCK (&conf->layout_lock);
-        {
-                inode_ctx_put (inode, this, (uint64_t)(long)layout);
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol) {
+            ret = i;
+            break;
         }
-        UNLOCK (&conf->layout_lock);
+    }
 
-        ret = 0;
-out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-linkfile.c b/xlators/cluster/dht/src/dht-linkfile.c
index 803f344aff1..89ec6cca56e 100644
--- a/xlators/cluster/dht/src/dht-linkfile.c
+++ b/xlators/cluster/dht/src/dht-linkfile.c
@@ -8,183 +8,321 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <glusterfs/compat.h>
+#include "dht-common.h"
 
+static int
+dht_linkfile_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
+{
+    char is_linkfile = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (op_ret)
+        goto out;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    if (!is_linkfile)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NOT_LINK_FILE_ERROR,
+                "name=%s", prev->name, "path=%s", local->loc.path, "gfid=%s",
+                gfid, NULL);
+out:
+    local->linkfile.linkfile_cbk(frame, cookie, this, op_ret, op_errno, inode,
+                                 stbuf, postparent, postparent, xattr);
+    return 0;
+}
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "compat.h"
-#include "dht-common.h"
+static int
+dht_linkfile_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    dict_t *xattrs = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
 
+    local = frame->local;
 
+    if (!op_ret)
+        local->linked = _gf_true;
 
-int
-dht_linkfile_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, inode_t *inode,
-                         struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-        local = frame->local;
+    if (op_ret && (op_errno == EEXIST)) {
+        conf = this->private;
+        subvol = cookie;
+        if (!subvol)
+            goto out;
+        xattrs = dict_new();
+        if (!xattrs)
+            goto out;
+        ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                    "mame=%s", conf->link_xattr_name, NULL);
+            goto out;
+        }
 
-        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-                                      inode, stbuf, preparent, postparent,
-                                      xdata);
+        STACK_WIND_COOKIE(frame, dht_linkfile_lookup_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->linkfile.loc, xattrs);
+        if (xattrs)
+            dict_unref(xattrs);
         return 0;
+    }
+out:
+    local->linkfile.linkfile_cbk(frame, cookie, this, op_ret, op_errno, inode,
+                                 stbuf, preparent, postparent, xdata);
+    if (xattrs)
+        dict_unref(xattrs);
+    return 0;
 }
 
-
 int
-dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
-                     xlator_t *tovol, xlator_t *fromvol, loc_t *loc)
+dht_linkfile_create(call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
+                    xlator_t *this, xlator_t *tovol, xlator_t *fromvol,
+                    loc_t *loc)
 {
-        dht_local_t *local = NULL;
-        dict_t      *dict = NULL;
-        int          need_unref = 0;
-        int          ret = 0;
-
-        local = frame->local;
-        local->linkfile.linkfile_cbk = linkfile_cbk;
-        local->linkfile.srcvol = tovol;
-
-        dict = local->params;
-        if (!dict) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-                need_unref = 1;
-        }
-
-        if (!uuid_is_null (local->gfid)) {
-                ret = dict_set_static_bin (dict, "gfid-req", local->gfid, 16);
-                if (ret)
-                        gf_log ("dht-linkfile", GF_LOG_INFO,
-                                "%s: gfid set failed", loc->path);
-        }
-
-        ret = dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    int need_unref = 0;
+    int ret = 0;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    local->linkfile.linkfile_cbk = linkfile_cbk;
+    local->linkfile.srcvol = tovol;
+    loc_copy(&local->linkfile.loc, loc);
+
+    local->linked = _gf_false;
+
+    dict = local->params;
+    if (!dict) {
+        dict = dict_new();
+        if (!dict)
+            goto out;
+        need_unref = 1;
+    }
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid);
+
+        ret = dict_set_gfuuid(dict, "gfid-req", local->gfid, true);
         if (ret)
-                gf_log ("dht-linkfile", GF_LOG_INFO,
-                        "%s: internal-fop set failed", loc->path);
+            gf_smsg("dht-linkfile", GF_LOG_INFO, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", loc->path, "gfid=%s", gfid, NULL);
+    } else {
+        gf_uuid_unparse(loc->gfid, gfid);
+    }
+
+    ret = dict_set_str(dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    if (ret)
+        gf_smsg("dht-linkfile", GF_LOG_INFO, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", GLUSTERFS_INTERNAL_FOP_KEY,
+                "gfid=%s", gfid, NULL);
+
+    ret = dict_set_str(dict, conf->link_xattr_name, tovol->name);
+
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, DHT_MSG_CREATE_LINK_FAILED,
+                "path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    local->link_subvol = fromvol;
+    /* Always create as root:root. dht_linkfile_attr_heal fixes the
+     * ownsership */
+    FRAME_SU_DO(frame, dht_local_t);
+    STACK_WIND_COOKIE(frame, dht_linkfile_create_cbk, fromvol, fromvol,
+                      fromvol->fops->mknod, loc, S_IFREG | DHT_LINKFILE_MODE, 0,
+                      0, dict);
+
+    if (need_unref && dict)
+        dict_unref(dict);
+
+    return 0;
+out:
+    local->linkfile.linkfile_cbk(frame, frame->this, frame->this, -1, ENOMEM,
+                                 loc->inode, NULL, NULL, NULL, NULL);
 
-        ret = dict_set_str (dict, "trusted.glusterfs.dht.linkto",
-                                   tovol->name);
+    if (need_unref && dict)
+        dict_unref(dict);
 
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_INFO,
-                        "%s: failed to initialize linkfile data",
-                        loc->path);
-                goto out;
-        }
+    return 0;
+}
 
-        STACK_WIND (frame, dht_linkfile_create_cbk,
-                    fromvol, fromvol->fops->mknod, loc,
-                    S_IFREG | DHT_LINKFILE_MODE, 0, 0, dict);
+int
+dht_linkfile_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preparent, struct iatt *postparent,
+                        dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        if (need_unref && dict)
-                dict_unref (dict);
+    local = frame->local;
+    subvol = cookie;
 
-        return 0;
-out:
-        local->linkfile.linkfile_cbk (frame, NULL, frame->this, -1, ENOMEM,
-                                      loc->inode, NULL, NULL, NULL, NULL);
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_UNLINK_FAILED,
+                "path=%s", local->loc.path, "gfid=%s", gfid, "subvolume=%s",
+                subvol->name, NULL);
+    }
 
-        if (need_unref && dict)
-                dict_unref (dict);
+    DHT_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno,
-                         struct iatt *preparent, struct iatt *postparent,
-                         dict_t *xdata)
+dht_linkfile_unlink(call_frame_t *frame, xlator_t *this, xlator_t *subvol,
+                    loc_t *loc)
 {
-        dht_local_t   *local = NULL;
-        call_frame_t  *prev = NULL;
-        xlator_t      *subvol = NULL;
-
-        local = frame->local;
-        prev = cookie;
-        subvol = prev->this;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "unlinking linkfile %s on %s failed (%s)",
-                        local->loc.path, subvol->name, strerror (op_errno));
-        }
+    call_frame_t *unlink_frame = NULL;
+    dht_local_t *unlink_local = NULL;
 
-        DHT_STACK_DESTROY (frame);
+    unlink_frame = copy_frame(frame);
+    if (!unlink_frame) {
+        goto err;
+    }
 
-        return 0;
-}
+    /* Using non-fop value here, as anyways, 'local->fop' is not used in
+       this particular case */
+    unlink_local = dht_local_init(unlink_frame, loc, NULL, GF_FOP_MAXVALUE);
+    if (!unlink_local) {
+        goto err;
+    }
 
+    STACK_WIND_COOKIE(unlink_frame, dht_linkfile_unlink_cbk, subvol, subvol,
+                      subvol->fops->unlink, &unlink_local->loc, 0, NULL);
 
-int
-dht_linkfile_unlink (call_frame_t *frame, xlator_t *this,
-                     xlator_t *subvol, loc_t *loc)
+    return 0;
+err:
+    if (unlink_frame)
+        DHT_STACK_DESTROY(unlink_frame);
+
+    return -1;
+}
+
+xlator_t *
+dht_linkfile_subvol(xlator_t *this, inode_t *inode, struct iatt *stbuf,
+                    dict_t *xattr)
 {
-        call_frame_t *unlink_frame = NULL;
-        dht_local_t  *unlink_local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    void *volname = NULL;
+    int i = 0, ret = 0;
 
-        unlink_frame = copy_frame (frame);
-        if (!unlink_frame) {
-                goto err;
-        }
+    conf = this->private;
 
-        /* Using non-fop value here, as anyways, 'local->fop' is not used in
-           this particular case */
-        unlink_local = dht_local_init (unlink_frame, loc, NULL,
-                                       GF_FOP_MAXVALUE);
-        if (!unlink_local) {
-                goto err;
-        }
+    if (!xattr)
+        goto out;
 
-        STACK_WIND (unlink_frame, dht_linkfile_unlink_cbk,
-                    subvol, subvol->fops->unlink,
-                    &unlink_local->loc, 0, NULL);
+    ret = dict_get_ptr(xattr, conf->link_xattr_name, &volname);
 
-        return 0;
-err:
-        if (unlink_frame)
-                DHT_STACK_DESTROY (unlink_frame);
+    if ((-1 == ret) || !volname)
+        goto out;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (strcmp(conf->subvolumes[i]->name, (char *)volname) == 0) {
+            subvol = conf->subvolumes[i];
+            break;
+        }
+    }
 
-        return -1;
+out:
+    return subvol;
 }
 
+static int
+dht_linkfile_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *statpre,
+                         struct iatt *statpost, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
 
-xlator_t *
-dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct iatt *stbuf,
-                     dict_t *xattr)
+    local = frame->local;
+    loc = &local->loc;
+
+    if (op_ret)
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_SETATTR_FAILED,
+                "path=%s", (loc->path ? loc->path : "NULL"), "gfid=%s",
+                uuid_utoa(local->gfid), NULL);
+
+    DHT_STACK_DESTROY(frame);
+
+    return 0;
+}
+
+int
+dht_linkfile_attr_heal(call_frame_t *frame, xlator_t *this)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *subvol = NULL;
-        void       *volname = NULL;
-        int         i = 0, ret = 0;
+    int ret = -1;
+    call_frame_t *copy = NULL;
+    dht_local_t *local = NULL;
+    dht_local_t *copy_local = NULL;
+    xlator_t *subvol = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *xattr = NULL;
+
+    local = frame->local;
+
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    GF_VALIDATE_OR_GOTO("dht", local->link_subvol, out);
+
+    if (local->stbuf.ia_type == IA_INVAL)
+        return 0;
 
-        conf = this->private;
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-        if (!xattr)
-                goto out;
+    gf_uuid_copy(local->loc.gfid, local->stbuf.ia_gfid);
 
-        ret = dict_get_ptr (xattr, "trusted.glusterfs.dht.linkto", &volname);
+    copy = copy_frame(frame);
 
-        if ((-1 == ret) || !volname)
-                goto out;
+    if (!copy)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (strcmp (conf->subvolumes[i]->name, (char *)volname) == 0) {
-                        subvol = conf->subvolumes[i];
-                        break;
-                }
-        }
+    copy_local = dht_local_init(copy, &local->loc, NULL, 0);
+
+    if (!copy_local)
+        goto out;
+
+    stbuf = local->stbuf;
+    subvol = local->link_subvol;
+
+    copy->local = copy_local;
 
+    FRAME_SU_DO(copy, dht_local_t);
+
+    STACK_WIND(copy, dht_linkfile_setattr_cbk, subvol, subvol->fops->setattr,
+               &copy_local->loc, &stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+               xattr);
+    ret = 0;
 out:
-        return subvol;
+    if ((ret < 0) && (copy))
+        DHT_STACK_DESTROY(copy);
+
+    if (xattr)
+        dict_unref(xattr);
+
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-lock.c b/xlators/cluster/dht/src/dht-lock.c
new file mode 100644
index 00000000000..638821ccee5
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-lock.c
@@ -0,0 +1,1392 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "dht-lock.h"
+
+static char *
+dht_lock_asprintf(dht_lock_t *lock)
+{
+    char *lk_buf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    if (lock == NULL)
+        goto out;
+
+    uuid_utoa_r(lock->loc.gfid, gfid);
+
+    gf_asprintf(&lk_buf, "%s:%s", lock->xl->name, gfid);
+
+out:
+    return lk_buf;
+}
+
+static void
+dht_log_lk_array(char *name, gf_loglevel_t log_level, dht_lock_t **lk_array,
+                 int count)
+{
+    int i = 0;
+    char *lk_buf = NULL;
+
+    if ((lk_array == NULL) || (count == 0))
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        lk_buf = dht_lock_asprintf(lk_array[i]);
+        if (!lk_buf)
+            goto out;
+
+        gf_smsg(name, log_level, 0, DHT_MSG_LK_ARRAY_INFO, "index=%d", i,
+                "lk_buf=%s", lk_buf, NULL);
+        GF_FREE(lk_buf);
+    }
+
+out:
+    return;
+}
+
+static void
+dht_lock_stack_destroy(call_frame_t *lock_frame, dht_lock_type_t lk)
+{
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+
+    if (lk == DHT_INODELK) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+    } else {
+        local->lock[0].ns.directory_ns.locks = NULL;
+        local->lock[0].ns.directory_ns.lk_count = 0;
+    }
+
+    DHT_STACK_DESTROY(lock_frame);
+    return;
+}
+
+static void
+dht_lock_free(dht_lock_t *lock)
+{
+    if (lock == NULL)
+        goto out;
+
+    loc_wipe(&lock->loc);
+    GF_FREE(lock->domain);
+    GF_FREE(lock->basename);
+    mem_put(lock);
+
+out:
+    return;
+}
+
+static void
+dht_set_lkowner(dht_lock_t **lk_array, int count, gf_lkowner_t *lkowner)
+{
+    int i = 0;
+
+    if (!lk_array || !lkowner)
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        lk_array[i]->lk_owner = *lkowner;
+    }
+
+out:
+    return;
+}
+
+static int
+dht_lock_request_cmp(const void *val1, const void *val2)
+{
+    dht_lock_t *lock1 = NULL;
+    dht_lock_t *lock2 = NULL;
+    int ret = -1;
+
+    lock1 = *(dht_lock_t **)val1;
+    lock2 = *(dht_lock_t **)val2;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", lock1, out);
+    GF_VALIDATE_OR_GOTO("dht-locks", lock2, out);
+
+    ret = strcmp(lock1->xl->name, lock2->xl->name);
+
+    if (ret == 0) {
+        ret = gf_uuid_compare(lock1->loc.gfid, lock2->loc.gfid);
+    }
+
+out:
+    return ret;
+}
+
+static int
+dht_lock_order_requests(dht_lock_t **locks, int count)
+{
+    int ret = -1;
+
+    if (!locks || !count)
+        goto out;
+
+    qsort(locks, count, sizeof(*locks), dht_lock_request_cmp);
+    ret = 0;
+
+out:
+    return ret;
+}
+
+void
+dht_lock_array_free(dht_lock_t **lk_array, int count)
+{
+    int i = 0;
+    dht_lock_t *lock = NULL;
+
+    if (lk_array == NULL)
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        lock = lk_array[i];
+        lk_array[i] = NULL;
+        dht_lock_free(lock);
+    }
+
+out:
+    return;
+}
+
+int32_t
+dht_lock_count(dht_lock_t **lk_array, int lk_count)
+{
+    int i = 0, locked = 0;
+
+    if ((lk_array == NULL) || (lk_count == 0))
+        goto out;
+
+    for (i = 0; i < lk_count; i++) {
+        if (lk_array[i]->locked)
+            locked++;
+    }
+out:
+    return locked;
+}
+
+static call_frame_t *
+dht_lock_frame(call_frame_t *parent_frame)
+{
+    call_frame_t *lock_frame = NULL;
+
+    lock_frame = copy_frame(parent_frame);
+    if (lock_frame == NULL)
+        goto out;
+
+    set_lk_owner_from_ptr(&lock_frame->root->lk_owner, parent_frame->root);
+
+out:
+    return lock_frame;
+}
+
+dht_lock_t *
+dht_lock_new(xlator_t *this, xlator_t *xl, loc_t *loc, short type,
+             const char *domain, const char *basename,
+             dht_reaction_type_t do_on_failure)
+{
+    dht_conf_t *conf = NULL;
+    dht_lock_t *lock = NULL;
+
+    conf = this->private;
+
+    lock = mem_get0(conf->lock_pool);
+    if (lock == NULL)
+        goto out;
+
+    lock->xl = xl;
+    lock->type = type;
+    lock->do_on_failure = do_on_failure;
+
+    lock->domain = gf_strdup(domain);
+    if (lock->domain == NULL) {
+        dht_lock_free(lock);
+        lock = NULL;
+        goto out;
+    }
+
+    if (basename) {
+        lock->basename = gf_strdup(basename);
+        if (lock->basename == NULL) {
+            dht_lock_free(lock);
+            lock = NULL;
+            goto out;
+        }
+    }
+
+    /* Fill only inode and gfid.
+       posix and protocol/server give preference to pargfid/basename over
+       gfid/inode for resolution if all the three parameters of loc_t are
+       present. I want to avoid the following hypothetical situation:
+
+       1. rebalance did a lookup on a dentry and got a gfid.
+       2. rebalance acquires lock on loc_t which was filled with gfid and
+          path (pargfid/bname) from step 1.
+       3. somebody deleted and recreated the same file
+       4. rename on the same path acquires lock on loc_t which now points
+          to a different inode (and hence gets the lock).
+       5. rebalance continues to migrate file (note that not all fops done
+          by rebalance during migration are inode/gfid based Eg., unlink)
+       6. rename continues.
+    */
+    lock->loc.inode = inode_ref(loc->inode);
+    loc_gfid(loc, lock->loc.gfid);
+
+out:
+    return lock;
+}
+
+static int
+dht_local_entrylk_init(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                       fop_entrylk_cbk_t entrylk_cbk)
+{
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local == NULL) {
+        local = dht_local_init(frame, NULL, NULL, 0);
+    }
+
+    if (local == NULL) {
+        goto out;
+    }
+
+    local->lock[0].ns.directory_ns.entrylk_cbk = entrylk_cbk;
+    local->lock[0].ns.directory_ns.locks = lk_array;
+    local->lock[0].ns.directory_ns.lk_count = lk_count;
+
+    ret = dht_lock_order_requests(local->lock[0].ns.directory_ns.locks,
+                                  local->lock[0].ns.directory_ns.lk_count);
+    if (ret < 0)
+        goto out;
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static void
+dht_entrylk_done(call_frame_t *lock_frame)
+{
+    fop_entrylk_cbk_t entrylk_cbk = NULL;
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+    main_frame = local->main_frame;
+
+    local->lock[0].ns.directory_ns.locks = NULL;
+    local->lock[0].ns.directory_ns.lk_count = 0;
+
+    entrylk_cbk = local->lock[0].ns.directory_ns.entrylk_cbk;
+    local->lock[0].ns.directory_ns.entrylk_cbk = NULL;
+
+    entrylk_cbk(main_frame, NULL, main_frame->this,
+                local->lock[0].ns.directory_ns.op_ret,
+                local->lock[0].ns.directory_ns.op_errno, NULL);
+
+    dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
+    return;
+}
+
+static int32_t
+dht_unlock_entrylk_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    gf_uuid_unparse(local->lock[0].ns.directory_ns.locks[0]->loc.inode->gfid,
+                    gfid);
+
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                DHT_MSG_UNLOCK_GFID_FAILED, "gfid=%s", gfid,
+                "DHT_LAYOUT_HEAL_DOMAIN", NULL);
+    }
+
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
+
+static int32_t
+dht_unlock_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+
+    uuid_utoa_r(local->lock[0].ns.directory_ns.locks[lk_index]->loc.gfid, gfid);
+
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED,
+                "name=%s",
+                local->lock[0].ns.directory_ns.locks[lk_index]->xl->name,
+                "gfid=%s", gfid, NULL);
+    } else {
+        local->lock[0].ns.directory_ns.locks[lk_index]->locked = 0;
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        dht_entrylk_done(frame);
+    }
+
+    return 0;
+}
+
+static int32_t
+dht_unlock_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_entrylk_cbk_t entrylk_cbk)
+{
+    dht_local_t *local = NULL;
+    int ret = -1, i = 0;
+    call_frame_t *lock_frame = NULL;
+    int call_cnt = 0;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, entrylk_cbk, done);
+
+    call_cnt = dht_lock_count(lk_array, lk_count);
+    if (call_cnt == 0) {
+        ret = 0;
+        goto done;
+    }
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+        goto done;
+    }
+
+    ret = dht_local_entrylk_init(lock_frame, lk_array, lk_count, entrylk_cbk);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+
+        goto done;
+    }
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+    local->call_cnt = call_cnt;
+
+    for (i = 0; i < local->lock[0].ns.directory_ns.lk_count; i++) {
+        if (!local->lock[0].ns.directory_ns.locks[i]->locked)
+            continue;
+
+        lock_frame->root
+            ->lk_owner = local->lock[0].ns.directory_ns.locks[i]->lk_owner;
+        STACK_WIND_COOKIE(
+            lock_frame, dht_unlock_entrylk_cbk, (void *)(long)i,
+            local->lock[0].ns.directory_ns.locks[i]->xl,
+            local->lock[0].ns.directory_ns.locks[i]->xl->fops->entrylk,
+            local->lock[0].ns.directory_ns.locks[i]->domain,
+            &local->lock[0].ns.directory_ns.locks[i]->loc,
+            local->lock[0].ns.directory_ns.locks[i]->basename, ENTRYLK_UNLOCK,
+            ENTRYLK_WRLCK, NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
+
+done:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
+
+    /* no locks acquired, invoke entrylk_cbk */
+    if (ret == 0)
+        entrylk_cbk(frame, NULL, frame->this, 0, 0, NULL);
+
+    return ret;
+}
+
+int32_t
+dht_unlock_entrylk_wrapper(call_frame_t *frame, dht_elock_wrap_t *entrylk)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+
+    local = frame->local;
+
+    if (!entrylk || !entrylk->locks)
+        goto out;
+
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_COPY_FRAME_FAILED, "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, NULL, NULL, 0);
+    if (lock_local == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_CREATE_FAILED, "local", "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_frame->local = lock_local;
+
+    lock_local->lock[0].ns.directory_ns.locks = entrylk->locks;
+    lock_local->lock[0].ns.directory_ns.lk_count = entrylk->lk_count;
+    entrylk->locks = NULL;
+    entrylk->lk_count = 0;
+
+    ret = dht_unlock_entrylk(
+        lock_frame, lock_local->lock[0].ns.directory_ns.locks,
+        lock_local->lock[0].ns.directory_ns.lk_count, dht_unlock_entrylk_done);
+    if (ret)
+        goto done;
+
+    lock_frame = NULL;
+
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
+
+out:
+    return 0;
+}
+
+static int
+dht_entrylk_cleanup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_entrylk_done(frame);
+    return 0;
+}
+
+static void
+dht_entrylk_cleanup(call_frame_t *lock_frame)
+{
+    dht_lock_t **lk_array = NULL;
+    int lk_count = 0, lk_acquired = 0;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+
+    lk_array = local->lock[0].ns.directory_ns.locks;
+    lk_count = local->lock[0].ns.directory_ns.lk_count;
+
+    lk_acquired = dht_lock_count(lk_array, lk_count);
+    if (lk_acquired != 0) {
+        dht_unlock_entrylk(lock_frame, lk_array, lk_count,
+                           dht_entrylk_cleanup_cbk);
+    } else {
+        dht_entrylk_done(lock_frame);
+    }
+
+    return;
+}
+
+static int32_t
+dht_blocking_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    int lk_index = 0;
+    int i = 0;
+    dht_local_t *local = NULL;
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret == 0) {
+        local->lock[0].ns.directory_ns.locks[lk_index]->locked = _gf_true;
+    } else {
+        switch (op_errno) {
+            case ESTALE:
+            case ENOENT:
+                if (local->lock[0]
+                        .ns.directory_ns.locks[lk_index]
+                        ->do_on_failure != IGNORE_ENOENT_ESTALE) {
+                    local->lock[0].ns.directory_ns.op_ret = -1;
+                    local->lock[0].ns.directory_ns.op_errno = op_errno;
+                    goto cleanup;
+                }
+                break;
+            default:
+                local->lock[0].ns.directory_ns.op_ret = -1;
+                local->lock[0].ns.directory_ns.op_errno = op_errno;
+                goto cleanup;
+        }
+    }
+
+    if (lk_index == (local->lock[0].ns.directory_ns.lk_count - 1)) {
+        for (i = 0; (i < local->lock[0].ns.directory_ns.lk_count) &&
+                    (!local->lock[0].ns.directory_ns.locks[i]->locked);
+             i++)
+            ;
+
+        if (i == local->lock[0].ns.directory_ns.lk_count) {
+            local->lock[0].ns.directory_ns.op_ret = -1;
+            local->lock[0].ns.directory_ns.op_errno = op_errno;
+        }
+
+        dht_entrylk_done(frame);
+    } else {
+        dht_blocking_entrylk_rec(frame, ++lk_index);
+    }
+
+    return 0;
+
+cleanup:
+    dht_entrylk_cleanup(frame);
+
+    return 0;
+}
+
+void
+dht_blocking_entrylk_rec(call_frame_t *frame, int i)
+{
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_WIND_COOKIE(
+        frame, dht_blocking_entrylk_cbk, (void *)(long)i,
+        local->lock[0].ns.directory_ns.locks[i]->xl,
+        local->lock[0].ns.directory_ns.locks[i]->xl->fops->entrylk,
+        local->lock[0].ns.directory_ns.locks[i]->domain,
+        &local->lock[0].ns.directory_ns.locks[i]->loc,
+        local->lock[0].ns.directory_ns.locks[i]->basename, ENTRYLK_LOCK,
+        ENTRYLK_WRLCK, NULL);
+
+    return;
+}
+
+int
+dht_blocking_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_entrylk_cbk_t entrylk_cbk)
+{
+    int ret = -1;
+    call_frame_t *lock_frame = NULL;
+    dht_local_t *local = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, entrylk_cbk, out);
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL)
+        goto out;
+
+    ret = dht_local_entrylk_init(lock_frame, lk_array, lk_count, entrylk_cbk);
+    if (ret < 0) {
+        goto out;
+    }
+
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+
+    dht_blocking_entrylk_rec(lock_frame, 0);
+
+    return 0;
+out:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
+
+    return -1;
+}
+
+static int
+dht_local_inodelk_init(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                       fop_inodelk_cbk_t inodelk_cbk)
+{
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local == NULL) {
+        local = dht_local_init(frame, NULL, NULL, 0);
+    }
+
+    if (local == NULL) {
+        goto out;
+    }
+
+    local->lock[0].layout.my_layout.inodelk_cbk = inodelk_cbk;
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = lk_count;
+
+    ret = dht_lock_order_requests(local->lock[0].layout.my_layout.locks,
+                                  local->lock[0].layout.my_layout.lk_count);
+    if (ret < 0)
+        goto out;
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static void
+dht_inodelk_done(call_frame_t *lock_frame)
+{
+    fop_inodelk_cbk_t inodelk_cbk = NULL;
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+    main_frame = local->main_frame;
+
+    local->lock[0].layout.my_layout.locks = NULL;
+    local->lock[0].layout.my_layout.lk_count = 0;
+
+    inodelk_cbk = local->lock[0].layout.my_layout.inodelk_cbk;
+    local->lock[0].layout.my_layout.inodelk_cbk = NULL;
+
+    inodelk_cbk(main_frame, NULL, main_frame->this,
+                local->lock[0].layout.my_layout.op_ret,
+                local->lock[0].layout.my_layout.op_errno, NULL);
+
+    dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+    return;
+}
+
+static int32_t
+dht_unlock_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret < 0) {
+        uuid_utoa_r(local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid,
+                    gfid);
+
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED,
+                "name=%s",
+                local->lock[0].layout.my_layout.locks[lk_index]->xl->name,
+                "gfid=%s", gfid, NULL);
+    } else {
+        local->lock[0].layout.my_layout.locks[lk_index]->locked = 0;
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        dht_inodelk_done(frame);
+    }
+
+    return 0;
+}
+
+static int32_t
+dht_unlock_inodelk_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    gf_uuid_unparse(local->lock[0].layout.my_layout.locks[0]->loc.inode->gfid,
+                    gfid);
+
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                DHT_MSG_UNLOCK_GFID_FAILED, "DHT_LAYOUT_HEAL_DOMAIN gfid=%s",
+                gfid, NULL);
+    }
+
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
+
+int32_t
+dht_unlock_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_inodelk_cbk_t inodelk_cbk)
+{
+    dht_local_t *local = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = -1, i = 0;
+    call_frame_t *lock_frame = NULL;
+    int call_cnt = 0;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, done);
+
+    call_cnt = dht_lock_count(lk_array, lk_count);
+    if (call_cnt == 0) {
+        ret = 0;
+        goto done;
+    }
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+        goto done;
+    }
+
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+
+        goto done;
+    }
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+    local->call_cnt = call_cnt;
+
+    flock.l_type = F_UNLCK;
+
+    for (i = 0; i < local->lock[0].layout.my_layout.lk_count; i++) {
+        if (!local->lock[0].layout.my_layout.locks[i]->locked)
+            continue;
+
+        lock_frame->root
+            ->lk_owner = local->lock[0].layout.my_layout.locks[i]->lk_owner;
+        STACK_WIND_COOKIE(
+            lock_frame, dht_unlock_inodelk_cbk, (void *)(long)i,
+            local->lock[0].layout.my_layout.locks[i]->xl,
+            local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+            local->lock[0].layout.my_layout.locks[i]->domain,
+            &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLK, &flock,
+            NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
+
+done:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+
+    /* no locks acquired, invoke inodelk_cbk */
+    if (ret == 0)
+        inodelk_cbk(frame, NULL, frame->this, 0, 0, NULL);
+
+    return ret;
+}
+
+int32_t
+dht_unlock_inodelk_wrapper(call_frame_t *frame, dht_ilock_wrap_t *inodelk)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+
+    local = frame->local;
+
+    if (!inodelk || !inodelk->locks)
+        goto out;
+
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_COPY_FRAME_FAILED, "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, NULL, NULL, 0);
+    if (lock_local == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_CREATE_FAILED, "local", "gfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_frame->local = lock_local;
+
+    lock_local->lock[0].layout.my_layout.locks = inodelk->locks;
+    lock_local->lock[0].layout.my_layout.lk_count = inodelk->lk_count;
+    inodelk->locks = NULL;
+    inodelk->lk_count = 0;
+
+    ret = dht_unlock_inodelk(
+        lock_frame, lock_local->lock[0].layout.my_layout.locks,
+        lock_local->lock[0].layout.my_layout.lk_count, dht_unlock_inodelk_done);
+
+    if (ret)
+        goto done;
+
+    lock_frame = NULL;
+
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
+out:
+    return 0;
+}
+
+static int
+dht_inodelk_cleanup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_inodelk_done(frame);
+    return 0;
+}
+
+static void
+dht_inodelk_cleanup(call_frame_t *lock_frame)
+{
+    dht_lock_t **lk_array = NULL;
+    int lk_count = 0, lk_acquired = 0;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+
+    lk_array = local->lock[0].layout.my_layout.locks;
+    lk_count = local->lock[0].layout.my_layout.lk_count;
+
+    lk_acquired = dht_lock_count(lk_array, lk_count);
+    if (lk_acquired != 0) {
+        dht_unlock_inodelk(lock_frame, lk_array, lk_count,
+                           dht_inodelk_cleanup_cbk);
+    } else {
+        dht_inodelk_done(lock_frame);
+    }
+
+    return;
+}
+
+static int32_t
+dht_nonblocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    lk_index = (long)cookie;
+
+    if (op_ret == -1) {
+        local->lock[0].layout.my_layout.op_ret = -1;
+        local->lock[0].layout.my_layout.op_errno = op_errno;
+
+        if (local && local->lock[0].layout.my_layout.locks[lk_index]) {
+            uuid_utoa_r(local->lock[0]
+                            .layout.my_layout.locks[lk_index]
+                            ->loc.inode->gfid,
+                        gfid);
+
+            gf_msg_debug(
+                this->name, op_errno,
+                "inodelk failed on gfid: %s "
+                "subvolume: %s",
+                gfid,
+                local->lock[0].layout.my_layout.locks[lk_index]->xl->name);
+        }
+
+        goto out;
+    }
+
+    local->lock[0].layout.my_layout.locks[lk_index]->locked = _gf_true;
+
+out:
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->lock[0].layout.my_layout.op_ret < 0) {
+            dht_inodelk_cleanup(frame);
+            return 0;
+        }
+
+        dht_inodelk_done(frame);
+    }
+
+    return 0;
+}
+
+int
+dht_nonblocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array,
+                        int lk_count, fop_inodelk_cbk_t inodelk_cbk)
+{
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0, ret = 0;
+    dht_local_t *local = NULL;
+    call_frame_t *lock_frame = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, out);
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL)
+        goto out;
+
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        goto out;
+    }
+
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+
+    local->call_cnt = lk_count;
+
+    for (i = 0; i < lk_count; i++) {
+        flock.l_type = local->lock[0].layout.my_layout.locks[i]->type;
+
+        STACK_WIND_COOKIE(
+            lock_frame, dht_nonblocking_inodelk_cbk, (void *)(long)i,
+            local->lock[0].layout.my_layout.locks[i]->xl,
+            local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+            local->lock[0].layout.my_layout.locks[i]->domain,
+            &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLK, &flock,
+            NULL);
+    }
+
+    return 0;
+
+out:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+
+    return -1;
+}
+
+static int32_t
+dht_blocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    int lk_index = 0;
+    int i = 0;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+    dht_reaction_type_t reaction = 0;
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret == 0) {
+        local->lock[0].layout.my_layout.locks[lk_index]->locked = _gf_true;
+    } else {
+        switch (op_errno) {
+            case ESTALE:
+            case ENOENT:
+                reaction = local->lock[0]
+                               .layout.my_layout.locks[lk_index]
+                               ->do_on_failure;
+                if ((reaction != IGNORE_ENOENT_ESTALE) &&
+                    (reaction != IGNORE_ENOENT_ESTALE_EIO)) {
+                    gf_uuid_unparse(local->lock[0]
+                                        .layout.my_layout.locks[lk_index]
+                                        ->loc.gfid,
+                                    gfid);
+                    local->lock[0].layout.my_layout.op_ret = -1;
+                    local->lock[0].layout.my_layout.op_errno = op_errno;
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_INODELK_FAILED, "subvol=%s",
+                            local->lock[0]
+                                .layout.my_layout.locks[lk_index]
+                                ->xl->name,
+                            "gfid=%s", gfid, NULL);
+                    goto cleanup;
+                }
+                break;
+            case EIO:
+                reaction = local->lock[0]
+                               .layout.my_layout.locks[lk_index]
+                               ->do_on_failure;
+                if (reaction != IGNORE_ENOENT_ESTALE_EIO) {
+                    gf_uuid_unparse(local->lock[0]
+                                        .layout.my_layout.locks[lk_index]
+                                        ->loc.gfid,
+                                    gfid);
+                    local->lock[0].layout.my_layout.op_ret = -1;
+                    local->lock[0].layout.my_layout.op_errno = op_errno;
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_INODELK_FAILED, "subvol=%s",
+                            local->lock[0]
+                                .layout.my_layout.locks[lk_index]
+                                ->xl->name,
+                            "gfid=%s", gfid, NULL);
+                    goto cleanup;
+                }
+                break;
+
+            default:
+                gf_uuid_unparse(
+                    local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid,
+                    gfid);
+                local->lock[0].layout.my_layout.op_ret = -1;
+                local->lock[0].layout.my_layout.op_errno = op_errno;
+                gf_smsg(
+                    this->name, GF_LOG_ERROR, op_errno, DHT_MSG_INODELK_FAILED,
+                    "subvol=%s",
+                    local->lock[0].layout.my_layout.locks[lk_index]->xl->name,
+                    "gfid=%s", gfid, NULL);
+                goto cleanup;
+        }
+    }
+
+    if (lk_index == (local->lock[0].layout.my_layout.lk_count - 1)) {
+        for (i = 0; (i < local->lock[0].layout.my_layout.lk_count) &&
+                    (!local->lock[0].layout.my_layout.locks[i]->locked);
+             i++)
+            ;
+
+        if (i == local->lock[0].layout.my_layout.lk_count) {
+            local->lock[0].layout.my_layout.op_ret = -1;
+            local->lock[0].layout.my_layout.op_errno = op_errno;
+        }
+
+        dht_inodelk_done(frame);
+    } else {
+        dht_blocking_inodelk_rec(frame, ++lk_index);
+    }
+
+    return 0;
+
+cleanup:
+    dht_inodelk_cleanup(frame);
+
+    return 0;
+}
+
+void
+dht_blocking_inodelk_rec(call_frame_t *frame, int i)
+{
+    dht_local_t *local = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    local = frame->local;
+
+    flock.l_type = local->lock[0].layout.my_layout.locks[i]->type;
+
+    STACK_WIND_COOKIE(
+        frame, dht_blocking_inodelk_cbk, (void *)(long)i,
+        local->lock[0].layout.my_layout.locks[i]->xl,
+        local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+        local->lock[0].layout.my_layout.locks[i]->domain,
+        &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLKW, &flock, NULL);
+
+    return;
+}
+
+int
+dht_blocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t inodelk_cbk)
+{
+    int ret = -1;
+    call_frame_t *lock_frame = NULL;
+    dht_local_t *local = NULL;
+    dht_local_t *tmp_local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, out);
+
+    tmp_local = frame->local;
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_uuid_unparse(tmp_local->loc.gfid, gfid);
+        gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCK_FRAME_FAILED,
+                "gfid=%s", gfid, "path=%s", tmp_local->loc.path, NULL);
+        goto out;
+    }
+
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        gf_uuid_unparse(tmp_local->loc.gfid, gfid);
+        gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCAL_LOCK_INIT_FAILED,
+                "gfid=%s", gfid, "path=%s", tmp_local->loc.path, NULL);
+        goto out;
+    }
+
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+
+    dht_blocking_inodelk_rec(lock_frame, 0);
+
+    return 0;
+out:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+
+    return -1;
+}
+
+void
+dht_unlock_namespace(call_frame_t *frame, dht_dir_transaction_t *lock)
+{
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lock, out);
+
+    dht_unlock_entrylk_wrapper(frame, &lock->ns.directory_ns);
+    dht_unlock_inodelk_wrapper(frame, &lock->ns.parent_layout);
+
+out:
+    return;
+}
+
+static int32_t
+dht_protect_namespace_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+    if (op_ret != 0)
+        dht_unlock_inodelk_wrapper(frame, &local->current->ns.parent_layout);
+
+    local->current->ns.ns_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+dht_blocking_entrylk_after_inodelk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    loc_t *loc = NULL;
+    dht_lock_t **lk_array = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int count = 0;
+    dht_elock_wrap_t *entrylk = NULL;
+
+    local = frame->local;
+    entrylk = &local->current->ns.directory_ns;
+
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    loc = &entrylk->locks[0]->loc;
+    gf_uuid_unparse(loc->gfid, pgfid);
+
+    local->op_ret = 0;
+    lk_array = entrylk->locks;
+    count = entrylk->lk_count;
+
+    ret = dht_blocking_entrylk(frame, lk_array, count,
+                               dht_protect_namespace_cbk);
+
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_ENTRYLK_FAILED_AFT_INODELK, "fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "basename=%s",
+                entrylk->locks[0]->basename, NULL);
+        goto err;
+    }
+
+    return 0;
+
+err:
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+        entrylk->locks = NULL;
+        entrylk->lk_count = 0;
+    }
+
+    /* Unlock inodelk. No harm calling unlock twice */
+    dht_unlock_inodelk_wrapper(frame, &local->current->ns.parent_layout);
+    /* Call ns_cbk. It will take care of unwinding */
+    local->current->ns.ns_cbk(frame, NULL, this, local->op_ret, local->op_errno,
+                              NULL);
+    return 0;
+}
+
+/* Given the loc and the subvol, this routine takes the inodelk on
+ * the parent inode and entrylk on (parent, loc->name). This routine
+ * is specific as it supports only one subvol on which it takes inodelk
+ * and then entrylk serially.
+ */
+int
+dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol,
+                      struct dht_namespace *ns, fop_entrylk_cbk_t ns_cbk)
+{
+    dht_ilock_wrap_t *inodelk = NULL;
+    dht_elock_wrap_t *entrylk = NULL;
+    dht_lock_t **lk_array = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    loc_t parent = {
+        0,
+    };
+    int ret = -1;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t op_errno = 0;
+    int count = 1;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, loc->parent, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, subvol, out);
+
+    local = frame->local;
+    this = frame->this;
+
+    inodelk = &ns->parent_layout;
+    entrylk = &ns->directory_ns;
+
+    /* Initialize entrylk_cbk and parent loc */
+    ns->ns_cbk = ns_cbk;
+
+    ret = dht_build_parent_loc(this, &parent, loc, &op_errno);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_LOC_FAILED,
+                "gfid=%s", loc->gfid, "name=%s", loc->name, "path=%s",
+                loc->path, NULL);
+        goto out;
+    }
+    gf_uuid_unparse(parent.gfid, pgfid);
+
+    /* Alloc inodelk */
+    inodelk->locks = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (inodelk->locks == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_CALLOC_FAILED, "fop=%s", gf_fop_list[local->fop],
+                "pgfid=%s", pgfid, "name=%s", loc->name, "path=%s", loc->path,
+                NULL);
+        goto out;
+    }
+
+    inodelk->locks[0] = dht_lock_new(this, subvol, &parent, F_RDLCK,
+                                     DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                                     FAIL_ON_ANY_ERROR);
+    if (inodelk->locks[0] == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_LOCK_ALLOC_FAILED, "inodelk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+        goto err;
+    }
+    inodelk->lk_count = count;
+
+    /* Allock entrylk */
+    entrylk->locks = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (entrylk->locks == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_CALLOC_FAILED, "entrylk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+
+        goto err;
+    }
+
+    entrylk->locks[0] = dht_lock_new(this, subvol, &parent, F_WRLCK,
+                                     DHT_ENTRY_SYNC_DOMAIN, loc->name,
+                                     FAIL_ON_ANY_ERROR);
+    if (entrylk->locks[0] == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_LOCK_ALLOC_FAILED, "entrylk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+
+        goto err;
+    }
+    entrylk->lk_count = count;
+
+    /* Take read inodelk on parent. If it is successful, take write entrylk
+     * on name in cbk.
+     */
+    lk_array = inodelk->locks;
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_blocking_entrylk_after_inodelk);
+    if (ret < 0) {
+        local->op_errno = EIO;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_BLOCK_INODELK_FAILED, "fop=%s", gf_fop_list[local->fop],
+                "pgfid=%s", pgfid, "name=%s", loc->name, "path=%s", loc->path,
+                NULL);
+
+        goto err;
+    }
+
+    loc_wipe(&parent);
+
+    return 0;
+err:
+    if (entrylk->locks != NULL) {
+        dht_lock_array_free(entrylk->locks, count);
+        GF_FREE(entrylk->locks);
+        entrylk->locks = NULL;
+        entrylk->lk_count = 0;
+    }
+
+    if (inodelk->locks != NULL) {
+        dht_lock_array_free(inodelk->locks, count);
+        GF_FREE(inodelk->locks);
+        inodelk->locks = NULL;
+        inodelk->lk_count = 0;
+    }
+
+    loc_wipe(&parent);
+out:
+    return -1;
+}
diff --git a/xlators/cluster/dht/src/dht-lock.h b/xlators/cluster/dht/src/dht-lock.h
new file mode 100644
index 00000000000..6485c03fb6e
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-lock.h
@@ -0,0 +1,91 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _DHT_LOCK_H
+#define _DHT_LOCK_H
+
+#include "dht-common.h"
+
+void
+dht_lock_array_free(dht_lock_t **lk_array, int count);
+
+int32_t
+dht_lock_count(dht_lock_t **lk_array, int lk_count);
+
+dht_lock_t *
+dht_lock_new(xlator_t *this, xlator_t *xl, loc_t *loc, short type,
+             const char *domain, const char *basename,
+             dht_reaction_type_t do_on_failure);
+
+int32_t
+dht_unlock_entrylk_wrapper(call_frame_t *, dht_elock_wrap_t *);
+
+void
+dht_blocking_entrylk_rec(call_frame_t *frame, int i);
+
+int
+dht_blocking_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t entrylk_cbk);
+
+int32_t
+dht_unlock_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_inodelk_cbk_t inodelk_cbk);
+
+int32_t
+dht_unlock_inodelk_wrapper(call_frame_t *, dht_ilock_wrap_t *);
+
+/* Acquire non-blocking inodelk on a list of xlators.
+ *
+ * @lk_array: array of lock requests lock on.
+ *
+ * @lk_count: number of locks in @lk_array
+ *
+ * @inodelk_cbk: will be called after inodelk replies are received
+ *
+ * @retval: -1 if stack_winding inodelk fails. 0 otherwise.
+ *          inodelk_cbk is called with appropriate error on errors.
+ *          On failure to acquire lock on all members of list, successful
+ *          locks are unlocked before invoking cbk.
+ */
+
+int
+dht_nonblocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array,
+                        int lk_count, fop_inodelk_cbk_t inodelk_cbk);
+
+void
+dht_blocking_inodelk_rec(call_frame_t *frame, int i);
+
+/* same as dht_nonblocking_inodelk, but issues sequential blocking locks on
+ * @lk_array directly. locks are issued on some order which remains same
+ * for a list of xlators (irrespective of order of xlators within list).
+ */
+
+int
+dht_blocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t inodelk_cbk);
+
+int32_t
+dht_blocking_entrylk_after_inodelk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, dict_t *xdata);
+
+int32_t
+dht_blocking_entrylk_after_inodelk_rename(call_frame_t *frame, void *cookie,
+                                          xlator_t *this, int32_t op_ret,
+                                          int32_t op_errno, dict_t *xdata);
+
+void
+dht_unlock_namespace(call_frame_t *, dht_dir_transaction_t *);
+
+int
+dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol,
+                      struct dht_namespace *ns, fop_entrylk_cbk_t ns_cbk);
+
+#endif /* _DHT_LOCK_H */
diff --git a/xlators/cluster/dht/src/dht-mem-types.h b/xlators/cluster/dht/src/dht-mem-types.h
index 4f3a90e914a..e3c4471334a 100644
--- a/xlators/cluster/dht/src/dht-mem-types.h
+++ b/xlators/cluster/dht/src/dht-mem-types.h
@@ -8,26 +8,31 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __DHT_MEM_TYPES_H__
 #define __DHT_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_dht_mem_types_ {
-        gf_dht_mt_dht_du_t = gf_common_mt_end + 1,
-        gf_dht_mt_dht_conf_t,
-        gf_dht_mt_char,
-        gf_dht_mt_int32_t,
-        gf_dht_mt_xlator_t,
-        gf_dht_mt_dht_layout_t,
-        gf_switch_mt_dht_conf_t,
-        gf_switch_mt_dht_du_t,
-        gf_switch_mt_switch_sched_array,
-        gf_switch_mt_switch_struct,
-        gf_dht_mt_subvol_time,
-        gf_dht_mt_loc_t,
-        gf_defrag_info_mt,
-        gf_dht_mt_end
+    gf_dht_mt_dht_du_t = gf_common_mt_end + 1,
+    gf_dht_mt_dht_conf_t,
+    gf_dht_mt_char,
+    gf_dht_mt_int32_t,
+    gf_dht_mt_xlator_t,
+    gf_dht_mt_dht_layout_t,
+    gf_switch_mt_switch_sched_array,
+    gf_switch_mt_switch_struct,
+    gf_dht_mt_subvol_time,
+    gf_dht_mt_loc_t,
+    gf_defrag_info_mt,
+    gf_dht_mt_inode_ctx_t,
+    gf_dht_mt_dirent_t,
+    gf_dht_mt_container_t,
+    gf_dht_mt_octx_t,
+    gf_dht_mt_miginfo_t,
+    gf_dht_mt_fd_ctx_t,
+    gf_dht_ret_cache_t,
+    gf_dht_nodeuuids_t,
+    gf_dht_mt_end
 };
 #endif
diff --git a/xlators/cluster/dht/src/dht-messages.h b/xlators/cluster/dht/src/dht-messages.h
new file mode 100644
index 00000000000..601f8dad78b
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-messages.h
@@ -0,0 +1,386 @@
+/*Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _DHT_MESSAGES_H_
+#define _DHT_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    DHT, DHT_MSG_CACHED_SUBVOL_GET_FAILED, DHT_MSG_CREATE_LINK_FAILED,
+    DHT_MSG_DICT_SET_FAILED, DHT_MSG_DIR_ATTR_HEAL_FAILED,
+    DHT_MSG_DIR_SELFHEAL_FAILED, DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
+    DHT_MSG_FILE_ON_MULT_SUBVOL, DHT_MSG_FILE_TYPE_MISMATCH,
+    DHT_MSG_GFID_MISMATCH, DHT_MSG_GFID_NULL, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+    DHT_MSG_INIT_FAILED, DHT_MSG_INVALID_CONFIGURATION,
+    DHT_MSG_INVALID_DISK_LAYOUT, DHT_MSG_INVALID_OPTION,
+    DHT_MSG_LAYOUT_FIX_FAILED, DHT_MSG_LAYOUT_MERGE_FAILED,
+    DHT_MSG_LAYOUT_MISMATCH, DHT_MSG_LAYOUT_NULL, DHT_MSG_MIGRATE_DATA_COMPLETE,
+    DHT_MSG_MIGRATE_DATA_FAILED, DHT_MSG_MIGRATE_FILE_COMPLETE,
+    DHT_MSG_MIGRATE_FILE_FAILED, DHT_MSG_NO_MEMORY, DHT_MSG_OPENDIR_FAILED,
+    DHT_MSG_REBALANCE_FAILED, DHT_MSG_REBALANCE_START_FAILED,
+    DHT_MSG_REBALANCE_STATUS, DHT_MSG_REBALANCE_STOPPED, DHT_MSG_RENAME_FAILED,
+    DHT_MSG_SETATTR_FAILED, DHT_MSG_SUBVOL_INSUFF_INODES,
+    DHT_MSG_SUBVOL_INSUFF_SPACE, DHT_MSG_UNLINK_FAILED,
+    DHT_MSG_LAYOUT_SET_FAILED, DHT_MSG_LOG_FIXED_LAYOUT,
+    DHT_MSG_GET_XATTR_FAILED, DHT_MSG_FILE_LOOKUP_FAILED,
+    DHT_MSG_OPEN_FD_FAILED, DHT_MSG_SET_INODE_CTX_FAILED,
+    DHT_MSG_UNLOCKING_FAILED, DHT_MSG_DISK_LAYOUT_NULL, DHT_MSG_SUBVOL_INFO,
+    DHT_MSG_CHUNK_SIZE_INFO, DHT_MSG_LAYOUT_FORM_FAILED, DHT_MSG_SUBVOL_ERROR,
+    DHT_MSG_LAYOUT_SORT_FAILED, DHT_MSG_REGEX_INFO, DHT_MSG_FOPEN_FAILED,
+    DHT_MSG_SET_HOSTNAME_FAILED, DHT_MSG_BRICK_ERROR, DHT_MSG_SYNCOP_FAILED,
+    DHT_MSG_MIGRATE_INFO, DHT_MSG_SOCKET_ERROR, DHT_MSG_CREATE_FD_FAILED,
+    DHT_MSG_READDIR_ERROR, DHT_MSG_CHILD_LOC_BUILD_FAILED,
+    DHT_MSG_SET_SWITCH_PATTERN_ERROR, DHT_MSG_COMPUTE_HASH_FAILED,
+    DHT_MSG_FIND_LAYOUT_ANOMALIES_ERROR, DHT_MSG_ANOMALIES_INFO,
+    DHT_MSG_LAYOUT_INFO, DHT_MSG_INODE_LK_ERROR, DHT_MSG_RENAME_INFO,
+    DHT_MSG_DATA_NULL, DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
+    DHT_MSG_UNLINK_LOOKUP_INFO, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+    DHT_MSG_OPERATION_NOT_SUP, DHT_MSG_NOT_LINK_FILE_ERROR, DHT_MSG_CHILD_DOWN,
+    DHT_MSG_UUID_PARSE_ERROR, DHT_MSG_GET_DISK_INFO_ERROR,
+    DHT_MSG_INVALID_VALUE, DHT_MSG_SWITCH_PATTERN_INFO,
+    DHT_MSG_SUBVOL_OP_FAILED, DHT_MSG_LAYOUT_PRESET_FAILED,
+    DHT_MSG_INVALID_LINKFILE, DHT_MSG_FIX_LAYOUT_INFO,
+    DHT_MSG_GET_HOSTNAME_FAILED, DHT_MSG_WRITE_FAILED,
+    DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED, DHT_MSG_FSYNC_FAILED,
+    DHT_MSG_SUBVOL_DECOMMISSION_INFO, DHT_MSG_BRICK_QUERY_FAILED,
+    DHT_MSG_SUBVOL_NO_LAYOUT_INFO, DHT_MSG_OPEN_FD_ON_DST_FAILED,
+    DHT_MSG_SUBVOL_NOT_FOUND, DHT_MSG_FILE_LOOKUP_ON_DST_FAILED,
+    DHT_MSG_DISK_LAYOUT_MISSING, DHT_MSG_DICT_GET_FAILED,
+    DHT_MSG_REVALIDATE_CBK_INFO, DHT_MSG_UPGRADE_BRICKS, DHT_MSG_LK_ARRAY_INFO,
+    DHT_MSG_RENAME_NOT_LOCAL, DHT_MSG_RECONFIGURE_INFO,
+    DHT_MSG_INIT_LOCAL_SUBVOL_FAILED, DHT_MSG_SYS_CALL_GET_TIME_FAILED,
+    DHT_MSG_NO_DISK_USAGE_STATUS, DHT_MSG_SUBVOL_DOWN_ERROR,
+    DHT_MSG_REBAL_THROTTLE_INFO, DHT_MSG_COMMIT_HASH_INFO,
+    DHT_MSG_REBAL_STRUCT_SET, DHT_MSG_HAS_MIGINFO, DHT_MSG_SETTLE_HASH_FAILED,
+    DHT_MSG_DEFRAG_PROCESS_DIR_FAILED, DHT_MSG_FD_CTX_SET_FAILED,
+    DHT_MSG_STALE_LOOKUP, DHT_MSG_PARENT_LAYOUT_CHANGED,
+    DHT_MSG_LOCK_MIGRATION_FAILED, DHT_MSG_LOCK_INODE_UNREF_FAILED,
+    DHT_MSG_ASPRINTF_FAILED, DHT_MSG_DIR_LOOKUP_FAILED, DHT_MSG_INODELK_FAILED,
+    DHT_MSG_LOCK_FRAME_FAILED, DHT_MSG_LOCAL_LOCK_INIT_FAILED,
+    DHT_MSG_ENTRYLK_ERROR, DHT_MSG_INODELK_ERROR, DHT_MSG_LOC_FAILED,
+    DHT_MSG_UNKNOWN_FOP, DHT_MSG_MIGRATE_FILE_SKIPPED,
+    DHT_MSG_DIR_XATTR_HEAL_FAILED, DHT_MSG_HASHED_SUBVOL_DOWN,
+    DHT_MSG_NON_HASHED_SUBVOL_DOWN, DHT_MSG_SYNCTASK_CREATE_FAILED,
+    DHT_MSG_DIR_HEAL_ABORT, DHT_MSG_MIGRATE_SKIP, DHT_MSG_FD_CREATE_FAILED,
+    DHT_MSG_DICT_NEW_FAILED, DHT_MSG_FAILED_TO_OPEN, DHT_MSG_CREATE_FAILED,
+    DHT_MSG_FILE_NOT_EXIST, DHT_MSG_CHOWN_FAILED, DHT_MSG_FALLOCATE_FAILED,
+    DHT_MSG_FTRUNCATE_FAILED, DHT_MSG_STATFS_FAILED, DHT_MSG_WRITE_CROSS,
+    DHT_MSG_NEW_TARGET_FOUND, DHT_MSG_INSUFF_MEMORY, DHT_MSG_SET_XATTR_FAILED,
+    DHT_MSG_SET_MODE_FAILED, DHT_MSG_FILE_EXISTS_IN_DEST,
+    DHT_MSG_SYMLINK_FAILED, DHT_MSG_LINKFILE_DEL_FAILED, DHT_MSG_MKNOD_FAILED,
+    DHT_MSG_MIGRATE_CLEANUP_FAILED, DHT_MSG_LOCK_MIGRATE,
+    DHT_MSG_PARENT_BUILD_FAILED, DHT_MSG_HASHED_SUBVOL_NOT_FOUND,
+    DHT_MSG_ACQUIRE_ENTRYLK_FAILED, DHT_MSG_CREATE_DST_FAILED,
+    DHT_MSG_MIGRATION_EXIT, DHT_MSG_CHANGED_DST, DHT_MSG_TRACE_FAILED,
+    DHT_MSG_WRITE_LOCK_FAILED, DHT_MSG_GETACTIVELK_FAILED, DHT_MSG_STAT_FAILED,
+    DHT_MSG_UNLINK_PERFORM_FAILED, DHT_MSG_CLANUP_SOURCE_FILE_FAILED,
+    DHT_MSG_UNLOCK_FILE_FAILED, DHT_MSG_REMOVE_XATTR_FAILED,
+    DHT_MSG_DATA_MIGRATE_ABORT, DHT_MSG_DEFRAG_NULL, DHT_MSG_PARENT_NULL,
+    DHT_MSG_GFID_NOT_PRESENT, DHT_MSG_CHILD_LOC_FAILED,
+    DHT_MSG_SET_LOOKUP_FAILED, DHT_MSG_DIR_REMOVED, DHT_MSG_FIX_NOT_COMP,
+    DHT_MSG_SUBVOL_DETER_FAILED, DHT_MSG_LOCAL_SUBVOL, DHT_MSG_NODE_UUID,
+    DHT_MSG_SIZE_FILE, DHT_MSG_GET_DATA_SIZE_FAILED,
+    DHT_MSG_PTHREAD_JOIN_FAILED, DHT_MSG_COUNTER_THREAD_CREATE_FAILED,
+    DHT_MSG_MIGRATION_INIT_QUEUE_FAILED, DHT_MSG_PAUSED_TIMEOUT, DHT_MSG_WOKE,
+    DHT_MSG_ABORT_REBALANCE, DHT_MSG_CREATE_TASK_REBAL_FAILED,
+    DHT_MSG_REBAL_ESTIMATE_NOT_AVAIL, DHT_MSG_ADD_CHOICES_ERROR,
+    DHT_MSG_GET_CHOICES_ERROR, DHT_MSG_PREPARE_STATUS_ERROR,
+    DHT_MSG_SET_CHOICE_FAILED, DHT_MSG_SET_HASHED_SUBVOL_FAILED,
+    DHT_MSG_XATTR_HEAL_NOT_POSS, DHT_MSG_LINKTO_FILE_FAILED,
+    DHT_MSG_STALE_LINKFILE_DELETE, DHT_MSG_NO_SUBVOL_FOR_LINKTO,
+    DHT_MSG_SUBVOL_RETURNED, DHT_MSG_UNKNOWN_LOCAL_XSEL, DHT_MSG_GET_XATTR_ERR,
+    DHT_MSG_ALLOC_OR_FILL_FAILED, DHT_MSG_GET_REAL_NAME_FAILED,
+    DHT_MSG_COPY_UUID_FAILED, DHT_MSG_MDS_DETER_FAILED,
+    DHT_MSG_CREATE_REBAL_FAILED, DHT_MSG_LINK_LAYOUT_FAILED,
+    DHT_MSG_NO_SUBVOL_IN_LAYOUT, DHT_MSG_MEM_ALLOC_FAILED,
+    DHT_MSG_SET_IN_PARAMS_DICT_FAILED, DHT_MSG_LOC_COPY_FAILED,
+    DHT_MSG_PARENT_LOC_FAILED, DHT_MSG_CREATE_LOCK_FAILED,
+    DHT_MSG_PREV_ATTEMPT_FAILED, DHT_MSG_REFRESH_ATTEMPT,
+    DHT_MSG_ACQUIRE_LOCK_FAILED, DHT_MSG_CREATE_STUB_FAILED,
+    DHT_MSG_WIND_LOCK_REQ_FAILED, DHT_MSG_REFRESH_FAILED,
+    DHT_MSG_CACHED_SUBVOL_ERROR, DHT_MSG_NO_LINK_SUBVOL, DHT_MSG_SET_KEY_FAILED,
+    DHT_MSG_REMOVE_LINKTO_FAILED, DHT_MSG_LAYOUT_DICT_SET_FAILED,
+    DHT_MSG_XATTR_DICT_NULL, DHT_MSG_DUMMY_ALLOC_FAILED, DHT_MSG_DICT_IS_NULL,
+    DHT_MSG_LINK_INODE_FAILED, DHT_MSG_SELFHEAL_FAILED, DHT_MSG_NO_MDS_SUBVOL,
+    DHT_MSG_LIST_XATTRS_FAILED, DHT_MSG_RESET_INTER_XATTR_FAILED,
+    DHT_MSG_MDS_DOWN_UNABLE_TO_SET, DHT_MSG_WIND_UNLOCK_FAILED,
+    DHT_MSG_COMMIT_HASH_FAILED, DHT_MSG_UNLOCK_GFID_FAILED,
+    DHT_MSG_UNLOCK_FOLLOW_ENTRYLK, DHT_MSG_COPY_FRAME_FAILED,
+    DHT_MSG_UNLOCK_FOLLOW_LOCKS, DHT_MSG_ENTRYLK_FAILED_AFT_INODELK,
+    DHT_MSG_CALLOC_FAILED, DHT_MSG_LOCK_ALLOC_FAILED,
+    DHT_MSG_BLOCK_INODELK_FAILED,
+    DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+    DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+    DHT_MSG_DST_NULL_SET_FAILED);
+
+#define DHT_MSG_FD_CTX_SET_FAILED_STR "Failed to set fd ctx"
+#define DHT_MSG_INVALID_VALUE_STR "Different dst found in the fd ctx"
+#define DHT_MSG_UNKNOWN_FOP_STR "Unknown FOP on file"
+#define DHT_MSG_OPEN_FD_ON_DST_FAILED_STR "Failed to open the fd on file"
+#define DHT_MSG_SYNCTASK_CREATE_FAILED_STR "Failed to create synctask"
+#define DHT_MSG_ASPRINTF_FAILED_STR                                            \
+    "asprintf failed while fetching subvol from the id"
+#define DHT_MSG_HAS_MIGINFO_STR "Found miginfo in the inode ctx"
+#define DHT_MSG_FILE_LOOKUP_FAILED_STR "failed to lookup the file"
+#define DHT_MSG_INVALID_LINKFILE_STR                                           \
+    "linkto target is different from cached-subvol. treating as destination "  \
+    "subvol"
+#define DHT_MSG_GFID_MISMATCH_STR "gfid different on the target file"
+#define DHT_MSG_GET_XATTR_FAILED_STR "failed to get 'linkto' xattr"
+#define DHT_MSG_SET_INODE_CTX_FAILED_STR "failed to set inode-ctx target file"
+#define DHT_MSG_DIR_SELFHEAL_FAILED_STR "Healing of path failed"
+#define DHT_MSG_DIR_HEAL_ABORT_STR                                             \
+    "Failed to get path from subvol. Aborting directory healing"
+#define DHT_MSG_DIR_XATTR_HEAL_FAILED_STR "xattr heal failed for directory"
+#define DHT_MSG_LOCK_INODE_UNREF_FAILED_STR                                    \
+    "Found a NULL inode. Failed to unref the inode"
+#define DHT_MSG_DICT_SET_FAILED_STR "Failed to set dictionary value"
+#define DHT_MSG_NOT_LINK_FILE_ERROR_STR "got non-linkfile"
+#define DHT_MSG_CREATE_LINK_FAILED_STR "failed to initialize linkfile data"
+#define DHT_MSG_UNLINK_FAILED_STR "Unlinking linkfile on subvolume failed"
+#define DHT_MSG_MIGRATE_FILE_FAILED_STR "Migrate file failed"
+#define DHT_MSG_NO_MEMORY_STR "could not allocate memory for dict"
+#define DHT_MSG_SUBVOL_ERROR_STR "Failed to get linkto subvol"
+#define DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED_STR "link failed on subvol"
+#define DHT_MSG_MIGRATE_FILE_SKIPPED_STR "Migration skipped"
+#define DHT_MSG_FD_CREATE_FAILED_STR "fd create failed"
+#define DHT_MSG_DICT_NEW_FAILED_STR "dict_new failed"
+#define DHT_MSG_FAILED_TO_OPEN_STR "failed to open"
+#define DHT_MSG_CREATE_FAILED_STR "failed to create"
+#define DHT_MSG_FILE_NOT_EXIST_STR "file does not exist"
+#define DHT_MSG_CHOWN_FAILED_STR "chown failed"
+#define DHT_MSG_FALLOCATE_FAILED_STR "fallocate failed"
+#define DHT_MSG_FTRUNCATE_FAILED_STR "ftruncate failed"
+#define DHT_MSG_STATFS_FAILED_STR "failed to get statfs"
+#define DHT_MSG_WRITE_CROSS_STR                                                \
+    "write will cross min-fre-disk for file on subvol. looking for new subvol"
+#define DHT_MSG_SUBVOL_INSUFF_SPACE_STR                                        \
+    "Could not find any subvol with space accommodating the file. Cosider "    \
+    "adding bricks"
+#define DHT_MSG_NEW_TARGET_FOUND_STR "New target found for file"
+#define DHT_MSG_INSUFF_MEMORY_STR "insufficient memory"
+#define DHT_MSG_SET_XATTR_FAILED_STR "failed to set xattr"
+#define DHT_MSG_SET_MODE_FAILED_STR "failed to set mode"
+#define DHT_MSG_FILE_EXISTS_IN_DEST_STR "file exists in destination"
+#define DHT_MSG_LINKFILE_DEL_FAILED_STR "failed to delete the linkfile"
+#define DHT_MSG_SYMLINK_FAILED_STR "symlink failed"
+#define DHT_MSG_MKNOD_FAILED_STR "mknod failed"
+#define DHT_MSG_SETATTR_FAILED_STR "failed to perform setattr"
+#define DHT_MSG_MIGRATE_CLEANUP_FAILED_STR                                     \
+    "Migrate file cleanup failed: failed to fstat file"
+#define DHT_MSG_LOCK_MIGRATE_STR "locks will be migrated for file"
+#define DHT_MSG_PARENT_BUILD_FAILED_STR                                        \
+    "failed to build parent loc, which is needed to acquire entrylk to "       \
+    "synchronize with renames on this path. Skipping migration"
+#define DHT_MSG_HASHED_SUBVOL_NOT_FOUND_STR                                    \
+    "cannot find hashed subvol which is needed to synchronize with renames "   \
+    "on this path. Skipping migration"
+#define DHT_MSG_ACQUIRE_ENTRYLK_FAILED_STR "failed to acquire entrylk on subvol"
+#define DHT_MSG_CREATE_DST_FAILED_STR "create dst failed for file"
+#define DHT_MSG_MIGRATION_EXIT_STR "Exiting migration"
+#define DHT_MSG_CHANGED_DST_STR "destination changed fo file"
+#define DHT_MSG_TRACE_FAILED_STR "Trace failed"
+#define DHT_MSG_WRITE_LOCK_FAILED_STR "write lock failed"
+#define DHT_MSG_GETACTIVELK_FAILED_STR "getactivelk failed for file"
+#define DHT_MSG_STAT_FAILED_STR "failed to do a stat"
+#define DHT_MSG_UNLINK_PERFORM_FAILED_STR "failed to perform unlink"
+#define DHT_MSG_MIGRATE_FILE_COMPLETE_STR "completed migration"
+#define DHT_MSG_CLANUP_SOURCE_FILE_FAILED_STR "failed to cleanup source file"
+#define DHT_MSG_UNLOCK_FILE_FAILED_STR "failed to unlock file"
+#define DHT_MSG_REMOVE_XATTR_FAILED_STR "remove xattr failed"
+#define DHT_MSG_SOCKET_ERROR_STR "Failed to unlink listener socket"
+#define DHT_MSG_HASHED_SUBVOL_GET_FAILED_STR "Failed to get hashed subvolume"
+#define DHT_MSG_CACHED_SUBVOL_GET_FAILED_STR "Failed to get cached subvolume"
+#define DHT_MSG_MIGRATE_DATA_FAILED_STR "migrate-data failed"
+#define DHT_MSG_DEFRAG_NULL_STR "defrag is NULL"
+#define DHT_MSG_DATA_MIGRATE_ABORT_STR                                         \
+    "Readdirp failed. Aborting data migration for dict"
+#define DHT_MSG_LAYOUT_FIX_FAILED_STR "fix layout failed"
+#define DHT_MSG_PARENT_NULL_STR "parent is NULL"
+#define DHT_MSG_GFID_NOT_PRESENT_STR "gfid not present"
+#define DHT_MSG_CHILD_LOC_FAILED_STR "Child loc build failed"
+#define DHT_MSG_SET_LOOKUP_FAILED_STR "Failed to set lookup"
+#define DHT_MSG_DIR_LOOKUP_FAILED_STR "lookup failed"
+#define DHT_MSG_DIR_REMOVED_STR "Dir renamed or removed. Skipping"
+#define DHT_MSG_READDIR_ERROR_STR "readdir failed, Aborting fix-layout"
+#define DHT_MSG_SETTLE_HASH_FAILED_STR "Settle hash failed"
+#define DHT_MSG_DEFRAG_PROCESS_DIR_FAILED_STR "gf_defrag_process_dir failed"
+#define DHT_MSG_FIX_NOT_COMP_STR                                               \
+    "Unable to retrieve fixlayout xattr. Assume background fix layout not "    \
+    "complete"
+#define DHT_MSG_SUBVOL_DETER_FAILED_STR                                        \
+    "local subvolume determination failed with error"
+#define DHT_MSG_LOCAL_SUBVOL_STR "local subvol"
+#define DHT_MSG_NODE_UUID_STR "node uuid"
+#define DHT_MSG_SIZE_FILE_STR "Total size files"
+#define DHT_MSG_GET_DATA_SIZE_FAILED_STR                                       \
+    "Failed to get the total data size. Unable to estimate time to complete "  \
+    "rebalance"
+#define DHT_MSG_PTHREAD_JOIN_FAILED_STR                                        \
+    "file_counter_thread: pthread_join failed"
+#define DHT_MSG_COUNTER_THREAD_CREATE_FAILED_STR                               \
+    "Failed to create the file counter thread"
+#define DHT_MSG_MIGRATION_INIT_QUEUE_FAILED_STR                                \
+    "Failed to initialise migration queue"
+#define DHT_MSG_REBALANCE_STOPPED_STR "Received stop command on rebalance"
+#define DHT_MSG_PAUSED_TIMEOUT_STR "Request pause timer timeout"
+#define DHT_MSG_WOKE_STR "woken"
+#define DHT_MSG_ABORT_REBALANCE_STR "Aborting rebalance"
+#define DHT_MSG_REBALANCE_START_FAILED_STR                                     \
+    "Failed to start rebalance: look up on / failed"
+#define DHT_MSG_CREATE_TASK_REBAL_FAILED_STR                                   \
+    "Could not create task for rebalance"
+#define DHT_MSG_REBAL_ESTIMATE_NOT_AVAIL_STR                                   \
+    "Rebalance estimates will not be available"
+#define DHT_MSG_REBALANCE_STATUS_STR "Rebalance status"
+#define DHT_MSG_DATA_NULL_STR "data value is NULL"
+#define DHT_MSG_ADD_CHOICES_ERROR_STR "Error to add choices in buffer"
+#define DHT_MSG_GET_CHOICES_ERROR_STR "Error to get choices"
+#define DHT_MSG_PREPARE_STATUS_ERROR_STR "Error to prepare status"
+#define DHT_MSG_SET_CHOICE_FAILED_STR "Failed to set full choice"
+#define DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED_STR                               \
+    "Failed to aggregate quota xattr"
+#define DHT_MSG_FILE_TYPE_MISMATCH_STR                                         \
+    "path exists as a file on one subvolume and directory on another. Please " \
+    "fix it manually"
+#define DHT_MSG_LAYOUT_SET_FAILED_STR "failed to set layout for subvolume"
+#define DHT_MSG_LAYOUT_MERGE_FAILED_STR "failed to merge layouts for subvolume"
+#define DHT_MSG_SET_HASHED_SUBVOL_FAILED_STR "Failed to set hashed subvolume"
+#define DHT_MSG_XATTR_HEAL_NOT_POSS_STR                                        \
+    "No gfid exists for path. so healing xattr is not possible"
+#define DHT_MSG_REVALIDATE_CBK_INFO_STR "Revalidate: subvolume returned -1"
+#define DHT_MSG_LAYOUT_MISMATCH_STR "Mismatching layouts"
+#define DHT_MSG_UNLINK_LOOKUP_INFO_STR "lookup_unlink retuened"
+#define DHT_MSG_LINKTO_FILE_FAILED_STR                                         \
+    "Could not unlink the linkto file as either fd is open and/or linkto "     \
+    "xattr is set"
+#define DHT_MSG_LAYOUT_PRESET_FAILED_STR                                       \
+    "Could not set pre-set layout for subvolume"
+#define DHT_MSG_FILE_ON_MULT_SUBVOL_STR                                        \
+    "multiple subvolumes have file (preferably rename the file in the "        \
+    "backend, and do a fresh lookup"
+#define DHT_MSG_STALE_LINKFILE_DELETE_STR                                      \
+    "attempting deletion of stale linkfile"
+#define DHT_MSG_LINK_FILE_LOOKUP_INFO_STR "Lookup on following linkfile"
+#define DHT_MSG_NO_SUBVOL_FOR_LINKTO_STR "No link subvolume for linkto"
+#define DHT_MSG_SUBVOL_RETURNED_STR "Subvolume returned -1"
+#define DHT_MSG_UNKNOWN_LOCAL_XSEL_STR "Unknown local->xsel"
+#define DHT_MSG_DICT_GET_FAILED_STR "Failed to get"
+#define DHT_MSG_UUID_PARSE_ERROR_STR "Failed to parse uuid"
+#define DHT_MSG_GET_XATTR_ERR_STR "getxattr err for dir"
+#define DHT_MSG_ALLOC_OR_FILL_FAILED_STR "alloc or fill failed"
+#define DHT_MSG_UPGRADE_BRICKS_STR                                             \
+    "At least one of the bricks does not support this operation. Please "      \
+    "upgrade all bricks"
+#define DHT_MSG_GET_REAL_NAME_FAILED_STR "Failed to get real filename"
+#define DHT_MSG_LAYOUT_NULL_STR "Layout is NULL"
+#define DHT_MSG_COPY_UUID_FAILED_STR "Failed to copy node uuid key"
+#define DHT_MSG_MDS_DETER_FAILED_STR                                           \
+    "Cannot determine MDS, fetching xattr randomly from a subvol"
+#define DHT_MSG_HASHED_SUBVOL_DOWN_STR                                         \
+    "MDS is down for path, so fetching xattr randomly from subvol"
+#define DHT_MSG_CREATE_REBAL_FAILED_STR                                        \
+    "failed to create a new rebalance synctask"
+#define DHT_MSG_FIX_LAYOUT_INFO_STR "fixing the layout"
+#define DHT_MSG_OPERATION_NOT_SUP_STR "wrong directory-spread-count value"
+#define DHT_MSG_LINK_LAYOUT_FAILED_STR "failed to link the layout in inode"
+#define DHT_MSG_NO_SUBVOL_IN_LAYOUT_STR "no subvolume in layout for path"
+#define DHT_MSG_INODE_LK_ERROR_STR "mknod lock failed for file"
+#define DHT_MSG_MEM_ALLOC_FAILED_STR "mem allocation failed"
+#define DHT_MSG_PARENT_LAYOUT_CHANGED_STR                                      \
+    "extracting in-memory layout of parent failed"
+#define DHT_MSG_SET_IN_PARAMS_DICT_FAILED_STR                                  \
+    "setting in params dictionary failed"
+#define DHT_MSG_LOC_COPY_FAILED_STR "loc_copy failed"
+#define DHT_MSG_LOC_FAILED_STR "parent loc build failed"
+#define DHT_MSG_PARENT_LOC_FAILED_STR "locking parent failed"
+#define DHT_MSG_CREATE_LOCK_FAILED_STR "Create lock failed"
+#define DHT_MSG_PREV_ATTEMPT_FAILED_STR                                        \
+    "mkdir loop detected. parent layout didn't change even though previous "   \
+    "attempt of mkdir failed because of in-memory layout not matching with "   \
+    "that on disk."
+#define DHT_MSG_REFRESH_ATTEMPT_STR                                            \
+    "mkdir parent layout changed. Attempting a refresh and then a retry"
+#define DHT_MSG_ACQUIRE_LOCK_FAILED_STR                                        \
+    "Acquiring lock on parent to guard against layout-change failed"
+#define DHT_MSG_CREATE_STUB_FAILED_STR "creating stub failed"
+#define DHT_MSG_WIND_LOCK_REQ_FAILED_STR                                       \
+    "cannot wind lock request to guard parent layout"
+#define DHT_MSG_REFRESH_FAILED_STR "refreshing parent layout failed."
+#define DHT_MSG_CACHED_SUBVOL_ERROR_STR "On cached subvol"
+#define DHT_MSG_NO_LINK_SUBVOL_STR "Linkfile does not have link subvolume"
+#define DHT_MSG_SET_KEY_FAILED_STR "failed to set key"
+#define DHT_MSG_CHILD_DOWN_STR "Received CHILD_DOWN. Exiting"
+#define DHT_MSG_LOG_FIXED_LAYOUT_STR "log layout fixed"
+#define DHT_MSG_REBAL_STRUCT_SET_STR "local->rebalance already set"
+#define DHT_MSG_REMOVE_LINKTO_FAILED_STR "Removal of linkto failed at subvol"
+#define DHT_MSG_LAYOUT_DICT_SET_FAILED_STR "dht layout dict set failed"
+#define DHT_MSG_SUBVOL_INFO_STR "creating subvolume"
+#define DHT_MSG_COMPUTE_HASH_FAILED_STR "hash computation failed"
+#define DHT_MSG_INVALID_DISK_LAYOUT_STR                                        \
+    "Invalid disk layout: Catastrophic error layout with unknown type found"
+#define DHT_MSG_LAYOUT_SORT_FAILED_STR "layout sort failed"
+#define DHT_MSG_ANOMALIES_INFO_STR "Found anomalies"
+#define DHT_MSG_XATTR_DICT_NULL_STR "xattr dictionary is NULL"
+#define DHT_MSG_DISK_LAYOUT_MISSING_STR "Disk layout missing"
+#define DHT_MSG_LAYOUT_INFO_STR "layout info"
+#define DHT_MSG_SUBVOL_NO_LAYOUT_INFO_STR "no pre-set layout for subvol"
+#define DHT_MSG_SELFHEAL_XATTR_FAILED_STR "layout setxattr failed"
+#define DHT_MSG_DIR_SELFHEAL_XATTR_FAILED_STR "Directory self heal xattr failed"
+#define DHT_MSG_DUMMY_ALLOC_FAILED_STR "failed to allocate dummy layout"
+#define DHT_MSG_DICT_IS_NULL_STR                                               \
+    "dict is NULL, need to make sure gfids are same"
+#define DHT_MSG_ENTRYLK_ERROR_STR "acquiring entrylk after inodelk failed"
+#define DHT_MSG_NO_DISK_USAGE_STATUS_STR "no du stats"
+#define DHT_MSG_LINK_INODE_FAILED_STR "linking inode failed"
+#define DHT_MSG_SELFHEAL_FAILED_STR "Directory selfheal failed"
+#define DHT_MSG_NO_MDS_SUBVOL_STR "No mds subvol"
+#define DHT_MSG_LIST_XATTRS_FAILED_STR "failed to list xattrs"
+#define DHT_MSG_RESET_INTER_XATTR_FAILED_STR "Failed to reset internal xattr"
+#define DHT_MSG_MDS_DOWN_UNABLE_TO_SET_STR                                     \
+    "mds subvol is down, unable to set xattr"
+#define DHT_MSG_DIR_ATTR_HEAL_FAILED_STR                                       \
+    "Directory attr heal failed. Failed to set uid/gid"
+#define DHT_MSG_WIND_UNLOCK_FAILED_STR                                         \
+    "Winding unlock failed: stale locks left on brick"
+#define DHT_MSG_COMMIT_HASH_FAILED_STR "Directory commit hash updaten failed"
+#define DHT_MSG_LK_ARRAY_INFO_STR "lk info"
+#define DHT_MSG_UNLOCK_GFID_FAILED_STR                                         \
+    "unlock failed on gfid: stale lock might be left"
+#define DHT_MSG_UNLOCKING_FAILED_STR "unlocking failed"
+#define DHT_MSG_UNLOCK_FOLLOW_ENTRYLK_STR "not unlocking following entrylks"
+#define DHT_MSG_COPY_FRAME_FAILED_STR "copy frame failed"
+#define DHT_MSG_UNLOCK_FOLLOW_LOCKS_STR "not unlocking following locks"
+#define DHT_MSG_INODELK_FAILED_STR "inodelk failed on subvol"
+#define DHT_MSG_LOCK_FRAME_FAILED_STR "memory allocation failed for lock_frame"
+#define DHT_MSG_LOCAL_LOCK_INIT_FAILED_STR "dht_local_lock_init failed"
+#define DHT_MSG_ENTRYLK_FAILED_AFT_INODELK_STR                                 \
+    "dht_blocking_entrylk failed after taking inodelk"
+#define DHT_MSG_BLOCK_INODELK_FAILED_STR "dht_blocking_inodelk failed"
+#define DHT_MSG_CALLOC_FAILED_STR "calloc failed"
+#define DHT_MSG_LOCK_ALLOC_FAILED_STR "lock allocation failed"
+#define DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS_STR        \
+    "cannot allocate a frame, not unlocking following entrylks"
+#define DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK_STR       \
+    "storing locks in local failed, not unlocking following entrylks"
+#define DHT_MSG_DST_NULL_SET_FAILED_STR                                        \
+    "src or dst is NULL, Failed to set dictionary value"
+
+#endif /* _DHT_MESSAGES_H_ */
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 8f2f0adb2fb..8ba8082bd86 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -8,606 +8,1457 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
+#include "dht-common.h"
+#include <glusterfs/syscall.h>
+#include <fnmatch.h>
+#include <signal.h>
+#include <glusterfs/events.h>
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
+
+#define GF_DISK_SECTOR_SIZE 512
+#define DHT_REBALANCE_PID 4242        /* Change it if required */
+#define DHT_REBALANCE_BLKSIZE 1048576 /* 1 MB */
+#define MAX_MIGRATE_QUEUE_COUNT 500
+#define MIN_MIGRATE_QUEUE_COUNT 200
+#define MAX_REBAL_TYPE_SIZE 16
+#define FILE_CNT_INTERVAL 600       /* 10 mins */
+#define ESTIMATE_START_INTERVAL 600 /* 10 mins */
+#define HARDLINK_MIG_INPROGRESS -2
+#define SKIP_MIGRATION_FD_POSITIVE -3
+#ifndef MAX
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
 #endif
 
-#include "dht-common.h"
-#include "xlator.h"
+#define GF_CRAWL_INDEX_MOVE(idx, sv_cnt)                                       \
+    {                                                                          \
+        idx++;                                                                 \
+        idx %= sv_cnt;                                                         \
+    }
 
-#define GF_DISK_SECTOR_SIZE             512
-#define DHT_REBALANCE_PID               4242 /* Change it if required */
-#define DHT_REBALANCE_BLKSIZE           (128 * 1024)
+uint64_t g_totalfiles = 0;
+uint64_t g_totalsize = 0;
 
-static int
-dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,
-                      int32_t size, off_t offset, struct iobref *iobref)
+void
+gf_defrag_free_dir_dfmeta(struct dir_dfmeta *meta, int local_subvols_cnt)
 {
-        int i            = 0;
-        int ret          = -1;
-        int start_idx    = 0;
-        int tmp_offset   = 0;
-        int write_needed = 0;
-        int buf_len      = 0;
-        int size_pending = 0;
-        char *buf        = NULL;
-
-        /* loop through each vector */
-        for (i = 0; i < count; i++) {
-                buf = vec[i].iov_base;
-                buf_len = vec[i].iov_len;
-
-                for (start_idx = 0; (start_idx + GF_DISK_SECTOR_SIZE) <= buf_len;
-                     start_idx += GF_DISK_SECTOR_SIZE) {
-
-                        if (mem_0filled (buf + start_idx, GF_DISK_SECTOR_SIZE) != 0) {
-                                write_needed = 1;
-                                continue;
-                        }
+    int i = 0;
+
+    if (meta) {
+        for (i = 0; i < local_subvols_cnt; i++) {
+            if (meta->equeue)
+                gf_dirent_free(&meta->equeue[i]);
+            if (meta->lfd && meta->lfd[i])
+                fd_unref(meta->lfd[i]);
+        }
 
-                        if (write_needed) {
-                                ret = syncop_write (to, fd, (buf + tmp_offset),
-                                                    (start_idx - tmp_offset),
-                                                    (offset + tmp_offset),
-                                                    iobref, 0);
-                                /* 'path' will be logged in calling function */
-                                if (ret < 0) {
-                                        gf_log (THIS->name, GF_LOG_WARNING,
-                                                "failed to write (%s)",
-                                                strerror (errno));
-                                        goto out;
-                                }
-
-                                write_needed = 0;
-                        }
-                        tmp_offset = start_idx + GF_DISK_SECTOR_SIZE;
-                }
+        GF_FREE(meta->equeue);
+        GF_FREE(meta->head);
+        GF_FREE(meta->iterator);
+        GF_FREE(meta->offset_var);
+        GF_FREE(meta->fetch_entries);
+        GF_FREE(meta->lfd);
+        GF_FREE(meta);
+    }
+}
 
-                if ((start_idx < buf_len) || write_needed) {
-                        /* This means, last chunk is not yet written.. write it */
-                        ret = syncop_write (to, fd, (buf + tmp_offset),
-                                            (buf_len - tmp_offset),
-                                            (offset + tmp_offset), iobref, 0);
-                        if (ret < 0) {
-                                /* 'path' will be logged in calling function */
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "failed to write (%s)",
-                                        strerror (errno));
-                                goto out;
-                        }
-                }
+void
+gf_defrag_free_container(struct dht_container *container)
+{
+    if (container) {
+        gf_dirent_entry_free(container->df_entry);
 
-                size_pending = (size - buf_len);
-                if (!size_pending)
-                        break;
+        if (container->parent_loc) {
+            loc_wipe(container->parent_loc);
         }
 
-        ret = size;
-out:
-        return ret;
+        GF_FREE(container->parent_loc);
 
+        GF_FREE(container);
+    }
 }
 
-int32_t
-gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
-                           struct iatt *stbuf)
+void
+dht_set_global_defrag_error(gf_defrag_info_t *defrag, int ret)
 {
-        int32_t                 ret             = -1;
-        xlator_t               *cached_subvol   = NULL;
-        xlator_t               *hashed_subvol   = NULL;
-        xlator_t               *linkto_subvol   = NULL;
-        data_t                 *data            = NULL;
-        struct iatt             iatt            = {0,};
-        int32_t                 op_errno        = 0;
-
-        GF_VALIDATE_OR_GOTO ("defrag", loc, out);
-        GF_VALIDATE_OR_GOTO ("defrag", loc->name, out);
-        GF_VALIDATE_OR_GOTO ("defrag", stbuf, out);
-        GF_VALIDATE_OR_GOTO ("defrag", this, out);
-        GF_VALIDATE_OR_GOTO ("defrag", xattrs, out);
-
-        if (uuid_is_null (loc->pargfid)) {
-                gf_log ("", GF_LOG_ERROR, "loc->pargfid is NULL for "
-                        "%s", loc->path);
-                goto out;
-        }
+    LOCK(&defrag->lock);
+    {
+        defrag->global_error = ret;
+    }
+    UNLOCK(&defrag->lock);
+    return;
+}
 
-        if (uuid_is_null (loc->gfid)) {
-                gf_log ("", GF_LOG_ERROR, "loc->gfid is NULL for "
-                        "%s", loc->path);
-                goto out;
-        }
+static int
+dht_send_rebalance_event(xlator_t *this, int cmd, gf_defrag_status_t status)
+{
+    int ret = -1;
+    char *volname = NULL;
+    char *tmpstr = NULL;
+    char *ptr = NULL;
+    char *suffix = "-dht";
+    int len = 0;
 
-        cached_subvol = dht_subvol_get_cached (this, loc->inode);
-        if (!cached_subvol) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get cached subvol"
-                        " for %s on %s", loc->name, this->name);
-                goto out;
-        }
+    eventtypes_t event = EVENT_LAST;
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (!hashed_subvol) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get hashed subvol"
-                        " for %s on %s", loc->name, this->name);
-                goto out;
+    switch (status) {
+        case GF_DEFRAG_STATUS_COMPLETE:
+            event = EVENT_VOLUME_REBALANCE_COMPLETE;
+            break;
+        case GF_DEFRAG_STATUS_FAILED:
+            event = EVENT_VOLUME_REBALANCE_FAILED;
+            break;
+        case GF_DEFRAG_STATUS_STOPPED:
+            event = EVENT_VOLUME_REBALANCE_STOP;
+            break;
+        default:
+            break;
+    }
+
+    /* DHT volume */
+    len = strlen(this->name) - strlen(suffix);
+    tmpstr = gf_strdup(this->name);
+    if (tmpstr) {
+        ptr = tmpstr + len;
+        if (!strcmp(ptr, suffix)) {
+            tmpstr[len] = '\0';
+            volname = tmpstr;
         }
+    }
 
-        gf_log (this->name, GF_LOG_INFO, "Attempting to migrate hardlink %s "
-                "with gfid %s from %s -> %s", loc->name, uuid_utoa (loc->gfid),
-                cached_subvol->name, hashed_subvol->name);
-        data = dict_get (xattrs, DHT_LINKFILE_KEY);
-        /* set linkto on cached -> hashed if not present, else link it */
-        if (!data) {
-                ret = dict_set_str (xattrs, DHT_LINKFILE_KEY,
-                                    hashed_subvol->name);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                                "linkto xattr in dict for %s", loc->name);
-                        goto out;
-                }
+    if (!volname) {
+        /* Better than nothing */
+        volname = this->name;
+    }
 
-                ret = syncop_setxattr (cached_subvol, loc, xattrs, 0);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Linkto setxattr "
-                                "failed %s -> %s (%s)", cached_subvol->name,
-                                loc->name, strerror (errno));
-                        goto out;
-                }
-                goto out;
-        } else {
-                linkto_subvol = dht_linkfile_subvol (this, NULL, NULL, xattrs);
-                if (!linkto_subvol) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                "linkto subvol for %s", loc->name);
-                } else {
-                        hashed_subvol = linkto_subvol;
-                }
+    if (event != EVENT_LAST) {
+        gf_event(event, "volume=%s", volname);
+    }
 
-                ret = syncop_link (hashed_subvol, loc, loc);
-                if  (ret) {
-                        op_errno = errno;
-                        gf_log (this->name, GF_LOG_ERROR, "link of %s -> %s"
-                                " failed on  subvol %s (%s)", loc->name,
-                                uuid_utoa(loc->gfid),
-                                hashed_subvol->name, strerror (op_errno));
-                        if (op_errno != EEXIST)
-                                goto out;
-                }
-        }
-        ret = syncop_lookup (hashed_subvol, loc, NULL, &iatt, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed lookup %s on %s (%s)"
-                        , loc->name, hashed_subvol->name, strerror (errno));
-                goto out;
-        }
+    GF_FREE(tmpstr);
+    return ret;
+}
 
-        if (iatt.ia_nlink == stbuf->ia_nlink) {
-                ret = dht_migrate_file (this, loc, cached_subvol, hashed_subvol,
-                                        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS);
-                if (ret)
-                        goto out;
-        }
-        ret = 0;
-out:
-        return ret;
+static void
+dht_strip_out_acls(dict_t *dict)
+{
+    if (dict) {
+        dict_del(dict, "trusted.SGI_ACL_FILE");
+        dict_del(dict, POSIX_ACL_ACCESS_XATTR);
+    }
 }
 
+/*
+   return values:
+   -1 : failure
+   -2 : success
+
+Hard link migration is carried out in three stages.
+
+(Say there are n hardlinks)
+Stage 1: Setting the new hashed subvol information on the 1st hardlink
+         encountered (linkto setxattr)
+
+Stage 2: Creating hardlinks on new hashed subvol for the 2nd to (n-1)th
+         hardlink
+
+Stage 3: Physical migration of the data file for nth hardlink
+
+Why to deem "-2" as success and not "0":
+
+   dht_migrate_file expects return value "0" from _is_file_migratable if
+the file has to be migrated.
+
+   _is_file_migratable returns zero only when it is called with the
+flag "GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS".
 
-static inline int
-__is_file_migratable (xlator_t *this, loc_t *loc,
-                      struct iatt *stbuf, dict_t *xattrs, int flags)
+   gf_defrag_handle_hardlink calls dht_migrate_file for physical migration
+of the data file with the flag "GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS"
+
+Hence, gf_defrag_handle_hardlink returning "0" for success will force
+"dht_migrate_file" to migrate each of the hardlink which is not intended.
+
+For each of the three stage mentioned above "-2" will be returned and will
+be converted to "0" in dht_migrate_file.
+
+*/
+
+int32_t
+gf_defrag_handle_hardlink(xlator_t *this, loc_t *loc, int *fop_errno)
 {
-        int ret = -1;
+    int32_t ret = -1;
+    xlator_t *cached_subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *linkto_subvol = NULL;
+    data_t *data = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    int32_t op_errno = 0;
+    dht_conf_t *conf = NULL;
+    gf_loglevel_t loglevel = 0;
+    dict_t *link_xattr = NULL;
+    dict_t *dict = NULL;
+    dict_t *xattr_rsp = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+
+    *fop_errno = EINVAL;
+
+    GF_VALIDATE_OR_GOTO("defrag", loc, out);
+    GF_VALIDATE_OR_GOTO("defrag", loc->name, out);
+    GF_VALIDATE_OR_GOTO("defrag", this, out);
+    GF_VALIDATE_OR_GOTO("defrag", this->private, out);
+
+    conf = this->private;
+
+    if (gf_uuid_is_null(loc->pargfid)) {
+        gf_msg("", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "loc->pargfid is NULL for %s",
+               loc->path);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    if (gf_uuid_is_null(loc->gfid)) {
+        gf_msg("", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "loc->gfid is NULL for %s",
+               loc->path);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
 
-        if (IA_ISDIR (stbuf->ia_type)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: migrate-file called on directory", loc->path);
-                ret = -1;
-                goto out;
+    link_xattr = dict_new();
+    if (!link_xattr) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        goto out;
+    }
+
+    /*
+      Parallel migration can lead to migration of the hard link multiple
+      times which can lead to data loss. Hence, adding a fresh lookup to
+      decide whether migration is required or not.
+
+      Elaborating the scenario for let say 10 hardlinks [link{1..10}]:
+          Let say the first hard link "link1"  does the setxattr of the
+      new hashed subvolume info on the cached file. As there are multiple
+      threads working, we might have already all the links created on the
+      new hashed by the time we reach hardlink let say link5. Now the
+      number of links on hashed is equal to that of cached. Hence, file
+      migration will happen for link6.
+
+             Cached                                 Hashed
+      --------T link6                        rwxrwxrwx   link6
+
+      Now post above state all the link file on the cached will be zero
+      byte linkto files. Hence, if we still do migration for the following
+      files link{7..10}, we will end up migrating 0 data leading to data
+      loss.
+            Hence, a lookup can make sure whether we need to migrate the
+      file or not.
+    */
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "could not allocate memory for dict");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to set 'linkto' key in dict",
+               loc->path);
+        goto out;
+    }
+
+    ret = syncop_lookup(this, loc, &stbuf, NULL, dict, &xattr_rsp);
+    if (ret) {
+        /*Ignore ENOENT and ESTALE as file might have been
+          migrated already*/
+        if (-ret == ENOENT || -ret == ESTALE) {
+            ret = -2;
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:%s lookup failed with ret = %d", loc->path,
+               ret);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    cached_subvol = dht_subvol_get_cached(this, loc->inode);
+    if (!cached_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "Failed to get cached subvol"
+               " for %s on %s",
+               loc->name, this->name);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (!hashed_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "Failed to get hashed subvol"
+               " for %s on %s",
+               loc->name, this->name);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    /* Hardlink migration happens only with remove-brick. So this condition will
+     * be true only when the migration has happened. In case hardlinks are
+     * migrated for rebalance case, remove this check. Having this check here
+     * avoid redundant calls below*/
+    if (hashed_subvol == cached_subvol) {
+        ret = -2;
+        goto out;
+    }
+
+    gf_log(this->name, GF_LOG_INFO,
+           "Attempting to migrate hardlink %s "
+           "with gfid %s from %s -> %s",
+           loc->name, uuid_utoa(loc->gfid), cached_subvol->name,
+           hashed_subvol->name);
+
+    data = dict_get(xattr_rsp, conf->link_xattr_name);
+    /* set linkto on cached -> hashed if not present, else link it */
+    if (!data) {
+        ret = dict_set_str(link_xattr, conf->link_xattr_name,
+                           hashed_subvol->name);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed :"
+                   "Failed to set dictionary value:"
+                   " key = %s for %s",
+                   conf->link_xattr_name, loc->name);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) {
-                ret = 0;
-                goto out;
+        ret = syncop_setxattr(cached_subvol, loc, link_xattr, 0, NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed :"
+                   "Linkto setxattr failed %s -> %s",
+                   cached_subvol->name, loc->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
-        if (stbuf->ia_nlink > 1) {
-                /* support for decomission */
-                if (flags == GF_DHT_MIGRATE_HARDLINK) {
-                        ret = gf_defrag_handle_hardlink (this, loc,
-                                                         xattrs, stbuf);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "%s: failed to migrate file with link",
-                                        loc->path);
-                        }
-                } else {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: file has hardlinks", loc->path);
-                }
-                ret = ENOTSUP;
+
+        gf_msg_debug(this->name, 0,
+                     "hardlink target subvol created on %s "
+                     ",cached %s, file %s",
+                     hashed_subvol->name, cached_subvol->name, loc->path);
+
+        ret = -2;
+        goto out;
+    } else {
+        linkto_subvol = dht_linkfile_subvol(this, NULL, NULL, xattr_rsp);
+        if (!linkto_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_ERROR,
+                   "Failed to get "
+                   "linkto subvol for %s",
+                   loc->name);
+        } else {
+            hashed_subvol = linkto_subvol;
+        }
+
+        ret = syncop_link(hashed_subvol, loc, loc, &iatt, NULL, NULL);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+
+            loglevel = (op_errno == EEXIST) ? GF_LOG_DEBUG : GF_LOG_ERROR;
+            gf_msg(this->name, loglevel, op_errno,
+                   DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED,
+                   "link of %s -> %s"
+                   " failed on  subvol %s",
+                   loc->name, uuid_utoa(loc->gfid), hashed_subvol->name);
+            if (op_errno != EEXIST) {
+                *fop_errno = op_errno;
                 goto out;
+            }
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "syncop_link successful for"
+                         " hardlink %s on subvol %s, cached %s",
+                         loc->path, hashed_subvol->name, cached_subvol->name);
         }
+    }
 
-        ret = 0;
+    ret = syncop_lookup(hashed_subvol, loc, &iatt, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :Failed lookup %s on %s ", loc->name,
+               hashed_subvol->name);
 
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* There is a race where on the target subvol for the hardlink
+     * (note: hash subvol for the hardlink might differ from this), some
+     * other client(non-rebalance) would have created a linkto file for that
+     * hardlink as part of lookup. So let say there are 10 hardlinks, on the
+     * 5th hardlink it self the hardlinks might have migrated. Now for
+     * (6..10th) hardlinks the cached and target would be same as the file
+     * has already migrated. Hence this check is needed  */
+    if (cached_subvol == hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "source %s and destination %s "
+                     "for hardlink %s are same",
+                     cached_subvol->name, hashed_subvol->name, loc->path);
+        ret = -2;
+        goto out;
+    }
+
+    if (iatt.ia_nlink == stbuf.ia_nlink) {
+        ret = dht_migrate_file(this, loc, cached_subvol, hashed_subvol,
+                               GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS, fop_errno);
+        if (ret) {
+            goto out;
+        }
+    }
+    ret = -2;
 out:
-        return ret;
+    if (link_xattr)
+        dict_unref(link_xattr);
+
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
+
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
 }
 
-static inline int
-__dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struct iatt *stbuf,
-                                 dict_t *dict, fd_t **dst_fd)
+static int
+__check_file_has_hardlink(xlator_t *this, loc_t *loc, struct iatt *stbuf,
+                          dict_t *xattrs, int flags, gf_defrag_info_t *defrag,
+                          dht_conf_t *conf, int *fop_errno)
 {
-        xlator_t *this = NULL;
-        int       ret  = -1;
-        fd_t     *fd   = NULL;
-        struct iatt new_stbuf = {0,};
+    int ret = 0;
 
-        this = THIS;
-
-        ret = dict_set_static_bin (dict, "gfid-req", stbuf->ia_gfid, 16);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set gfid in dict for create", loc->path);
-                goto out;
+    if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) {
+        ret = 0;
+        return ret;
+    }
+    if (stbuf->ia_nlink > 1) {
+        /* support for decomission */
+        if (flags == GF_DHT_MIGRATE_HARDLINK) {
+            synclock_lock(&conf->link_lock);
+            ret = gf_defrag_handle_hardlink(this, loc, fop_errno);
+            synclock_unlock(&conf->link_lock);
+            /*
+            Returning zero will force the file to be remigrated.
+            Checkout gf_defrag_handle_hardlink for more information.
+            */
+            if (ret && ret != -2) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "Migrate file failed:"
+                       "%s: failed to migrate file with link",
+                       loc->path);
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migration skipped for:"
+                   "%s: file has hardlinks",
+                   loc->path);
+            *fop_errno = ENOTSUP;
+            ret = 1;
         }
+    }
+
+    return ret;
+}
 
-        ret = dict_set_str (dict, DHT_LINKFILE_KEY, from->name);
+/*
+     return values
+     0 : File will be migrated
+    -2 : File will not be migrated
+         (This is the return value from gf_defrag_handle_hardlink. Checkout
+         gf_defrag_handle_hardlink for description of "returning -2")
+    -1 : failure
+*/
+static int
+__is_file_migratable(xlator_t *this, loc_t *loc, struct iatt *stbuf,
+                     dict_t *xattrs, int flags, gf_defrag_info_t *defrag,
+                     dht_conf_t *conf, int *fop_errno)
+{
+    int ret = -1;
+    int lock_count = 0;
+
+    if (IA_ISDIR(stbuf->ia_type)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: migrate-file called on directory",
+               loc->path);
+        *fop_errno = EISDIR;
+        ret = -1;
+        goto out;
+    }
+
+    if (!conf->lock_migration_enabled) {
+        ret = dict_get_int32(xattrs, GLUSTERFS_POSIXLK_COUNT, &lock_count);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set gfid in dict for create", loc->path);
-                goto out;
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: Unable to get lock count for file",
+                   loc->path);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
         }
 
-        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: fd create failed (destination) (%s)",
-                        loc->path, strerror (errno));
-                ret = -1;
-                goto out;
+        if (lock_count) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: %s: File has locks."
+                   " Skipping file migration",
+                   loc->path);
+            *fop_errno = ENOTSUP;
+            ret = 1;
+            goto out;
         }
+    }
 
-        ret = syncop_lookup (to, loc, NULL, &new_stbuf, NULL, NULL);
-        if (!ret) {
-                /* File exits in the destination, check if gfid matches */
-                if (uuid_compare (stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "file %s exits in %s with different gfid",
-                                loc->path, to->name);
-                        fd_unref (fd);
-                        goto out;
-                }
+    /* Check if file has hardlink*/
+    ret = __check_file_has_hardlink(this, loc, stbuf, xattrs, flags, defrag,
+                                    conf, fop_errno);
+out:
+    return ret;
+}
+
+static int
+__dht_rebalance_create_dst_file(xlator_t *this, xlator_t *to, xlator_t *from,
+                                loc_t *loc, struct iatt *stbuf, fd_t **dst_fd,
+                                int *fop_errno, int file_has_holes)
+{
+    int ret = -1;
+    int ret2 = -1;
+    fd_t *fd = NULL;
+    struct iatt new_stbuf = {
+        0,
+    };
+    struct iatt check_stbuf = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+    dict_t *dict = NULL;
+    dict_t *xdata = NULL;
+
+    conf = this->private;
+
+    dict = dict_new();
+    if (!dict) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dictionary allocation failed for"
+               "path:%s",
+               loc->path);
+        goto out;
+    }
+    ret = dict_set_gfuuid(dict, "gfid-req", stbuf->ia_gfid, true);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = gfid-req", loc->path);
+        goto out;
+    }
+
+    ret = dict_set_str(dict, conf->link_xattr_name, from->name);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = %s ", loc->path,
+               conf->link_xattr_name);
+        goto out;
+    }
+
+    fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+    if (!fd) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: fd create failed (destination)", loc->path);
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: dict_new failed)", loc->path);
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(xdata, GF_CLEAN_WRITE_PROTECTION, 1);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = %s ", loc->path,
+               GF_CLEAN_WRITE_PROTECTION);
+        goto out;
+    }
+
+    ret = syncop_lookup(to, loc, &new_stbuf, NULL, xdata, NULL);
+    if (!ret) {
+        /* File exits in the destination, check if gfid matches */
+        if (gf_uuid_compare(stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH,
+                   "file %s exists in %s with different gfid", loc->path,
+                   to->name);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
         }
-        if ((ret == -1) && (errno != ENOENT)) {
-                /* File exists in destination, but not accessible */
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "%s: failed to lookup file (%s)",
-                        loc->path, strerror (errno));
-                goto out;
+    }
+    if ((ret < 0) && (-ret != ENOENT)) {
+        /* File exists in destination, but not accessible */
+        gf_msg(THIS->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to lookup file", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* Create the destination with LINKFILE mode, and linkto xattr,
+       if the linkfile already exists, just open the file */
+    if (!ret) {
+        /*
+         * File already present, just open the file.
+         */
+        ret = syncop_open(to, loc, O_RDWR, fd, NULL, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to open %s on %s", loc->path, to->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
-
-        /* Create the destination with LINKFILE mode, and linkto xattr,
-           if the linkfile already exists, it will just open the file */
-        ret = syncop_create (to, loc, O_RDWR, DHT_LINKFILE_MODE, fd,
-                             dict);
+    } else {
+        ret = syncop_create(to, loc, O_RDWR, DHT_LINKFILE_MODE, fd, &new_stbuf,
+                            dict, NULL);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create %s on %s (%s)",
-                        loc->path, to->name, strerror (errno));
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to create %s on %s", loc->path, to->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    }
+
+    fd_bind(fd);
+
+    /*Reason of doing lookup after create again:
+     *In the create, there is some time-gap between opening fd at the
+     *server (posix_layer) and binding it in server (incrementing fd count),
+     *so if in that time-gap, if other process sends unlink considering it
+     *as a linkto file, because inode->fd count will be 0, so file will be
+     *unlinked at the backend. And because further operations are performed
+     *on fd, so though migration will be done but will end with no file
+     *at  the backend.
+     */
+
+    ret = syncop_lookup(to, loc, &check_stbuf, NULL, NULL, NULL);
+    if (!ret) {
+        if (gf_uuid_compare(stbuf->ia_gfid, check_stbuf.ia_gfid) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH,
+                   "file %s exists in %s with different gfid,"
+                   "found in lookup after create",
+                   loc->path, to->name);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (-ret == ENOENT) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: file does not exist"
+               "on %s",
+               loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_fsetattr(to, fd, stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                          NULL, NULL, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "chown failed for %s on %s", loc->path, to->name);
+    }
+
+    /* No need to bother about 0 byte size files */
+    if (stbuf->ia_size > 0) {
+        if (conf->use_fallocate && !file_has_holes) {
+            ret = syncop_fallocate(to, fd, 0, 0, stbuf->ia_size, NULL, NULL);
+            if (ret < 0) {
+                if (ret == -EOPNOTSUPP || ret == -EINVAL || ret == -ENOSYS) {
+                    conf->use_fallocate = _gf_false;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_MIGRATE_FILE_FAILED,
+                           "fallocate failed for %s on %s", loc->path,
+                           to->name);
+
+                    *fop_errno = -ret;
+
+                    /* fallocate does not release the space
+                     * in some cases
+                     */
+                    ret2 = syncop_ftruncate(to, fd, 0, NULL, NULL, NULL, NULL);
+                    if (ret2 < 0) {
+                        gf_msg(this->name, GF_LOG_WARNING, -ret2,
+                               DHT_MSG_MIGRATE_FILE_FAILED,
+                               "ftruncate failed for "
+                               "%s on %s",
+                               loc->path, to->name);
+                    }
+                    goto out;
+                }
+            }
+        } else {
+            ret = syncop_ftruncate(to, fd, stbuf->ia_size, NULL, NULL, NULL,
+                                   NULL);
+            if (ret < 0) {
+                *fop_errno = -ret;
+                gf_msg(this->name, GF_LOG_WARNING, -ret,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "ftruncate failed for %s on %s", loc->path, to->name);
+            }
         }
+    }
 
-        if (dst_fd)
-                *dst_fd = fd;
+    /* success */
+    ret = 0;
 
-        /* success */
-        ret = 0;
+    if (dst_fd)
+        *dst_fd = fd;
 
 out:
-        return ret;
-}
+    if (ret) {
+        if (fd) {
+            fd_unref(fd);
+        }
+    }
+    if (dict)
+        dict_unref(dict);
 
-static inline int
-__dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
-                        struct iatt *stbuf, int flag)
-{
-        struct statvfs  src_statfs = {0,};
-        struct statvfs  dst_statfs = {0,};
-        int             ret        = -1;
-        xlator_t       *this       = NULL;
+    if (xdata)
+        dict_unref(xdata);
 
-        this = THIS;
+    return ret;
+}
 
-        ret = syncop_statfs (from, loc, &src_statfs);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get statfs of %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
-                goto out;
+static int
+__dht_check_free_space(xlator_t *this, xlator_t *to, xlator_t *from, loc_t *loc,
+                       struct iatt *stbuf, int flag, dht_conf_t *conf,
+                       gf_boolean_t *target_changed, xlator_t **new_subvol,
+                       int *fop_errno)
+{
+    struct statvfs src_statfs = {
+        0,
+    };
+    struct statvfs dst_statfs = {
+        0,
+    };
+    int ret = -1;
+    dict_t *xdata = NULL;
+    dht_layout_t *layout = NULL;
+    uint64_t src_statfs_blocks = 1;
+    uint64_t dst_statfs_blocks = 1;
+    double dst_post_availspacepercent = 0;
+    double src_post_availspacepercent = 0;
+    uint64_t file_blocks = 0;
+    uint64_t src_total_blocks = 0;
+    uint64_t dst_total_blocks = 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "failed to allocate dictionary");
+        goto out;
+    }
+
+    ret = dict_set_int8(xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to set " GF_INTERNAL_IGNORE_DEEM_STATFS " in dict");
+        ret = -1;
+        *fop_errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_statfs(from, loc, &src_statfs, xdata, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to get statfs of %s on %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_statfs(to, loc, &dst_statfs, xdata, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to get statfs of %s on %s", loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "min_free_disk - %f , block available - %" PRId64
+                 ", block size - %lu",
+                 conf->min_free_disk, dst_statfs.f_bavail, dst_statfs.f_bsize);
+
+    dst_statfs_blocks = dst_statfs.f_bavail *
+                        (dst_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    src_statfs_blocks = src_statfs.f_bavail *
+                        (src_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    dst_total_blocks = dst_statfs.f_blocks *
+                       (dst_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    src_total_blocks = src_statfs.f_blocks *
+                       (src_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    /* if force option is given, do not check for space @ dst.
+     * Check only if space is avail for the file */
+    if (flag != GF_DHT_MIGRATE_DATA)
+        goto check_avail_space;
+
+    /* Check:
+       During rebalance `migrate-data` - Destination subvol experiences
+       a `reduction` in 'blocks' of free space, at the same time source
+       subvol gains certain 'blocks' of free space. A valid check is
+       necessary here to avoid erroneous move to destination where
+       the space could be scantily available.
+       With heterogeneous brick support, an actual space comparison could
+       prevent any files being migrated to newly added bricks if they are
+       smaller then the free space available on the existing bricks.
+     */
+    if (!conf->use_fallocate) {
+        file_blocks = stbuf->ia_size + GF_DISK_SECTOR_SIZE - 1;
+        file_blocks /= GF_DISK_SECTOR_SIZE;
+
+        if (file_blocks >= dst_statfs_blocks) {
+            dst_statfs_blocks = 0;
+        } else {
+            dst_statfs_blocks -= file_blocks;
         }
+    }
+
+    src_post_availspacepercent = ((src_statfs_blocks + file_blocks) * 100) /
+                                 src_total_blocks;
+
+    dst_post_availspacepercent = (dst_statfs_blocks * 100) / dst_total_blocks;
+
+    if (dst_post_availspacepercent < src_post_availspacepercent) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "data movement of file "
+               "{blocks:%" PRIu64
+               " name:(%s)} would result in "
+               "dst node (%s:%" PRIu64
+               ") having lower disk "
+               "space than the source node (%s:%" PRIu64
+               ")"
+               ".Skipping file.",
+               stbuf->ia_blocks, loc->path, to->name, dst_statfs_blocks,
+               from->name, src_statfs_blocks);
+
+        /* this is not a 'failure', but we don't want to
+           consider this as 'success' too :-/ */
+        *fop_errno = ENOSPC;
+        ret = 1;
+        goto out;
+    }
 
-        ret = syncop_statfs (to, loc, &dst_statfs);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get statfs of %s on %s (%s)",
-                        loc->path, to->name, strerror (errno));
-                goto out;
+check_avail_space:
+    if (conf->disk_unit == 'p' && dst_statfs.f_blocks) {
+        dst_post_availspacepercent = (dst_statfs_blocks * 100) /
+                                     dst_total_blocks;
+
+        gf_msg_debug(this->name, 0,
+                     "file : %s, post_availspacepercent"
+                     " : %lf f_bavail : %" PRIu64 " min-free-disk: %lf",
+                     loc->path, dst_post_availspacepercent, dst_statfs.f_bavail,
+                     conf->min_free_disk);
+
+        if (dst_post_availspacepercent < conf->min_free_disk) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "Write will cross min-free-disk for "
+                   "file - %s on subvol - %s. Looking "
+                   "for new subvol",
+                   loc->path, to->name);
+
+            goto find_new_subvol;
+        } else {
+            ret = 0;
+            goto out;
         }
+    }
 
-        /* if force option is given, do not check for space @ dst.
-         * Check only if space is avail for the file */
-        if (flag != GF_DHT_MIGRATE_DATA)
-                goto check_avail_space;
+    if (conf->disk_unit != 'p') {
+        if ((dst_statfs_blocks * GF_DISK_SECTOR_SIZE) < conf->min_free_disk) {
+            gf_msg_debug(this->name, 0,
+                         "file : %s,  destination frsize: %lu "
+                         "f_bavail : %" PRIu64 " min-free-disk: %lf",
+                         loc->path, dst_statfs.f_frsize, dst_statfs.f_bavail,
+                         conf->min_free_disk);
 
-        if (((dst_statfs.f_bavail *
-              dst_statfs.f_bsize) / GF_DISK_SECTOR_SIZE) <
-            (((src_statfs.f_bavail * src_statfs.f_bsize) /
-              GF_DISK_SECTOR_SIZE) - stbuf->ia_blocks)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "data movement attempted from node (%s) with"
-                        " higher disk space to a node (%s) with "
-                        "lesser disk space (%s)", from->name,
-                        to->name, loc->path);
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "write will"
+                   " cross min-free-disk for file - %s on subvol -"
+                   " %s. looking for new subvol",
+                   loc->path, to->name);
 
-                /* this is not a 'failure', but we don't want to
-                   consider this as 'success' too :-/ */
-                ret = 1;
-                goto out;
-        }
+            goto find_new_subvol;
 
-check_avail_space:
-        if (((dst_statfs.f_bavail * dst_statfs.f_bsize) /
-              GF_DISK_SECTOR_SIZE) < stbuf->ia_blocks) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "data movement attempted from node (%s) with "
-                        "to node (%s) which does not have required free space"
-                        " for %s", from->name, to->name, loc->path);
-                ret = 1;
-                goto out;
+        } else {
+            ret = 0;
+            goto out;
         }
+    }
 
+find_new_subvol:
+    layout = dht_layout_get(this, loc->parent);
+    if (!layout) {
+        gf_log(this->name, GF_LOG_ERROR, "Layout is NULL");
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    *new_subvol = dht_subvol_with_free_space_inodes(this, to, from, layout,
+                                                    stbuf->ia_size);
+    if ((!(*new_subvol)) || (*new_subvol == from)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+               "Could not find any subvol"
+               " with space accommodating the file - %s. Consider "
+               "adding bricks",
+               loc->path);
+
+        *target_changed = _gf_false;
+        *fop_errno = ENOSPC;
+        ret = -1;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "new target found - %s"
+               " for file - %s",
+               (*new_subvol)->name, loc->path);
+        *target_changed = _gf_true;
         ret = 0;
+    }
+
 out:
-        return ret;
+    if (xdata)
+        dict_unref(xdata);
+    return ret;
 }
 
-static inline int
-__dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,
-                             uint64_t ia_size, int hole_exists)
+static int
+__dht_rebalance_migrate_data(xlator_t *this, gf_defrag_info_t *defrag,
+                             xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,
+                             uint64_t ia_size, int hole_exists, int *fop_errno)
 {
-        int            ret    = 0;
-        int            count  = 0;
-        off_t          offset = 0;
-        struct iovec  *vector = NULL;
-        struct iobref *iobref = NULL;
-        uint64_t       total  = 0;
-        size_t         read_size = 0;
-
-        /* if file size is '0', no need to enter this loop */
-        while (total < ia_size) {
-                read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE) ?
-                             DHT_REBALANCE_BLKSIZE : (ia_size - total));
-                ret = syncop_readv (from, src, read_size,
-                                    offset, 0, &vector, &count, &iobref);
-                if (!ret || (ret < 0)) {
-                        break;
+    int ret = 0;
+    int count = 0;
+    off_t offset = 0;
+    off_t data_offset = 0;
+    off_t hole_offset = 0;
+    struct iovec *vector = NULL;
+    struct iobref *iobref = NULL;
+    uint64_t total = 0;
+    size_t read_size = 0;
+    size_t data_block_size = 0;
+    dict_t *xdata = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    /* if file size is '0', no need to enter this loop */
+    while (total < ia_size) {
+        /* This is a regular file - read it sequentially */
+        if (!hole_exists) {
+            read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE)
+                             ? DHT_REBALANCE_BLKSIZE
+                             : (ia_size - total));
+        } else {
+            /* This is a sparse file - read only the data segments in the file
+             */
+
+            /* If the previous data block is fully copied, find the next data
+             * segment
+             * starting at the offset of the last read and written byte,  */
+            if (data_block_size <= 0) {
+                ret = syncop_seek(from, src, offset, GF_SEEK_DATA, NULL,
+                                  &data_offset);
+                if (ret) {
+                    if (ret == -ENXIO)
+                        ret = 0; /* No more data segments */
+                    else
+                        *fop_errno = -ret; /* Error occurred */
+
+                    break;
                 }
 
-                if (hole_exists)
-                        ret = dht_write_with_holes (to, dst, vector, count,
-                                                    ret, offset, iobref);
-                else
-                        ret = syncop_writev (to, dst, vector, count,
-                                             offset, iobref, 0);
-                if (ret < 0) {
+                /* If the position of the current data segment is greater than
+                 * the position of the next hole, find the next hole in order to
+                 * calculate the length of the new data segment */
+                if (data_offset > hole_offset) {
+                    /* Starting at the offset of the last data segment, find the
+                     * next hole */
+                    ret = syncop_seek(from, src, data_offset, GF_SEEK_HOLE,
+                                      NULL, &hole_offset);
+                    if (ret) {
+                        /* If an error occurred here it's a real error because
+                         * if the seek for a data segment was successful then
+                         * necessarily another hole must exist (EOF is a hole)
+                         */
+                        *fop_errno = -ret;
                         break;
-                }
-                offset += ret;
-                total += ret;
+                    }
 
-                GF_FREE (vector);
-                if (iobref)
-                        iobref_unref (iobref);
-                iobref = NULL;
-                vector = NULL;
+                    /* Calculate the total size of the current data block */
+                    data_block_size = hole_offset - data_offset;
+                }
+            } else {
+                /* There is still data in the current segment, move the
+                 * data_offset to the position of the last written byte */
+                data_offset = offset;
+            }
+
+            /* Calculate how much data needs to be read and written. If the data
+             * segment's length is bigger than DHT_REBALANCE_BLKSIZE, read and
+             * write DHT_REBALANCE_BLKSIZE data length and the rest in the
+             * next iteration(s) */
+            read_size = ((data_block_size > DHT_REBALANCE_BLKSIZE)
+                             ? DHT_REBALANCE_BLKSIZE
+                             : data_block_size);
+
+            /* Calculate the remaining size of the data block - maybe there's no
+             * need to seek for data in the next iteration */
+            data_block_size -= read_size;
+
+            /* Set offset to the offset of the data segment so read and write
+             * will have the correct position */
+            offset = data_offset;
         }
-        if (iobref)
-                iobref_unref (iobref);
-        GF_FREE (vector);
-
-        if (ret >= 0)
-                ret = 0;
-
-        return ret;
-}
 
+        ret = syncop_readv(from, src, read_size, offset, 0, &vector, &count,
+                           &iobref, NULL, NULL, NULL);
 
-static inline int
-__dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
-                               struct iatt *stbuf, fd_t **src_fd)
-{
-        int          ret  = 0;
-        fd_t        *fd   = NULL;
-        dict_t      *dict = NULL;
-        xlator_t    *this = NULL;
-        struct iatt  iatt = {0,};
+        if (!ret || (ret < 0)) {
+            if (!ret) {
+                /* File was probably truncated*/
+                ret = -1;
+                *fop_errno = ENOSPC;
+            } else {
+                *fop_errno = -ret;
+            }
+            break;
+        }
 
-        this = THIS;
+        if (!conf->force_migration) {
+            if (!xdata) {
+                xdata = dict_new();
+                if (!xdata) {
+                    gf_msg("dht", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                           "insufficient memory");
+                    ret = -1;
+                    *fop_errno = ENOMEM;
+                    break;
+                }
 
-        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: fd create failed (source)", loc->path);
-                ret = -1;
-                goto out;
+                /* Fail this write and abort rebalance if we
+                 * detect a write from client since migration of
+                 * this file started. This is done to avoid
+                 * potential data corruption due to out of order
+                 * writes from rebalance and client to the same
+                 * region (as compared between src and dst
+                 * files). See
+                 * https://github.com/gluster/glusterfs/issues/308
+                 * for more details.
+                 */
+                ret = dict_set_int32_sizen(xdata, GF_AVOID_OVERWRITE, 1);
+                if (ret) {
+                    gf_msg("dht", GF_LOG_ERROR, 0, ENOMEM,
+                           "failed to set dict");
+                    ret = -1;
+                    *fop_errno = ENOMEM;
+                    break;
+                }
+            }
         }
 
-        ret = syncop_open (from, loc, O_RDWR, fd);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to open file %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
-                goto out;
+        ret = syncop_writev(to, dst, vector, count, offset, iobref, 0, NULL,
+                            NULL, xdata, NULL);
+        if (ret < 0) {
+            *fop_errno = -ret;
+            break;
         }
 
+        offset += ret;
+        total += ret;
+
+        GF_FREE(vector);
+        if (iobref)
+            iobref_unref(iobref);
+        iobref = NULL;
+        vector = NULL;
+    }
+    if (iobref)
+        iobref_unref(iobref);
+    GF_FREE(vector);
+
+    if (ret >= 0)
+        ret = 0;
+    else
         ret = -1;
-        dict = dict_new ();
-        if (!dict)
-                goto out;
 
-        ret = dict_set_str (dict, DHT_LINKFILE_KEY, to->name);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to set xattr in dict for %s (linkto:%s)",
-                        loc->path, to->name);
-                goto out;
-        }
+    if (xdata) {
+        dict_unref(xdata);
+    }
 
-        /* Once the migration starts, the source should have 'linkto' key set
-           to show which is the target, so other clients can work around it */
-        ret = syncop_setxattr (from, loc, dict, 0);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to set xattr on %s in %s (%s)",
-                        loc->path, from->name, strerror (errno));
-                goto out;
-        }
+    return ret;
+}
 
-        /* mode should be (+S+T) to indicate migration is in progress */
-        iatt.ia_prot = stbuf->ia_prot;
-        iatt.ia_type = stbuf->ia_type;
-        iatt.ia_prot.sticky = 1;
-        iatt.ia_prot.sgid = 1;
+static int
+__dht_rebalance_open_src_file(xlator_t *this, xlator_t *from, xlator_t *to,
+                              loc_t *loc, struct iatt *stbuf, fd_t **src_fd,
+                              gf_boolean_t *clean_src, int *fop_errno)
+{
+    int ret = 0;
+    fd_t *fd = NULL;
+    dict_t *dict = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    *clean_src = _gf_false;
+
+    fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+    if (!fd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: fd create failed (source)", loc->path);
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_open(from, loc, O_RDWR, fd, NULL, NULL);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to open file %s on %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        ret = syncop_setattr (from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to set mode on %s in %s (%s)",
-                        loc->path, from->name, strerror (errno));
-                goto out;
-        }
+    fd_bind(fd);
 
-        if (src_fd)
-                *src_fd = fd;
+    if (src_fd)
+        *src_fd = fd;
 
-        /* success */
-        ret = 0;
+    ret = -1;
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: Could not allocate memory for dict", loc->path);
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, conf->link_xattr_name, to->name);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set xattr in dict for %s (linkto:%s)", loc->path,
+               to->name);
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* Once the migration starts, the source should have 'linkto' key set
+       to show which is the target, so other clients can work around it */
+    ret = syncop_setxattr(from, loc, dict, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to set xattr on %s in %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* Reset source mode/xattr if migration fails*/
+    *clean_src = _gf_true;
+
+    /* mode should be (+S+T) to indicate migration is in progress */
+    iatt.ia_prot = stbuf->ia_prot;
+    iatt.ia_type = stbuf->ia_type;
+    iatt.ia_prot.sticky = 1;
+    iatt.ia_prot.sgid = 1;
+
+    ret = syncop_setattr(from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL, NULL,
+                         NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to set mode on %s in %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* success */
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int
-migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
-                       struct iatt *buf)
+migrate_special_files(xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
+                      struct iatt *buf, int *fop_errno)
 {
-        int          ret      = -1;
-        dict_t      *rsp_dict = NULL;
-        dict_t      *dict     = NULL;
-        char        *link     = NULL;
-        struct iatt  stbuf    = {0,};
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = -1;
+    dict_t *rsp_dict = NULL;
+    dict_t *dict = NULL;
+    char *link = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    dht_conf_t *conf = this->private;
+
+    dict = dict_new();
+    if (!dict) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set 'linkto' key in dict", loc->path);
+        goto out;
+    }
+
+    /* check in the destination if the file is link file */
+    ret = syncop_lookup(to, loc, &stbuf, NULL, dict, &rsp_dict);
+    if ((ret < 0) && (-ret != ENOENT)) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: lookup failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* we no more require this key */
+    dict_del(dict, conf->link_xattr_name);
+
+    /* file exists in target node, only if it is 'linkfile' its valid,
+       otherwise, error out */
+    if (!ret) {
+        if (!check_is_linkfile(loc->inode, &stbuf, rsp_dict,
+                               conf->link_xattr_name)) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: file exists in destination", loc->path);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
+        }
 
-        ret = dict_set_int32 (dict, DHT_LINKFILE_KEY, 256);
+        /* as file is linkfile, delete it */
+        ret = syncop_unlink(to, loc, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set 'linkto' key in dict", loc->path);
-                goto out;
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to delete the linkfile", loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
+    }
 
-        /* check in the destination if the file is link file */
-        ret = syncop_lookup (to, loc, dict, &stbuf, &rsp_dict, NULL);
-        if ((ret == -1) && (errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_WARNING, "%s: lookup failed (%s)",
-                        loc->path, strerror (errno));
-                goto out;
+    /* Set the gfid of the source file in dict */
+    ret = dict_set_gfuuid(dict, "gfid-req", buf->ia_gfid, true);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set gfid in dict for create", loc->path);
+        goto out;
+    }
+
+    /* Create the file in target */
+    if (IA_ISLNK(buf->ia_type)) {
+        /* Handle symlinks separately */
+        ret = syncop_readlink(from, loc, &link, buf->ia_size, NULL, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: readlink on symlink failed", loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
 
-        /* we no more require this key */
-        dict_del (dict, DHT_LINKFILE_KEY);
+        ret = syncop_symlink(to, loc, link, 0, dict, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED, "%s: creating symlink failed",
+                   loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
 
-        /* file exists in target node, only if it is 'linkfile' its valid,
-           otherwise, error out */
-        if (!ret) {
-                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: file exists in destination", loc->path);
-                        ret = -1;
-                        goto out;
-                }
+        goto done;
+    }
 
-                /* as file is linkfile, delete it */
-                ret = syncop_unlink (to, loc);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to delete the linkfile (%s)",
-                                loc->path, strerror (errno));
-                        goto out;
-                }
-        }
+    ret = syncop_mknod(to, loc, st_mode_from_ia(buf->ia_prot, buf->ia_type),
+                       makedev(ia_major(buf->ia_rdev), ia_minor(buf->ia_rdev)),
+                       0, dict, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: mknod failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        /* Set the gfid of the source file in dict */
-        ret = dict_set_static_bin (dict, "gfid-req", buf->ia_gfid, 16);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set gfid in dict for create", loc->path);
-                goto out;
-        }
+done:
+    ret = syncop_setattr(to, loc, buf,
+                         (GF_SET_ATTR_MTIME | GF_SET_ATTR_UID |
+                          GF_SET_ATTR_GID | GF_SET_ATTR_MODE),
+                         NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to perform setattr on %s", loc->path, to->name);
+        *fop_errno = -ret;
+    }
+
+    ret = syncop_unlink(from, loc, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: unlink failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+    }
 
-        /* Create the file in target */
-        if (IA_ISLNK (buf->ia_type)) {
-                /* Handle symlinks separately */
-                ret = syncop_readlink (from, loc, &link, buf->ia_size);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: readlink on symlink failed (%s)",
-                                loc->path, strerror (errno));
-                        goto out;
-                }
+out:
+    GF_FREE(link);
+    if (dict)
+        dict_unref(dict);
 
-                ret = syncop_symlink (to, loc, link, dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: creating symlink failed (%s)",
-                                loc->path, strerror (errno));
-                        goto out;
-                }
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-                goto done;
-        }
+    return ret;
+}
 
-        ret = syncop_mknod (to, loc, st_mode_from_ia (buf->ia_prot,
-                                                      buf->ia_type),
-                            makedev (ia_major (buf->ia_rdev),
-                                     ia_minor (buf->ia_rdev)), dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "%s: mknod failed (%s)",
-                        loc->path, strerror (errno));
-                goto out;
-        }
+static int
+__dht_migration_cleanup_src_file(xlator_t *this, loc_t *loc, fd_t *fd,
+                                 xlator_t *from, ia_prot_t *src_ia_prot)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    struct iatt new_stbuf = {
+        0,
+    };
+
+    if (!this || !fd || !from || !src_ia_prot) {
+        goto out;
+    }
+
+    conf = this->private;
+
+    /*Revert source mode and xattr changes*/
+    ret = syncop_fstat(from, fd, &new_stbuf, NULL, NULL);
+    if (ret < 0) {
+        /* Failed to get the stat info */
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file cleanup failed: failed to fstat "
+               "file %s on %s ",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
 
-done:
-        ret = syncop_unlink (from, loc);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "%s: unlink failed (%s)",
-                        loc->path, strerror (errno));
+    /* Remove the sticky bit and sgid bit set, reset it to 0*/
+    if (!src_ia_prot->sticky)
+        new_stbuf.ia_prot.sticky = 0;
 
-out:
-        if (dict)
-                dict_unref (dict);
+    if (!src_ia_prot->sgid)
+        new_stbuf.ia_prot.sgid = 0;
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    ret = syncop_fsetattr(from, fd, &new_stbuf,
+                          (GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL, NULL,
+                          NULL, NULL);
 
-        return ret;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file cleanup failed:"
+               "%s: failed to perform fsetattr on %s ",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_fremovexattr(from, fd, conf->link_xattr_name, 0, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to remove linkto xattr on %s (%s)", loc->path,
+               from->name, strerror(-ret));
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
+
+out:
+    return ret;
 }
 
 /*
@@ -618,1067 +1469,3234 @@ out:
     1 : not a failure, but we can't migrate data as of now
 */
 int
-dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
-                  int flag)
+dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                 int flag, int *fop_errno)
 {
-        int             ret            = -1;
-        struct iatt     new_stbuf      = {0,};
-        struct iatt     stbuf          = {0,};
-        struct iatt     empty_iatt     = {0,};
-        ia_prot_t       src_ia_prot    = {0,};
-        fd_t           *src_fd         = NULL;
-        fd_t           *dst_fd         = NULL;
-        dict_t         *dict           = NULL;
-        dict_t         *xattr          = NULL;
-        dict_t         *xattr_rsp      = NULL;
-        int             file_has_holes = 0;
-
-        gf_log (this->name, GF_LOG_INFO, "%s: attempting to move from %s to %s",
-                loc->path, from->name, to->name);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = -1;
+    struct iatt new_stbuf = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt empty_iatt = {
+        0,
+    };
+    ia_prot_t src_ia_prot = {
+        0,
+    };
+    fd_t *src_fd = NULL;
+    fd_t *dst_fd = NULL;
+    dict_t *dict = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xattr_rsp = NULL;
+    int file_has_holes = 0;
+    dht_conf_t *conf = this->private;
+    int rcvd_enoent_from_src = 0;
+    struct gf_flock flock = {
+        0,
+    };
+    struct gf_flock plock = {
+        0,
+    };
+    loc_t tmp_loc = {
+        0,
+    };
+    loc_t parent_loc = {
+        0,
+    };
+    gf_boolean_t inodelk_locked = _gf_false;
+    gf_boolean_t entrylk_locked = _gf_false;
+    gf_boolean_t p_locked = _gf_false;
+    int lk_ret = -1;
+    gf_defrag_info_t *defrag = NULL;
+    gf_boolean_t clean_src = _gf_false;
+    gf_boolean_t clean_dst = _gf_false;
+    int log_level = GF_LOG_INFO;
+    gf_boolean_t delete_src_linkto = _gf_true;
+    lock_migration_info_t locklist;
+    dict_t *meta_dict = NULL;
+    gf_boolean_t meta_locked = _gf_false;
+    gf_boolean_t target_changed = _gf_false;
+    xlator_t *new_target = NULL;
+    xlator_t *old_target = NULL;
+    xlator_t *hashed_subvol = NULL;
+    fd_t *linkto_fd = NULL;
+    dict_t *xdata = NULL;
+
+    if (from == to) {
+        gf_msg_debug(this->name, 0,
+                     "destination and source are same. file %s"
+                     " might have migrated already",
+                     loc->path);
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, DHT_LINKFILE_KEY, 256);
+    gf_log(this->name, log_level, "%s: attempting to move from %s to %s",
+           loc->path, from->name, to->name);
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "Could not allocate memory for dict");
+        goto out;
+    }
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to set 'linkto' key in dict",
+               loc->path);
+        goto out;
+    }
+
+    /* Do not migrate file in case lock migration is not enabled on the
+     * volume*/
+    if (!conf->lock_migration_enabled) {
+        ret = dict_set_int32(dict, GLUSTERFS_POSIXLK_COUNT, sizeof(int32_t));
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set 'linkto' key in dict", loc->path);
-                goto out;
+            *fop_errno = ENOMEM;
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: %s: failed to "
+                   "set " GLUSTERFS_POSIXLK_COUNT " key in dict",
+                   loc->path);
+            goto out;
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "locks will be migrated"
+               " for file: %s",
+               loc->path);
+    }
+
+    /* The file is locked to prevent a rename during a migration. Renames
+     * and migrations on the file at the same time can lead to data loss.
+     */
+
+    ret = dht_build_parent_loc(this, &parent_loc, loc, fop_errno);
+    if (ret < 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to build parent loc, which is needed to "
+               "acquire entrylk to synchronize with renames on this "
+               "path. Skipping migration",
+               loc->path);
+        goto out;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: cannot find hashed subvol which is needed to "
+               "synchronize with renames on this path. "
+               "Skipping migration",
+               loc->path);
+        goto out;
+    }
+
+    flock.l_type = F_WRLCK;
+
+    tmp_loc.inode = inode_ref(loc->inode);
+    gf_uuid_copy(tmp_loc.gfid, loc->gfid);
+    tmp_loc.path = gf_strdup(loc->path);
+
+    /* this inodelk happens with flock.owner being zero. But to synchronize
+     * hardlink migration we need to have different lkowner for each migration
+     * Filed a bug here: https://bugzilla.redhat.com/show_bug.cgi?id=1468202 to
+     * track the fix for this. Currently synclock takes care of synchronizing
+     * hardlink migration. Once this bug is fixed we can avoid taking synclock
+     */
+    ret = syncop_inodelk(from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc, F_SETLKW,
+                         &flock, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "migrate file failed: "
+               "%s: failed to lock file on %s",
+               loc->path, from->name);
+        goto out;
+    }
+
+    inodelk_locked = _gf_true;
+
+    /* dht_rename has changed to use entrylk on hashed subvol for
+     * synchronization. So, rebalance too has to acquire an entrylk on
+     * hashed subvol.
+     */
+    ret = syncop_entrylk(hashed_subvol, DHT_ENTRY_SYNC_DOMAIN, &parent_loc,
+                         loc->name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to acquire entrylk on subvol %s", loc->path,
+               hashed_subvol->name);
+        goto out;
+    }
+
+    entrylk_locked = _gf_true;
+
+    /* Phase 1 - Data migration is in progress from now on */
+    ret = syncop_lookup(from, loc, &stbuf, NULL, dict, &xattr_rsp);
+    if (ret) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: lookup failed on %s",
+               loc->path, from->name);
+        goto out;
+    }
+
+    /* preserve source mode, so set the same to the destination */
+    src_ia_prot = stbuf.ia_prot;
+
+    /* Check if file can be migrated */
+    ret = __is_file_migratable(this, loc, &stbuf, xattr_rsp, flag, defrag, conf,
+                               fop_errno);
+    if (ret) {
+        if (ret == HARDLINK_MIG_INPROGRESS)
+            ret = 0;
+        goto out;
+    }
+
+    /* Take care of the special files */
+    if (!IA_ISREG(stbuf.ia_type)) {
+        /* Special files */
+        ret = migrate_special_files(this, from, to, loc, &stbuf, fop_errno);
+        goto out;
+    }
+
+    /* Try to preserve 'holes' while migrating data */
+    if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE))
+        file_has_holes = 1;
+
+    /* create the destination, with required modes/xattr */
+    ret = __dht_rebalance_create_dst_file(this, to, from, loc, &stbuf, &dst_fd,
+                                          fop_errno, file_has_holes);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "Create dst failed"
+               " on - %s for file - %s",
+               to->name, loc->path);
+        goto out;
+    }
+
+    clean_dst = _gf_true;
+
+    ret = __dht_check_free_space(this, to, from, loc, &stbuf, flag, conf,
+                                 &target_changed, &new_target, fop_errno);
+    if (target_changed) {
+        /* Can't handle for hardlinks. Marking this as failure */
+        if (flag == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS || stbuf.ia_nlink > 1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+                   "Exiting migration for"
+                   " file - %s. flag - %d, stbuf.ia_nlink - %d",
+                   loc->path, flag, stbuf.ia_nlink);
+            ret = -1;
+            goto out;
         }
 
-        /* Phase 1 - Data migration is in progress from now on */
-        ret = syncop_lookup (from, loc, dict, &stbuf, &xattr_rsp, NULL);
+        ret = syncop_ftruncate(to, dst_fd, 0, NULL, NULL, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: lookup failed on %s (%s)",
-                        loc->path, from->name, strerror (errno));
-                goto out;
+            gf_log(this->name, GF_LOG_WARNING,
+                   "%s: failed to perform truncate on %s (%s)", loc->path,
+                   to->name, strerror(-ret));
         }
 
-        /* we no more require this key */
-        dict_del (dict, DHT_LINKFILE_KEY);
+        syncop_close(dst_fd);
+        dst_fd = NULL;
 
-        /* preserve source mode, so set the same to the destination */
-        src_ia_prot = stbuf.ia_prot;
+        old_target = to;
+        to = new_target;
 
-        /* Check if file can be migrated */
-        ret = __is_file_migratable (this, loc, &stbuf, xattr_rsp, flag);
-        if (ret)
-                goto out;
+        clean_dst = _gf_false;
 
-        /* Take care of the special files */
-        if (!IA_ISREG (stbuf.ia_type)) {
-                /* Special files */
-                ret = migrate_special_files (this, from, to, loc, &stbuf);
-                goto out;
+        /* if the file migration is successful to this new target, then
+         * update the xattr on the old destination to point the new
+         * destination. We need to do update this only post migration
+         * as in case of failure the linkto needs to point to the source
+         * subvol */
+        ret = __dht_rebalance_create_dst_file(
+            this, to, from, loc, &stbuf, &dst_fd, fop_errno, file_has_holes);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Create dst failed"
+                   " on - %s for file - %s",
+                   to->name, loc->path);
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                   "destination for file "
+                   "- %s is changed to - %s",
+                   loc->path, to->name);
+            clean_dst = _gf_true;
         }
+    }
+
+    if (ret) {
+        goto out;
+    }
+
+    /* Open the source, and also update mode/xattr */
+    ret = __dht_rebalance_open_src_file(this, from, to, loc, &stbuf, &src_fd,
+                                        &clean_src, fop_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: failed to open %s on %s", loc->path,
+               from->name);
+        goto out;
+    }
+
+    /* TODO: move all xattr related operations to fd based operations */
+    ret = syncop_listxattr(from, loc, &xattr, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to get xattr from %s",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
+
+    /* Copying posix acls to the linkto file messes up the permissions*/
+    dht_strip_out_acls(xattr);
+
+    /* Remove the linkto xattr as we don't want to overwrite the value
+     * set on the dst.
+     */
+    dict_del(xattr, conf->link_xattr_name);
+
+    /* We need to error out if this fails as having the wrong shard xattrs
+     * set on the dst could cause data corruption
+     */
+    ret = syncop_fsetxattr(to, dst_fd, xattr, 0, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to set xattr on %s", loc->path, to->name);
+        ret = -1;
+        goto out;
+    }
 
-        /* create the destination, with required modes/xattr */
-        ret = __dht_rebalance_create_dst_file (to, from, loc, &stbuf,
-                                               dict, &dst_fd);
-        if (ret)
-                goto out;
+    if (xattr_rsp) {
+        /* we no more require this key */
+        dict_del(dict, conf->link_xattr_name);
+        dict_unref(xattr_rsp);
+    }
+
+    ret = syncop_fstat(from, src_fd, &stbuf, dict, &xattr_rsp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:failed to lookup %s on %s ", loc->path,
+               from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* Check again if file has hardlink */
+    ret = __check_file_has_hardlink(this, loc, &stbuf, xattr_rsp, flag, defrag,
+                                    conf, fop_errno);
+    if (ret) {
+        if (ret == HARDLINK_MIG_INPROGRESS)
+            ret = 0;
+        goto out;
+    }
+
+    ret = __dht_rebalance_migrate_data(this, defrag, from, to, src_fd, dst_fd,
+                                       stbuf.ia_size, file_has_holes,
+                                       fop_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: %s: failed to migrate data", loc->path);
 
-        ret = __dht_check_free_space (to, from, loc, &stbuf, flag);
-        if (ret) {
-                goto out;
+        ret = -1;
+        goto out;
+    }
+
+    /* TODO: Sync the locks */
+
+    xdata = dict_new();
+    if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set last-fsync flag on "
+               "%s (%s)",
+               loc->path, to->name, strerror(ENOMEM));
+    }
+
+    ret = syncop_fsync(to, dst_fd, 0, NULL, NULL, xdata, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING, "%s: failed to fsync on %s (%s)",
+               loc->path, to->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
+
+    ret = syncop_fstat(from, src_fd, &new_stbuf, NULL, NULL);
+    if (ret < 0) {
+        /* Failed to get the stat info */
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: failed to fstat file %s on %s ", loc->path,
+               from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* Lock the entire source file to prevent clients from taking a
+       lock on it as dht_lk does not handle file migration.
+
+       This still leaves a small window where conflicting locks can
+       be granted to different clients. If client1 requests a blocking
+       lock on the src file, it will be granted after the migrating
+       process releases its lock. If client2 requests a lock on the dst
+       data file, it will also be granted, but all FOPs will be redirected
+       to the dst data file.
+    */
+
+    /* Take meta lock  */
+
+    if (conf->lock_migration_enabled) {
+        meta_dict = dict_new();
+        if (!meta_dict) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "dict_new failed");
+
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* Open the source, and also update mode/xattr */
-        ret = __dht_rebalance_open_src_file (from, to, loc, &stbuf, &src_fd);
+        ret = dict_set_str(meta_dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to open %s on %s",
-                        loc->path, from->name);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value: key = %s,"
+                   " path = %s",
+                   GLUSTERFS_INTERNAL_FOP_KEY, loc->path);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        ret = syncop_fstat (from, src_fd, &stbuf);
+        ret = dict_set_int32(meta_dict, GF_META_LOCK_KEY, 1);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to lookup %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* Try to preserve 'holes' while migrating data */
-        if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE))
-                file_has_holes = 1;
-
-        /* All I/O happens in this function */
-        ret = __dht_rebalance_migrate_data (from, to, src_fd, dst_fd,
-					    stbuf.ia_size, file_has_holes);
+        ret = syncop_setxattr(from, loc, meta_dict, 0, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: failed to migrate data",
-                        loc->path);
-                /* reset the destination back to 0 */
-                ret = syncop_ftruncate (to, dst_fd, 0);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s: failed to reset target size back to 0 (%s)",
-                                loc->path, strerror (errno));
-                }
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace syncop_setxattr metalock failed");
 
-                ret = -1;
-                goto out;
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        } else {
+            meta_locked = _gf_true;
         }
+    }
 
-        /* TODO: move all xattr related operations to fd based operations */
-        ret = syncop_listxattr (from, loc, &xattr);
-        if (ret == -1)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to get xattr from %s (%s)",
-                        loc->path, from->name, strerror (errno));
+    if (!conf->lock_migration_enabled) {
+        plock.l_type = F_WRLCK;
+        plock.l_start = 0;
+        plock.l_len = 0;
+        plock.l_whence = SEEK_SET;
 
-        ret = syncop_setxattr (to, loc, xattr, 0);
-        if (ret == -1)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set xattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+        ret = syncop_lk(from, src_fd, F_SETLK, &plock, NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: Failed to lock on %s",
+                   loc->path, from->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
 
-        /* TODO: Sync the locks */
+        p_locked = _gf_true;
 
-        ret = syncop_fsync (to, dst_fd);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to fsync on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+    } else {
+        INIT_LIST_HEAD(&locklist.list);
 
+        ret = syncop_getactivelk(from, loc, &locklist, NULL, NULL);
+        if (ret == 0) {
+            gf_log(this->name, GF_LOG_INFO, "No active locks on:%s", loc->path);
 
-        /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
+        } else if (ret > 0) {
+            ret = syncop_setactivelk(to, loc, &locklist, NULL, NULL);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, -ret,
+                       DHT_MSG_LOCK_MIGRATION_FAILED, "write lock failed on:%s",
+                       loc->path);
 
-        ret = syncop_fstat (from, src_fd, &new_stbuf);
-        if (ret < 0) {
-                /* Failed to get the stat info */
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to fstat file %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
-                goto out;
+                *fop_errno = -ret;
+                ret = -1;
+                goto metaunlock;
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret,
+                   DHT_MSG_LOCK_MIGRATION_FAILED,
+                   "getactivelk failed for file: %s", loc->path);
+            *fop_errno = -ret;
         }
-
-        /* source would have both sticky bit and sgid bit set, reset it to 0,
-           and set the source permission on destination, if it was not set
-           prior to setting rebalance-modes in source  */
-        if (!src_ia_prot.sticky)
-                new_stbuf.ia_prot.sticky = 0;
-
-        if (!src_ia_prot.sgid)
-                new_stbuf.ia_prot.sgid = 0;
-
-        /* TODO: if the source actually had sticky bit, or sgid bit set,
-           we are not handling it */
-
-        ret = syncop_fsetattr (to, dst_fd, &new_stbuf,
-                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID |
-                                GF_SET_ATTR_MODE), NULL, NULL);
+    }
+
+    /* source would have both sticky bit and sgid bit set, reset it to 0,
+       and set the source permission on destination, if it was not set
+       prior to setting rebalance-modes in source  */
+    if (!src_ia_prot.sticky)
+        new_stbuf.ia_prot.sticky = 0;
+
+    if (!src_ia_prot.sgid)
+        new_stbuf.ia_prot.sgid = 0;
+
+    /* TODO: if the source actually had sticky bit, or sgid bit set,
+       we are not handling it */
+
+    ret = syncop_fsetattr(
+        to, dst_fd, &new_stbuf,
+        (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL, NULL,
+        NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to perform setattr on %s ",
+               loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto metaunlock;
+    }
+
+    /* Because 'futimes' is not portable */
+    ret = syncop_setattr(to, loc, &new_stbuf,
+                         (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME), NULL, NULL,
+                         NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform setattr on %s ", loc->path, to->name);
+        *fop_errno = -ret;
+    }
+
+    if (target_changed) {
+        dict_del(dict, GLUSTERFS_POSIXLK_COUNT);
+        ret = dict_set_str(dict, conf->link_xattr_name, to->name);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform setattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set xattr in dict for %s (linkto:%s)", loc->path,
+                   to->name);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
+        }
+
+        ret = syncop_setxattr(old_target, loc, dict, 0, NULL, NULL);
+        if (ret && -ret != ESTALE && -ret != ENOENT) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to set xattr on %s in %s", loc->path,
+                   old_target->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        } else if (-ret == ESTALE || -ret == ENOENT) {
+            /* The failure ESTALE indicates that the linkto
+             * file on the hashed subvol might have been deleted.
+             * In this case will create a linkto file with new target
+             * as linkto xattr value*/
+            linkto_fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+            if (!linkto_fd) {
+                gf_msg(this->name, GF_LOG_ERROR, errno,
+                       DHT_MSG_MIGRATE_FILE_FAILED, "%s: fd create failed",
+                       loc->path);
+                *fop_errno = ENOMEM;
+                ret = -1;
+                goto out;
+            }
+            ret = syncop_create(old_target, loc, O_RDWR, DHT_LINKFILE_MODE,
+                                linkto_fd, NULL, dict, NULL);
+            if (ret != 0 && -ret != EEXIST && -ret != ESTALE) {
+                *fop_errno = -ret;
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, -ret,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "failed to create linkto file on %s in %s", loc->path,
+                       old_target->name);
                 goto out;
+            } else if (ret == 0) {
+                ret = syncop_fsetattr(old_target, linkto_fd, &stbuf,
+                                      (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL,
+                                      NULL, NULL, NULL);
+                if (ret < 0) {
+                    *fop_errno = -ret;
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_MIGRATE_FILE_FAILED,
+                           "chown failed for %s on %s", loc->path,
+                           old_target->name);
+                }
+            }
+        }
+    }
+
+    clean_dst = _gf_false;
+
+    /* Posix acls are not set on DHT linkto files as part of the initial
+     * initial xattrs set on the dst file, so these need
+     * to be set on the dst file after the linkto attrs are removed.
+     * TODO: Optimize this.
+     */
+    if (xattr) {
+        dict_unref(xattr);
+        xattr = NULL;
+    }
+
+    /* Set only the Posix ACLs this time */
+    ret = syncop_getxattr(from, loc, &xattr, POSIX_ACL_ACCESS_XATTR, NULL,
+                          NULL);
+    if (ret < 0) {
+        if ((-ret != ENODATA) && (-ret != ENOATTR)) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: failed to get xattr from %s",
+                   loc->path, from->name);
+            *fop_errno = -ret;
+        }
+    } else {
+        ret = syncop_setxattr(to, loc, xattr, 0, NULL, NULL);
+        if (ret < 0) {
+            /* Potential problem here where Posix ACLs will
+             * not be set on the target file */
+
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: failed to set xattr on %s",
+                   loc->path, to->name);
+            *fop_errno = -ret;
+        }
+    }
+
+    /* The src file is being unlinked after this so we don't need
+       to clean it up */
+    clean_src = _gf_false;
+
+    /* Make the source as a linkfile first before deleting it */
+    empty_iatt.ia_prot.sticky = 1;
+    ret = syncop_fsetattr(from, src_fd, &empty_iatt, GF_SET_ATTR_MODE, NULL,
+                          NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to perform setattr on %s ",
+               loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto metaunlock;
+    }
+
+    /* Free up the data blocks on the source node, as the whole
+        file is migrated */
+    ret = syncop_ftruncate(from, src_fd, 0, NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform truncate on %s (%s)", loc->path,
+               from->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* remove the 'linkto' xattr from the destination */
+    ret = syncop_fremovexattr(to, dst_fd, conf->link_xattr_name, 0, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform removexattr on %s (%s)", loc->path,
+               to->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* Do a stat and check the gfid before unlink */
+
+    /*
+     * Cached file changes its state from non-linkto to linkto file after
+     * migrating data. If lookup from any other mount-point is performed,
+     * converted-linkto-cached file will be treated as a stale and will be
+     * unlinked. But by this time, file is already migrated. So further
+     * failure because of ENOENT should  not be treated as error
+     */
+
+    ret = syncop_stat(from, loc, &empty_iatt, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to do a stat on %s", loc->path, from->name);
+
+        if (-ret != ENOENT) {
+            *fop_errno = -ret;
+            ret = -1;
+            goto metaunlock;
         }
 
-        /* Because 'futimes' is not portable */
-        ret = syncop_setattr (to, loc, &new_stbuf,
-                              (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME),
-                              NULL, NULL);
+        rcvd_enoent_from_src = 1;
+    }
+
+    if ((gf_uuid_compare(empty_iatt.ia_gfid, loc->gfid) == 0) &&
+        (!rcvd_enoent_from_src) && delete_src_linkto) {
+        /* take out the source from namespace */
+        ret = syncop_unlink(from, loc, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform setattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to perform unlink on %s", loc->path, from->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto metaunlock;
         }
+    }
+
+    ret = syncop_lookup(this, loc, NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "%s: failed to lookup the file on subvolumes", loc->path);
+        *fop_errno = -ret;
+    }
+
+    gf_msg(this->name, log_level, 0, DHT_MSG_MIGRATE_FILE_COMPLETE,
+           "completed migration of %s from subvolume %s to %s", loc->path,
+           from->name, to->name);
 
-        /* Make the source as a linkfile first before deleting it */
-        empty_iatt.ia_prot.sticky = 1;
-        ret = syncop_fsetattr (from, src_fd, &empty_iatt,
-                               GF_SET_ATTR_MODE, NULL, NULL);
+    ret = 0;
+
+metaunlock:
+
+    if (conf->lock_migration_enabled && meta_locked) {
+        dict_del(meta_dict, GF_META_LOCK_KEY);
+
+        ret = dict_set_int32(meta_dict, GF_META_UNLOCK_KEY, 1);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,             \
-                        "%s: failed to perform setattr on %s (%s)",
-                        loc->path, from->name, strerror (errno));
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
+
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* Do a stat and check the gfid before unlink */
-        ret = syncop_stat (from, loc, &empty_iatt);
+        if (clean_dst == _gf_false)
+            ret = dict_set_int32(meta_dict, "status", 1);
+        else
+            ret = dict_set_int32(meta_dict, "status", 0);
+
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to do a stat on %s (%s)",
-                        loc->path, from->name, strerror (errno));
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
 
-        if (uuid_compare (empty_iatt.ia_gfid, loc->gfid) == 0) {
-                /* take out the source from namespace */
-                ret = syncop_unlink (from, loc);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to perform unlink on %s (%s)",
-                                loc->path, from->name, strerror (errno));
-                        goto out;
-                }
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* Free up the data blocks on the source node, as the whole
-           file is migrated */
-        ret = syncop_ftruncate (from, src_fd, 0);
+        ret = syncop_setxattr(from, loc, meta_dict, 0, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform truncate on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace syncop_setxattr meta unlock failed");
+
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
+    }
 
-        /* remove the 'linkto' xattr from the destination */
-        ret = syncop_fremovexattr (to, dst_fd, DHT_LINKFILE_KEY);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform removexattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+out:
+    if (clean_src) {
+        /* Revert source mode and xattr changes*/
+        lk_ret = __dht_migration_cleanup_src_file(this, loc, src_fd, from,
+                                                  &src_ia_prot);
+        if (lk_ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to cleanup source file on %s", loc->path,
+                   from->name);
         }
+    }
+
+    /* reset the destination back to 0 */
+    if (clean_dst) {
+        lk_ret = syncop_ftruncate(to, dst_fd, 0, NULL, NULL, NULL, NULL);
+        if (lk_ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: "
+                   "%s: failed to reset target size back to 0",
+                   loc->path);
+        }
+    }
 
-        ret = syncop_lookup (this, loc, NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%s: failed to lookup the file on subvolumes (%s)",
-                        loc->path, strerror (errno));
+    if (inodelk_locked) {
+        flock.l_type = F_UNLCK;
+
+        lk_ret = syncop_inodelk(from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc,
+                                F_SETLK, &flock, NULL, NULL);
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock file on %s", loc->path, from->name);
+        }
+    }
+
+    if (entrylk_locked) {
+        lk_ret = syncop_entrylk(hashed_subvol, DHT_ENTRY_SYNC_DOMAIN,
+                                &parent_loc, loc->name, ENTRYLK_UNLOCK,
+                                ENTRYLK_UNLOCK, NULL, NULL);
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock entrylk on %s", loc->path,
+                   hashed_subvol->name);
         }
+    }
 
-        gf_log (this->name, GF_LOG_INFO,
-                "completed migration of %s from subvolume %s to %s",
-                loc->path, from->name, to->name);
+    if (p_locked) {
+        plock.l_type = F_UNLCK;
+        lk_ret = syncop_lk(from, src_fd, F_SETLK, &plock, NULL, NULL);
 
-        ret = 0;
-out:
-        if (dict)
-                dict_unref (dict);
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock file on %s", loc->path, from->name);
+        }
+    }
 
-        if (xattr)
-                dict_unref (xattr);
-        if (xattr_rsp)
-                dict_unref (xattr_rsp);
+    lk_ret = syncop_removexattr(to, loc, GF_PROTECT_FROM_EXTERNAL_WRITES, NULL,
+                                NULL);
+    if (lk_ret && (lk_ret != -ENODATA) && (lk_ret != -ENOATTR)) {
+        gf_msg(this->name, GF_LOG_WARNING, -lk_ret, 0,
+               "%s: removexattr failed key %s", loc->path,
+               GF_PROTECT_FROM_EXTERNAL_WRITES);
+    }
 
-        if (dst_fd)
-                syncop_close (dst_fd);
-        if (src_fd)
-                syncop_close (src_fd);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
-}
+    if (xattr)
+        dict_unref(xattr);
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
 
-static int
-rebalance_task (void *data)
-{
-        int           ret   = -1;
-        dht_local_t  *local = NULL;
-        call_frame_t *frame = NULL;
+    if (dst_fd)
+        syncop_close(dst_fd);
 
-        frame = data;
+    if (src_fd)
+        syncop_close(src_fd);
+    if (linkto_fd)
+        syncop_close(linkto_fd);
 
-        local = frame->local;
+    if (xdata)
+        dict_unref(xdata);
 
-        /* This function is 'synchrounous', hence if it returns,
-           we are done with the task */
-        ret = dht_migrate_file (THIS, &local->loc, local->rebalance.from_subvol,
-                                local->rebalance.target_node, local->flags);
+    loc_wipe(&tmp_loc);
+    loc_wipe(&parent_loc);
 
-        return ret;
+    return ret;
 }
 
 static int
-rebalance_task_completion (int op_ret, call_frame_t *sync_frame, void *data)
+rebalance_task(void *data)
 {
-        int           ret        = -1;
-        uint64_t      layout_int = 0;
-        dht_layout_t *layout     = 0;
-        xlator_t     *this       = NULL;
-        dht_local_t  *local      = NULL;
-        int32_t       op_errno   = EINVAL;
-
-        this = THIS;
-        local = sync_frame->local;
-
-        if (!op_ret) {
-                /* Make sure we have valid 'layout' in inode ctx
-                   after the operation */
-                ret = inode_ctx_del (local->loc.inode, this, &layout_int);
-                if (!ret && layout_int) {
-                        layout = (dht_layout_t *)(long)layout_int;
-                        dht_layout_unref (this, layout);
-                }
+    int ret = -1;
+    dht_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int fop_errno = 0;
 
-                ret = dht_layout_preset (this, local->rebalance.target_node,
-                                         local->loc.inode);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to set inode ctx", local->loc.path);
-        }
+    frame = data;
 
-        if (op_ret == -1) {
-                /* Failure of migration process, mostly due to write process.
-                   as we can't preserve the exact errno, lets say there was
-                   no space to migrate-data
-                */
-                op_errno = ENOSPC;
-        }
+    local = frame->local;
 
-        if (op_ret == 1) {
-                /* migration didn't happen, but is not a failure, let the user
-                   understand that he doesn't have permission to migrate the
-                   file.
-                */
-                op_ret = -1;
-                op_errno = EPERM;
-        }
+    /* This function is 'synchrounous', hence if it returns,
+       we are done with the task */
+    ret = dht_migrate_file(THIS, &local->loc, local->rebalance.from_subvol,
+                           local->rebalance.target_node, local->flags,
+                           &fop_errno);
 
-        DHT_STACK_UNWIND (setxattr, sync_frame, op_ret, op_errno, NULL);
-        return 0;
+    return ret;
+}
+
+static int
+rebalance_task_completion(int op_ret, call_frame_t *sync_frame, void *data)
+{
+    int32_t op_errno = EINVAL;
+
+    if (op_ret == -1) {
+        /* Failure of migration process, mostly due to write process.
+           as we can't preserve the exact errno, lets say there was
+           no space to migrate-data
+        */
+        op_errno = ENOSPC;
+    } else if (op_ret == 1) {
+        /* migration didn't happen, but is not a failure, let the user
+           understand that he doesn't have permission to migrate the
+           file.
+        */
+        op_ret = -1;
+        op_errno = EPERM;
+    } else if (op_ret != 0) {
+        op_errno = -op_ret;
+        op_ret = -1;
+    }
+
+    DHT_STACK_UNWIND(setxattr, sync_frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 int
-dht_start_rebalance_task (xlator_t *this, call_frame_t *frame)
+dht_start_rebalance_task(xlator_t *this, call_frame_t *frame)
 {
-        int         ret     = -1;
+    int ret = -1;
 
-        ret = synctask_new (this->ctx->env, rebalance_task,
-                            rebalance_task_completion,
-                            frame, frame);
-        return ret;
+    ret = synctask_new(this->ctx->env, rebalance_task,
+                       rebalance_task_completion, frame, frame);
+    return ret;
 }
 
 int
-gf_listener_stop (xlator_t *this)
+gf_listener_stop(xlator_t *this)
 {
-        glusterfs_ctx_t  *ctx = NULL;
-        cmd_args_t       *cmd_args = NULL;
-        int              ret = 0;
-
-        ctx = this->ctx;
-        GF_ASSERT (ctx);
-        cmd_args = &ctx->cmd_args;
-        if (cmd_args->sock_file) {
-                ret = unlink (cmd_args->sock_file);
-                if (ret && (ENOENT == errno)) {
-                        ret = 0;
-                }
+    glusterfs_ctx_t *ctx = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
+
+    ctx = this->ctx;
+    GF_ASSERT(ctx);
+    cmd_args = &ctx->cmd_args;
+    if (cmd_args->sock_file) {
+        ret = sys_unlink(cmd_args->sock_file);
+        if (ret && (ENOENT == errno)) {
+            ret = 0;
         }
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to unlink listener "
-                        "socket %s, error: %s", cmd_args->sock_file,
-                        strerror (errno));
-        }
-        return ret;
+    }
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, DHT_MSG_SOCKET_ERROR,
+               "Failed to unlink listener "
+               "socket %s",
+               cmd_args->sock_file);
+    }
+    return ret;
 }
 
 void
-dht_build_root_inode (xlator_t *this, inode_t **inode)
+dht_build_root_inode(xlator_t *this, inode_t **inode)
 {
-        inode_table_t    *itable        = NULL;
-        uuid_t            root_gfid     = {0, };
+    inode_table_t *itable = NULL;
+    static uuid_t root_gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
 
-        itable = inode_table_new (0, this);
-        if (!itable)
-                return;
+    itable = inode_table_new(0, this);
+    if (!itable)
+        return;
 
-        root_gfid[15] = 1;
-        *inode = inode_find (itable, root_gfid);
+    *inode = inode_find(itable, root_gfid);
 }
 
 void
-dht_build_root_loc (inode_t *inode, loc_t *loc)
+dht_build_root_loc(inode_t *inode, loc_t *loc)
 {
-        loc->path = "/";
-        loc->inode = inode;
-        loc->inode->ia_type = IA_IFDIR;
-        memset (loc->gfid, 0, 16);
-        loc->gfid[15] = 1;
+    loc->path = "/";
+    loc->inode = inode;
+    loc->inode->ia_type = IA_IFDIR;
+    memset(loc->gfid, 0, 16);
+    loc->gfid[15] = 1;
 }
 
-
 /* return values: 1 -> error, bug ignore and continue
                   0 -> proceed
                  -1 -> error, handle it */
 int32_t
-gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
+gf_defrag_handle_migrate_error(int32_t op_errno, gf_defrag_info_t *defrag)
 {
-        /* if errno is not ENOSPC or ENOTCONN, we can still continue
-           with rebalance process */
-        if ((errno != ENOSPC) || (errno != ENOTCONN))
-                return 1;
-
-        if (errno == ENOTCONN) {
-                /* Most probably mount point went missing (mostly due
-                   to a brick down), say rebalance failure to user,
-                   let him restart it if everything is fine */
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                return -1;
+    int ret = 0;
+    /* if errno is not ENOTCONN, we can still continue
+       with rebalance process */
+    if (op_errno != ENOTCONN) {
+        ret = 1;
+        goto out;
+    }
+
+    if (op_errno == ENOTCONN) {
+        /* Most probably mount point went missing (mostly due
+           to a brick down), say rebalance failure to user,
+           let him restart it if everything is fine */
+        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+        ret = -1;
+        goto out;
+    }
+
+out:
+    return ret;
+}
+
+static gf_boolean_t
+gf_defrag_pattern_match(gf_defrag_info_t *defrag, char *name, uint64_t size)
+{
+    gf_defrag_pattern_list_t *trav = NULL;
+    gf_boolean_t match = _gf_false;
+    gf_boolean_t ret = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("dht", defrag, out);
+
+    trav = defrag->defrag_pattern;
+    while (trav) {
+        if (!fnmatch(trav->path_pattern, name, FNM_NOESCAPE)) {
+            match = _gf_true;
+            break;
         }
+        trav = trav->next;
+    }
 
-        if (errno == ENOSPC) {
-                /* rebalance process itself failed, may be
-                   remote brick went down, or write failed due to
-                   disk full etc etc.. */
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                return -1;
+    if ((match == _gf_true) && (size >= trav->size))
+        ret = _gf_true;
+
+out:
+    return ret;
+}
+
+int
+dht_dfreaddirp_done(dht_dfoffset_ctx_t *offset_var, int cnt)
+{
+    int i;
+    int result = 1;
+
+    for (i = 0; i < cnt; i++) {
+        if (offset_var[i].readdir_done == 0) {
+            result = 0;
+            break;
         }
+    }
+    return result;
+}
 
-        return 0;
+int static gf_defrag_ctx_subvols_init(dht_dfoffset_ctx_t *offset_var,
+                                      xlator_t *this)
+{
+    int i;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf)
+        return -1;
+
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        offset_var[i].this = conf->local_subvols[i];
+        offset_var[i].offset = (off_t)0;
+        offset_var[i].readdir_done = 0;
+    }
+
+    return 0;
+}
+
+static int
+dht_get_first_non_null_index(subvol_nodeuuids_info_t *entry)
+{
+    int i = 0;
+    int index = 0;
+
+    for (i = 0; i < entry->count; i++) {
+        if (!gf_uuid_is_null(entry->elements[i].uuid)) {
+            index = i;
+            goto out;
+        }
+    }
+
+    if (i == entry->count) {
+        index = -1;
+    }
+out:
+    return index;
 }
 
-/* We do a depth first traversal of directories. But before we move into
- * subdirs, we complete the data migration of those directories whose layouts
- * have been fixed
+/* Return value
+ * 0 : this node does not migrate the file
+ * 1 : this node migrates the file
+ *
+ * Use the hash value of the gfid to determine which node will migrate files.
+ * Using the gfid instead of the name also ensures that the same node handles
+ * all hardlinks.
  */
 
-int
-gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
-                        dict_t *migrate_data)
+gf_boolean_t
+gf_defrag_should_i_migrate(xlator_t *this, int local_subvol_index, uuid_t gfid)
 {
-        int                      ret            = -1;
-        loc_t                    entry_loc      = {0,};
-        fd_t                    *fd             = NULL;
-        gf_dirent_t              entries;
-        gf_dirent_t             *tmp            = NULL;
-        gf_dirent_t             *entry          = NULL;
-        gf_boolean_t             free_entries   = _gf_false;
-        off_t                    offset         = 0;
-        dict_t                  *dict           = NULL;
-        struct iatt              iatt           = {0,};
-        int32_t                  op_errno       = 0;
-        char                    *uuid_str       = NULL;
-        uuid_t                   node_uuid      = {0,};
-        int                      readdir_operrno = 0;
-        struct timeval           dir_start      = {0,};
-        struct timeval           end            = {0,};
-        double                   elapsed        = {0,};
-        struct timeval           start          = {0,};
-
-        gf_log (this->name, GF_LOG_INFO, "migrate data called on %s",
-                loc->path);
-        gettimeofday (&dir_start, NULL);
-
-        fd = fd_create (loc->inode, defrag->pid);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
-                goto out;
+    gf_boolean_t ret = _gf_false;
+    int i = local_subvol_index;
+    char *str = NULL;
+    uint32_t hashval = 0;
+    int32_t index = 0;
+    dht_conf_t *conf = NULL;
+    char buf[UUID_CANONICAL_FORM_LEN + 1] = {
+        0,
+    };
+    subvol_nodeuuids_info_t *entry = NULL;
+
+    conf = this->private;
+
+    /* Pure distribute. A subvol in this case
+        will be handled by only one node */
+
+    entry = &(conf->local_nodeuuids[i]);
+    if (entry->count == 1) {
+        return 1;
+    }
+
+    str = uuid_utoa_r(gfid, buf);
+    if (dht_hash_compute(this, 0, str, &hashval) == 0) {
+        index = (hashval % entry->count);
+        if (entry->elements[index].info == REBAL_NODEUUID_MINE) {
+            /* Index matches this node's nodeuuid.*/
+            ret = _gf_true;
+            goto out;
         }
 
-        ret = syncop_opendir (this, loc, fd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
-                        loc->path);
+        /* Brick down - some other node has to migrate these files*/
+        if (gf_uuid_is_null(entry->elements[index].uuid)) {
+            /* Fall back to the first non-null index */
+            index = dht_get_first_non_null_index(entry);
+
+            if (index == -1) {
+                /* None of the bricks in the subvol are up.
+                 * CHILD_DOWN will kill the process soon */
+
+                return _gf_false;
+            }
+
+            if (entry->elements[index].info == REBAL_NODEUUID_MINE) {
+                /* Index matches this node's nodeuuid.*/
+                ret = _gf_true;
                 goto out;
+            }
         }
+    }
+out:
+    return ret;
+}
 
-        INIT_LIST_HEAD (&entries.list);
-
-        while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
-                                       &entries)) != 0) {
-                if (ret < 0)
-                        break;
+int
+gf_defrag_migrate_single_file(void *opaque)
+{
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    int ret = 0;
+    gf_dirent_t *entry = NULL;
+    struct timeval start = {
+        0,
+    };
+    loc_t entry_loc = {
+        0,
+    };
+    loc_t *loc = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *migrate_data = NULL;
+    struct timeval end = {
+        0,
+    };
+    double elapsed = {
+        0,
+    };
+    struct dht_container *rebal_entry = NULL;
+    inode_t *inode = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    call_frame_t *statfs_frame = NULL;
+    xlator_t *old_THIS = NULL;
+    data_t *tmp = NULL;
+    int fop_errno = 0;
+    gf_dht_migrate_data_type_t rebal_type = GF_DHT_MIGRATE_DATA;
+    char value[MAX_REBAL_TYPE_SIZE] = {
+        0,
+    };
+    struct iatt *iatt_ptr = NULL;
+    gf_boolean_t update_skippedcount = _gf_true;
+    int i = 0;
+    gf_boolean_t should_i_migrate = 0;
+
+    rebal_entry = (struct dht_container *)opaque;
+    if (!rebal_entry) {
+        gf_log("DHT", GF_LOG_ERROR, "rebal_entry is NULL");
+        ret = -1;
+        goto out;
+    }
 
-                /* Need to keep track of ENOENT errno, that means, there is no
-                   need to send more readdirp() */
-                readdir_operrno = errno;
+    this = rebal_entry->this;
 
-                free_entries = _gf_true;
+    conf = this->private;
 
-                if (list_empty (&entries.list))
-                        break;
-                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
-                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                                ret = 1;
-                                goto out;
-                        }
+    defrag = conf->defrag;
 
-                        offset = entry->d_off;
+    loc = rebal_entry->parent_loc;
 
-                        if (!strcmp (entry->d_name, ".") ||
-                            !strcmp (entry->d_name, ".."))
-                                continue;
+    migrate_data = rebal_entry->migrate_data;
 
-                        if (IA_ISDIR (entry->d_stat.ia_type))
-                                continue;
+    entry = rebal_entry->df_entry;
+    iatt_ptr = &entry->d_stat;
 
-                        defrag->num_files_lookedup++;
-                        if (defrag->stats == _gf_true) {
-                                gettimeofday (&start, NULL);
-                        }
-                        loc_wipe (&entry_loc);
-                        ret =dht_build_child_loc (this, &entry_loc, loc,
-                                                  entry->d_name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
-                                        " build failed");
-                                goto out;
-                        }
+    if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+        ret = -1;
+        goto out;
+    }
 
-                        if (uuid_is_null (entry->d_stat.ia_gfid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        " gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
+    if (defrag->stats == _gf_true) {
+        gettimeofday(&start, NULL);
+    }
 
-                        uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
+    if (defrag->defrag_pattern &&
+        (gf_defrag_pattern_match(defrag, entry->d_name,
+                                 entry->d_stat.ia_size) == _gf_false)) {
+        gf_log(this->name, GF_LOG_ERROR, "pattern_match failed");
+        goto out;
+    }
 
-                        if (uuid_is_null (loc->gfid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        " gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
+    memset(&entry_loc, 0, sizeof(entry_loc));
 
-                        uuid_copy (entry_loc.pargfid, loc->gfid);
+    ret = dht_build_child_loc(this, &entry_loc, loc, entry->d_name);
+    if (ret) {
+        LOCK(&defrag->lock);
+        {
+            defrag->total_failures += 1;
+        }
+        UNLOCK(&defrag->lock);
 
-                        entry_loc.inode->ia_type = entry->d_stat.ia_type;
+        ret = 0;
 
-                        ret = syncop_lookup (this, &entry_loc, NULL, &iatt,
-                                             NULL, NULL);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s"
-                                        " lookup failed", entry_loc.path);
-                                continue;
-                        }
+        gf_log(this->name, GF_LOG_ERROR, "Child loc build failed");
 
-                        ret = syncop_getxattr (this, &entry_loc, &dict,
-                                               GF_XATTR_NODE_UUID_KEY);
-                        if(ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "get node-uuid for %s", entry_loc.path);
-                                continue;
-                        }
+        goto out;
+    }
 
-                        ret = dict_get_str (dict, GF_XATTR_NODE_UUID_KEY,
-                                            &uuid_str);
-                        if(ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "get node-uuid from dict for %s",
-                                        entry_loc.path);
-                                continue;
-                        }
+    should_i_migrate = gf_defrag_should_i_migrate(
+        this, rebal_entry->local_subvol_index, entry->d_stat.ia_gfid);
 
-                        if (uuid_parse (uuid_str, node_uuid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "uuid_parse "
-                                        "failed for %s", entry_loc.path);
-                                continue;
-                        }
+    gf_uuid_copy(entry_loc.gfid, entry->d_stat.ia_gfid);
 
-                        /* if file belongs to different node, skip migration
-                         * the other node will take responsibility of migration
-                         */
-                        if (uuid_compare (node_uuid, defrag->node_uuid)) {
-                                gf_log (this->name, GF_LOG_TRACE, "%s does not"
-                                        "belong to this node", entry_loc.path);
-                                continue;
-                        }
+    gf_uuid_copy(entry_loc.pargfid, loc->gfid);
 
-                        uuid_str = NULL;
+    ret = syncop_lookup(this, &entry_loc, &iatt, NULL, NULL, NULL);
 
-                        dict_del (dict, GF_XATTR_NODE_UUID_KEY);
+    if (!should_i_migrate) {
+        /* this node isn't supposed to migrate the file. suppressing any
+         * potential error from lookup as this file is under migration by
+         * another node */
+        if (ret) {
+            gf_msg_debug(this->name, -ret,
+                         "Ignoring lookup failure: node isn't migrating %s",
+                         entry_loc.path);
+            ret = 0;
+        }
+        gf_msg_debug(this->name, 0, "Don't migrate %s ", entry_loc.path);
+        goto out;
+    }
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: %s lookup failed", entry_loc.path);
+
+        /* Increase failure count only for remove-brick op, so that
+         * user is warned to check the removed-brick for any files left
+         * unmigrated
+         */
+        if (conf->decommission_subvols_cnt) {
+            LOCK(&defrag->lock);
+            {
+                defrag->total_failures += 1;
+            }
+            UNLOCK(&defrag->lock);
+        }
 
+        ret = 0;
+        goto out;
+    }
 
-                        /* if distribute is present, it will honor this key.
-                         * -1 is returned if distribute is not present or file
-                         * doesn't have a link-file. If file has link-file, the
-                         * path of link-file will be the value, and also that
-                         * guarantees that file has to be mostly migrated */
+    iatt_ptr = &iatt;
 
-                        ret = syncop_getxattr (this, &entry_loc, &dict,
-                                               GF_XATTR_LINKINFO_KEY);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_TRACE, "failed to "
-                                        "get link-to key for %s",
-                                        entry_loc.path);
-                                continue;
-                        }
+    hashed_subvol = dht_subvol_get_hashed(this, &entry_loc);
+    if (!hashed_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "Failed to get hashed subvol for %s", entry_loc.path);
+        ret = 0;
+        goto out;
+    }
 
-                        ret = syncop_setxattr (this, &entry_loc, migrate_data,
-                                               0);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "migrate-data"
-                                        " failed for %s", entry_loc.path);
-                                defrag->total_failures +=1;
-                        }
+    cached_subvol = dht_subvol_get_cached(this, entry_loc.inode);
+    if (!cached_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CACHED_SUBVOL_GET_FAILED,
+               "Failed to get cached subvol for %s", entry_loc.path);
 
-                        if (ret == -1) {
-                                op_errno = errno;
-                                ret = gf_defrag_handle_migrate_error (op_errno,
-                                                                      defrag);
-
-                                if (!ret)
-                                        gf_log (this->name, GF_LOG_DEBUG,
-                                                "migrate-data on %s failed: %s",
-                                                entry_loc.path,
-                                                strerror (op_errno));
-                                else if (ret == 1)
-                                        continue;
-                                else if (ret == -1)
-                                        goto out;
-                        }
+        ret = 0;
+        goto out;
+    }
 
-                        LOCK (&defrag->lock);
+    if (hashed_subvol == cached_subvol) {
+        ret = 0;
+        goto out;
+    }
+
+    inode = inode_link(entry_loc.inode, entry_loc.parent, entry->d_name, &iatt);
+    inode_unref(entry_loc.inode);
+    /* use the inode returned by inode_link */
+    entry_loc.inode = inode;
+
+    old_THIS = THIS;
+    THIS = this;
+    statfs_frame = create_frame(this, this->ctx->pool);
+    if (!statfs_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+               "Insufficient memory. Frame creation failed");
+        ret = -1;
+        goto out;
+    }
+
+    /* async statfs information for honoring min-free-disk */
+    dht_get_du_info(statfs_frame, this, loc);
+    THIS = old_THIS;
+
+    tmp = dict_get(migrate_data, GF_XATTR_FILE_MIGRATE_KEY);
+    if (tmp) {
+        memcpy(value, tmp->data, tmp->len);
+        if (strcmp(value, "force") == 0)
+            rebal_type = GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+
+        if (conf->decommission_in_progress)
+            rebal_type = GF_DHT_MIGRATE_HARDLINK;
+    }
+
+    ret = dht_migrate_file(this, &entry_loc, cached_subvol, hashed_subvol,
+                           rebal_type, &fop_errno);
+    if (ret == 1) {
+        if (fop_errno == ENOSPC) {
+            gf_msg_debug(this->name, 0,
+                         "migrate-data skipped for"
+                         " %s due to space constraints",
+                         entry_loc.path);
+
+            /* For remove-brick case if the source is not one of the
+             * removed-brick, do not mark the error as failure */
+            if (conf->decommission_subvols_cnt) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (conf->decommissioned_bricks[i] == cached_subvol) {
+                        LOCK(&defrag->lock);
                         {
-                                defrag->total_files += 1;
-                                defrag->total_data += iatt.ia_size;
+                            defrag->total_failures += 1;
+                            update_skippedcount = _gf_false;
                         }
-                        UNLOCK (&defrag->lock);
-                        if (defrag->stats == _gf_true) {
-                                gettimeofday (&end, NULL);
-                                elapsed = (end.tv_sec - start.tv_sec) * 1e6 +
-                                          (end.tv_usec - start.tv_usec);
-                                gf_log (this->name, GF_LOG_INFO, "Migration of "
-                                        "file:%s size:%"PRIu64" bytes took %.2f"
-                                        "secs", entry_loc.path, iatt.ia_size,
-                                         elapsed/1e6);
-                        }
-                }
-
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-                INIT_LIST_HEAD (&entries.list);
+                        UNLOCK(&defrag->lock);
 
-                if (readdir_operrno == ENOENT)
                         break;
+                    }
+                }
+            }
+
+            if (update_skippedcount) {
+                LOCK(&defrag->lock);
+                {
+                    defrag->skipped += 1;
+                }
+                UNLOCK(&defrag->lock);
+
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_MIGRATE_FILE_SKIPPED,
+                       "File migration skipped for %s.", entry_loc.path);
+            }
+
+        } else if (fop_errno == ENOTSUP) {
+            gf_msg_debug(this->name, 0,
+                         "migrate-data skipped for"
+                         " hardlink %s ",
+                         entry_loc.path);
+            LOCK(&defrag->lock);
+            {
+                defrag->skipped += 1;
+            }
+            UNLOCK(&defrag->lock);
+
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_MIGRATE_FILE_SKIPPED,
+                   "File migration skipped for %s.", entry_loc.path);
         }
 
-        gettimeofday (&end, NULL);
-        elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 +
-                  (end.tv_usec - dir_start.tv_usec);
-        gf_log (this->name, GF_LOG_INFO, "Migration operation on dir %s took "
-                "%.2f secs", loc->path, elapsed/1e6);
         ret = 0;
-out:
-        if (free_entries)
-                gf_dirent_free (&entries);
+        goto out;
+    } else if (ret < 0) {
+        if (fop_errno != EEXIST) {
+            gf_msg(this->name, GF_LOG_ERROR, fop_errno,
+                   DHT_MSG_MIGRATE_FILE_FAILED, "migrate-data failed for %s",
+                   entry_loc.path);
+
+            LOCK(&defrag->lock);
+            {
+                defrag->total_failures += 1;
+            }
+            UNLOCK(&defrag->lock);
+        }
 
-        loc_wipe (&entry_loc);
+        ret = gf_defrag_handle_migrate_error(fop_errno, defrag);
 
-        if (dict)
-                dict_unref(dict);
+        if (!ret) {
+            gf_msg(this->name, GF_LOG_ERROR, fop_errno,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "migrate-data on %s failed:", entry_loc.path);
+        } else if (ret == 1) {
+            ret = 0;
+        }
 
-        if (fd)
-                fd_unref (fd);
-        return ret;
+        goto out;
+    }
+
+    LOCK(&defrag->lock);
+    {
+        defrag->total_files += 1;
+        defrag->total_data += iatt.ia_size;
+    }
+    UNLOCK(&defrag->lock);
+
+    if (defrag->stats == _gf_true) {
+        gettimeofday(&end, NULL);
+        elapsed = gf_tvdiff(&start, &end);
+        gf_log(this->name, GF_LOG_INFO,
+               "Migration of "
+               "file:%s size:%" PRIu64
+               " bytes took %.2f"
+               "secs and ret: %d",
+               entry_loc.name, iatt.ia_size, elapsed / 1e6, ret);
+    }
 
-}
+out:
+    if (statfs_frame) {
+        STACK_DESTROY(statfs_frame->root);
+    }
+
+    if (iatt_ptr) {
+        LOCK(&defrag->lock);
+        {
+            defrag->size_processed += iatt_ptr->ia_size;
+        }
+        UNLOCK(&defrag->lock);
+    }
+    loc_wipe(&entry_loc);
 
+    return ret;
+}
 
-int
-gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
-                  dict_t *fix_layout, dict_t *migrate_data)
+void *
+gf_defrag_task(void *opaque)
 {
-        int                      ret            = -1;
-        loc_t                    entry_loc      = {0,};
-        fd_t                    *fd             = NULL;
-        gf_dirent_t              entries;
-        gf_dirent_t             *tmp            = NULL;
-        gf_dirent_t             *entry          = NULL;
-        gf_boolean_t             free_entries   = _gf_false;
-        dict_t                  *dict           = NULL;
-        off_t                    offset         = 0;
-        struct iatt              iatt           = {0,};
-
-        ret = syncop_lookup (this, loc, NULL, &iatt, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Lookup failed on %s",
-                        loc->path);
-                goto out;
+    struct list_head *q_head = NULL;
+    struct dht_container *iterator = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    int ret = 0;
+    pid_t pid = GF_CLIENT_PID_DEFRAG;
+
+    defrag = (gf_defrag_info_t *)opaque;
+    if (!defrag) {
+        gf_msg("dht", GF_LOG_ERROR, 0, 0, "defrag is NULL");
+        goto out;
+    }
+
+    syncopctx_setfspid(&pid);
+
+    q_head = &(defrag->queue[0].list);
+
+    /* The following while loop will dequeue one entry from the defrag->queue
+       under lock. We will update the defrag->global_error only when there
+       is an error which is critical to stop the rebalance process. The stop
+       message will be intimated to other migrator threads by setting the
+       defrag->defrag_status to GF_DEFRAG_STATUS_FAILED.
+
+       In defrag->queue, a low watermark (MIN_MIGRATE_QUEUE_COUNT) is
+       maintained so that crawler does not starve the file migration
+       workers and a high watermark (MAX_MIGRATE_QUEUE_COUNT) so that
+       crawler does not go far ahead in filling up the queue.
+     */
+
+    while (_gf_true) {
+        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+            pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
+            pthread_cond_broadcast(&defrag->parallel_migration_cond);
+            goto out;
         }
 
-        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
-                ret = gf_defrag_migrate_data (this, defrag, loc, migrate_data);
-                if (ret)
-                        goto out;
+        pthread_mutex_lock(&defrag->dfq_mutex);
+        {
+            /*Throttle down:
+              If the reconfigured count is less than current thread
+              count, then the current thread will sleep */
+
+            /*TODO: Need to refactor the following block to work
+             *under defrag->lock. For now access
+             * defrag->current_thread_count and rthcount under
+             * dfq_mutex lock */
+            while (!defrag->crawl_done && (defrag->recon_thread_count <
+                                           defrag->current_thread_count)) {
+                defrag->current_thread_count--;
+                gf_msg_debug("DHT", 0,
+                             "Thread sleeping. "
+                             "current thread count: %d",
+                             defrag->current_thread_count);
+
+                pthread_cond_wait(&defrag->df_wakeup_thread,
+                                  &defrag->dfq_mutex);
+
+                defrag->current_thread_count++;
+                gf_msg_debug("DHT", 0,
+                             "Thread wokeup. "
+                             "current thread count: %d",
+                             defrag->current_thread_count);
+            }
+
+            if (defrag->q_entry_count) {
+                iterator = list_entry(q_head->next, typeof(*iterator), list);
+
+                gf_msg_debug("DHT", 0,
+                             "picking entry "
+                             "%s",
+                             iterator->df_entry->d_name);
+
+                list_del_init(&(iterator->list));
+
+                defrag->q_entry_count--;
+
+                if ((defrag->q_entry_count < MIN_MIGRATE_QUEUE_COUNT) &&
+                    defrag->wakeup_crawler) {
+                    pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
+                }
+                pthread_mutex_unlock(&defrag->dfq_mutex);
+                ret = gf_defrag_migrate_single_file((void *)iterator);
+
+                /*Critical errors: ENOTCONN and ENOSPACE*/
+                if (ret) {
+                    dht_set_global_defrag_error(defrag, ret);
+
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+
+                    pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
+
+                    pthread_cond_broadcast(&defrag->parallel_migration_cond);
+
+                    goto out;
+                }
+
+                gf_defrag_free_container(iterator);
+
+                continue;
+            } else {
+                /* defrag->crawl_done flag is set means crawling
+                 file system is done and hence a list_empty when
+                 the above flag is set indicates there are no more
+                 entries to be added to the queue and rebalance is
+                 finished */
+
+                if (!defrag->crawl_done) {
+                    defrag->current_thread_count--;
+                    gf_msg_debug("DHT", 0,
+                                 "Thread "
+                                 "sleeping while  waiting "
+                                 "for migration entries. "
+                                 "current thread  count:%d",
+                                 defrag->current_thread_count);
+
+                    pthread_cond_wait(&defrag->parallel_migration_cond,
+                                      &defrag->dfq_mutex);
+                }
+
+                if (defrag->crawl_done && !defrag->q_entry_count) {
+                    defrag->current_thread_count++;
+                    gf_msg_debug("DHT", 0, "Exiting thread");
+
+                    pthread_cond_broadcast(&defrag->parallel_migration_cond);
+                    goto unlock;
+                } else {
+                    defrag->current_thread_count++;
+                    gf_msg_debug("DHT", 0,
+                                 "Thread woke up"
+                                 " as found migrating entries. "
+                                 "current thread count:%d",
+                                 defrag->current_thread_count);
+
+                    pthread_mutex_unlock(&defrag->dfq_mutex);
+                    continue;
+                }
+            }
         }
+    unlock:
+        pthread_mutex_unlock(&defrag->dfq_mutex);
+        break;
+    }
+out:
+    return NULL;
+}
 
-        gf_log (this->name, GF_LOG_TRACE, "fix layout called on %s", loc->path);
+int static gf_defrag_get_entry(xlator_t *this, int i,
+                               struct dht_container **container, loc_t *loc,
+                               dht_conf_t *conf, gf_defrag_info_t *defrag,
+                               fd_t *fd, dict_t *migrate_data,
+                               struct dir_dfmeta *dir_dfmeta, dict_t *xattr_req,
+                               int *perrno)
+{
+    int ret = 0;
+    char is_linkfile = 0;
+    gf_dirent_t *df_entry = NULL;
+    struct dht_container *tmp_container = NULL;
+
+    if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+        ret = -1;
+        goto out;
+    }
 
-        fd = fd_create (loc->inode, defrag->pid);
+    if (dir_dfmeta->offset_var[i].readdir_done == 1) {
+        ret = 0;
+        goto out;
+    }
+
+    if (dir_dfmeta->fetch_entries[i] == 1) {
         if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
-                ret = -1;
-                goto out;
+            dir_dfmeta->fetch_entries[i] = 0;
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
         }
 
-        ret = syncop_opendir (this, loc, fd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
-                        loc->path);
-                ret = -1;
-                goto out;
+        ret = syncop_readdirp(conf->local_subvols[i], fd, 131072,
+                              dir_dfmeta->offset_var[i].offset,
+                              &(dir_dfmeta->equeue[i]), xattr_req, NULL);
+        if (ret == 0) {
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
         }
 
-        INIT_LIST_HEAD (&entries.list);
-        while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
-                &entries)) != 0)
-        {
-                if ((ret < 0) || (ret && (errno == ENOENT)))
-                        break;
-                free_entries = _gf_true;
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_DATA_FAILED,
+                   "Readdirp failed. Aborting data migration for "
+                   "directory: %s",
+                   loc->path);
+            *perrno = -ret;
+            ret = -1;
+            goto out;
+        }
 
-                if (list_empty (&entries.list))
-                        break;
-                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
-                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                                ret = 1;
-                                goto out;
-                        }
+        if (list_empty(&(dir_dfmeta->equeue[i].list))) {
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
+        }
 
-                        offset = entry->d_off;
+        dir_dfmeta->fetch_entries[i] = 0;
+    }
 
-                        if (!strcmp (entry->d_name, ".") ||
-                            !strcmp (entry->d_name, ".."))
-                                continue;
+    while (1) {
+        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+            ret = -1;
+            goto out;
+        }
 
-                        if (!IA_ISDIR (entry->d_stat.ia_type))
-                                continue;
+        df_entry = list_entry(dir_dfmeta->iterator[i]->next, typeof(*df_entry),
+                              list);
 
-                        loc_wipe (&entry_loc);
-                        ret =dht_build_child_loc (this, &entry_loc, loc,
-                                                  entry->d_name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
-                                        " build failed");
-                                goto out;
-                        }
+        if (&df_entry->list == dir_dfmeta->head[i]) {
+            gf_dirent_free(&(dir_dfmeta->equeue[i]));
+            INIT_LIST_HEAD(&(dir_dfmeta->equeue[i].list));
+            dir_dfmeta->fetch_entries[i] = 1;
+            dir_dfmeta->iterator[i] = dir_dfmeta->head[i];
+            ret = 0;
+            goto out;
+        }
 
-                        if (uuid_is_null (entry->d_stat.ia_gfid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        "gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
+        dir_dfmeta->iterator[i] = dir_dfmeta->iterator[i]->next;
 
-                        entry_loc.inode->ia_type = entry->d_stat.ia_type;
+        dir_dfmeta->offset_var[i].offset = df_entry->d_off;
+        if (!strcmp(df_entry->d_name, ".") || !strcmp(df_entry->d_name, ".."))
+            continue;
 
-                        uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
-                        if (uuid_is_null (loc->gfid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        "gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
+        if (IA_ISDIR(df_entry->d_stat.ia_type)) {
+            defrag->size_processed += df_entry->d_stat.ia_size;
+            continue;
+        }
 
-                        uuid_copy (entry_loc.pargfid, loc->gfid);
+        defrag->num_files_lookedup++;
 
-                        ret = syncop_lookup (this, &entry_loc, NULL, &iatt,
-                                             NULL, NULL);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s"
-                                        " lookup failed", entry_loc.path);
-                                continue;
-                        }
+        if (defrag->defrag_pattern &&
+            (gf_defrag_pattern_match(defrag, df_entry->d_name,
+                                     df_entry->d_stat.ia_size) == _gf_false)) {
+            defrag->size_processed += df_entry->d_stat.ia_size;
+            continue;
+        }
 
-                        ret = syncop_setxattr (this, &entry_loc, fix_layout,
-                                               0);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Setxattr "
-                                        "failed for %s", entry_loc.path);
-                                defrag->defrag_status =
-                                GF_DEFRAG_STATUS_FAILED;
-                                goto out;
-                        }
-                        ret = gf_defrag_fix_layout (this, defrag, &entry_loc,
-                                                    fix_layout, migrate_data);
+        is_linkfile = check_is_linkfile(NULL, &df_entry->d_stat, df_entry->dict,
+                                        conf->link_xattr_name);
 
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Fix layout "
-                                        "failed for %s", entry_loc.path);
-                                goto out;
-                        }
+        if (is_linkfile) {
+            /* No need to add linkto file to the queue for
+               migration. Only the actual data file need to
+               be checked for migration criteria.
+            */
 
-                }
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-                INIT_LIST_HEAD (&entries.list);
+            gf_msg_debug(this->name, 0,
+                         "Skipping linkfile"
+                         " %s on subvol: %s",
+                         df_entry->d_name, conf->local_subvols[i]->name);
+            continue;
         }
 
-        ret = 0;
-out:
-        if (free_entries)
-                gf_dirent_free (&entries);
+        /*Build Container Structure */
+
+        tmp_container = GF_CALLOC(1, sizeof(struct dht_container),
+                                  gf_dht_mt_container_t);
+        if (!tmp_container) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for container");
+            ret = -1;
+            goto out;
+        }
+        tmp_container->df_entry = gf_dirent_for_name(df_entry->d_name);
+        if (!tmp_container->df_entry) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for df_entry");
+            ret = -1;
+            goto out;
+        }
 
-        loc_wipe (&entry_loc);
+        tmp_container->local_subvol_index = i;
 
-        if (dict)
-                dict_unref(dict);
+        tmp_container->df_entry->d_stat = df_entry->d_stat;
 
-        if (fd)
-                fd_unref (fd);
+        tmp_container->df_entry->d_ino = df_entry->d_ino;
 
-        return ret;
+        tmp_container->df_entry->d_type = df_entry->d_type;
 
-}
+        tmp_container->df_entry->d_len = df_entry->d_len;
 
+        tmp_container->parent_loc = GF_CALLOC(1, sizeof(*loc), gf_dht_mt_loc_t);
+        if (!tmp_container->parent_loc) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for loc");
+            ret = -1;
+            goto out;
+        }
+
+        ret = loc_copy(tmp_container->parent_loc, loc);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "loc_copy failed");
+            ret = -1;
+            goto out;
+        }
+
+        tmp_container->migrate_data = migrate_data;
+
+        tmp_container->this = this;
+
+        if (df_entry->dict)
+            tmp_container->df_entry->dict = dict_ref(df_entry->dict);
+
+        /*Build Container Structure >> END*/
+
+        ret = 0;
+        goto out;
+    }
+
+out:
+    if (ret == 0) {
+        *container = tmp_container;
+    } else {
+        if (tmp_container) {
+            gf_defrag_free_container(tmp_container);
+        }
+    }
+
+    return ret;
+}
 
 int
-gf_defrag_start_crawl (void *data)
+gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                      dict_t *migrate_data, int *perrno)
 {
-        xlator_t                *this   = NULL;
-        dht_conf_t              *conf   = NULL;
-        gf_defrag_info_t        *defrag = NULL;
-        int                      ret    = -1;
-        loc_t                    loc    = {0,};
-        struct iatt              iatt   = {0,};
-        struct iatt              parent = {0,};
-        dict_t                  *fix_layout = NULL;
-        dict_t                  *migrate_data = NULL;
-        dict_t                  *status = NULL;
-        glusterfs_ctx_t         *ctx = NULL;
-
-        this = data;
-        if (!this)
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    gf_dirent_t entries;
+    dict_t *xattr_req = NULL;
+    struct timeval dir_start = {
+        0,
+    };
+    struct timeval end = {
+        0,
+    };
+    double elapsed = {
+        0,
+    };
+    int local_subvols_cnt = 0;
+    int i = 0;
+    int j = 0;
+    struct dht_container *container = NULL;
+    int ldfq_count = 0;
+    int dfc_index = 0;
+    int throttle_up = 0;
+    struct dir_dfmeta *dir_dfmeta = NULL;
+    xlator_t *old_THIS = NULL;
+
+    gf_log(this->name, GF_LOG_INFO, "migrate data called on %s", loc->path);
+    gettimeofday(&dir_start, NULL);
+
+    conf = this->private;
+    local_subvols_cnt = conf->local_subvols_cnt;
+
+    if (!local_subvols_cnt) {
+        ret = 0;
+        goto out;
+    }
+
+    old_THIS = THIS;
+    THIS = this;
+
+    dir_dfmeta = GF_CALLOC(1, sizeof(*dir_dfmeta), gf_common_mt_pointer);
+    if (!dir_dfmeta) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->lfd = GF_CALLOC(local_subvols_cnt, sizeof(fd_t *),
+                                gf_common_mt_pointer);
+    if (!dir_dfmeta->lfd) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_INSUFF_MEMORY,
+                "for dir_dfmeta", NULL);
+        ret = -1;
+        *perrno = ENOMEM;
+        goto out;
+    }
+
+    for (i = 0; i < local_subvols_cnt; i++) {
+        dir_dfmeta->lfd[i] = fd_create(loc->inode, defrag->pid);
+        if (!dir_dfmeta->lfd[i]) {
+            gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_FD_CREATE_FAILED,
+                    NULL);
+            *perrno = ENOMEM;
+            ret = -1;
+            goto out;
+        }
+
+        ret = syncop_opendir(conf->local_subvols[i], loc, dir_dfmeta->lfd[i],
+                             NULL, NULL);
+        if (ret) {
+            fd_unref(dir_dfmeta->lfd[i]);
+            dir_dfmeta->lfd[i] = NULL;
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FAILED_TO_OPEN,
+                    "dir: %s", loc->path, "subvol: %s",
+                    conf->local_subvols[i]->name, NULL);
+
+            if (conf->decommission_in_progress) {
+                *perrno = -ret;
+                ret = -1;
                 goto out;
+            }
+        } else {
+            fd_bind(dir_dfmeta->lfd[i]);
+        }
+    }
+
+    dir_dfmeta->head = GF_CALLOC(local_subvols_cnt, sizeof(*(dir_dfmeta->head)),
+                                 gf_common_mt_pointer);
+    if (!dir_dfmeta->head) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->head is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->iterator = GF_CALLOC(local_subvols_cnt,
+                                     sizeof(*(dir_dfmeta->iterator)),
+                                     gf_common_mt_pointer);
+    if (!dir_dfmeta->iterator) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->iterator is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->equeue = GF_CALLOC(local_subvols_cnt, sizeof(entries),
+                                   gf_dht_mt_dirent_t);
+    if (!dir_dfmeta->equeue) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->equeue is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->offset_var = GF_CALLOC(
+        local_subvols_cnt, sizeof(dht_dfoffset_ctx_t), gf_dht_mt_octx_t);
+    if (!dir_dfmeta->offset_var) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->offset_var is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_defrag_ctx_subvols_init(dir_dfmeta->offset_var, this);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "dht_dfoffset_ctx_t"
+               "initialization failed");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->fetch_entries = GF_CALLOC(local_subvols_cnt, sizeof(int),
+                                          gf_common_mt_int);
+    if (!dir_dfmeta->fetch_entries) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_INSUFF_MEMORY,
+                "for dir_dfmeta->fetch_entries", NULL);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < local_subvols_cnt; i++) {
+        INIT_LIST_HEAD(&(dir_dfmeta->equeue[i].list));
+        dir_dfmeta->head[i] = &(dir_dfmeta->equeue[i].list);
+        dir_dfmeta->iterator[i] = dir_dfmeta->head[i];
+        dir_dfmeta->fetch_entries[i] = 1;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set dict for "
+               "key: %s",
+               conf->link_xattr_name);
+        ret = -1;
+        goto out;
+    }
+
+    /*
+     Job: Read entries from each local subvol and store the entries
+          in equeue array of linked list. Now pick one entry from the
+          equeue array in a round robin basis and add them to defrag Queue.
+    */
+
+    while (!dht_dfreaddirp_done(dir_dfmeta->offset_var, local_subvols_cnt)) {
+        pthread_mutex_lock(&defrag->dfq_mutex);
+        {
+            /*Throttle up: If reconfigured count is higher than
+              current thread count, wake up the sleeping threads
+              TODO: Need to refactor this. Instead of making the
+              thread sleep and wake, we should terminate and spawn
+              threads on-demand*/
+
+            if (defrag->recon_thread_count > defrag->current_thread_count) {
+                throttle_up = (defrag->recon_thread_count -
+                               defrag->current_thread_count);
+                for (j = 0; j < throttle_up; j++) {
+                    pthread_cond_signal(&defrag->df_wakeup_thread);
+                }
+            }
+
+            while (defrag->q_entry_count > MAX_MIGRATE_QUEUE_COUNT) {
+                defrag->wakeup_crawler = 1;
+                pthread_cond_wait(&defrag->rebalance_crawler_alarm,
+                                  &defrag->dfq_mutex);
+            }
+
+            ldfq_count = defrag->q_entry_count;
+
+            if (defrag->wakeup_crawler) {
+                defrag->wakeup_crawler = 0;
+            }
+        }
+        pthread_mutex_unlock(&defrag->dfq_mutex);
+
+        while (
+            ldfq_count <= MAX_MIGRATE_QUEUE_COUNT &&
+            !dht_dfreaddirp_done(dir_dfmeta->offset_var, local_subvols_cnt)) {
+            ret = gf_defrag_get_entry(this, dfc_index, &container, loc, conf,
+                                      defrag, dir_dfmeta->lfd[dfc_index],
+                                      migrate_data, dir_dfmeta, xattr_req,
+                                      perrno);
 
-        ctx = this->ctx;
-        if (!ctx)
+            if (defrag->defrag_status == GF_DEFRAG_STATUS_STOPPED) {
                 goto out;
+            }
 
-        conf = this->private;
-        if (!conf)
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Found "
+                       "error from gf_defrag_get_entry");
+
+                ret = -1;
                 goto out;
+            }
+
+            /* Check if we got an entry, else we need to move the
+               index to the next subvol */
+            if (!container) {
+                GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt);
+                continue;
+            }
+
+            /* Q this entry in the dfq */
+            pthread_mutex_lock(&defrag->dfq_mutex);
+            {
+                list_add_tail(&container->list, &(defrag->queue[0].list));
+                defrag->q_entry_count++;
+                ldfq_count = defrag->q_entry_count;
+
+                gf_msg_debug(this->name, 0,
+                             "added "
+                             "file:%s parent:%s to the queue ",
+                             container->df_entry->d_name,
+                             container->parent_loc->path);
+
+                pthread_cond_signal(&defrag->parallel_migration_cond);
+            }
+            pthread_mutex_unlock(&defrag->dfq_mutex);
+
+            GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt);
+        }
+    }
+
+    gettimeofday(&end, NULL);
+    elapsed = gf_tvdiff(&dir_start, &end);
+    gf_log(this->name, GF_LOG_INFO,
+           "Migration operation on dir %s took "
+           "%.2f secs",
+           loc->path, elapsed / 1e6);
+    ret = 0;
+out:
+    THIS = old_THIS;
+    gf_defrag_free_dir_dfmeta(dir_dfmeta, local_subvols_cnt);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    /* It does not matter if it errored out - this number is
+     * used to calculate rebalance estimated time to complete.
+     * No locking required as dirs are processed by a single thread.
+     */
+    defrag->num_dirs_processed++;
+    return ret;
+}
+
+int
+gf_defrag_settle_hash(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                      dict_t *fix_layout)
+{
+    int ret;
+    dht_conf_t *conf = NULL;
+    /*
+     * Now we're ready to update the directory commit hash for the volume
+     * root, so that hash miscompares and broadcast lookups can stop.
+     * However, we want to skip that if fix-layout is all we did.  In
+     * that case, we want the miscompares etc. to continue until a real
+     * rebalance is complete.
+     */
+    if (defrag->cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX ||
+        defrag->cmd == GF_DEFRAG_CMD_DETACH_START) {
+        return 0;
+    }
+
+    conf = this->private;
+    if (!conf) {
+        /*Uh oh
+         */
+        return -1;
+    }
+
+    if (conf->local_subvols_cnt == 0 || !conf->lookup_optimize) {
+        /* Commit hash updates are only done on local subvolumes and
+         * only when lookup optimization is needed (for older client
+         * support)
+         */
+        return 0;
+    }
+
+    ret = dict_set_uint32(fix_layout, "new-commit-hash",
+                          defrag->new_commit_hash);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to set new-commit-hash");
+        return -1;
+    }
+
+    ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+               "fix layout on %s failed", loc->path);
+
+        if (-ret == ENOENT || -ret == ESTALE) {
+            /* Dir most likely is deleted */
+            return 0;
+        }
+
+        return -1;
+    }
+
+    /* TBD: find more efficient solution than adding/deleting every time */
+    dict_del(fix_layout, "new-commit-hash");
+
+    return 0;
+}
+
+int
+gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                     dict_t *fix_layout, dict_t *migrate_data)
+{
+    int ret = -1;
+    loc_t entry_loc = {
+        0,
+    };
+    fd_t *fd = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *tmp = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_boolean_t free_entries = _gf_false;
+    off_t offset = 0;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *linked_inode = NULL, *inode = NULL;
+    dht_conf_t *conf = NULL;
+    int perrno = 0;
+
+    conf = this->private;
+    if (!conf) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_lookup(this, loc, &iatt, NULL, NULL, NULL);
+    if (ret) {
+        if (strcmp(loc->path, "/") == 0) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "lookup failed for:%s", loc->path);
+
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
+        }
+
+        if (-ret == ENOENT || -ret == ESTALE) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "Dir:%s renamed or removed. Skipping", loc->path);
+            if (conf->decommission_subvols_cnt) {
+                defrag->total_failures++;
+            }
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "lookup failed for:%s", loc->path);
+
+            defrag->total_failures++;
+            goto out;
+        }
+    }
 
-        defrag = conf->defrag;
-        if (!defrag)
+    fd = fd_create(loc->inode, defrag->pid);
+    if (!fd) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to create fd");
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_opendir(this, loc, fd, NULL, NULL);
+    if (ret) {
+        if (-ret == ENOENT || -ret == ESTALE) {
+            if (conf->decommission_subvols_cnt) {
+                defrag->total_failures++;
+            }
+            ret = 0;
+            goto out;
+        }
+
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to open dir %s, "
+               "err:%d",
+               loc->path, -ret);
+
+        ret = -1;
+        goto out;
+    }
+
+    fd_bind(fd);
+    INIT_LIST_HEAD(&entries.list);
+
+    while ((ret = syncop_readdirp(this, fd, 131072, offset, &entries, NULL,
+                                  NULL)) != 0) {
+        if (ret < 0) {
+            if (-ret == ENOENT || -ret == ESTALE) {
+                if (conf->decommission_subvols_cnt) {
+                    defrag->total_failures++;
+                }
+                ret = 0;
                 goto out;
+            }
+
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_READDIR_ERROR,
+                   "readdirp failed for "
+                   "path %s. Aborting fix-layout",
+                   loc->path);
+
+            ret = -1;
+            goto out;
+        }
 
-        gettimeofday (&defrag->start_time, NULL);
-        dht_build_root_inode (this, &defrag->root_inode);
-        if (!defrag->root_inode)
+        if (list_empty(&entries.list))
+            break;
+
+        free_entries = _gf_true;
+
+        list_for_each_entry_safe(entry, tmp, &entries.list, list)
+        {
+            if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+                ret = 1;
                 goto out;
+            }
 
-        dht_build_root_loc (defrag->root_inode, &loc);
+            offset = entry->d_off;
 
-        /* fix-layout on '/' first */
+            if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                continue;
+            if (!IA_ISDIR(entry->d_stat.ia_type)) {
+                continue;
+            }
+            loc_wipe(&entry_loc);
 
-        ret = syncop_lookup (this, &loc, NULL, &iatt, NULL, &parent);
+            ret = dht_build_child_loc(this, &entry_loc, loc, entry->d_name);
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "Child loc"
+                       " build failed for entry: %s",
+                       entry->d_name);
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "look up on / failed");
+                if (conf->decommission_in_progress) {
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+
+                    goto out;
+                } else {
+                    continue;
+                }
+            }
+
+            if (gf_uuid_is_null(entry->d_stat.ia_gfid)) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "%s/%s"
+                       " gfid not present",
+                       loc->path, entry->d_name);
+                continue;
+            }
+
+            gf_uuid_copy(entry_loc.gfid, entry->d_stat.ia_gfid);
+
+            /*In case the gfid stored in the inode by inode_link
+             * and the gfid obtained in the lookup differs, then
+             * client3_3_lookup_cbk will return ESTALE and proper
+             * error will be captured
+             */
+
+            linked_inode = inode_link(entry_loc.inode, loc->inode,
+                                      entry->d_name, &entry->d_stat);
+
+            inode = entry_loc.inode;
+            entry_loc.inode = linked_inode;
+            inode_unref(inode);
+
+            if (gf_uuid_is_null(loc->gfid)) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "%s/%s"
+                       " gfid not present",
+                       loc->path, entry->d_name);
+                continue;
+            }
+
+            gf_uuid_copy(entry_loc.pargfid, loc->gfid);
+
+            ret = syncop_lookup(this, &entry_loc, &iatt, NULL, NULL, NULL);
+            if (ret) {
+                if (-ret == ENOENT || -ret == ESTALE) {
+                    gf_msg(this->name, GF_LOG_INFO, -ret,
+                           DHT_MSG_DIR_LOOKUP_FAILED,
+                           "Dir:%s renamed or removed. "
+                           "Skipping",
+                           loc->path);
+                    ret = 0;
+                    if (conf->decommission_subvols_cnt) {
+                        defrag->total_failures++;
+                    }
+                    continue;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_DIR_LOOKUP_FAILED, "lookup failed for:%s",
+                           entry_loc.path);
+
+                    defrag->total_failures++;
+
+                    if (conf->decommission_in_progress) {
+                        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+                        ret = -1;
+                        goto out;
+                    } else {
+                        continue;
+                    }
+                }
+            }
+
+            /* A return value of 2 means, either process_dir or
+             * lookup of a dir failed. Hence, don't commit hash
+             * for the current directory*/
+
+            ret = gf_defrag_fix_layout(this, defrag, &entry_loc, fix_layout,
+                                       migrate_data);
+
+            if (defrag->defrag_status == GF_DEFRAG_STATUS_STOPPED ||
+                defrag->defrag_status == GF_DEFRAG_STATUS_FAILED) {
                 goto out;
+            }
+
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_FIX_FAILED,
+                       "Fix layout failed for %s", entry_loc.path);
+
+                defrag->total_failures++;
+
+                if (conf->decommission_in_progress) {
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+
+                    goto out;
+                } else {
+                    /* Let's not commit-hash if
+                     * gf_defrag_fix_layout failed*/
+                    continue;
+                }
+            }
         }
 
-        fix_layout = dict_new ();
-        if (!fix_layout) {
+        gf_dirent_free(&entries);
+        free_entries = _gf_false;
+        INIT_LIST_HEAD(&entries.list);
+    }
+
+    /* A directory layout is fixed only after its subdirs are healed to
+     * any newly added bricks. If the layout is fixed before subdirs are
+     * healed, the newly added brick will get a non-null layout.
+     * Any subdirs which hash to that layout will no longer show up
+     * in a directory listing until they are healed.
+     */
+
+    ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL);
+
+    /* In case of a race where the directory is deleted just before
+     * layout setxattr, the errors are updated in the layout structure.
+     * We can use this information to make a decision whether the directory
+     * is deleted entirely.
+     */
+    if (ret == 0) {
+        ret = dht_dir_layout_error_check(this, loc->inode);
+        ret = -ret;
+    }
+
+    if (ret) {
+        if (-ret == ENOENT || -ret == ESTALE) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+                   "Setxattr failed. Dir %s "
+                   "renamed or removed",
+                   loc->path);
+            if (conf->decommission_subvols_cnt) {
+                defrag->total_failures++;
+            }
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+                   "Setxattr failed for %s", loc->path);
+
+            defrag->total_failures++;
+
+            if (conf->decommission_in_progress) {
+                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
                 ret = -1;
                 goto out;
+            }
         }
+    }
 
-        ret = dict_set_str (fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes");
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set dict str");
-                goto out;
-        }
+    if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+        ret = gf_defrag_process_dir(this, defrag, loc, migrate_data, &perrno);
 
-        ret = syncop_setxattr (this, &loc, fix_layout, 0);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "fix layout on %s failed",
-                        loc.path);
+            if (perrno == ENOENT || perrno == ESTALE) {
+                ret = 0;
                 goto out;
-        }
+            } else {
+                defrag->total_failures++;
 
-        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
-                migrate_data = dict_new ();
-                if (!migrate_data) {
-                        ret = -1;
-                        goto out;
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       DHT_MSG_DEFRAG_PROCESS_DIR_FAILED,
+                       "gf_defrag_process_dir failed for "
+                       "directory: %s",
+                       loc->path);
+
+                if (conf->decommission_in_progress) {
+                    goto out;
                 }
-                if (defrag->cmd == GF_DEFRAG_CMD_START_FORCE)
-                        ret = dict_set_str (migrate_data,
-                                            "distribute.migrate-data", "force");
-                else
-                        ret = dict_set_str (migrate_data,
-                                            "distribute.migrate-data",
-                                            "non-force");
-                if (ret)
-                        goto out;
+            }
         }
-        ret = gf_defrag_fix_layout (this, defrag, &loc, fix_layout,
-                                    migrate_data);
-        if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
-            (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
-                defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+    }
+
+    gf_msg_trace(this->name, 0, "fix layout called on %s", loc->path);
+
+    if (gf_defrag_settle_hash(this, defrag, loc, fix_layout) != 0) {
+        defrag->total_failures++;
+
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SETTLE_HASH_FAILED,
+               "Settle hash failed for %s", loc->path);
+
+        ret = -1;
+
+        if (conf->decommission_in_progress) {
+            defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+            goto out;
         }
+    }
+
+    ret = 0;
+out:
+    if (free_entries)
+        gf_dirent_free(&entries);
 
+    loc_wipe(&entry_loc);
 
+    if (fd)
+        fd_unref(fd);
+
+    return ret;
+}
+
+int
+dht_init_local_subvols_and_nodeuuids(xlator_t *this, dht_conf_t *conf,
+                                     loc_t *loc)
+{
+    dict_t *dict = NULL;
+    uuid_t *uuid_ptr = NULL;
+    int ret = -1;
+    int i = 0;
+    int j = 0;
+
+    /* Find local subvolumes */
+    ret = syncop_getxattr(this, loc, &dict, GF_REBAL_FIND_LOCAL_SUBVOL, NULL,
+                          NULL);
+    if (ret && (ret != -ENODATA)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+               "local "
+               "subvolume determination failed with error: %d",
+               -ret);
+        ret = -1;
+        goto out;
+    }
+
+    if (!ret)
+        goto out;
+
+    ret = syncop_getxattr(this, loc, &dict, GF_REBAL_OLD_FIND_LOCAL_SUBVOL,
+                          NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+               "local "
+               "subvolume determination failed with error: %d",
+               -ret);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 
 out:
-        LOCK (&defrag->lock);
-        {
-                status = dict_new ();
-                gf_defrag_status_get (defrag, status);
-                if (ctx->notify)
-                        ctx->notify (GF_EN_DEFRAG_STATUS, status);
-                if (status)
-                        dict_unref (status);
-                defrag->is_exiting = 1;
+    if (ret) {
+        return ret;
+    }
+
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "local subvol: "
+               "%s",
+               conf->local_subvols[i]->name);
+
+        for (j = 0; j < conf->local_nodeuuids[i].count; j++) {
+            uuid_ptr = &(conf->local_nodeuuids[i].elements[j].uuid);
+            gf_msg(this->name, GF_LOG_INFO, 0, 0, "node uuid : %s",
+                   uuid_utoa(*uuid_ptr));
         }
-        UNLOCK (&defrag->lock);
+    }
+
+    return ret;
+}
+
+/* Functions for the rebalance estimates feature */
+
+uint64_t
+gf_defrag_subvol_file_size(xlator_t *this, loc_t *root_loc)
+{
+    int ret = -1;
+    struct statvfs buf = {
+        0,
+    };
+
+    ret = syncop_statfs(this, root_loc, &buf, NULL, NULL);
+    if (ret) {
+        /* Aargh! */
+        return 0;
+    }
+    return ((buf.f_blocks - buf.f_bfree) * buf.f_frsize);
+}
+
+uint64_t
+gf_defrag_total_file_size(xlator_t *this, loc_t *root_loc)
+{
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    uint64_t size_files = 0;
+    uint64_t total_size = 0;
 
-        if (defrag) {
-                GF_FREE (defrag);
-                conf->defrag = NULL;
+    conf = this->private;
+    if (!conf) {
+        return 0;
+    }
+
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        size_files = gf_defrag_subvol_file_size(conf->local_subvols[i],
+                                                root_loc);
+        total_size += size_files;
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "local subvol: %s,"
+               "cnt = %" PRIu64,
+               conf->local_subvols[i]->name, size_files);
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, 0, "Total size files = %" PRIu64,
+           total_size);
+
+    return total_size;
+}
+
+static void *
+dht_file_counter_thread(void *args)
+{
+    gf_defrag_info_t *defrag = NULL;
+    loc_t root_loc = {
+        0,
+    };
+    struct timespec time_to_wait = {
+        0,
+    };
+    uint64_t tmp_size = 0;
+
+    if (!args)
+        return NULL;
+
+    defrag = (gf_defrag_info_t *)args;
+    dht_build_root_loc(defrag->root_inode, &root_loc);
+
+    while (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) {
+        timespec_now(&time_to_wait);
+        time_to_wait.tv_sec += 600;
+
+        pthread_mutex_lock(&defrag->fc_mutex);
+        pthread_cond_timedwait(&defrag->fc_wakeup_cond, &defrag->fc_mutex,
+                               &time_to_wait);
+
+        pthread_mutex_unlock(&defrag->fc_mutex);
+
+        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED)
+            break;
+
+        tmp_size = gf_defrag_total_file_size(defrag->this, &root_loc);
+
+        gf_log("dht", GF_LOG_INFO, "tmp data size =%" PRIu64, tmp_size);
+
+        if (!tmp_size) {
+            gf_msg("dht", GF_LOG_ERROR, 0, 0,
+                   "Failed to get "
+                   "the total data size. Unable to estimate "
+                   "time to complete rebalance.");
+        } else {
+            g_totalsize = tmp_size;
+            gf_msg_debug("dht", 0, "total data size =%" PRIu64, g_totalsize);
         }
+    }
 
-        return ret;
+    return NULL;
 }
 
+int
+gf_defrag_estimates_cleanup(xlator_t *this, gf_defrag_info_t *defrag,
+                            pthread_t filecnt_thread)
+{
+    int ret = -1;
+
+    /* Wake up the filecounter thread.
+     * By now the defrag status will no longer be
+     * GF_DEFRAG_STATUS_STARTED so the thread will exit the loop.
+     */
+    pthread_mutex_lock(&defrag->fc_mutex);
+    {
+        pthread_cond_broadcast(&defrag->fc_wakeup_cond);
+    }
+    pthread_mutex_unlock(&defrag->fc_mutex);
+
+    ret = pthread_join(filecnt_thread, NULL);
+    if (ret) {
+        gf_msg("dht", GF_LOG_ERROR, ret, 0,
+               "file_counter_thread: pthread_join failed.");
+        ret = -1;
+    }
+    return ret;
+}
 
-static int
-gf_defrag_done  (int ret, call_frame_t *sync_frame, void *data)
+int
+gf_defrag_estimates_init(xlator_t *this, loc_t *loc, pthread_t *filecnt_thread)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+
+    conf = this->private;
+    defrag = conf->defrag;
+
+    g_totalsize = gf_defrag_total_file_size(this, loc);
+    if (!g_totalsize) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "Failed to get "
+               "the total data size. Unable to estimate "
+               "time to complete rebalance.");
+        goto out;
+    }
+
+    ret = gf_thread_create(filecnt_thread, NULL, dht_file_counter_thread,
+                           (void *)defrag, "dhtfcnt");
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, 0,
+               "Failed to "
+               "create the file counter thread ");
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+out:
+    return ret;
+}
+
+/* Init and cleanup functions for parallel file migration*/
+int
+gf_defrag_parallel_migration_init(xlator_t *this, gf_defrag_info_t *defrag,
+                                  pthread_t **tid_array, int *thread_index)
 {
-        gf_listener_stop (sync_frame->this);
+    int ret = -1;
+    int thread_spawn_count = 0;
+    int index = 0;
+    pthread_t *tid = NULL;
 
-        STACK_DESTROY (sync_frame->root);
-        kill (getpid(), SIGTERM);
-        return 0;
+    if (!defrag)
+        goto out;
+
+    /* Initialize global entry queue */
+    defrag->queue = GF_CALLOC(1, sizeof(struct dht_container),
+                              gf_dht_mt_container_t);
+
+    if (!defrag->queue) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+               "Failed to initialise migration queue");
+        ret = -1;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&(defrag->queue[0].list));
+
+    thread_spawn_count = MAX(MAX_REBAL_THREADS, 4);
+
+    gf_msg_debug(this->name, 0, "thread_spawn_count: %d", thread_spawn_count);
+
+    tid = GF_CALLOC(thread_spawn_count, sizeof(pthread_t),
+                    gf_common_mt_pthread_t);
+    if (!tid) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+               "Failed to create migration threads");
+        ret = -1;
+        goto out;
+    }
+    defrag->current_thread_count = thread_spawn_count;
+
+    /*Spawn Threads Here*/
+    while (index < thread_spawn_count) {
+        ret = gf_thread_create(&(tid[index]), NULL, gf_defrag_task,
+                               (void *)defrag, "dhtmig%d", (index + 1) & 0x3ff);
+        if (ret != 0) {
+            gf_msg("DHT", GF_LOG_ERROR, ret, 0, "Thread[%d] creation failed. ",
+                   index);
+            ret = -1;
+            goto out;
+        } else {
+            gf_log("DHT", GF_LOG_INFO,
+                   "Thread[%d] "
+                   "creation successful",
+                   index);
+        }
+        index++;
+    }
+
+    ret = 0;
+out:
+    *thread_index = index;
+    *tid_array = tid;
+
+    return ret;
 }
 
-void *
-gf_defrag_start (void *data)
+int
+gf_defrag_parallel_migration_cleanup(gf_defrag_info_t *defrag,
+                                     pthread_t *tid_array, int thread_index)
 {
-        int                      ret    = -1;
-        call_frame_t            *frame  = NULL;
-        dht_conf_t              *conf   = NULL;
-        gf_defrag_info_t        *defrag = NULL;
-        xlator_t                *this  = NULL;
-
-        this = data;
-        conf = this->private;
-        if (!conf)
-                goto out;
+    int ret = -1;
+    int i = 0;
 
-        defrag = conf->defrag;
-        if (!defrag)
-                goto out;
+    if (!defrag)
+        goto out;
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+    /* Wake up all migration threads */
+    pthread_mutex_lock(&defrag->dfq_mutex);
+    {
+        defrag->crawl_done = 1;
 
-        frame->root->pid = GF_CLIENT_PID_DEFRAG;
+        pthread_cond_broadcast(&defrag->parallel_migration_cond);
+        pthread_cond_broadcast(&defrag->df_wakeup_thread);
+    }
+    pthread_mutex_unlock(&defrag->dfq_mutex);
 
-        defrag->pid = frame->root->pid;
+    /*Wait for all the threads to complete their task*/
+    for (i = 0; i < thread_index; i++) {
+        pthread_join(tid_array[i], NULL);
+    }
 
-        defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
+    GF_FREE(tid_array);
 
-        ret = synctask_new (this->ctx->env, gf_defrag_start_crawl,
-                            gf_defrag_done, frame, this);
+    /* Cleanup the migration queue */
+    if (defrag->queue) {
+        gf_dirent_free(defrag->queue[0].df_entry);
+        INIT_LIST_HEAD(&(defrag->queue[0].list));
+    }
 
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Could not create"
-                        " task for rebalance");
+    GF_FREE(defrag->queue);
+
+    ret = 0;
 out:
-        return NULL;
+    return ret;
 }
 
 int
-gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
+gf_defrag_start_crawl(void *data)
 {
-        int      ret    = 0;
-        uint64_t files  = 0;
-        uint64_t size   = 0;
-        uint64_t lookup = 0;
-        uint64_t failures = 0;
-        char     *status = "";
-        double   elapsed = 0;
-        struct timeval end = {0,};
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    dict_t *fix_layout = NULL;
+    dict_t *migrate_data = NULL;
+    dict_t *status = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    call_frame_t *statfs_frame = NULL;
+    xlator_t *old_THIS = NULL;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    struct iatt parent = {
+        0,
+    };
+    int thread_index = 0;
+    pthread_t *tid = NULL;
+    pthread_t filecnt_thread;
+    gf_boolean_t fc_thread_started = _gf_false;
+
+    this = data;
+    if (!this)
+        goto exit;
+
+    ctx = this->ctx;
+    if (!ctx)
+        goto exit;
+
+    conf = this->private;
+    if (!conf)
+        goto exit;
+
+    defrag = conf->defrag;
+    if (!defrag)
+        goto exit;
+
+    defrag->start_time = gf_time();
+
+    dht_build_root_inode(this, &defrag->root_inode);
+    if (!defrag->root_inode)
+        goto out;
+
+    dht_build_root_loc(defrag->root_inode, &loc);
+
+    /* fix-layout on '/' first */
+
+    ret = syncop_lookup(this, &loc, &iatt, &parent, NULL, NULL);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_START_FAILED,
+               "Failed to start rebalance: look up on / failed");
+        ret = -1;
+        goto out;
+    }
 
+    old_THIS = THIS;
+    THIS = this;
 
-        if (!defrag)
-                goto out;
+    statfs_frame = create_frame(this, this->ctx->pool);
+    if (!statfs_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+               "Insufficient memory. Frame creation failed");
+        ret = -1;
+        goto out;
+    }
 
-        ret = 0;
-        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
-                goto out;
+    /* async statfs update for honoring min-free-disk */
+    dht_get_du_info(statfs_frame, this, &loc);
+    THIS = old_THIS;
 
-        files  = defrag->total_files;
-        size   = defrag->total_data;
-        lookup = defrag->num_files_lookedup;
-        failures = defrag->total_failures;
+    fix_layout = dict_new();
+    if (!fix_layout) {
+        ret = -1;
+        goto out;
+    }
+
+    /*
+     * Unfortunately, we can't do special xattrs (like fix.layout) and
+     * real ones in the same call currently, and changing it seems
+     * riskier than just doing two calls.
+     */
+
+    gf_log(this->name, GF_LOG_INFO, "%s using commit hash %u", __func__,
+           conf->vol_commit_hash);
+
+    ret = dict_set_uint32(fix_layout, conf->commithash_xattr_name,
+                          conf->vol_commit_hash);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to set %s",
+               conf->commithash_xattr_name);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_setxattr(this, &loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to set commit hash on %s. "
+               "Rebalance cannot proceed.",
+               loc.path);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    /* We now return to our regularly scheduled program. */
+
+    ret = dict_set_str(fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+               "Failed to start rebalance:"
+               "Failed to set dictionary value: key = %s",
+               GF_XATTR_FIX_LAYOUT_KEY);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
 
-        gettimeofday (&end, NULL);
+    defrag->new_commit_hash = conf->vol_commit_hash;
 
-        elapsed = end.tv_sec - defrag->start_time.tv_sec;
+    ret = syncop_setxattr(this, &loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_FAILED,
+               "fix layout on %s failed", loc.path);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
 
-        if (!dict)
-                goto log;
+    if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+        /* We need to migrate files */
 
-        ret = dict_set_uint64 (dict, "files", files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set file count");
+        migrate_data = dict_new();
+        if (!migrate_data) {
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_str(
+            migrate_data, GF_XATTR_FILE_MIGRATE_KEY,
+            (defrag->cmd == GF_DEFRAG_CMD_START_FORCE) ? "force" : "non-force");
+        if (ret) {
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
+        }
 
-        ret = dict_set_uint64 (dict, "size", size);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set size of xfer");
+        ret = dht_init_local_subvols_and_nodeuuids(this, conf, &loc);
+        if (ret) {
+            ret = -1;
+            goto out;
+        }
 
-        ret = dict_set_uint64 (dict, "lookups", lookup);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set lookedup file count");
+        /* Initialise the structures required for parallel migration */
+        ret = gf_defrag_parallel_migration_init(this, defrag, &tid,
+                                                &thread_index);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Aborting rebalance.");
+            goto out;
+        }
 
-        ret = dict_set_int32 (dict, "status", defrag->defrag_status);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set status");
-        if (elapsed) {
-                ret = dict_set_double (dict, "run-time", elapsed);
-                if (ret)
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "failed to set run-time");
+        ret = gf_defrag_estimates_init(this, &loc, &filecnt_thread);
+        if (ret) {
+            /* Not a fatal error. Allow the rebalance to proceed*/
+            ret = 0;
+        } else {
+            fc_thread_started = _gf_true;
         }
+    }
+
+    ret = gf_defrag_fix_layout(this, defrag, &loc, fix_layout, migrate_data);
+    if (ret) {
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    if (gf_defrag_settle_hash(this, defrag, &loc, fix_layout) != 0) {
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    gf_log("DHT", GF_LOG_INFO, "crawling file-system completed");
+out:
+
+    /* We are here means crawling the entire file system is done
+       or something failed. Set defrag->crawl_done flag to intimate
+       the migrator threads to exhaust the defrag->queue and terminate*/
+
+    if (ret) {
+        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+    }
+
+    gf_defrag_parallel_migration_cleanup(defrag, tid, thread_index);
+
+    if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
+        (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
+        defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+    }
+
+    if (fc_thread_started) {
+        gf_defrag_estimates_cleanup(this, defrag, filecnt_thread);
+    }
+
+    dht_send_rebalance_event(this, defrag->cmd, defrag->defrag_status);
+
+    status = dict_new();
+    LOCK(&defrag->lock);
+    {
+        gf_defrag_status_get(conf, status);
+        if (ctx && ctx->notify)
+            ctx->notify(GF_EN_DEFRAG_STATUS, status);
+        if (status)
+            dict_unref(status);
+        defrag->is_exiting = 1;
+    }
+    UNLOCK(&defrag->lock);
+
+    GF_FREE(defrag);
+    conf->defrag = NULL;
+
+    if (migrate_data)
+        dict_unref(migrate_data);
+
+    if (statfs_frame) {
+        STACK_DESTROY(statfs_frame->root);
+    }
+exit:
+    return ret;
+}
+
+static int
+gf_defrag_done(int ret, call_frame_t *sync_frame, void *data)
+{
+    gf_listener_stop(sync_frame->this);
+
+    STACK_DESTROY(sync_frame->root);
+    kill(getpid(), SIGTERM);
+    return 0;
+}
+
+void *
+gf_defrag_start(void *data)
+{
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    xlator_t *this = NULL;
+    xlator_t *old_THIS = NULL;
+
+    this = data;
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    defrag = conf->defrag;
+    if (!defrag)
+        goto out;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    frame->root->pid = GF_CLIENT_PID_DEFRAG;
+
+    defrag->pid = frame->root->pid;
+
+    defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
+
+    old_THIS = THIS;
+    THIS = this;
+    ret = synctask_new(this->ctx->env, gf_defrag_start_crawl, gf_defrag_done,
+                       frame, this);
+
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+               "Could not create task for rebalance");
+    THIS = old_THIS;
+out:
+    return NULL;
+}
+
+uint64_t
+gf_defrag_get_estimates_based_on_size(dht_conf_t *conf)
+{
+    gf_defrag_info_t *defrag = NULL;
+    double rate_processed = 0;
+    uint64_t total_processed = 0;
+    uint64_t tmp_count = 0;
+    uint64_t time_to_complete = 0;
+    double elapsed = 0;
+
+    defrag = conf->defrag;
+
+    if (!g_totalsize)
+        goto out;
+
+    elapsed = gf_time() - defrag->start_time;
+
+    /* Don't calculate the estimates for the first 10 minutes.
+     * It is unlikely to be accurate and estimates are not required
+     * if the process finishes in less than 10 mins.
+     */
+
+    if (elapsed < ESTIMATE_START_INTERVAL) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, 0,
+               "Rebalance estimates will not be available for the "
+               "first %d seconds.",
+               ESTIMATE_START_INTERVAL);
+
+        goto out;
+    }
+
+    total_processed = defrag->size_processed;
+
+    /* rate at which files processed */
+    rate_processed = (total_processed) / elapsed;
+
+    tmp_count = g_totalsize;
+
+    if (rate_processed) {
+        time_to_complete = (tmp_count) / rate_processed;
+
+    } else {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, 0,
+               "Unable to calculate estimated time for rebalance");
+    }
+
+    gf_log(THIS->name, GF_LOG_INFO,
+           "TIME: (size) total_processed=%" PRIu64 " tmp_cnt = %" PRIu64
+           ","
+           "rate_processed=%f, elapsed = %f",
+           total_processed, tmp_count, rate_processed, elapsed);
+
+out:
+    return time_to_complete;
+}
+
+int
+gf_defrag_status_get(dht_conf_t *conf, dict_t *dict)
+{
+    int ret = 0;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookup = 0;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    char *status = "";
+    double elapsed = 0;
+    uint64_t time_to_complete = 0;
+    uint64_t time_left = 0;
+    gf_defrag_info_t *defrag = conf->defrag;
+
+    if (!defrag)
+        goto out;
+
+    ret = 0;
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
+        goto out;
+
+    files = defrag->total_files;
+    size = defrag->total_data;
+    lookup = defrag->num_files_lookedup;
+    failures = defrag->total_failures;
+    skipped = defrag->skipped;
+
+    elapsed = gf_time() - defrag->start_time;
+
+    /* The rebalance is still in progress */
+
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) {
+        time_to_complete = gf_defrag_get_estimates_based_on_size(conf);
+
+        if (time_to_complete && (time_to_complete > elapsed))
+            time_left = time_to_complete - elapsed;
+
+        gf_log(THIS->name, GF_LOG_INFO,
+               "TIME: Estimated total time to complete (size)= %" PRIu64
+               " seconds, seconds left = %" PRIu64 "",
+               time_to_complete, time_left);
+    }
+
+    if (!dict)
+        goto log;
+
+    ret = dict_set_uint64(dict, "files", files);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set file count");
+
+    ret = dict_set_uint64(dict, "size", size);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set size of xfer");
+
+    ret = dict_set_uint64(dict, "lookups", lookup);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set lookedup file count");
+
+    ret = dict_set_int32(dict, "status", defrag->defrag_status);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set status");
+
+    ret = dict_set_double(dict, "run-time", elapsed);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set run-time");
+
+    ret = dict_set_uint64(dict, "failures", failures);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set failure count");
+
+    ret = dict_set_uint64(dict, "skipped", skipped);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set skipped file count");
+
+    ret = dict_set_uint64(dict, "time-left", time_left);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set time-left");
 
-        ret = dict_set_uint64 (dict, "failures", failures);
 log:
-        switch (defrag->defrag_status) {
+    switch (defrag->defrag_status) {
         case GF_DEFRAG_STATUS_NOT_STARTED:
-                status = "not started";
-                break;
+            status = "not started";
+            break;
         case GF_DEFRAG_STATUS_STARTED:
-                status = "in progress";
-                break;
+            status = "in progress";
+            break;
         case GF_DEFRAG_STATUS_STOPPED:
-                status = "stopped";
-                break;
+            status = "stopped";
+            break;
         case GF_DEFRAG_STATUS_COMPLETE:
-                status = "completed";
-                break;
+            status = "completed";
+            break;
         case GF_DEFRAG_STATUS_FAILED:
-                status = "failed";
-                break;
-        }
-
-        gf_log (THIS->name, GF_LOG_INFO, "Rebalance is %s. Time taken is %.2f "
-                "secs", status, elapsed);
-        gf_log (THIS->name, GF_LOG_INFO, "Files migrated: %"PRIu64", size: %"
-                PRIu64", lookups: %"PRIu64", failures: %"PRIu64, files, size,
-                lookup, failures);
-
-
+            status = "failed";
+            break;
+        default:
+            break;
+    }
+
+    gf_msg(THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
+           "Rebalance is %s. Time taken is %.2f secs", status, elapsed);
+    gf_msg(THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
+           "Files migrated: %" PRIu64 ", size: %" PRIu64 ", lookups: %" PRIu64
+           ", failures: %" PRIu64
+           ", skipped: "
+           "%" PRIu64,
+           files, size, lookup, failures, skipped);
 out:
-        return 0;
+    return 0;
 }
 
 int
-gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output)
+gf_defrag_stop(dht_conf_t *conf, gf_defrag_status_t status, dict_t *output)
 {
-        /* TODO: set a variable 'stop_defrag' here, it should be checked
-           in defrag loop */
-        int     ret = -1;
-        GF_ASSERT (defrag);
+    /* TODO: set a variable 'stop_defrag' here, it should be checked
+       in defrag loop */
+    int ret = -1;
+    gf_defrag_info_t *defrag = conf->defrag;
 
-        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
-                goto out;
-        }
+    GF_ASSERT(defrag);
 
-        gf_log ("", GF_LOG_INFO, "Recieved stop command on rebalance");
-        defrag->defrag_status = GF_DEFRAG_STATUS_STOPPED;
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
+        goto out;
+    }
 
-        if (output)
-                gf_defrag_status_get (defrag, output);
-        ret = 0;
+    gf_msg("", GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STOPPED,
+           "Received stop command on rebalance");
+    defrag->defrag_status = status;
+
+    if (output)
+        gf_defrag_status_get(conf, output);
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("", 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index f1a4364df20..d9dbf50492f 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -11,852 +11,1987 @@
 /* TODO: link(oldpath, newpath) fails if newpath already exists. DHT should
  *       delete the newpath if it gets EEXISTS from link() call.
  */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "defaults.h"
+#include "dht-lock.h"
+#include <glusterfs/defaults.h>
 
+int
+dht_rename_unlock(call_frame_t *frame, xlator_t *this);
+int32_t
+dht_rename_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int
-dht_rename_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        if (op_ret == -1) {
-                /* TODO: undo the damage */
-
-                gf_log (this->name, GF_LOG_INFO,
-                        "rename %s -> %s on %s failed (%s)",
-                        local->loc.path, local->loc2.path,
-                        prev->this->name, strerror (op_errno));
-
-                local->op_ret   = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
-        /* TODO: construct proper stbuf for dir */
-        /*
-         * FIXME: is this the correct way to build stbuf and
-         * parent bufs?
-         */
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->preparent, prenewparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postparent, postnewparent,
-                        prev->this);
+dht_rename_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
 
+    local = frame->local;
 
-unwind:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                WIPE (&local->preoldparent);
-                WIPE (&local->postoldparent);
-                WIPE (&local->preparent);
-                WIPE (&local->postparent);
-
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf, &local->preoldparent,
-                                  &local->postoldparent,
-                                  &local->preparent, &local->postparent, xdata);
-        }
+    dht_set_fixed_dir_stat(&local->preoldparent);
+    dht_set_fixed_dir_stat(&local->postoldparent);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        return 0;
+    if (IA_ISREG(local->stbuf.ia_type))
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+
+    DHT_STACK_UNWIND(rename, frame, local->op_ret, local->op_errno,
+                     &local->stbuf, &local->preoldparent, &local->postoldparent,
+                     &local->preparent, &local->postparent, local->xattr);
+    return 0;
 }
 
+static void
+dht_rename_dir_unlock_src(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+    dht_unlock_namespace(frame, &local->lock[0]);
+    return;
+}
+
+static void
+dht_rename_dir_unlock_dst(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+    int op_ret = -1;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[1].ns.directory_ns);
+
+    /* Unlock inodelk */
+    op_ret = dht_unlock_inodelk(frame, local->lock[1].ns.parent_layout.locks,
+                                local->lock[1].ns.parent_layout.lk_count,
+                                dht_rename_unlock_cbk);
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        if (IA_ISREG(local->stbuf.ia_type))
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s:%s %s:%s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->src_cached->name,
+                   local->loc2.path, dst_gfid,
+                   local->dst_cached ? local->dst_cached->name : NULL);
+        else
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s %s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->loc2.path, dst_gfid);
+
+        dht_rename_unlock_cbk(frame, NULL, this, 0, 0, NULL);
+    }
+
+    return;
+}
 
+static int
+dht_rename_dir_unlock(call_frame_t *frame, xlator_t *this)
+{
+    dht_rename_dir_unlock_src(frame, this);
+    dht_rename_dir_unlock_dst(frame, this);
+    return 0;
+}
 int
-dht_rename_hashed_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                           struct iatt *preoldparent,
-                           struct iatt *postoldparent,
-                           struct iatt *prenewparent,
-                           struct iatt *postnewparent, dict_t *xdata)
-{
-        dht_conf_t   *conf = NULL;
-        dht_local_t  *local = NULL;
-        int           call_cnt = 0;
-        call_frame_t *prev = NULL;
-        int           i = 0;
-
-        conf = this->private;
-        local = frame->local;
-        prev = cookie;
-
-        if (op_ret == -1) {
-                /* TODO: undo the damage */
-
-                gf_log (this->name, GF_LOG_INFO,
-                        "rename %s -> %s on %s failed (%s)",
-                        local->loc.path, local->loc2.path,
-                        prev->this->name, strerror (op_errno));
-
-                local->op_ret   = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
-        /* TODO: construct proper stbuf for dir */
-        /*
-         * FIXME: is this the correct way to build stbuf and
-         * parent bufs?
+dht_rename_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
+{
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int subvol_cnt = -1;
+
+    conf = this->private;
+    local = frame->local;
+    prev = cookie;
+    subvol_cnt = dht_subvol_cnt(this, prev);
+    local->ret_cache[subvol_cnt] = op_ret;
+
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
+
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+               "Rename %s -> %s on %s failed, (gfid = %s)", local->loc.path,
+               local->loc2.path, prev->name, gfid);
+
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+    /* TODO: construct proper stbuf for dir */
+    /*
+     * FIXME: is this the correct way to build stbuf and
+     * parent bufs?
+     */
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preoldparent, preoldparent);
+    dht_iatt_merge(this, &local->postoldparent, postoldparent);
+    dht_iatt_merge(this, &local->preparent, prenewparent);
+    dht_iatt_merge(this, &local->postparent, postnewparent);
+
+unwind:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /* We get here with local->call_cnt == 0. Which means
+         * we are the only one executing this code, there is
+         * no contention. Therefore it's safe to manipulate or
+         * deref local->call_cnt directly (without locking).
          */
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->preparent, prenewparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postparent, postnewparent,
-                        prev->this);
-
-        call_cnt = local->call_cnt = conf->subvolume_cnt - 1;
-
-        if (!local->call_cnt)
-                goto unwind;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == local->dst_hashed)
+        if (local->ret_cache[conf->subvolume_cnt] == 0) {
+            /* count errant subvols in last field of ret_cache */
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (local->ret_cache[i] != 0)
+                    ++local->ret_cache[conf->subvolume_cnt];
+            }
+            if (local->ret_cache[conf->subvolume_cnt]) {
+                /* undoing the damage:
+                 * for all subvolumes, where rename
+                 * succeeded, we perform the reverse operation
+                 */
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (local->ret_cache[i] == 0)
+                        ++local->call_cnt;
+                }
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (local->ret_cache[i])
                         continue;
-                STACK_WIND (frame, dht_rename_dir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->rename,
-                            &local->loc, &local->loc2, NULL);
-                if (!--call_cnt)
-                        break;
-        }
 
+                    STACK_WIND(frame, dht_rename_dir_cbk, conf->subvolumes[i],
+                               conf->subvolumes[i]->fops->rename, &local->loc2,
+                               &local->loc, NULL);
+                }
 
-        return 0;
-unwind:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+                return 0;
+            }
+        }
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                          &local->stbuf, &local->preoldparent,
-                          &local->postoldparent,
-                          &local->preparent, &local->postparent, NULL);
+        WIPE(&local->preoldparent);
+        WIPE(&local->postoldparent);
+        WIPE(&local->preparent);
+        WIPE(&local->postparent);
 
-        return 0;
+        dht_rename_dir_unlock(frame, this);
+    }
+
+    return 0;
 }
 
+int
+dht_rename_hashed_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                          struct iatt *preoldparent, struct iatt *postoldparent,
+                          struct iatt *prenewparent, struct iatt *postnewparent,
+                          dict_t *xdata)
+{
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int call_cnt = 0;
+    xlator_t *prev = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    conf = this->private;
+    local = frame->local;
+    prev = cookie;
+
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
+
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+               "rename %s -> %s on %s failed, (gfid = %s) ", local->loc.path,
+               local->loc2.path, prev->name, gfid);
+
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+    /* TODO: construct proper stbuf for dir */
+    /*
+     * FIXME: is this the correct way to build stbuf and
+     * parent bufs?
+     */
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preoldparent, preoldparent);
+    dht_iatt_merge(this, &local->postoldparent, postoldparent);
+    dht_iatt_merge(this, &local->preparent, prenewparent);
+    dht_iatt_merge(this, &local->postparent, postnewparent);
+
+    call_cnt = local->call_cnt = conf->subvolume_cnt - 1;
+
+    if (!local->call_cnt)
+        goto unwind;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == local->dst_hashed)
+            continue;
+        STACK_WIND_COOKIE(
+            frame, dht_rename_dir_cbk, conf->subvolumes[i], conf->subvolumes[i],
+            conf->subvolumes[i]->fops->rename, &local->loc, &local->loc2, NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
+unwind:
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
+
+    dht_rename_dir_unlock(frame, this);
+    return 0;
+}
 
 int
-dht_rename_dir_do (call_frame_t *frame, xlator_t *this)
+dht_rename_dir_do(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret == -1)
-                goto err;
+    if (local->op_ret == -1)
+        goto err;
 
-        local->op_ret = 0;
+    local->op_ret = 0;
 
-        STACK_WIND (frame, dht_rename_hashed_dir_cbk,
-                    local->dst_hashed,
-                    local->dst_hashed->fops->rename,
-                    &local->loc, &local->loc2, NULL);
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_rename_hashed_dir_cbk, local->dst_hashed,
+                      local->dst_hashed, local->dst_hashed->fops->rename,
+                      &local->loc, &local->loc2, NULL);
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno, NULL, NULL,
-                          NULL, NULL, NULL, NULL);
-        return 0;
+    dht_rename_dir_unlock(frame, this);
+    return 0;
 }
 
-
 int
-dht_rename_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries,
-                        dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-
-        if (op_ret > 2) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "readdir on %s for %s returned %d entries",
-                        prev->this->name, local->loc.path, op_ret);
-                local->op_ret = -1;
-                local->op_errno = ENOTEMPTY;
-        }
+dht_rename_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
 
-        this_call_cnt = dht_frame_return (frame);
+    local = frame->local;
+    prev = cookie;
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rename_dir_do (frame, this);
-        }
+    if (op_ret > 2) {
+        gf_msg_trace(this->name, 0, "readdir on %s for %s returned %d entries",
+                     prev->name, local->loc.path, op_ret);
+        local->op_ret = -1;
+        local->op_errno = ENOTEMPTY;
+    }
 
-        return 0;
-}
+    this_call_cnt = dht_frame_return(frame);
 
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_dir_do(frame, this);
+    }
+
+    return 0;
+}
 
 int
-dht_rename_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+dht_rename_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
+    local = frame->local;
+    prev = cookie;
 
-        local = frame->local;
-        prev  = cookie;
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_OPENDIR_FAILED,
+               "opendir on %s for %s failed,(gfid = %s) ", prev->name,
+               local->loc.path, gfid);
+        goto err;
+    }
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "opendir on %s for %s failed (%s)",
-                        prev->this->name, local->loc.path,
-                        strerror (op_errno));
-                goto err;
-        }
-
-        STACK_WIND (frame, dht_rename_readdir_cbk,
-                    prev->this, prev->this->fops->readdir,
-                    local->fd, 4096, 0, NULL);
+    fd_bind(fd);
+    STACK_WIND_COOKIE(frame, dht_rename_readdir_cbk, prev, prev,
+                      prev->fops->readdir, local->fd, 4096, 0, NULL);
 
-        return 0;
+    return 0;
 
 err:
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rename_dir_do (frame, this);
-        }
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_dir_do(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rename_dir (call_frame_t *frame, xlator_t *this)
+dht_rename_dir_lock2_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_conf_t  *conf = NULL;
-        dht_local_t *local = NULL;
-        int          i = 0;
-        int          op_errno = -1;
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_conf_t *conf = NULL;
+    int i = 0;
+
+    local = frame->local;
+    conf = this->private;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    local->fd = fd_create(local->loc.inode, frame->root->pid);
+    if (!local->fd) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->op_ret = 0;
+
+    if (!local->dst_cached) {
+        dht_rename_dir_do(frame, this);
+        return 0;
+    }
 
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_rename_opendir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->opendir, &local->loc2,
+                          local->fd, NULL);
+    }
 
-        conf = frame->this->private;
-        local = frame->local;
+    return 0;
 
-        local->call_cnt = conf->subvolume_cnt;
+err:
+    /* No harm in calling an extra unlock */
+    dht_rename_dir_unlock(frame, this);
+    return 0;
+}
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->subvolume_status[i]) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "one of the subvolumes down (%s)",
-                                conf->subvolumes[i]->name);
-                        op_errno = ENOTCONN;
-                        goto err;
-                }
-        }
+int
+dht_rename_dir_lock1_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    if (local->current == &local->lock[0]) {
+        loc = &local->loc2;
+        subvol = local->dst_hashed;
+        local->current = &local->lock[1];
+    } else {
+        loc = &local->loc;
+        subvol = local->src_hashed;
+        local->current = &local->lock[0];
+    }
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_dir_lock2_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
+err:
+    /* No harm in calling an extra unlock */
+    dht_rename_dir_unlock(frame, this);
+    return 0;
+}
 
-        local->fd = fd_create (local->loc.inode, frame->root->pid);
-        if (!local->fd) {
-                op_errno = ENOMEM;
-                goto err;
+/*
+ * If the hashed subvolumes of both source and dst are the different,
+ * lock in dictionary order of hashed subvol->name. This is important
+ * in case the parent directory is the same for both src and dst to
+ * prevent inodelk deadlocks when racing with a fix-layout op on the parent.
+ *
+ * If the hashed subvols are the same, use the gfid/name to determine
+ * the order of taking locks to prevent entrylk deadlocks when the parent
+ * dirs are the same.
+ *
+ */
+static int
+dht_order_rename_lock(call_frame_t *frame, loc_t **loc, xlator_t **subvol)
+{
+    int ret = 0;
+    int op_ret = 0;
+    dht_local_t *local = NULL;
+    char *src = NULL;
+    char *dst = NULL;
+
+    local = frame->local;
+
+    if (local->src_hashed->name == local->dst_hashed->name) {
+        ret = 0;
+    } else {
+        ret = strcmp(local->src_hashed->name, local->dst_hashed->name);
+    }
+
+    if (ret == 0) {
+        /* hashed subvols are the same for src and dst */
+        /* Entrylks need to be ordered*/
+
+        src = alloca(GF_UUID_BNAME_BUF_SIZE + strlen(local->loc.name) + 1);
+        if (!src) {
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, 0,
+                   "Insufficient memory for src");
+            op_ret = -1;
+            goto out;
         }
 
-        local->op_ret = 0;
+        if (!gf_uuid_is_null(local->loc.pargfid))
+            uuid_utoa_r(local->loc.pargfid, src);
+        else if (local->loc.parent)
+            uuid_utoa_r(local->loc.parent->gfid, src);
+        else
+            src[0] = '\0';
 
-        if (!local->dst_cached) {
-                dht_rename_dir_do (frame, this);
-                return 0;
-        }
+        strcat(src, local->loc.name);
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_rename_opendir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->opendir,
-                            &local->loc2, local->fd, NULL);
+        dst = alloca(GF_UUID_BNAME_BUF_SIZE + strlen(local->loc2.name) + 1);
+        if (!dst) {
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, 0,
+                   "Insufficient memory for dst");
+            op_ret = -1;
+            goto out;
         }
 
-        return 0;
+        if (!gf_uuid_is_null(local->loc2.pargfid))
+            uuid_utoa_r(local->loc2.pargfid, dst);
+        else if (local->loc2.parent)
+            uuid_utoa_r(local->loc2.parent->gfid, dst);
+        else
+            dst[0] = '\0';
+
+        strcat(dst, local->loc2.name);
+        ret = strcmp(src, dst);
+    }
+
+    if (ret <= 0) {
+        /*inodelk in dictionary order of hashed subvol names*/
+        /*entrylk in dictionary order of gfid/basename */
+        local->current = &local->lock[0];
+        *loc = &local->loc;
+        *subvol = local->src_hashed;
+
+    } else {
+        local->current = &local->lock[1];
+        *loc = &local->loc2;
+        *subvol = local->dst_hashed;
+    }
+
+    op_ret = 0;
+
+out:
+    return op_ret;
+}
+
+int
+dht_rename_dir(call_frame_t *frame, xlator_t *this)
+{
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+    int i = 0;
+    int ret = 0;
+    int op_errno = -1;
+
+    conf = frame->this->private;
+    local = frame->local;
+
+    local->ret_cache = GF_CALLOC(conf->subvolume_cnt + 1, sizeof(int),
+                                 gf_dht_ret_cache_t);
+
+    if (local->ret_cache == NULL) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = conf->subvolume_cnt;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+                   "Rename dir failed: subvolume down (%s)",
+                   conf->subvolumes[i]->name);
+            op_errno = ENOTCONN;
+            goto err;
+        }
+    }
+
+    /* Locks on src and dst needs to ordered which otherwise might cause
+     * deadlocks when rename (src, dst) and rename (dst, src) is done from
+     * two different clients
+     */
+    ret = dht_order_rename_lock(frame, &loc, &subvol);
+    if (ret) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* Rename must take locks on src to avoid lookup selfheal from
+     * recreating src on those subvols where the rename was successful.
+     * The locks can't be issued parallel as two different clients might
+     * attempt same rename command and be in dead lock.
+     */
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_dir_lock1_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
+static int
+dht_rename_track_for_changelog(xlator_t *this, dict_t *xattr, loc_t *oldloc,
+                               loc_t *newloc)
+{
+    int ret = -1;
+    dht_changelog_rename_info_t *info = NULL;
+    char *name = NULL;
+    int len1 = 0;
+    int len2 = 0;
+    int size = 0;
+
+    if (!xattr || !oldloc || !newloc || !this)
+        return ret;
+
+    len1 = strlen(oldloc->name) + 1;
+    len2 = strlen(newloc->name) + 1;
+    size = sizeof(dht_changelog_rename_info_t) + len1 + len2;
+
+    info = GF_CALLOC(size, sizeof(char), gf_common_mt_char);
+    if (!info) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to calloc memory");
+        return ret;
+    }
+
+    gf_uuid_copy(info->old_pargfid, oldloc->pargfid);
+    gf_uuid_copy(info->new_pargfid, newloc->pargfid);
+
+    info->oldname_len = len1;
+    info->newname_len = len2;
+    strncpy(info->buffer, oldloc->name, len1);
+    name = info->buffer + len1;
+    strncpy(name, newloc->name, len2);
+
+    ret = dict_set_bin(xattr, DHT_CHANGELOG_RENAME_OP_KEY, info, size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s,"
+               " path = %s",
+               DHT_CHANGELOG_RENAME_OP_KEY, oldloc->name);
+        GF_FREE(info);
+    }
+
+    return ret;
+}
+
+#define DHT_MARKER_DONT_ACCOUNT(xattr)                                         \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr)                                                        \
+                break;                                                         \
+        }                                                                      \
+        tmp = dict_set_str(xattr, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY, "yes");   \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   GLUSTERFS_MARKER_DONT_ACCOUNT_KEY, local->loc.path);        \
+        }                                                                      \
+    } while (0)
+
+#define DHT_CHANGELOG_TRACK_AS_RENAME(xattr, oldloc, newloc)                   \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr) {                                                      \
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,   \
+                       "Failed to create dictionary to "                       \
+                       "track rename");                                        \
+                break;                                                         \
+            }                                                                  \
+        }                                                                      \
+                                                                               \
+        tmp = dht_rename_track_for_changelog(this, xattr, oldloc, newloc);     \
+                                                                               \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   DHT_CHANGELOG_RENAME_OP_KEY, (oldloc)->path);               \
+        }                                                                      \
+    } while (0)
 
 int
-dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                       struct iatt *postparent, dict_t *xdata)
+dht_rename_unlock(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    int op_ret = -1;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_ilock_wrap_t inodelk_wrapper = {
+        0,
+    };
+
+    local = frame->local;
+    inodelk_wrapper.locks = local->rename_inodelk_backward_compatible;
+    inodelk_wrapper.lk_count = local->rename_inodelk_bc_count;
+
+    op_ret = dht_unlock_inodelk_wrapper(frame, &inodelk_wrapper);
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        if (IA_ISREG(local->stbuf.ia_type))
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s:%s %s:%s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->src_cached->name,
+                   local->loc2.path, dst_gfid,
+                   local->dst_cached ? local->dst_cached->name : NULL);
+        else
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s %s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->loc2.path, dst_gfid);
+    }
+
+    dht_unlock_namespace(frame, &local->lock[0]);
+    dht_unlock_namespace(frame, &local->lock[1]);
+
+    dht_rename_unlock_cbk(frame, NULL, this, local->op_ret, local->op_errno,
+                          NULL);
+    return 0;
+}
 
-        local = frame->local;
-        prev  = cookie;
+int
+dht_rename_done(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
 
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "!local, should not happen");
-                goto out;
-        }
+    local = frame->local;
 
-        this_call_cnt = dht_frame_return (frame);
+    if (local->linked == _gf_true) {
+        local->linked = _gf_false;
+        dht_linkfile_attr_heal(frame, this);
+    }
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: unlink on %s failed (%s)",
-                        local->loc.path, prev->this->name, strerror (op_errno));
-        }
+    dht_rename_unlock(frame, this);
+    return 0;
+}
 
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
-
-        if (is_last_call (this_call_cnt)) {
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf, &local->preoldparent,
-                                  &local->postoldparent, &local->preparent,
-                                  &local->postparent, NULL);
-        }
+int
+dht_rename_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    FRAME_SU_UNDO(frame, dht_local_t);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_VALUE,
+               "!local, should not happen");
+        goto out;
+    }
+
+    this_call_cnt = dht_frame_return(frame);
+
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLINK_FAILED,
+               "%s: Rename: unlink on %s failed ", local->loc.path, prev->name);
+    }
+
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
+
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_done(frame, this);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rename_cleanup (call_frame_t *frame)
+dht_rename_cleanup(call_frame_t *frame)
 {
-        dht_local_t *local = NULL;
-        xlator_t    *this = NULL;
-        xlator_t    *src_hashed = NULL;
-        xlator_t    *src_cached = NULL;
-        xlator_t    *dst_hashed = NULL;
-        xlator_t    *dst_cached = NULL;
-        int          call_cnt = 0;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    int call_cnt = 0;
+    dict_t *xattr = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
+    local = frame->local;
+    this = frame->this;
 
-        local = frame->local;
-        this  = frame->this;
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
+    if (src_cached == dst_cached)
+        goto nolinks;
 
-        if (src_cached == dst_cached)
-                goto nolinks;
+    if (local->linked && (dst_hashed != src_hashed) &&
+        (dst_hashed != src_cached)) {
+        call_cnt++;
+    }
 
-        if (dst_hashed != src_hashed && dst_hashed != src_cached)
-                call_cnt++;
+    if (local->added_link && (src_cached != dst_hashed)) {
+        call_cnt++;
+    }
 
-        if (src_cached != dst_hashed)
-                call_cnt++;
+    local->call_cnt = call_cnt;
 
-        local->call_cnt = call_cnt;
+    if (!call_cnt)
+        goto nolinks;
 
-        if (!call_cnt)
-                goto nolinks;
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-        if (dst_hashed != src_hashed && dst_hashed != src_cached) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "unlinking linkfile %s @ %s => %s",
-                        local->loc.path, dst_hashed->name, src_cached->name);
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            dst_hashed, dst_hashed->fops->unlink,
-                            &local->loc, 0, NULL);
-        }
+    gf_uuid_unparse(local->loc.inode->gfid, gfid);
 
-        if (src_cached != dst_hashed) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "unlinking link %s => %s (%s)", local->loc.path,
-                        local->loc2.path, src_cached->name);
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            src_cached, src_cached->fops->unlink,
-                            &local->loc2, 0, NULL);
-        }
+    if (local->linked && (dst_hashed != src_hashed) &&
+        (dst_hashed != src_cached)) {
+        dict_t *xattr_new = NULL;
 
-        return 0;
+        gf_msg_trace(this->name, 0,
+                     "unlinking linkfile %s @ %s => %s, (gfid = %s)",
+                     local->loc.path, dst_hashed->name, src_cached->name, gfid);
 
-nolinks:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                          &local->stbuf, &local->preoldparent,
-                          &local->postoldparent, &local->preparent,
-                          &local->postparent, NULL);
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-        return 0;
-}
+        FRAME_SU_DO(frame, dht_local_t);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, dst_hashed, dst_hashed,
+                          dst_hashed->fops->unlink, &local->loc, 0, xattr_new);
 
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-int
-dht_rename_links_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             inode_t *inode, struct iatt *stbuf,
-                             struct iatt *preparent, struct iatt *postparent,
-                             dict_t *xdata)
-{
-        call_frame_t *prev = NULL;
-        dht_local_t  *local = NULL;
-
-        prev = cookie;
-        local = frame->local;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "link/file %s on %s failed (%s)",
-                        local->loc.path, prev->this->name, strerror (op_errno));
+    if (local->added_link && (src_cached != dst_hashed)) {
+        dict_t *xattr_new = NULL;
+
+        gf_msg_trace(this->name, 0, "unlinking link %s => %s (%s), (gfid = %s)",
+                     local->loc.path, local->loc2.path, src_cached->name, gfid);
+
+        xattr_new = dict_copy_with_ref(xattr, NULL);
+
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
         }
+        /* *
+         * The link to file is created using root permission.
+         * Hence deletion should happen using root. Otherwise
+         * it will fail.
+         */
+        FRAME_SU_DO(frame, dht_local_t);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_cached, src_cached,
+                          src_cached->fops->unlink, &local->loc2, 0, xattr_new);
 
-        DHT_STACK_DESTROY (frame);
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-        return 0;
-}
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
+
+nolinks:
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
 
+    dht_rename_unlock(frame, this);
+    return 0;
+}
 
 int
-dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                struct iatt *preoldparent, struct iatt *postoldparent,
-                struct iatt *prenewparent, struct iatt *postnewparent,
-                dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *src_hashed = NULL;
-        xlator_t     *src_cached = NULL;
-        xlator_t     *dst_hashed = NULL;
-        xlator_t     *dst_cached = NULL;
-        xlator_t     *rename_subvol = NULL;
-        call_frame_t *link_frame = NULL;
-        dht_local_t *link_local = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: rename on %s failed (%s)", local->loc.path,
-                        prev->this->name, strerror (op_errno));
-                local->op_ret   = op_ret;
-                local->op_errno = op_errno;
-                goto cleanup;
-        }
+dht_rename_unlink(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *rename_subvol = NULL;
+    dict_t *xattr = NULL;
 
-        if ((src_cached == dst_cached) && (dst_hashed != dst_cached)) {
-                link_frame = copy_frame (frame);
-                if (!link_frame) {
-                        goto err;
-                }
+    local = frame->local;
 
-                /* fop value sent as maxvalue because it is not used
-                   anywhere in this case */
-                link_local = dht_local_init (link_frame, &local->loc2, NULL,
-                                             GF_FOP_MAXVALUE);
-                if (!link_local) {
-                        goto err;
-                }
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-                if (link_local->loc.inode)
-                        inode_unref (link_local->loc.inode);
-                link_local->loc.inode = inode_ref (local->loc.inode);
-                uuid_copy (link_local->gfid, local->loc.inode->gfid);
+    local->call_cnt = 0;
 
-                dht_linkfile_create (link_frame, dht_rename_links_create_cbk,
-                                     src_cached, dst_hashed, &link_local->loc);
-        }
+    /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
+     * is called. since rename has already happened on rename_subvol,
+     * unlink shouldn't be sent for oldpath (either linkfile or cached-file)
+     * on rename_subvol. */
+    if (src_cached == dst_cached)
+        rename_subvol = src_cached;
+    else
+        rename_subvol = dst_hashed;
 
-err:
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent, prev->this);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent, prev->this);
-        dht_iatt_merge (this, &local->preparent, prenewparent, prev->this);
-        dht_iatt_merge (this, &local->postparent, postnewparent, prev->this);
-
-        /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
-         *       is called. since rename has already happened on rename_subvol,
-         *       unlink should not be sent for oldpath (either linkfile or cached-file)
-         *       on rename_subvol. */
-        if (src_cached == dst_cached)
-                rename_subvol = src_cached;
-        else
-                rename_subvol = dst_hashed;
+    /* TODO: delete files in background */
 
-        /* TODO: delete files in background */
+    if (src_cached != dst_hashed && src_cached != dst_cached)
+        local->call_cnt++;
 
-        if (src_cached != dst_hashed && src_cached != dst_cached)
-                local->call_cnt++;
+    if (src_hashed != rename_subvol && src_hashed != src_cached)
+        local->call_cnt++;
 
-        if (src_hashed != rename_subvol && src_hashed != src_cached)
-                local->call_cnt++;
+    if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
+        local->call_cnt++;
 
-        if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
-                local->call_cnt++;
+    if (local->call_cnt == 0)
+        goto unwind;
 
-        if (local->call_cnt == 0)
-                goto unwind;
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-        if (src_cached != dst_hashed && src_cached != dst_cached) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "deleting old src datafile %s @ %s",
-                        local->loc.path, src_cached->name);
+    if (src_cached != dst_hashed && src_cached != dst_cached) {
+        dict_t *xattr_new = NULL;
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            src_cached, src_cached->fops->unlink,
-                            &local->loc, 0, NULL);
-        }
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-        if (src_hashed != rename_subvol && src_hashed != src_cached) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "deleting old src linkfile %s @ %s",
-                        local->loc.path, src_hashed->name);
+        gf_msg_trace(this->name, 0, "deleting old src datafile %s @ %s",
+                     local->loc.path, src_cached->name);
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            src_hashed, src_hashed->fops->unlink,
-                            &local->loc, 0, NULL);
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
         }
 
-        if (dst_cached
-            && (dst_cached != dst_hashed)
-            && (dst_cached != src_cached)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "deleting old dst datafile %s @ %s",
-                        local->loc2.path, dst_cached->name);
+        DHT_CHANGELOG_TRACK_AS_RENAME(xattr_new, &local->loc, &local->loc2);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_cached, src_cached,
+                          src_cached->fops->unlink, &local->loc, 0, xattr_new);
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            dst_cached, dst_cached->fops->unlink,
-                            &local->loc2, 0, NULL);
-        }
-        return 0;
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-unwind:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+    if (src_hashed != rename_subvol && src_hashed != src_cached) {
+        dict_t *xattr_new = NULL;
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                          &local->stbuf, &local->preoldparent,
-                          &local->postoldparent, &local->preparent,
-                          &local->postparent, NULL);
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-        return 0;
+        gf_msg_trace(this->name, 0, "deleting old src linkfile %s @ %s",
+                     local->loc.path, src_hashed->name);
 
-cleanup:
-        dht_rename_cleanup (frame);
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_hashed, src_hashed,
+                          src_hashed->fops->unlink, &local->loc, 0, xattr_new);
+
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
+
+    if (dst_cached && (dst_cached != dst_hashed) &&
+        (dst_cached != src_cached)) {
+        gf_msg_trace(this->name, 0, "deleting old dst datafile %s @ %s",
+                     local->loc2.path, dst_cached->name);
+
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, dst_cached, dst_cached,
+                          dst_cached->fops->unlink, &local->loc2, 0, xattr);
+    }
+    if (xattr)
+        dict_unref(xattr);
+    return 0;
+
+unwind:
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
+
+    dht_rename_done(frame, this);
+
+    return 0;
 }
 
+int
+dht_rename_links_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *stbuf, struct iatt *preparent,
+                            struct iatt *postparent, dict_t *xdata)
+{
+    xlator_t *prev = NULL;
+    dht_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
+
+    prev = cookie;
+    local = frame->local;
+    main_frame = local->main_frame;
+
+    /* TODO: Handle this case in lookup-optimize */
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_CREATE_LINK_FAILED,
+               "link/file %s on %s failed", local->loc.path, prev->name);
+    }
+
+    if (local->linked == _gf_true) {
+        local->linked = _gf_false;
+        dht_linkfile_attr_heal(frame, this);
+    }
+
+    dht_rename_unlink(main_frame, this);
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
 int
-dht_do_rename (call_frame_t *frame)
+dht_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+               struct iatt *preoldparent, struct iatt *postoldparent,
+               struct iatt *prenewparent, struct iatt *postnewparent,
+               dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        xlator_t    *dst_hashed = NULL;
-        xlator_t    *src_cached = NULL;
-        xlator_t    *dst_cached = NULL;
-        xlator_t    *this = NULL;
-        xlator_t    *rename_subvol = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    call_frame_t *link_frame = NULL;
+    dht_local_t *link_local = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
+
+    if (local->linked == _gf_true)
+        FRAME_SU_UNDO(frame, dht_local_t);
+
+    /* It is a critical failure iff we fail to rename the cached file
+     * if the rename of the linkto failed, it is not a critical failure,
+     * and we do not want to lose the created hard link for the new
+     * name as that could have been read by other clients.
+     *
+     * NOTE: If another client is attempting the same oldname -> newname
+     * rename, and finds both file names as existing, and are hard links
+     * to each other, then FUSE would send in an unlink for oldname. In
+     * this time duration if we treat the linkto as a critical error and
+     * unlink the newname we created, we would have effectively lost the
+     * file to rename operations.
+     *
+     * Repercussions of treating this as a non-critical error is that
+     * we could leave behind a stale linkto file and/or not create the new
+     * linkto file, the second case would be rectified by a subsequent
+     * lookup, the first case by a rebalance, like for all stale linkto
+     * files */
+
+    if (op_ret == -1) {
+        /* Critical failure: unable to rename the cached file */
+        if (prev == src_cached) {
+            gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_RENAME_FAILED,
+                   "%s: Rename on %s failed, (gfid = %s) ", local->loc.path,
+                   prev->name,
+                   local->loc.inode ? uuid_utoa(local->loc.inode->gfid) : "");
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            goto cleanup;
+        } else {
+            /* Non-critical failure, unable to rename the linkto
+             * file
+             */
+            gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+                   "%s: Rename (linkto file) on %s failed, "
+                   "(gfid = %s) ",
+                   local->loc.path, prev->name,
+                   local->loc.inode ? uuid_utoa(local->loc.inode->gfid) : "");
+        }
+    }
+    if (xdata) {
+        if (!local->xattr)
+            local->xattr = dict_ref(xdata);
+        else
+            local->xattr = dict_copy_with_ref(xdata, local->xattr);
+    }
+
+    /* Merge attrs only from src_cached. In case there of src_cached !=
+     * dst_hashed, this ignores linkfile attrs. */
+    if (prev == src_cached) {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->preoldparent, preoldparent);
+        dht_iatt_merge(this, &local->postoldparent, postoldparent);
+        dht_iatt_merge(this, &local->preparent, prenewparent);
+        dht_iatt_merge(this, &local->postparent, postnewparent);
+    }
+
+    /* Create the linkto file for the dst file */
+    if ((src_cached == dst_cached) && (dst_hashed != dst_cached)) {
+        link_frame = copy_frame(frame);
+        if (!link_frame) {
+            goto unlink;
+        }
 
+        /* fop value sent as maxvalue because it is not used
+         * anywhere in this case */
+        link_local = dht_local_init(link_frame, &local->loc2, NULL,
+                                    GF_FOP_MAXVALUE);
+        if (!link_local) {
+            goto unlink;
+        }
 
-        local = frame->local;
-        this  = frame->this;
+        if (link_local->loc.inode)
+            inode_unref(link_local->loc.inode);
+        link_local->loc.inode = inode_ref(local->loc.inode);
+        link_local->main_frame = frame;
+        link_local->stbuf = local->stbuf;
+        gf_uuid_copy(link_local->gfid, local->loc.inode->gfid);
 
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
-        src_cached = local->src_cached;
+        dht_linkfile_create(link_frame, dht_rename_links_create_cbk, this,
+                            src_cached, dst_hashed, &link_local->loc);
+        return 0;
+    }
 
-        if (src_cached == dst_cached)
-                rename_subvol = src_cached;
-        else
-                rename_subvol = dst_hashed;
+unlink:
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "renaming %s => %s (%s)",
-                local->loc.path, local->loc2.path, rename_subvol->name);
+    if (link_frame) {
+        DHT_STACK_DESTROY(link_frame);
+    }
+    dht_rename_unlink(frame, this);
+    return 0;
 
-        STACK_WIND (frame, dht_rename_cbk,
-                    rename_subvol, rename_subvol->fops->rename,
-                    &local->loc, &local->loc2, NULL);
+cleanup:
+    dht_rename_cleanup(frame);
 
-        return 0;
+    return 0;
 }
 
+int
+dht_do_rename(call_frame_t *frame)
+{
+    dht_local_t *local = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *this = NULL;
+    xlator_t *rename_subvol = NULL;
+
+    local = frame->local;
+    this = frame->this;
+
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
+    src_cached = local->src_cached;
+
+    if (src_cached == dst_cached)
+        rename_subvol = src_cached;
+    else
+        rename_subvol = dst_hashed;
+
+    if ((src_cached != dst_hashed) && (rename_subvol == dst_hashed)) {
+        DHT_MARKER_DONT_ACCOUNT(local->xattr_req);
+    }
+
+    if (rename_subvol == src_cached) {
+        DHT_CHANGELOG_TRACK_AS_RENAME(local->xattr_req, &local->loc,
+                                      &local->loc2);
+    }
+
+    gf_msg_trace(this->name, 0, "renaming %s => %s (%s)", local->loc.path,
+                 local->loc2.path, rename_subvol->name);
+
+    if (local->linked == _gf_true)
+        FRAME_SU_DO(frame, dht_local_t);
+    STACK_WIND_COOKIE(frame, dht_rename_cbk, rename_subvol, rename_subvol,
+                      rename_subvol->fops->rename, &local->loc, &local->loc2,
+                      local->xattr_req);
+    return 0;
+}
 
 int
-dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           this_call_cnt  = 0;
-
-
-        local = frame->local;
-        prev = cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "link/file on %s failed (%s)",
-                        prev->this->name, strerror (op_errno));
-                local->op_ret   = -1;
-                if (op_errno != ENOENT)
-                        local->op_errno = op_errno;
-        }
+dht_rename_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                if (local->op_ret == -1)
-                        goto cleanup;
+    local = frame->local;
+    prev = cookie;
 
-                dht_do_rename (frame);
-        }
+    if (op_ret == -1) {
+        gf_msg_debug(this->name, 0, "link/file on %s failed (%s)", prev->name,
+                     strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        local->added_link = _gf_false;
+    } else
+        dht_iatt_merge(this, &local->stbuf, stbuf);
 
-        return 0;
+    if (local->op_ret == -1)
+        goto cleanup;
+
+    dht_do_rename(frame);
+
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rename_unlink_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *preparent, struct iatt *postparent,
-                             dict_t *xdata)
+dht_rename_linkto_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *stbuf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_cached = NULL;
+    dict_t *xattr = NULL;
 
+    local = frame->local;
+    DHT_MARK_FOP_INTERNAL(xattr);
+    prev = cookie;
+    src_cached = local->src_cached;
 
-	local = frame->local;
-	prev = cookie;
+    if (op_ret == -1) {
+        gf_msg_debug(this->name, 0, "link/file on %s failed (%s)", prev->name,
+                     strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
 
-	if ((op_ret == -1) && (op_errno != ENOENT)) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"unlink of %s on %s failed (%s)",
-			local->loc2.path, prev->this->name,
-                        strerror (op_errno));
-		local->op_ret   = -1;
-		local->op_errno = op_errno;
-	}
+    /* If linkto creation failed move to failure cleanup code,
+     * instead of continuing with creating the link file */
+    if (local->op_ret != 0) {
+        goto cleanup;
+    }
 
-        if (local->op_ret == -1)
-                goto cleanup;
+    gf_msg_trace(this->name, 0, "link %s => %s (%s)", local->loc.path,
+                 local->loc2.path, src_cached->name);
+    if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+        DHT_MARKER_DONT_ACCOUNT(xattr);
+    }
 
-        dht_do_rename (frame);
+    local->added_link = _gf_true;
 
-	return 0;
+    STACK_WIND_COOKIE(frame, dht_rename_link_cbk, src_cached, src_cached,
+                      src_cached->fops->link, &local->loc, &local->loc2, xattr);
+
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
+
+    if (xattr)
+        dict_unref(xattr);
 
-	return 0;
+    return 0;
 }
 
+int
+dht_rename_unlink_links_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *preparent, struct iatt *postparent,
+                            dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if ((op_ret == -1) && (op_errno != ENOENT)) {
+        gf_msg_debug(this->name, 0, "unlink of %s on %s failed (%s)",
+                     local->loc2.path, prev->name, strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
+
+    if (local->op_ret == -1)
+        goto cleanup;
+
+    dht_do_rename(frame);
+
+    return 0;
+
+cleanup:
+    dht_rename_cleanup(frame);
+
+    return 0;
+}
 
 int
-dht_rename_create_links (call_frame_t *frame)
+dht_rename_create_links(call_frame_t *frame)
 {
-        dht_local_t *local = NULL;
-        xlator_t    *this = NULL;
-        xlator_t    *src_hashed = NULL;
-        xlator_t    *src_cached = NULL;
-        xlator_t    *dst_hashed = NULL;
-        xlator_t    *dst_cached = NULL;
-        int          call_cnt = 0;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    int call_cnt = 0;
+    dict_t *xattr = NULL;
 
+    local = frame->local;
+    this = frame->this;
 
-        local = frame->local;
-        this  = frame->this;
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
+    DHT_MARK_FOP_INTERNAL(xattr);
 
+    if (src_cached == dst_cached) {
+        dict_t *xattr_new = NULL;
 
-        if (src_cached == dst_cached) {
-                if (dst_hashed == dst_cached)
-                        goto nolinks;
+        if (dst_hashed == dst_cached)
+            goto nolinks;
 
-		gf_log (this->name, GF_LOG_TRACE,
-			"unlinking dst linkfile %s @ %s",
-			local->loc2.path, dst_hashed->name);
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-		STACK_WIND (frame, dht_rename_unlink_links_cbk,
-			    dst_hashed, dst_hashed->fops->unlink,
-			    &local->loc2, 0, NULL);
-                return 0;
-        }
+        gf_msg_trace(this->name, 0, "unlinking dst linkfile %s @ %s",
+                     local->loc2.path, dst_hashed->name);
 
-        if (dst_hashed != src_hashed && dst_hashed != src_cached)
-                call_cnt++;
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-        if (src_cached != dst_hashed)
-                call_cnt++;
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_links_cbk, dst_hashed,
+                          dst_hashed, dst_hashed->fops->unlink, &local->loc2, 0,
+                          xattr_new);
 
-        local->call_cnt = call_cnt;
+        dict_unref(xattr_new);
+        if (xattr)
+            dict_unref(xattr);
 
-        if (dst_hashed != src_hashed && dst_hashed != src_cached) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "linkfile %s @ %s => %s",
-                        local->loc.path, dst_hashed->name, src_cached->name);
-                memcpy (local->gfid, local->loc.inode->gfid, 16);
-		dht_linkfile_create (frame, dht_rename_links_cbk,
-				     src_cached, dst_hashed, &local->loc);
-	}
+        return 0;
+    }
+
+    if (src_cached != dst_hashed) {
+        /* needed to create the link file */
+        call_cnt++;
+        if (dst_hashed != src_hashed)
+            /* needed to create the linkto file */
+            call_cnt++;
+    }
+
+    /* We should not have any failures post the link creation, as this
+     * introduces the newname into the namespace. Clients could have cached
+     * the existence of the newname and may start taking actions based on
+     * the same. Hence create the linkto first, and then attempt the link.
+     *
+     * NOTE: If another client is attempting the same oldname -> newname
+     * rename, and finds both file names as existing, and are hard links
+     * to each other, then FUSE would send in an unlink for oldname. In
+     * this time duration if we treat the linkto as a critical error and
+     * unlink the newname we created, we would have effectively lost the
+     * file to rename operations. */
+    if (dst_hashed != src_hashed && src_cached != dst_hashed) {
+        gf_msg_trace(this->name, 0, "linkfile %s @ %s => %s", local->loc.path,
+                     dst_hashed->name, src_cached->name);
+
+        memcpy(local->gfid, local->loc.inode->gfid, 16);
+        dht_linkfile_create(frame, dht_rename_linkto_cbk, this, src_cached,
+                            dst_hashed, &local->loc);
+    } else if (src_cached != dst_hashed) {
+        dict_t *xattr_new = NULL;
+
+        xattr_new = dict_copy_with_ref(xattr, NULL);
+
+        gf_msg_trace(this->name, 0, "link %s => %s (%s)", local->loc.path,
+                     local->loc2.path, src_cached->name);
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
+        }
 
-	if (src_cached != dst_hashed) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"link %s => %s (%s)", local->loc.path,
-			local->loc2.path, src_cached->name);
-		STACK_WIND (frame, dht_rename_links_cbk,
-			    src_cached, src_cached->fops->link,
-			    &local->loc, &local->loc2, NULL);
-	}
+        local->added_link = _gf_true;
+
+        STACK_WIND_COOKIE(frame, dht_rename_link_cbk, src_cached, src_cached,
+                          src_cached->fops->link, &local->loc, &local->loc2,
+                          xattr_new);
+
+        dict_unref(xattr_new);
+    }
 
 nolinks:
-        if (!call_cnt) {
-                /* skip to next step */
-                dht_do_rename (frame);
+    if (!call_cnt) {
+        /* skip to next step */
+        dht_do_rename(frame);
+    }
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
+}
+
+int
+dht_rename_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, dict_t *xattr,
+                      struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_server[GF_UUID_BUF_SIZE] = {0};
+    int child_index = -1;
+    gf_boolean_t is_src = _gf_false;
+    loc_t *loc = NULL;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+    conf = this->private;
+
+    is_src = (child_index == 0);
+    if (is_src)
+        loc = &local->loc;
+    else
+        loc = &local->loc2;
+
+    if (op_ret >= 0) {
+        if (is_src)
+            local->src_cached = dht_subvol_get_cached(this, local->loc.inode);
+        else {
+            if (loc->inode)
+                gf_uuid_unparse(loc->inode->gfid, gfid_local);
+
+            gf_msg_debug(this->name, 0,
+                         "dst_cached before lookup: %s, "
+                         "(path:%s)(gfid:%s),",
+                         local->loc2.path,
+                         local->dst_cached ? local->dst_cached->name : NULL,
+                         local->dst_cached ? gfid_local : NULL);
+
+            local->dst_cached = dht_subvol_get_cached(this,
+                                                      local->loc2_copy.inode);
+
+            gf_uuid_unparse(stbuf->ia_gfid, gfid_local);
+
+            gf_msg_debug(this->name, GF_LOG_WARNING,
+                         "dst_cached after lookup: %s, "
+                         "(path:%s)(gfid:%s)",
+                         local->loc2.path,
+                         local->dst_cached ? local->dst_cached->name : NULL,
+                         local->dst_cached ? gfid_local : NULL);
+
+            if ((local->loc2.inode == NULL) ||
+                gf_uuid_compare(stbuf->ia_gfid, local->loc2.inode->gfid)) {
+                if (local->loc2.inode != NULL) {
+                    inode_unlink(local->loc2.inode, local->loc2.parent,
+                                 local->loc2.name);
+                    inode_unref(local->loc2.inode);
+                }
+
+                local->loc2.inode = inode_link(local->loc2_copy.inode,
+                                               local->loc2_copy.parent,
+                                               local->loc2_copy.name, stbuf);
+                gf_uuid_copy(local->loc2.gfid, stbuf->ia_gfid);
+            }
+        }
+    }
+
+    if (op_ret < 0) {
+        if (is_src) {
+            /* The meaning of is_linkfile is overloaded here. For locking
+             * to work properly both rebalance and rename should acquire
+             * lock on datafile. The reason for sending this lookup is to
+             * find out whether we've acquired a lock on data file.
+             * Between the lookup before rename and this rename, the
+             * file could be migrated by a rebalance process and now this
+             * file this might be a linkto file. We verify that by sending
+             * this lookup. However, if this lookup fails we cannot really
+             * say whether we've acquired lock on a datafile or linkto file.
+             * So, we act conservatively and _assume_
+             * that this is a linkfile and fail the rename operation.
+             */
+            local->is_linkfile = _gf_true;
+            local->op_errno = op_errno;
+        } else {
+            if (local->dst_cached)
+                gf_msg_debug(this->name, op_errno,
+                             "file %s (gfid:%s) was present "
+                             "(hashed-subvol=%s, "
+                             "cached-subvol=%s) before rename,"
+                             " but lookup failed",
+                             local->loc2.path,
+                             uuid_utoa(local->loc2.inode->gfid),
+                             local->dst_hashed->name, local->dst_cached->name);
+            if (dht_inode_missing(op_errno))
+                local->dst_cached = NULL;
+        }
+    } else if (is_src && xattr &&
+               check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        local->is_linkfile = _gf_true;
+        /* Found linkto file instead of data file, passdown ENOENT
+         * based on the above comment */
+        local->op_errno = ENOENT;
+    }
+
+    if (!local->is_linkfile && (op_ret >= 0) &&
+        gf_uuid_compare(loc->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(loc->gfid, gfid_local);
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_server);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "path:%s, received a different gfid, local_gfid= %s"
+               " server_gfid: %s",
+               local->loc.path, gfid_local, gfid_server);
+
+        /* Will passdown ENOENT anyway since the file we sent on
+         * rename is replaced with a different file */
+        local->op_errno = ENOENT;
+        /* Since local->is_linkfile is used here to detect failure,
+         * marking this to true */
+        local->is_linkfile = _gf_true;
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->is_linkfile) {
+            local->op_ret = -1;
+            goto fail;
         }
 
-        return 0;
-}
+        dht_rename_create_links(frame);
+    }
 
+    return 0;
+fail:
+    dht_rename_unlock(frame, this);
+    return 0;
+}
 
 int
-dht_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        xlator_t    *src_cached = NULL;
-        xlator_t    *src_hashed = NULL;
-        xlator_t    *dst_cached = NULL;
-        xlator_t    *dst_hashed = NULL;
-        int          op_errno = -1;
-        int          ret = -1;
-        dht_local_t *local = NULL;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-        src_hashed = dht_subvol_get_hashed (this, oldloc);
-        if (!src_hashed) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no subvolume in layout for path=%s",
-                        oldloc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+dht_rename_file_lock1_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "protecting namespace of %s failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->current == &local->lock[0] ? local->loc.path
+                                                 : local->loc2.path,
+               local->loc.path, src_gfid, local->src_hashed->name,
+               local->loc2.path, dst_gfid,
+               local->dst_hashed ? local->dst_hashed->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    if (local->current == &local->lock[0]) {
+        loc = &local->loc2;
+        subvol = local->dst_hashed;
+        local->current = &local->lock[1];
+    } else {
+        loc = &local->loc;
+        subvol = local->src_hashed;
+        local->current = &local->lock[0];
+    }
+
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_lock_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
+err:
+    /* No harm in calling an extra unlock */
+    dht_rename_unlock(frame, this);
+    return 0;
+}
 
-        src_cached = dht_subvol_get_cached (this, oldloc->inode);
-        if (!src_cached) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no cached subvolume for path=%s", oldloc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+int32_t
+dht_rename_file_protect_namespace(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int32_t op_ret,
+                                  int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring inodelk failed "
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+
+        goto err;
+    }
+
+    /* Locks on src and dst needs to ordered which otherwise might cause
+     * deadlocks when rename (src, dst) and rename (dst, src) is done from
+     * two different clients
+     */
+    ret = dht_order_rename_lock(frame, &loc, &subvol);
+    if (ret) {
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_file_lock1_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
 
-        dst_hashed = dht_subvol_get_hashed (this, newloc);
-        if (!dst_hashed) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no subvolume in layout for path=%s",
-                        newloc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+err:
+    /* Its fine to call unlock even when no locks are acquired, as we check
+     * for lock->locked before winding a unlock call.
+     */
+    dht_rename_unlock(frame, this);
 
-        if (newloc->inode)
-                dst_cached = dht_subvol_get_cached (this, newloc->inode);
+    return 0;
+}
 
-        local = dht_local_init (frame, oldloc, NULL, GF_FOP_RENAME);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        /* cached_subvol will be set from dht_local_init, reset it to NULL,
-           as the logic of handling rename is different  */
-        local->cached_subvol = NULL;
-
-        ret = loc_copy (&local->loc2, newloc);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto err;
+int32_t
+dht_rename_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dict_t *xattr_req = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *subvol = NULL;
+    dht_lock_t *lock = NULL;
+
+    local = frame->local;
+    conf = this->private;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "protecting namespace of %s failed. "
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->current == &local->lock[0] ? local->loc.path
+                                                 : local->loc2.path,
+               local->loc.path, src_gfid, local->src_hashed->name,
+               local->loc2.path, dst_gfid,
+               local->dst_hashed ? local->dst_hashed->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+
+        goto done;
+    }
+
+    xattr_req = dict_new();
+    if (xattr_req == NULL) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto done;
+    }
+
+    op_ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = -op_ret;
+        goto done;
+    }
+
+    /* dst_cached might've changed. This normally happens for two reasons:
+     * 1. rebalance migrated dst
+     * 2. Another parallel rename was done overwriting dst
+     *
+     * Doing a lookup on local->loc2 when dst exists, but is associated
+     * with a different gfid will result in an ESTALE error. So, do a fresh
+     * lookup with a new inode on dst-path and handle change of dst-cached
+     * in the cbk. Also, to identify dst-cached changes we do a lookup on
+     * "this" rather than the subvol.
+     */
+    loc_copy(&local->loc2_copy, &local->loc2);
+    inode_unref(local->loc2_copy.inode);
+    local->loc2_copy.inode = inode_new(local->loc.inode->table);
+
+    /* Why not use local->lock.locks[?].loc for lookup post lock phase
+     * ---------------------------------------------------------------
+     * "layout.parent_layout.locks[?].loc" does not have the name and pargfid
+     * populated.
+     * Reason: If we had populated the name and pargfid, server might
+     * resolve to a successful lookup even if there is a file with same name
+     * with a different gfid(unlink & create) as server does name based
+     * resolution on first priority. And this can result in operating on a
+     * different inode entirely.
+     *
+     * Now consider a scenario where source file was renamed by some other
+     * client to a new name just before this lock was granted. So if a
+     * lookup would be done on local->lock[0].layout.parent_layout.locks[?].loc,
+     * server will send success even if the entry was renamed (since server will
+     * do a gfid based resolution). So once a lock is granted, make sure the
+     * file exists with the name that the client requested with.
+     * */
+
+    local->call_cnt = 2;
+    for (i = 0; i < 2; i++) {
+        if (i == 0) {
+            lock = local->rename_inodelk_backward_compatible[0];
+            if (gf_uuid_compare(local->loc.gfid, lock->loc.gfid) == 0)
+                subvol = lock->xl;
+            else {
+                lock = local->rename_inodelk_backward_compatible[1];
+                subvol = lock->xl;
+            }
+        } else {
+            subvol = this;
         }
 
-        local->src_hashed = src_hashed;
-        local->src_cached = src_cached;
-        local->dst_hashed = dst_hashed;
-        local->dst_cached = dst_cached;
+        STACK_WIND_COOKIE(frame, dht_rename_lookup_cbk, (void *)(long)i, subvol,
+                          subvol->fops->lookup,
+                          (i == 0) ? &local->loc : &local->loc2_copy,
+                          xattr_req);
+    }
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "renaming %s (hash=%s/cache=%s) => %s (hash=%s/cache=%s)",
-                oldloc->path, src_hashed->name, src_cached->name,
-                newloc->path, dst_hashed->name,
-                dst_cached ? dst_cached->name : "<nul>");
+    dict_unref(xattr_req);
+    return 0;
 
-        if (IA_ISDIR (oldloc->inode->ia_type)) {
-                dht_rename_dir (frame, this);
-        } else {
-                local->op_ret = 0;
-                dht_rename_create_links (frame);
+done:
+    /* Its fine to call unlock even when no locks are acquired, as we check
+     * for lock->locked before winding a unlock call.
+     */
+    dht_rename_unlock(frame, this);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    return 0;
+}
+
+int
+dht_rename_lock(call_frame_t *frame)
+{
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
+
+    local = frame->local;
+
+    if (local->dst_cached)
+        count++;
+
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (lk_array == NULL)
+        goto err;
+
+    lk_array[0] = dht_lock_new(frame->this, local->src_cached, &local->loc,
+                               F_WRLCK, DHT_FILE_MIGRATE_DOMAIN, NULL,
+                               FAIL_ON_ANY_ERROR);
+    if (lk_array[0] == NULL)
+        goto err;
+
+    if (local->dst_cached) {
+        /* dst might be removed by the time inodelk reaches bricks,
+         * which can result in ESTALE errors. POSIX imposes no
+         * restriction for dst to be present for renames to be
+         * successful. So, we'll ignore ESTALE errors. As far as
+         * synchronization on dst goes, we'll achieve the same by
+         * holding entrylk on parent directory of dst in the namespace
+         * of basename(dst). Also, there might not be quorum in cluster
+         * xlators like EC/disperse on errno, in which case they return
+         * EIO. For eg., in a disperse (4 + 2), 3 might return success
+         * and three might return ESTALE. Disperse, having no Quorum
+         * unwinds inodelk with EIO. So, ignore EIO too.
+         */
+        lk_array[1] = dht_lock_new(frame->this, local->dst_cached, &local->loc2,
+                                   F_WRLCK, DHT_FILE_MIGRATE_DOMAIN, NULL,
+                                   IGNORE_ENOENT_ESTALE_EIO);
+        if (lk_array[1] == NULL)
+            goto err;
+    }
+
+    local->rename_inodelk_backward_compatible = lk_array;
+    local->rename_inodelk_bc_count = count;
+
+    /* retaining inodelks for the sake of backward compatibility. Please
+     * make sure to remove this inodelk once all of 3.10, 3.12 and 3.13
+     * reach EOL. Better way of getting synchronization would be to acquire
+     * entrylks on src and dst parent directories in the namespace of
+     * basenames of src and dst
+     */
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_rename_file_protect_namespace);
+    if (ret < 0) {
+        local->rename_inodelk_backward_compatible = NULL;
+        local->rename_inodelk_bc_count = 0;
+        goto err;
+    }
+
+    return 0;
+err:
+    if (lk_array != NULL) {
+        int tmp_count = 0, i = 0;
+
+        for (i = 0; (i < count) && (lk_array[i]); i++, tmp_count++)
+            ;
+
+        dht_lock_array_free(lk_array, tmp_count);
+        GF_FREE(lk_array);
+    }
+
+    return -1;
+}
+
+int
+dht_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+    xlator_t *src_cached = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    int op_errno = -1;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    char newgfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(oldloc, err);
+    VALIDATE_OR_GOTO(newloc, err);
+
+    gf_uuid_unparse(oldloc->inode->gfid, gfid);
+
+    src_hashed = dht_subvol_get_hashed(this, oldloc);
+    if (!src_hashed) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No hashed subvolume in layout for path=%s,"
+               "(gfid = %s)",
+               oldloc->path, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    src_cached = dht_subvol_get_cached(this, oldloc->inode);
+    if (!src_cached) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No cached subvolume for path = %s,"
+               "(gfid = %s)",
+               oldloc->path, gfid);
+
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    dst_hashed = dht_subvol_get_hashed(this, newloc);
+    if (!dst_hashed) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No hashed subvolume in layout for path=%s", newloc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (newloc->inode)
+        dst_cached = dht_subvol_get_cached(this, newloc->inode);
+
+    local = dht_local_init(frame, oldloc, NULL, GF_FOP_RENAME);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    /* cached_subvol will be set from dht_local_init, reset it to NULL,
+       as the logic of handling rename is different  */
+    local->cached_subvol = NULL;
+
+    ret = loc_copy(&local->loc2, newloc);
+    if (ret == -1) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->src_hashed = src_hashed;
+    local->src_cached = src_cached;
+    local->dst_hashed = dst_hashed;
+    local->dst_cached = dst_cached;
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (newloc->inode)
+        gf_uuid_unparse(newloc->inode->gfid, newgfid);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_INFO,
+           "renaming %s (%s) (hash=%s/cache=%s) => %s (%s) "
+           "(hash=%s/cache=%s) ",
+           oldloc->path, gfid, src_hashed->name, src_cached->name, newloc->path,
+           newloc->inode ? newgfid : NULL, dst_hashed->name,
+           dst_cached ? dst_cached->name : "<nul>");
+
+    if (IA_ISDIR(oldloc->inode->ia_type)) {
+        dht_rename_dir(frame, this);
+    } else {
+        local->op_ret = 0;
+        ret = dht_rename_lock(frame);
+        if (ret < 0) {
+            op_errno = ENOMEM;
+            goto err;
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
+}
+
+int
+dht_pt_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
+{
+    gf_boolean_t free_xdata = _gf_false;
+
+    /* Just a pass through */
+    if (!IA_ISDIR(oldloc->inode->ia_type)) {
+        if (!xdata) {
+            free_xdata = _gf_true;
+        }
+        DHT_CHANGELOG_TRACK_AS_RENAME(xdata, oldloc, newloc);
+    }
+    default_rename(frame, this, oldloc, newloc, xdata);
+    if (free_xdata && xdata) {
+        dict_unref(xdata);
+        xdata = NULL;
+    }
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index 8b7d15ce520..3e24065227c 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -8,920 +8,2593 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "dht-common.h"
-
-#define DHT_SET_LAYOUT_RANGE(layout,i,srt,chunk,cnt,path)    do {       \
-                layout->list[i].start = srt;                            \
-                layout->list[i].stop  = srt + chunk - 1;                \
-                                                                        \
-                gf_log (this->name, GF_LOG_TRACE,                       \
-                        "gave fix: %u - %u on %s for %s",               \
-                        layout->list[i].start, layout->list[i].stop,    \
-                        layout->list[i].xlator->name, path);            \
-        } while (0)
-
-static inline uint32_t
-dht_find_overlap (int idx, int cnk_idx, uint32_t start, uint32_t stop,
-                  uint32_t chunk_size)
-{
-        uint32_t overlap = 0;
-        uint32_t chunk_begin = 0;
-
-        chunk_begin = cnk_idx * chunk_size;
-
-        /* There is no chance of overlap */
-        if ((chunk_begin > stop) ||
-            ((chunk_begin + chunk_size) < start))
-                goto out;
+#include "dht-lock.h"
+
+#define DHT_SET_LAYOUT_RANGE(layout, i, srt, chunk, path)                      \
+    do {                                                                       \
+        layout->list[i].start = srt;                                           \
+        layout->list[i].stop = srt + chunk - 1;                                \
+        layout->list[i].commit_hash = layout->commit_hash;                     \
+                                                                               \
+        gf_msg_trace(this->name, 0,                                            \
+                     "gave fix: 0x%x - 0x%x, with commit-hash 0x%x"            \
+                     " on %s for %s",                                          \
+                     layout->list[i].start, layout->list[i].stop,              \
+                     layout->list[i].commit_hash,                              \
+                     layout->list[i].xlator->name, path);                      \
+    } while (0)
+
+#define DHT_RESET_LAYOUT_RANGE(layout)                                         \
+    do {                                                                       \
+        int cnt = 0;                                                           \
+        for (cnt = 0; cnt < layout->cnt; cnt++) {                              \
+            layout->list[cnt].start = 0;                                       \
+            layout->list[cnt].stop = 0;                                        \
+        }                                                                      \
+    } while (0)
+
+static int
+dht_selfheal_layout_lock(call_frame_t *frame, dht_layout_t *layout,
+                         gf_boolean_t newdir, dht_selfheal_layout_t healer,
+                         dht_need_heal_t should_heal);
+
+static uint32_t
+dht_overlap_calc(dht_layout_t *old, int o, dht_layout_t *new, int n)
+{
+    if (o >= old->cnt || n >= new->cnt)
+        return 0;
 
-        if ((chunk_begin <= start) &&
-            ((chunk_begin + chunk_size) <= stop)) {
-                overlap = ((chunk_begin + chunk_size) - start);
-                goto out;
-        }
+    if (old->list[o].err > 0 || new->list[n].err > 0)
+        return 0;
 
-        if ((chunk_begin <= start) &&
-            ((chunk_begin + chunk_size) >= stop)) {
-                overlap = (stop - start);
-                goto out;
-        }
+    if (old->list[o].start == old->list[o].stop) {
+        return 0;
+    }
 
-        if ((chunk_begin < stop) &&
-            ((chunk_begin + chunk_size) >= stop)) {
-                overlap = (stop - chunk_begin);
-                goto out;
-        }
+    if (new->list[n].start == new->list[n].stop) {
+        return 0;
+    }
 
-out:
-        return overlap;
+    if ((old->list[o].start > new->list[n].stop) ||
+        (old->list[o].stop < new->list[n].start))
+        return 0;
+
+    return min(old->list[o].stop, new->list[n].stop) -
+           max(old->list[o].start, new->list[n].start) + 1;
 }
 
 int
-dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
+dht_selfheal_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
-        local = frame->local;
-        local->selfheal.dir_cbk (frame, NULL, frame->this, ret,
-                                 local->op_errno, NULL);
+int
+dht_selfheal_dir_finish(call_frame_t *frame, xlator_t *this, int ret,
+                        int invoke_cbk)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
 
-        return 0;
-}
+    local = frame->local;
 
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[0].ns.directory_ns);
 
-int
-dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int op_ret, int op_errno, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *subvol = NULL;
-        int           i = 0;
-        dht_layout_t *layout = NULL;
-        int           err = 0;
-        int           this_call_cnt = 0;
-
-        local = frame->local;
-        layout = local->selfheal.layout;
-        prev = cookie;
-        subvol = prev->this;
-
-        if (op_ret == 0)
-                err = 0;
-        else
-                err = op_errno;
+    /* Unlock inodelk */
+    lock_count = dht_lock_count(local->lock[0].ns.parent_layout.locks,
+                                local->lock[0].ns.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == subvol) {
-                        layout->list[i].err = err;
-                        break;
-                }
-        }
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
 
-        this_call_cnt = dht_frame_return (frame);
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
 
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_dir_finish (frame, this, 0);
-        }
+    lock_local->lock[0].ns.parent_layout.locks = local->lock[0]
+                                                     .ns.parent_layout.locks;
+    lock_local->lock[0]
+        .ns.parent_layout.lk_count = local->lock[0].ns.parent_layout.lk_count;
 
-        return 0;
-}
+    local->lock[0].ns.parent_layout.locks = NULL;
+    local->lock[0].ns.parent_layout.lk_count = 0;
 
+    dht_unlock_inodelk(lock_frame, lock_local->lock[0].ns.parent_layout.locks,
+                       lock_local->lock[0].ns.parent_layout.lk_count,
+                       dht_selfheal_unlock_cbk);
+    lock_frame = NULL;
+
+done:
+    if (invoke_cbk)
+        local->selfheal.dir_cbk(frame, NULL, frame->this, ret, local->op_errno,
+                                NULL);
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
+
+    return 0;
+}
 
 int
-dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
-                                  dht_layout_t *layout, int i)
+dht_refresh_layout_done(call_frame_t *frame)
 {
-        xlator_t          *subvol = NULL;
-        dict_t            *xattr = NULL;
-        int                ret = 0;
-        xlator_t          *this = NULL;
-        int32_t           *disk_layout = NULL;
-        dht_local_t       *local = NULL;
+    int ret = -1;
+    dht_layout_t *refreshed = NULL, *heal = NULL;
+    dht_local_t *local = NULL;
+    dht_need_heal_t should_heal = NULL;
+    dht_selfheal_layout_t healer = NULL;
 
+    local = frame->local;
 
-        local = frame->local;
-        subvol = layout->list[i].xlator;
-        this = frame->this;
+    refreshed = local->selfheal.refreshed_layout;
+    heal = local->selfheal.layout;
 
-        GF_VALIDATE_OR_GOTO ("", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, layout, err);
-        GF_VALIDATE_OR_GOTO (this->name, local, err);
-        GF_VALIDATE_OR_GOTO (this->name, subvol, err);
+    healer = local->selfheal.healer;
+    should_heal = local->selfheal.should_heal;
 
-        xattr = get_new_dict ();
-        if (!xattr) {
-                goto err;
+    ret = dht_layout_sort(refreshed);
+    if (ret == -1) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LAYOUT_SORT_FAILED, NULL);
+        goto err;
+    }
+
+    if (should_heal(frame, &heal, &refreshed)) {
+        healer(frame, &local->loc, heal);
+    } else {
+        local->selfheal.layout = NULL;
+        local->selfheal.refreshed_layout = NULL;
+        local->selfheal.layout = refreshed;
+
+        dht_layout_unref(frame->this, heal);
+
+        dht_selfheal_dir_finish(frame, frame->this, 0, 1);
+    }
+
+    return 0;
+
+err:
+    dht_selfheal_dir_finish(frame, frame->this, -1, 1);
+    return 0;
+}
+
+int
+dht_refresh_layout_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, inode_t *inode,
+                       struct iatt *stbuf, dict_t *xattr,
+                       struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", this->private, err);
+
+    local = frame->local;
+    prev = cookie;
+
+    layout = local->selfheal.refreshed_layout;
+
+    LOCK(&frame->lock);
+    {
+        op_ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
+
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+
+        if (op_ret == -1) {
+            gf_uuid_unparse(local->loc.gfid, gfid);
+            local->op_errno = op_errno;
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    DHT_MSG_FILE_LOOKUP_FAILED, "path=%s", local->loc.path,
+                    "name=%s", prev->name, "gfid=%s", gfid, NULL);
+
+            goto unlock;
         }
 
-        ret = dht_disk_layout_extract (this, layout, i, &disk_layout);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: (subvol %s) failed to extract disk layout",
-                        loc->path, subvol->name);
-                goto err;
+        local->op_ret = 0;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        if (local->op_ret == 0) {
+            local->refresh_layout_done(frame);
+        } else {
+            goto err;
         }
+    }
 
-        ret = dict_set_bin (xattr, "trusted.glusterfs.dht",
-                            disk_layout, 4 * 4);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: (subvol %s) failed to set xattr dictionary",
-                        loc->path, subvol->name);
-                goto err;
+    return 0;
+
+err:
+    if (local) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    }
+    return 0;
+}
+
+int
+dht_refresh_layout(call_frame_t *frame)
+{
+    int call_cnt = 0;
+    int i = 0, ret = -1;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+
+    this = frame->this;
+    conf = this->private;
+    local = frame->local;
+
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
+    local->op_ret = -1;
+
+    if (local->selfheal.refreshed_layout) {
+        dht_layout_unref(this, local->selfheal.refreshed_layout);
+        local->selfheal.refreshed_layout = NULL;
+    }
+
+    local->selfheal.refreshed_layout = dht_layout_new(this,
+                                                      conf->subvolume_cnt);
+    if (!local->selfheal.refreshed_layout) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    if (local->xattr != NULL) {
+        dict_del(local->xattr, conf->xattr_name);
+    }
+
+    if (local->xattr_req == NULL) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        local->xattr_req = dict_new();
+        if (local->xattr_req == NULL) {
+            gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                    "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+            goto out;
         }
-        disk_layout = NULL;
+    }
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "setting hash range %u - %u (type %d) on subvolume %s for %s",
-                layout->list[i].start, layout->list[i].stop,
-                layout->type, subvol->name, loc->path);
+    if (dict_get(local->xattr_req, conf->xattr_name) == 0) {
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", local->loc.path, "key=%s", conf->xattr_name,
+                    NULL);
+    }
 
-        dict_ref (xattr);
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_refresh_layout_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, &local->loc,
+                          local->xattr_req);
+    }
 
-        if (!uuid_is_null (local->gfid))
-                uuid_copy (loc->gfid, local->gfid);
+    return 0;
 
-        STACK_WIND (frame, dht_selfheal_dir_xattr_cbk,
-                    subvol, subvol->fops->setxattr,
-                    loc, xattr, 0, NULL);
+out:
+    if (local) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    }
+    return 0;
+}
 
-        dict_unref (xattr);
+int32_t
+dht_selfheal_layout_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
 
-        return 0;
+    local = frame->local;
 
-err:
-        if (xattr)
-                dict_destroy (xattr);
+    if (!local) {
+        goto err;
+    }
 
-        GF_FREE (disk_layout);
+    if (op_ret < 0) {
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        dht_selfheal_dir_xattr_cbk (frame, subvol, frame->this,
-                                    -1, ENOMEM, NULL);
-        return 0;
+    local->refresh_layout_unlock = dht_selfheal_dir_finish;
+    local->refresh_layout_done = dht_refresh_layout_done;
+
+    dht_refresh_layout(frame);
+    return 0;
+
+err:
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
 }
 
-int
-dht_fix_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+gf_boolean_t
+dht_should_heal_layout(call_frame_t *frame, dht_layout_t **heal,
+                       dht_layout_t **ondisk)
 {
-        dht_local_t *local = NULL;
-        int          i = 0;
-        int          count = 0;
-        xlator_t    *this = NULL;
+    gf_boolean_t fixit = _gf_true;
+    dht_local_t *local = NULL;
+    int heal_missing_dirs = 0;
+
+    local = frame->local;
+
+    if ((heal == NULL) || (*heal == NULL) || (ondisk == NULL) ||
+        (*ondisk == NULL))
+        goto out;
+
+    dht_layout_anomalies(
+        frame->this, &local->loc, *ondisk, &local->selfheal.hole_cnt,
+        &local->selfheal.overlaps_cnt, &local->selfheal.missing_cnt,
+        &local->selfheal.down, &local->selfheal.misc, NULL);
+
+    /* Directories might've been created as part of this self-heal. We've to
+     * sync non-layout xattrs and set range 0-0 on new directories
+     */
+    heal_missing_dirs = local->selfheal.force_mkdir
+                            ? local->selfheal.force_mkdir
+                            : dht_layout_missing_dirs(*heal);
+
+    if ((local->selfheal.hole_cnt == 0) &&
+        (local->selfheal.overlaps_cnt == 0) && heal_missing_dirs) {
+        dht_layout_t *tmp = NULL;
+
+        /* Just added a brick and need to set 0-0 range on this brick.
+         * But ondisk layout is well-formed. So, swap layouts "heal" and
+         * "ondisk". Now "ondisk" layout will be used for healing
+         * xattrs. If there are any non-participating subvols in
+         * "ondisk" layout, dht_selfheal_dir_xattr_persubvol will set
+         * 0-0 and non-layout xattrs. This way we won't end up in
+         * "corrupting" already set and well-formed "ondisk" layout.
+         */
+        tmp = *heal;
+        *heal = *ondisk;
+        *ondisk = tmp;
+
+        /* Current selfheal code, heals non-layout xattrs only after
+         * an add-brick. In fact non-layout xattrs are considered as
+         * secondary citizens which are healed only if layout xattrs
+         * need to be healed. This is wrong, since for eg., quota can be
+         * set when layout is well-formed, but a node is down. Also,
+         * just for healing non-layout xattrs, we don't need locking.
+         * This issue is _NOT FIXED_ by this patch.
+         */
+    }
+
+    fixit = (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt ||
+             heal_missing_dirs);
 
-        local = frame->local;
-        this = frame->this;
+out:
+    return fixit;
+}
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "writing the new range for all subvolumes");
+int
+dht_layout_span(dht_layout_t *layout)
+{
+    int i = 0, count = 0;
 
-        local->call_cnt = count = layout->cnt;
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err)
+            continue;
 
-        for (i = 0; i < layout->cnt; i++) {
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i);
+        if (layout->list[i].start != layout->list[i].stop)
+            count++;
+    }
 
-                if (--count == 0)
-                        break;
-        }
-        return 0;
+    return count;
 }
 
 int
-dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+dht_decommissioned_bricks_in_layout(xlator_t *this, dht_layout_t *layout)
 {
-        dht_local_t *local = NULL;
-        int          missing_xattr = 0;
-        int          i = 0;
-        xlator_t    *this = NULL;
+    dht_conf_t *conf = NULL;
+    int count = 0, i = 0, j = 0;
 
-        local = frame->local;
-        this = frame->this;
+    if ((this == NULL) || (layout == NULL))
+        goto out;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop) {
-                        /* err != -1 would mean xattr present on the directory
-                         * or the directory is non existent.
-                         * !layout->list[i].stop would mean layout absent
-                         */
+    conf = this->private;
 
-                        continue;
-                }
-                missing_xattr++;
+    for (i = 0; i < layout->cnt; i++) {
+        for (j = 0; j < conf->subvolume_cnt; j++) {
+            if (conf->decommissioned_bricks[j] &&
+                conf->decommissioned_bricks[j] == layout->list[i].xlator) {
+                count++;
+            }
         }
+    }
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "%d subvolumes missing xattr for %s",
-                missing_xattr, loc->path);
+out:
+    return count;
+}
 
-        if (missing_xattr == 0) {
-                dht_selfheal_dir_finish (frame, this, 0);
-                return 0;
+dht_distribution_type_t
+dht_distribution_type(xlator_t *this, dht_layout_t *layout)
+{
+    dht_distribution_type_t type = GF_DHT_EQUAL_DISTRIBUTION;
+    int i = 0;
+    uint32_t start_range = 0, range = 0, diff = 0;
+
+    if ((this == NULL) || (layout == NULL) || (layout->cnt < 1)) {
+        goto out;
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (start_range == 0) {
+            start_range = layout->list[i].stop - layout->list[i].start;
+            continue;
         }
 
-        local->call_cnt = missing_xattr;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop)
-                        continue;
-
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i);
+        range = layout->list[i].stop - layout->list[i].start;
+        diff = (range >= start_range) ? range - start_range
+                                      : start_range - range;
 
-                if (--missing_xattr == 0)
-                        break;
+        if ((range != 0) && (diff > layout->cnt)) {
+            type = GF_DHT_WEIGHTED_DISTRIBUTION;
+            break;
         }
-        return 0;
+    }
+
+out:
+    return type;
 }
 
-int
-dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, struct iatt *statpre,
-                              struct iatt *statpost, dict_t *xdata)
+gf_boolean_t
+dht_should_fix_layout(call_frame_t *frame, dht_layout_t **inmem,
+                      dht_layout_t **ondisk)
 {
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
-        int            this_call_cnt = 0;
+    gf_boolean_t fixit = _gf_true;
 
-        local  = frame->local;
-        layout = local->selfheal.layout;
+    dht_local_t *local = NULL;
+    int layout_span = 0;
+    int decommissioned_bricks = 0;
+    dht_conf_t *conf = NULL;
+    dht_distribution_type_t inmem_dist_type = 0;
+    dht_distribution_type_t ondisk_dist_type = 0;
 
-        this_call_cnt = dht_frame_return (frame);
+    conf = frame->this->private;
 
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_dir_xattr (frame, &local->loc, layout);
-        }
+    local = frame->local;
 
-        return 0;
-}
+    if ((inmem == NULL) || (*inmem == NULL) || (ondisk == NULL) ||
+        (*ondisk == NULL))
+        goto out;
 
+    dht_layout_anomalies(frame->this, &local->loc, *ondisk,
+                         &local->selfheal.hole_cnt,
+                         &local->selfheal.overlaps_cnt, NULL,
+                         &local->selfheal.down, &local->selfheal.misc, NULL);
 
-int
-dht_selfheal_dir_setattr (call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
-                          int32_t valid, dht_layout_t *layout)
+    if (local->selfheal.down || local->selfheal.misc) {
+        fixit = _gf_false;
+        goto out;
+    }
+
+    if (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt)
+        goto out;
+
+    /* If commit hashes are being updated, let it through */
+    if ((*inmem)->commit_hash != (*ondisk)->commit_hash)
+        goto out;
+
+    layout_span = dht_layout_span(*ondisk);
+
+    decommissioned_bricks = dht_decommissioned_bricks_in_layout(frame->this,
+                                                                *ondisk);
+    inmem_dist_type = dht_distribution_type(frame->this, *inmem);
+    ondisk_dist_type = dht_distribution_type(frame->this, *ondisk);
+
+    if ((decommissioned_bricks == 0) &&
+        (layout_span ==
+         (conf->subvolume_cnt - conf->decommission_subvols_cnt)) &&
+        (inmem_dist_type == ondisk_dist_type))
+        fixit = _gf_false;
+
+out:
+
+    return fixit;
+}
+
+static int
+dht_selfheal_layout_lock(call_frame_t *frame, dht_layout_t *layout,
+                         gf_boolean_t newdir, dht_selfheal_layout_t healer,
+                         dht_need_heal_t should_heal)
 {
-        int           missing_attr = 0;
-        int           i     = 0;
-        dht_local_t  *local = NULL;
-        xlator_t     *this = NULL;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0;
+    dht_lock_t **lk_array = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *tmp = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local = frame->local;
-        this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == -1)
-                        missing_attr++;
+    local = frame->local;
+
+    conf = frame->this->private;
+
+    local->selfheal.healer = healer;
+    local->selfheal.should_heal = should_heal;
+
+    tmp = local->selfheal.layout;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
+    dht_layout_unref(frame->this, tmp);
+
+    if (!newdir) {
+        count = conf->subvolume_cnt;
+
+        lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+        if (lk_array == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
         }
 
-        if (missing_attr == 0) {
-                dht_selfheal_dir_xattr (frame, loc, layout);
-                return 0;
+        for (i = 0; i < count; i++) {
+            lk_array[i] = dht_lock_new(
+                frame->this, conf->subvolumes[i], &local->loc, F_WRLCK,
+                DHT_LAYOUT_HEAL_DOMAIN, NULL, FAIL_ON_ANY_ERROR);
+            if (lk_array[i] == NULL) {
+                gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+                gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM,
+                        DHT_MSG_MEM_ALLOC_FAILED, "lk_array-gfid=%s", gfid,
+                        "path=%s", local->loc.path, NULL);
+                goto err;
+            }
+        }
+    } else {
+        count = 1;
+        lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+        if (lk_array == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
+        }
+
+        lk_array[0] = dht_lock_new(frame->this, local->hashed_subvol,
+                                   &local->loc, F_WRLCK, DHT_LAYOUT_HEAL_DOMAIN,
+                                   NULL, FAIL_ON_ANY_ERROR);
+        if (lk_array[0] == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
         }
+    }
 
-        if (!uuid_is_null (local->gfid))
-                uuid_copy (loc->gfid, local->gfid);
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = count;
 
-        local->call_cnt = missing_attr;
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == -1) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "setattr for %s on subvol %s",
-                                loc->path, layout->list[i].xlator->name);
-
-                        STACK_WIND (frame, dht_selfheal_dir_setattr_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->setattr,
-                                    loc, stbuf, valid, NULL);
-                }
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_selfheal_layout_lock_cbk);
+    if (ret < 0) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+        goto err;
+    }
+
+    return 0;
+err:
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
+
+    return -1;
+}
+
+static int
+dht_selfheal_dir_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    struct iatt *stbuf = NULL;
+    int i = 0;
+    int ret = 0;
+    dht_layout_t *layout = NULL;
+    int err = 0;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    layout = local->selfheal.layout;
+    subvol = cookie;
+
+    if (op_ret == 0) {
+        err = 0;
+    } else {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "name=%s", subvol->name,
+                "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        err = op_errno;
+    }
+
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+    if (ret < 0) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_msg_debug(this->name, 0,
+                     "key = %s not present in dict"
+                     ", path:%s gfid:%s",
+                     DHT_IATT_IN_XDATA_KEY, local->loc.path, gfid);
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol) {
+            layout->list[i].err = err;
+            break;
         }
+    }
 
-        return 0;
+    LOCK(&frame->lock);
+    {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+    }
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        dht_selfheal_dir_finish(frame, this, 0, 1);
+    }
+
+    return 0;
 }
 
+/* Code is required to set user xattr to local->xattr
+ */
 int
-dht_selfheal_dir_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int op_ret, int op_errno,
-                            inode_t *inode, struct iatt *stbuf,
-                            struct iatt *preparent, struct iatt *postparent,
-                            dict_t *xdata)
-{
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
-        call_frame_t  *prev = NULL;
-        xlator_t      *subvol = NULL;
-        int            i = 0;
-        int            this_call_cnt = 0;
-
-
-        local  = frame->local;
-        layout = local->selfheal.layout;
-        prev   = cookie;
-        subvol = prev->this;
-
-        if ((op_ret == 0) || ((op_ret == -1) && (op_errno == EEXIST))) {
-                for (i = 0; i < layout->cnt; i++) {
-                        if (layout->list[i].xlator == subvol) {
-                                layout->list[i].err = -1;
-                                break;
-                        }
-                }
-        }
+dht_set_user_xattr(dict_t *dict, char *k, data_t *v, void *data)
+{
+    dict_t *set_xattr = data;
+    int ret = -1;
 
-        if (op_ret) {
-                gf_log (this->name, ((op_errno == EEXIST) ? GF_LOG_DEBUG :
-                                     GF_LOG_WARNING),
-                        "selfhealing directory %s failed: %s",
-                        local->loc.path, strerror (op_errno));
-                goto out;
+    ret = dict_set(set_xattr, k, v);
+    return ret;
+}
+
+static int
+dht_selfheal_dir_xattr_persubvol(call_frame_t *frame, loc_t *loc,
+                                 dht_layout_t *layout, int i,
+                                 xlator_t *req_subvol)
+{
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+    int32_t *disk_layout = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    data_t *data = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    if (req_subvol)
+        subvol = req_subvol;
+    else
+        subvol = layout->list[i].xlator;
+    this = frame->this;
+
+    GF_VALIDATE_OR_GOTO("", this, err);
+    GF_VALIDATE_OR_GOTO(this->name, layout, err);
+    GF_VALIDATE_OR_GOTO(this->name, local, err);
+    GF_VALIDATE_OR_GOTO(this->name, subvol, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        goto err;
+    }
+
+    xdata = dict_new();
+    if (!xdata)
+        goto err;
+
+    ret = dict_set_str(xdata, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", GLUSTERFS_INTERNAL_FOP_KEY,
+                "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    ret = dict_set_int8(xdata, DHT_IATT_IN_XDATA_KEY, 1);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", DHT_IATT_IN_XDATA_KEY,
+                "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    gf_uuid_unparse(loc->inode->gfid, gfid);
+
+    ret = dht_disk_layout_extract(this, layout, i, &disk_layout);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
+                "extract-disk-layout-failed, path=%s", loc->path, "subvol=%s",
+                subvol->name, "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    ret = dict_set_bin(xattr, conf->xattr_name, disk_layout, 4 * 4);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "path=%s", loc->path,
+                "subvol=%s", subvol->name,
+                "set-xattr-dictionary-failed"
+                "gfid=%s",
+                gfid, NULL);
+        goto err;
+    }
+    disk_layout = NULL;
+
+    gf_msg_trace(this->name, 0,
+                 "setting hash range 0x%x - 0x%x (type %d) on subvolume %s"
+                 " for %s",
+                 layout->list[i].start, layout->list[i].stop, layout->type,
+                 subvol->name, loc->path);
+
+    if (local->xattr) {
+        data = dict_get(local->xattr, QUOTA_LIMIT_KEY);
+        if (data) {
+            ret = dict_add(xattr, QUOTA_LIMIT_KEY, data);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", loc->path, "key=%s", QUOTA_LIMIT_KEY, NULL);
+            }
         }
+        data = dict_get(local->xattr, QUOTA_LIMIT_OBJECTS_KEY);
+        if (data) {
+            ret = dict_add(xattr, QUOTA_LIMIT_OBJECTS_KEY, data);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", loc->path, "key=%s", QUOTA_LIMIT_OBJECTS_KEY,
+                        NULL);
+            }
+        }
+    }
+
+    if (!gf_uuid_is_null(local->gfid))
+        gf_uuid_copy(loc->gfid, local->gfid);
+
+    STACK_WIND_COOKIE(frame, dht_selfheal_dir_xattr_cbk, (void *)subvol, subvol,
+                      subvol->fops->setxattr, loc, xattr, 0, xdata);
+
+    dict_unref(xattr);
+    dict_unref(xdata);
+
+    return 0;
+
+err:
+    if (xattr)
+        dict_unref(xattr);
+    if (xdata)
+        dict_unref(xdata);
+
+    GF_FREE(disk_layout);
 
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-        dht_iatt_merge (this, &local->postparent, postparent, prev->this);
+    dht_selfheal_dir_xattr_cbk(frame, (void *)subvol, frame->this, -1, ENOMEM,
+                               NULL);
+    return 0;
+}
+
+static int
+dht_fix_dir_xattr(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+{
+    dht_local_t *local = NULL;
+    int i = 0;
+    int count = 0;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *dummy = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    gf_msg_debug(this->name, 0, "%s: Writing the new range for all subvolumes",
+                 loc->path);
+
+    local->call_cnt = count = conf->subvolume_cnt;
+
+    if (gf_log_get_loglevel() >= GF_LOG_DEBUG)
+        dht_log_new_layout_for_dir_selfheal(this, loc, layout);
+
+    for (i = 0; i < layout->cnt; i++) {
+        dht_selfheal_dir_xattr_persubvol(frame, loc, layout, i, NULL);
+
+        if (--count == 0)
+            goto out;
+    }
+    /* if we are here, subvolcount > layout_count. subvols-per-directory
+     * option might be set here. We need to clear out layout from the
+     * non-participating subvolumes, else it will result in overlaps */
+    dummy = dht_layout_new(this, 1);
+    if (!dummy)
+        goto out;
+    dummy->commit_hash = layout->commit_hash;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            dht_selfheal_dir_xattr_persubvol(frame, loc, dummy, 0,
+                                             conf->subvolumes[i]);
+            if (--count == 0)
+                break;
+        }
+    }
 
+    dht_layout_unref(this, dummy);
 out:
-        this_call_cnt = dht_frame_return (frame);
+    return 0;
+}
 
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_dir_setattr (frame, &local->loc, &local->stbuf, 0xffffff, layout);
+static int
+dht_selfheal_dir_xattr(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+{
+    dht_local_t *local = NULL;
+    int missing_xattr = 0;
+    int i = 0;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *dummy = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err != -1 || !layout->list[i].stop) {
+            /* err != -1 would mean xattr present on the directory
+             * or the directory is non existent.
+             * !layout->list[i].stop would mean layout absent
+             */
+
+            continue;
         }
+        missing_xattr++;
+    }
+    /* Also account for subvolumes with no-layout. Used for zero'ing out
+     * the layouts and for setting quota key's if present */
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            missing_xattr++;
+        }
+    }
+    gf_msg_trace(this->name, 0, "%d subvolumes missing xattr for %s",
+                 missing_xattr, loc->path);
 
+    if (missing_xattr == 0) {
+        dht_selfheal_dir_finish(frame, this, 0, 1);
         return 0;
+    }
+
+    local->call_cnt = missing_xattr;
+
+    if (gf_log_get_loglevel() >= GF_LOG_DEBUG)
+        dht_log_new_layout_for_dir_selfheal(this, loc, layout);
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err != -1 || !layout->list[i].stop)
+            continue;
+
+        dht_selfheal_dir_xattr_persubvol(frame, loc, layout, i, NULL);
+
+        if (--missing_xattr == 0)
+            break;
+    }
+    dummy = dht_layout_new(this, 1);
+    if (!dummy) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DUMMY_ALLOC_FAILED,
+                "path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+    for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            dht_selfheal_dir_xattr_persubvol(frame, loc, dummy, 0,
+                                             conf->subvolumes[i]);
+            missing_xattr--;
+        }
+    }
+
+    dht_layout_unref(this, dummy);
+out:
+    return 0;
 }
 
-void
-dht_selfheal_dir_mkdir_setacl (dict_t *xattr, dict_t *dict)
+int
+dht_selfheal_dir_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, struct iatt *statpre,
+                             struct iatt *statpost, dict_t *xdata)
 {
-        data_t          *acl_default = NULL;
-        data_t          *acl_access = NULL;
-        xlator_t        *this = NULL;
-        int     ret = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int this_call_cnt = 0, ret = -1;
 
-        GF_ASSERT (xattr);
-        GF_ASSERT (dict);
+    local = frame->local;
+    layout = local->selfheal.layout;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this_call_cnt = dht_frame_return(frame);
 
-        acl_default = dict_get (xattr, POSIX_ACL_DEFAULT_XATTR);
+    if (is_last_call(this_call_cnt)) {
+        if (!local->heal_layout) {
+            gf_msg_trace(this->name, 0, "Skip heal layout for %s gfid = %s ",
+                         local->loc.path, uuid_utoa(local->gfid));
 
-        if (!acl_default) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "ACL_DEFAULT xattr not present");
-                goto cont;
+            dht_selfheal_dir_finish(frame, this, 0, 1);
+            return 0;
         }
-        ret = dict_set (dict, POSIX_ACL_DEFAULT_XATTR, acl_default);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Could not set ACL_DEFAULT xattr");
-cont:
-        acl_access = dict_get (xattr, POSIX_ACL_ACCESS_XATTR);
-        if (!acl_access) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "ACL_ACCESS xattr not present");
-                goto out;
+        ret = dht_selfheal_layout_lock(frame, layout, _gf_false,
+                                       dht_selfheal_dir_xattr,
+                                       dht_should_heal_layout);
+
+        if (ret < 0) {
+            dht_selfheal_dir_finish(frame, this, -1, 1);
         }
-        ret = dict_set (dict, POSIX_ACL_ACCESS_XATTR, acl_access);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Could not set ACL_ACCESS xattr");
+    }
 
-out:
-        return;
+    return 0;
 }
 
 int
-dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
-                        dht_layout_t *layout, int force)
+dht_selfheal_dir_setattr(call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
+                         int32_t valid, dht_layout_t *layout)
 {
-        int           missing_dirs = 0;
-        int           i     = 0;
-        int           ret   = -1;
-        dht_local_t  *local = NULL;
-        xlator_t     *this = NULL;
-        dict_t       *dict = NULL;
-
-        local = frame->local;
-        this = frame->this;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == ENOENT || force)
-                        missing_dirs++;
+    int missing_attr = 0;
+    int i = 0, ret = -1;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    int cnt = 0;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    /* We need to heal the attrs if:
+     * 1. Any directories were missing - the newly created dirs will need
+     *    to have the correct attrs set
+     * 2. An existing dir does not have the correct permissions -they may
+     *    have been changed when a brick was down.
+     */
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == -1)
+            missing_attr++;
+    }
+
+    if ((missing_attr == 0) && (local->need_attrheal == 0)) {
+        if (!local->heal_layout) {
+            gf_msg_trace(this->name, 0, "Skip heal layout for %s gfid = %s ",
+                         loc->path, uuid_utoa(loc->gfid));
+            dht_selfheal_dir_finish(frame, this, 0, 1);
+            return 0;
         }
+        ret = dht_selfheal_layout_lock(frame, layout, _gf_false,
+                                       dht_selfheal_dir_xattr,
+                                       dht_should_heal_layout);
 
-        if (missing_dirs == 0) {
-                dht_selfheal_dir_setattr (frame, loc, &local->stbuf, 0xffffffff, layout);
-                return 0;
+        if (ret < 0) {
+            dht_selfheal_dir_finish(frame, this, -1, 1);
         }
 
-        local->call_cnt = missing_dirs;
-        if (!uuid_is_null (local->gfid)) {
-                dict = dict_new ();
-                if (!dict)
-                        return -1;
+        return 0;
+    }
 
-                ret = dict_set_static_bin (dict, "gfid-req", local->gfid, 16);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to set gfid in dict", loc->path);
-        } else if (local->params) {
-                /* Send the dictionary from higher layers directly */
-                dict = dict_ref (local->params);
-        }
-        /* Set acls */
-        if (local->xattr && dict)
-                dht_selfheal_dir_mkdir_setacl (local->xattr, dict);
+    cnt = local->call_cnt = conf->subvolume_cnt;
 
-        if (!dict)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dict is NULL, need to make sure gfids are same");
+    for (i = 0; i < cnt; i++) {
+        STACK_WIND(frame, dht_selfheal_dir_setattr_cbk, layout->list[i].xlator,
+                   layout->list[i].xlator->fops->setattr, loc, stbuf, valid,
+                   NULL);
+    }
+
+    return 0;
+}
 
+static int
+dht_selfheal_dir_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, inode_t *inode,
+                           struct iatt *stbuf, struct iatt *preparent,
+                           struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *subvol = NULL;
+    int i = 0, ret = -1;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    layout = local->selfheal.layout;
+    prev = cookie;
+    subvol = prev;
+
+    if ((op_ret == 0) || ((op_ret == -1) && (op_errno == EEXIST))) {
         for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == ENOENT || force) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "creating directory %s on subvol %s",
-                                loc->path, layout->list[i].xlator->name);
-
-                        STACK_WIND (frame, dht_selfheal_dir_mkdir_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->mkdir,
-                                    loc,
-                                    st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type),
-                                    0, dict);
-                }
+            if (layout->list[i].xlator == subvol) {
+                layout->list[i].err = -1;
+                break;
+            }
         }
+    }
+
+    if (op_ret) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name,
+                ((op_errno == EEXIST) ? GF_LOG_DEBUG : GF_LOG_WARNING),
+                op_errno, DHT_MSG_DIR_SELFHEAL_FAILED, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+    dht_iatt_merge(this, &local->preparent, preparent);
+    dht_iatt_merge(this, &local->postparent, postparent);
+    ret = 0;
 
-        if (dict)
-                dict_unref (dict);
+out:
+    this_call_cnt = dht_frame_return(frame);
 
-        return 0;
-}
+    if (is_last_call(this_call_cnt)) {
+        dht_selfheal_dir_finish(frame, this, ret, 0);
+        dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf, 0xffffff,
+                                 layout);
+    }
 
+    return 0;
+}
 
-int
-dht_selfheal_layout_alloc_start (xlator_t *this, loc_t *loc,
-                                 dht_layout_t *layout)
+static int
+dht_selfheal_dir_mkdir_lookup_done(call_frame_t *frame, xlator_t *this)
 {
-        int           start = 0;
-        uint32_t      hashval = 0;
-        int           ret = 0;
+    dht_local_t *local = NULL;
+    int i = 0;
+    dict_t *dict = NULL;
+    dht_layout_t *layout = NULL;
+    loc_t *loc = NULL;
+    int cnt = 0;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(this->private, err);
+
+    local = frame->local;
+    layout = local->layout;
+    loc = &local->loc;
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        dict = dict_new();
+        if (!dict)
+            return -1;
 
-        ret = dht_hash_compute (layout->type, loc->path, &hashval);
-        if (ret == 0) {
-                start = (hashval % layout->cnt);
+        ret = dict_set_gfuuid(dict, "gfid-req", local->gfid, true);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", loc->path, "key=gfid-req", NULL);
+    } else if (local->params) {
+        /* Send the dictionary from higher layers directly */
+
+        dict = dict_ref(local->params);
+    }
+    /* Code to update all extended attributed from local->xattr
+       to dict
+    */
+    dht_dir_set_heal_xattr(this, local, dict, local->xattr, NULL, NULL);
+
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_IS_NULL, NULL);
+        dict = dict_new();
+        if (!dict)
+            return -1;
+    }
+    ret = dict_set_flag(dict, GF_INTERNAL_CTX_KEY, GF_DHT_HEAL_DIR);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, "key=%s",
+                GF_INTERNAL_CTX_KEY, "path=%s", loc->path, NULL);
+        /* We can still continue. As heal can still happen
+         * unless quota limits have reached for the dir.
+         */
+    }
+
+    cnt = layout->cnt;
+    for (i = 0; i < cnt; i++) {
+        if (layout->list[i].err == ESTALE || layout->list[i].err == ENOENT ||
+            local->selfheal.force_mkdir) {
+            gf_msg_debug(this->name, 0, "Creating directory %s on subvol %s",
+                         loc->path, layout->list[i].xlator->name);
+
+            STACK_WIND_COOKIE(
+                frame, dht_selfheal_dir_mkdir_cbk, layout->list[i].xlator,
+                layout->list[i].xlator, layout->list[i].xlator->fops->mkdir,
+                loc,
+                st_mode_from_ia(local->stbuf.ia_prot, local->stbuf.ia_type), 0,
+                dict);
         }
+    }
+
+    if (dict)
+        dict_unref(dict);
 
-        return start;
+    return 0;
+
+err:
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
 }
 
-static inline int
-dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
+static int
+dht_selfheal_dir_mkdir_lookup_cbk(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int op_ret, int op_errno,
+                                  inode_t *inode, struct iatt *stbuf,
+                                  dict_t *xattr, struct iatt *postparent)
 {
-        int i = 0;
-        int j = 0;
-        int err = 0;
-        int count = 0;
-        dht_conf_t *conf = NULL;
-
-        /* Gets in use only for replace-brick, remove-brick */
-        conf = this->private;
-        for (i = 0; i < layout->cnt; i++) {
-                for (j = 0; j < conf->subvolume_cnt; j++) {
-                        if (conf->decommissioned_bricks[j] &&
-                            conf->decommissioned_bricks[j] == layout->list[i].xlator) {
-                                layout->list[i].err = -EINVAL;
-                                break;
-                        }
-                }
+    dht_local_t *local = NULL;
+    int i = 0;
+    int this_call_cnt = 0;
+    int missing_dirs = 0;
+    dht_layout_t *layout = NULL;
+    xlator_t *prev = 0;
+    loc_t *loc = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    int index = -1;
+
+    VALIDATE_OR_GOTO(this->private, err);
+
+    local = frame->local;
+    layout = local->layout;
+    loc = &local->loc;
+    prev = cookie;
+
+    if (!gf_uuid_is_null(local->gfid))
+        gf_uuid_unparse(local->gfid, gfid_local);
+
+    LOCK(&frame->lock);
+    {
+        index = dht_layout_index_for_subvol(layout, prev);
+        if ((op_ret < 0) && (op_errno == ENOENT || op_errno == ESTALE)) {
+            local->selfheal.hole_cnt = !local->selfheal.hole_cnt
+                                           ? 1
+                                           : local->selfheal.hole_cnt + 1;
+            /* the status might have changed. Update the layout with the
+             * new status
+             */
+            if (index >= 0) {
+                layout->list[index].err = op_errno;
+            }
         }
 
-        for (i = 0; i < layout->cnt; i++) {
-                err = layout->list[i].err;
-                if (err == -1 || err == 0) {
-                        layout->list[i].err = -1;
-                        count++;
-                }
+        if (!op_ret) {
+            dht_iatt_merge(this, &local->stbuf, stbuf);
+            if (prev == local->mds_subvol) {
+                dict_unref(local->xattr);
+                local->xattr = dict_ref(xattr);
+            }
+            /* the status might have changed. Update the layout with the
+             * new status
+             */
+            if (index >= 0) {
+                layout->list[index].err = -1;
+            }
         }
+    }
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        if (local->selfheal.hole_cnt == layout->cnt) {
+            gf_msg_debug(this->name, op_errno,
+                         "Lookup failed, an rmdir could have "
+                         "deleted this entry %s",
+                         loc->name);
+            local->op_errno = op_errno;
+            goto err;
+        } else {
+            for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == ENOENT ||
+                    layout->list[i].err == ESTALE ||
+                    local->selfheal.force_mkdir)
+                    missing_dirs++;
+            }
+
+            if (missing_dirs == 0) {
+                dht_selfheal_dir_finish(frame, this, 0, 0);
+                dht_selfheal_dir_setattr(frame, loc, &local->stbuf, 0xffffffff,
+                                         layout);
+                return 0;
+            }
 
-        /* no subvolume has enough space, but can't stop directory creation */
-        if (!count || !new_layout) {
-                for (i = 0; i < layout->cnt; i++) {
-                        err = layout->list[i].err;
-                        if (err == ENOSPC) {
-                                layout->list[i].err = -1;
-                                count++;
-                        }
-                }
+            local->call_cnt = missing_dirs;
+            dht_selfheal_dir_mkdir_lookup_done(frame, this);
         }
+    }
 
-        count = ((layout->spread_cnt) ? layout->spread_cnt :
-                 ((count) ? count : 1));
+    return 0;
 
-        return count;
+err:
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
 }
 
-
-dht_layout_t *
-dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
-                             dht_layout_t *layout)
+static int
+dht_selfheal_dir_mkdir_lock_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
 {
-        uint32_t      chunk        = 0;
-        uint32_t      start        = 0;
-        uint32_t      stop         = 0;
-        uint32_t      overlap      = 0;
-        uint32_t      max_overlap  = 0;
-        uint32_t      chunk_begin  = 0;
-        int           count        = 0;
-        int           cnt          = 0;
-        int           i            = 0;
-        int           j            = 0;
-        int           k            = 0;
-        int           loop_cnt     = 0;
-        int           start_subvol = 0;
-        int          *fix_array    = NULL;
-        xlator_t     *this         = NULL;
-        dht_layout_t *new_layout   = NULL;
-        dht_conf_t   *priv         = NULL;
-        dht_local_t  *local        = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    int ret = -1;
+    xlator_t *mds_subvol = NULL;
 
-        this  = frame->this;
-        priv  = this->private;
-        local = frame->local;
+    VALIDATE_OR_GOTO(this->private, err);
 
-        if (layout->type == DHT_HASH_TYPE_DM_USER) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "leaving %s alone",
-                        loc->path);
-                goto done;
+    conf = this->private;
+    local = frame->local;
+    mds_subvol = local->mds_subvol;
+
+    local->call_cnt = conf->subvolume_cnt;
+
+    if (op_ret < 0) {
+        if (op_errno == EINVAL) {
+            local->call_cnt = 1;
+            dht_selfheal_dir_mkdir_lookup_done(frame, this);
+            return 0;
         }
 
-        count = cnt = dht_get_layout_count (this, layout, 0);
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_ENTRYLK_ERROR,
+                "path=%s", local->loc.path, NULL);
+
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    /* After getting locks, perform lookup again to ensure that the
+       directory was not deleted by a racing rmdir
+    */
+    if (!local->xattr_req)
+        local->xattr_req = dict_new();
+
+    ret = dict_set_int32(local->xattr_req, "list-xattr", 1);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, "path=%s",
+                local->loc.path, NULL);
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol && conf->subvolumes[i] == mds_subvol) {
+            STACK_WIND_COOKIE(frame, dht_selfheal_dir_mkdir_lookup_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        } else {
+            STACK_WIND_COOKIE(frame, dht_selfheal_dir_mkdir_lookup_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              NULL);
+        }
+    }
 
-        chunk = ((unsigned long) 0xffffffff) / ((cnt) ? cnt : 1);
+    return 0;
 
-        start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
+err:
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
+}
 
-        fix_array = GF_CALLOC (sizeof (int), layout->cnt, gf_common_mt_char);
-        if (!fix_array) {
-                /* No fix, use the existing layout itself */
-                goto done;
+static int
+dht_selfheal_dir_mkdir(call_frame_t *frame, loc_t *loc, dht_layout_t *layout,
+                       int force)
+{
+    int missing_dirs = 0;
+    int i = 0;
+    int op_errno = 0;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    local->selfheal.force_mkdir = force;
+    local->selfheal.hole_cnt = 0;
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == ENOENT || force)
+            missing_dirs++;
+    }
+
+    if (missing_dirs == 0) {
+        /* We don't need to create any directories. Proceed to heal the
+         * attrs and xattrs
+         */
+        if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+            if (local->need_xattr_heal) {
+                local->need_xattr_heal = 0;
+                ret = dht_dir_xattr_heal(this, local, &op_errno);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s",
+                            local->loc.path, "gfid=%s", local->gfid, NULL);
+                }
+            } else {
+                if (!gf_uuid_is_null(local->gfid))
+                    gf_uuid_copy(loc->gfid, local->gfid);
+
+                ret = dht_common_mark_mdsxattr(frame, NULL, 0);
+                if (!ret)
+                    return 0;
+
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_SET_XATTR_FAILED,
+                        "path=%s", local->loc.path, "gfid=%s", local->gfid,
+                        NULL);
+            }
+        }
+        dht_selfheal_dir_setattr(frame, loc, &local->stbuf, 0xffffffff, layout);
+        return 0;
+    }
+
+    /* MDS xattr is populated only while DHT is having more than one
+     subvol.In case of graph switch while adding more dht subvols need to
+     consider hash subvol as a MDS to avoid MDS check failure at the time
+     of running fop on directory
+    */
+    if (!dict_get(local->xattr, conf->mds_xattr_key) &&
+        (conf->subvolume_cnt > 1)) {
+        if (local->hashed_subvol == NULL) {
+            local->hashed_subvol = dht_subvol_get_hashed(this, loc);
+            if (local->hashed_subvol == NULL) {
+                local->op_errno = EINVAL;
+                gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                        DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s",
+                        loc->pargfid, "name=%s", loc->name, "path=%s",
+                        loc->path, NULL);
+                goto err;
+            }
+        }
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this,
+                                       local->hashed_subvol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set hashed subvol for %s on inode vol is %s",
+                   local->loc.path,
+                   local->hashed_subvol ? local->hashed_subvol->name : "NULL");
+            goto err;
         }
+    }
+
+    if (local->hashed_subvol == NULL) {
+        local->hashed_subvol = dht_subvol_get_hashed(this, loc);
+        if (local->hashed_subvol == NULL) {
+            local->op_errno = EINVAL;
+            gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                    DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s", loc->pargfid,
+                    "name=%s", loc->name, "path=%s", loc->path, NULL);
+            goto err;
+        }
+    }
+
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, loc, local->hashed_subvol,
+                                &local->current->ns,
+                                dht_selfheal_dir_mkdir_lock_cbk);
+
+    if (ret < 0)
+        goto err;
 
-        new_layout = dht_layout_new (this, priv->subvolume_cnt);
-        if (!new_layout)
-                goto done;
+    return 0;
+err:
+    return -1;
+}
 
-        for (i = 0; i < new_layout->cnt; i++) {
-                /* TODO: fix this in layout_alloc() itself */
-                new_layout->list[i].err = -ENOENT;
-                if (i < layout->cnt)
-                        new_layout->list[i].xlator = layout->list[i].xlator;
+static int
+dht_selfheal_layout_alloc_start(xlator_t *this, loc_t *loc,
+                                dht_layout_t *layout)
+{
+    int start = 0;
+    uint32_t hashval = 0;
+    int ret = 0;
+    const char *str = NULL;
+    dht_conf_t *conf = NULL;
+    char buf[UUID_CANONICAL_FORM_LEN + 1] = {
+        0,
+    };
+
+    conf = this->private;
+
+    if (conf->randomize_by_gfid) {
+        str = uuid_utoa_r(loc->gfid, buf);
+    } else {
+        str = loc->path;
+    }
+
+    ret = dht_hash_compute(this, layout->type, str, &hashval);
+    if (ret == 0) {
+        start = (hashval % layout->cnt);
+    }
+
+    return start;
+}
+
+static int
+dht_get_layout_count(xlator_t *this, dht_layout_t *layout, int new_layout)
+{
+    int i = 0;
+    int j = 0;
+    int err = 0;
+    int count = 0;
+    dht_conf_t *conf = NULL;
+
+    /* Gets in use only for replace-brick, remove-brick */
+    conf = this->private;
+    for (i = 0; i < layout->cnt; i++) {
+        for (j = 0; j < conf->subvolume_cnt; j++) {
+            if (conf->decommissioned_bricks[j] &&
+                conf->decommissioned_bricks[j] == layout->list[i].xlator) {
+                layout->list[i].err = EINVAL;
+                break;
+            }
+        }
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        err = layout->list[i].err;
+        if (err == -1 || err == 0 || err == ENOENT) {
+            /* Take this with a pinch of salt. The behaviour seems
+             * to be slightly different when this function is
+             * invoked from mkdir codepath. For eg., err == 0 in
+             * mkdir codepath means directory created but xattr
+             * is not set yet.
+             */
+
+            /* Setting list[i].err = -1 is an indication for
+               dht_selfheal_layout_new_directory() to assign
+               a range. We set it to -1 based on any one of
+               the three criteria:
+
+               - err == -1 already, which means directory
+                 existed but layout was not set on it.
+
+               - err == 0, which means directory exists and
+                 has an old layout piece which will be
+                 overwritten now.
+
+               - err == ENOENT, which means directory does
+                 not exist (possibly racing with mkdir or
+                 finishing half done mkdir). The missing
+                 directory will be attempted to be recreated.
+            */
+            count++;
+            if (!err)
+                layout->list[i].err = -1;
         }
+    }
 
-        /* Check if there are any overlap in layout, and give the proper fix */
+    /* no subvolume has enough space, but can't stop directory creation */
+    if (!count || !new_layout) {
         for (i = 0; i < layout->cnt; i++) {
-                /* No need to fix if 'err' is not '-1' */
-                if (layout->list[i].err != -1)
-                        continue;
-
-                /* If already existing layout is having no range, skip it */
-                start = layout->list[i].start;
-                stop  = layout->list[i].stop;
-                if ((stop - start) == 0)
-                        continue;
-
-                max_overlap = 0;
-
-                /* 'j' is used as starting point of each chunk */
-                for (j = 1; j <= count; j++) {
-                        /* if chunk is already used, don't use it again */
-                        for (k = 0; k < i; k++)
-                                if (j == fix_array[k])
-                                        break;
-                        if (k < i)
-                                continue;
-
-                        overlap = dht_find_overlap (i, (j-1), start, stop, chunk);
-                        if (max_overlap < overlap) {
-                                max_overlap = overlap;
-                                fix_array[i] = j;
-                        }
-                }
+            err = layout->list[i].err;
+            if (err == ENOSPC) {
+                layout->list[i].err = -1;
+                count++;
+            }
+        }
+    }
 
-                /* If we have any overlap, then use that itself as new
-                   layout for the subvolume */
-                if (fix_array[i]) {
-                        chunk_begin = chunk * (fix_array[i] - 1);
-                        new_layout->list[i].err = -1;
-                        DHT_SET_LAYOUT_RANGE (new_layout, i, chunk_begin,
-                                              chunk, cnt, loc->path);
-                        /* make sure to give (max - 1) as 'stop' range,
-                           if it is last chunk */
-                        if (fix_array[i] == count)
-                                new_layout->list[i].stop = 0xffffffff;
-                        if (--cnt == 0)
-                                goto done;
+    /* if layout->spread_cnt is set, check if it is <= available
+     * subvolumes (down brick and decommissioned bricks are considered
+     * un-available). Else return count (available up bricks) */
+    count = ((layout->spread_cnt && (layout->spread_cnt <= count))
+                 ? layout->spread_cnt
+                 : ((count) ? count : 1));
 
-                }
-        }
+    return count;
+}
 
-        /* Now, look for layouts which are not having any overlaps
-           and give it a fix */
-        for (loop_cnt = 0, i = start_subvol; loop_cnt < new_layout->cnt;
-             i++, loop_cnt++) {
-                if (i == new_layout->cnt)
-                        i = 0;
+void
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *new_layout);
 
-                /* If 'fix_array[i]' is set, the layout is already fixed. */
-                if (fix_array[i])
-                        continue;
+void
+dht_layout_range_swap(dht_layout_t *layout, int i, int j);
 
-                if (layout->list[i].err != -1) {
-                        new_layout->list[i].err = layout->list[i].err;
-                        continue;
-                }
+/*
+ * It's a bit icky using local variables in a macro, but it makes the rest
+ * of the code a lot clearer.
+ */
+#define OV_ENTRY(x, y) table[x * new->cnt + y]
+
+static void
+dht_selfheal_layout_maximize_overlap(call_frame_t *frame, loc_t *loc,
+                                     dht_layout_t *new, dht_layout_t *old)
+{
+    int i = 0;
+    int j = 0;
+    uint32_t curr_overlap = 0;
+    uint32_t max_overlap = 0;
+    int max_overlap_idx = -1;
+    uint32_t overlap = 0;
+    uint32_t *table = NULL;
+
+    dht_layout_sort_volname(old);
+    /* Now both old_layout->list[] and new_layout->list[]
+       are match the same xlators/subvolumes. i.e,
+       old_layout->[i] and new_layout->[i] are referring
+       to the same subvolumes
+    */
+
+    /* Build a table of overlaps between new[i] and old[j]. */
+    table = alloca(sizeof(overlap) * old->cnt * new->cnt);
+    if (!table) {
+        return;
+    }
+    memset(table, 0, sizeof(overlap) * old->cnt * new->cnt);
+    for (i = 0; i < new->cnt; ++i) {
+        for (j = 0; j < old->cnt; ++j) {
+            OV_ENTRY(i, j) = dht_overlap_calc(old, j, new, i);
+        }
+    }
+
+    for (i = 0; i < new->cnt; i++) {
+        if (new->list[i].err > 0) {
+            /* Subvol might be marked for decommission
+               with EINVAL, or some other serious error
+               marked with positive errno.
+            */
+            continue;
+        }
 
-                for (k = 1; k <= count; k++) {
-                        for (j = 0; j < new_layout->cnt; j++) {
-                                if (k == fix_array[j])
-                                        break;
-                        }
-                        /* Didn't find any of the list begining with 'k' */
-                        if (j == new_layout->cnt)
-                                break;
+        max_overlap = 0;
+        max_overlap_idx = i;
+        for (j = (i + 1); j < new->cnt; ++j) {
+            if (new->list[j].err > 0) {
+                /* Subvol might be marked for decommission
+                with EINVAL, or some other serious error
+                marked with positive errno.
+                */
+                continue;
+            }
+            /* Calculate the overlap now. */
+            curr_overlap = OV_ENTRY(i, i) + OV_ENTRY(j, j);
+            /* Calculate the overlap after the proposed swap. */
+            overlap = OV_ENTRY(i, j) + OV_ENTRY(j, i);
+            /* Are we better than status quo? */
+            if (overlap > curr_overlap) {
+                overlap -= curr_overlap;
+                /* Are we better than the previous choice? */
+                if (overlap > max_overlap) {
+                    max_overlap = overlap;
+                    max_overlap_idx = j;
                 }
+            }
+        }
 
-                fix_array[i] = k;
-                chunk_begin = (k - 1) * chunk;
-                new_layout->list[i].err = -1;
-                DHT_SET_LAYOUT_RANGE (new_layout, i, chunk_begin, chunk, cnt,
-                                      loc->path);
-                /* make sure to give (max - 1) as 'stop' range,
-                   if it is last chunk */
-                if (k == count)
-                        new_layout->list[i].stop = 0xffffffff;
-                if (--cnt == 0)
-                        goto done;
+        if (max_overlap_idx != i) {
+            dht_layout_range_swap(new, i, max_overlap_idx);
+            /* Need to swap the table values too. */
+            for (j = 0; j < old->cnt; ++j) {
+                overlap = OV_ENTRY(i, j);
+                OV_ENTRY(i, j) = OV_ENTRY(max_overlap_idx, j);
+                OV_ENTRY(max_overlap_idx, j) = overlap;
+            }
         }
+    }
+}
 
+static dht_layout_t *
+dht_fix_layout_of_directory(call_frame_t *frame, loc_t *loc,
+                            dht_layout_t *layout)
+{
+    int i = 0;
+    xlator_t *this = NULL;
+    dht_layout_t *new_layout = NULL;
+    dht_conf_t *priv = NULL;
+    dht_local_t *local = NULL;
+    uint32_t subvol_down = 0;
+    gf_boolean_t maximize_overlap = _gf_true;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    this = frame->this;
+    priv = this->private;
+    local = frame->local;
+
+    if (layout->type == DHT_HASH_TYPE_DM_USER) {
+        gf_msg_debug(THIS->name, 0, "leaving %s alone", loc->path);
+        goto done;
+    }
+
+    new_layout = dht_layout_new(this, priv->subvolume_cnt);
+    if (!new_layout) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                "new_layout, path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto done;
+    }
+
+    /* If a subvolume is down, do not re-write the layout. */
+    dht_layout_anomalies(this, loc, layout, NULL, NULL, NULL, &subvol_down,
+                         NULL, NULL);
+
+    if (subvol_down) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_FIX_FAILED,
+                "subvol-down=%u", subvol_down, "Skipping-fix-layout", "path=%s",
+                loc->path, "gfid=%s", gfid, NULL);
+        GF_FREE(new_layout);
+        return NULL;
+    }
+
+    for (i = 0; i < new_layout->cnt; i++) {
+        if (layout->list[i].err != ENOSPC)
+            new_layout->list[i].err = layout->list[i].err;
+        else
+            new_layout->list[i].err = -1;
+
+        new_layout->list[i].xlator = layout->list[i].xlator;
+    }
+
+    new_layout->commit_hash = layout->commit_hash;
+
+    if (priv->du_stats) {
+        for (i = 0; i < priv->subvolume_cnt; ++i) {
+            gf_smsg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_SUBVOL_INFO,
+                    "index=%d", i, "name=%s", priv->subvolumes[i]->name,
+                    "chunks=%u", priv->du_stats[i].chunks, "path=%s", loc->path,
+                    NULL);
+
+            /* Maximize overlap if the bricks are all the same
+             *  size.
+             * This is probably not going to be very common on
+             * live setups but will benefit our regression tests
+             */
+            if (i && (priv->du_stats[i].chunks != priv->du_stats[0].chunks)) {
+                maximize_overlap = _gf_false;
+            }
+        }
+    } else {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_DISK_USAGE_STATUS,
+                NULL);
+    }
+
+    /* First give it a layout as though it is a new directory. This
+       ensures rotation to kick in */
+    dht_layout_sort_volname(new_layout);
+    dht_selfheal_layout_new_directory(frame, loc, new_layout);
+
+    /* Maximize overlap if weighted-rebalance is disabled */
+    if (!priv->do_weighting)
+        maximize_overlap = _gf_true;
+
+    /* Now selectively re-assign ranges only when it helps */
+    if (maximize_overlap) {
+        dht_selfheal_layout_maximize_overlap(frame, loc, new_layout, layout);
+    }
 done:
-        if (new_layout) {
-                /* Now that the new layout has all the proper layout, change the
-                   inode context */
-                dht_layout_set (this, loc->inode, new_layout);
+    if (new_layout) {
+        /* Make sure the extra 'ref' for existing layout is removed */
+        dht_layout_unref(this, local->layout);
 
-                /* Make sure the extra 'ref' for existing layout is removed */
-                dht_layout_unref (this, local->layout);
+        local->layout = new_layout;
+    }
 
-                local->layout = new_layout;
-        }
+    return local->layout;
+}
 
-        GF_FREE (fix_array);
+/*
+ * Having to call this 2x for each entry in the layout is pretty horrible, but
+ * that's what all of this layout-sorting nonsense gets us.
+ */
+static uint32_t
+dht_get_chunks_from_xl(xlator_t *parent, xlator_t *child)
+{
+    dht_conf_t *priv = parent->private;
+    xlator_list_t *trav;
+    uint32_t index = 0;
 
-        return local->layout;
-}
+    if (!priv->du_stats) {
+        return 0;
+    }
 
+    for (trav = parent->children; trav; trav = trav->next) {
+        if (trav->xlator == child) {
+            return priv->du_stats[index].chunks;
+        }
+        ++index;
+    }
+
+    return 0;
+}
 
 void
-dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
-                                   dht_layout_t *layout)
-{
-        xlator_t    *this = NULL;
-        uint32_t     chunk = 0;
-        int          i = 0;
-        uint32_t     start = 0;
-        int          cnt = 0;
-        int          err = 0;
-        int          start_subvol = 0;
-
-        this = frame->this;
-
-        cnt = dht_get_layout_count (this, layout, 1);
-
-        chunk = ((unsigned long) 0xffffffff) / ((cnt) ? cnt : 1);
-
-        start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
-
-        for (i = start_subvol; i < layout->cnt; i++) {
-                err = layout->list[i].err;
-                if (err == -1) {
-                        DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
-                                             cnt, loc->path);
-                        if (--cnt == 0) {
-                                layout->list[i].stop = 0xffffffff;
-                                goto done;
-                        }
-                        start += chunk;
-                }
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *layout)
+{
+    xlator_t *this = NULL;
+    double chunk = 0;
+    int i = 0;
+    uint32_t start = 0;
+    int bricks_to_use = 0;
+    int err = 0;
+    int start_subvol = 0;
+    uint32_t curr_size;
+    uint32_t range_size;
+    uint64_t total_size = 0;
+    int real_i;
+    dht_conf_t *priv;
+    gf_boolean_t weight_by_size;
+    int bricks_used = 0;
+
+    this = frame->this;
+    priv = this->private;
+    weight_by_size = priv->do_weighting;
+
+    bricks_to_use = dht_get_layout_count(this, layout, 1);
+    GF_ASSERT(bricks_to_use > 0);
+
+    bricks_used = 0;
+    for (i = 0; i < layout->cnt; ++i) {
+        err = layout->list[i].err;
+        if ((err != -1) && (err != ENOENT)) {
+            continue;
         }
-
-        for (i = 0; i < start_subvol; i++) {
-                err = layout->list[i].err;
-                if (err == -1) {
-                        DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
-                                             cnt, loc->path);
-                        if (--cnt == 0) {
-                                layout->list[i].stop = 0xffffffff;
-                                goto done;
-                        }
-                        start += chunk;
-                }
+        curr_size = dht_get_chunks_from_xl(this, layout->list[i].xlator);
+        if (!curr_size) {
+            weight_by_size = _gf_false;
+            break;
+        }
+        total_size += curr_size;
+        if (++bricks_used >= bricks_to_use) {
+            break;
         }
+    }
+
+    if (weight_by_size && total_size) {
+        /* We know total_size is not zero. */
+        chunk = ((double)0xffffffff) / ((double)total_size);
+        gf_msg_debug(this->name, 0,
+                     "chunk size = 0xffffffff / %" PRIu64 " = %f", total_size,
+                     chunk);
+    } else {
+        weight_by_size = _gf_false;
+        chunk = ((unsigned long)0xffffffff) / bricks_to_use;
+    }
+
+    start_subvol = dht_selfheal_layout_alloc_start(this, loc, layout);
+
+    /* clear out the range, as we are re-computing here */
+    DHT_RESET_LAYOUT_RANGE(layout);
+
+    /*
+     * OK, what's this "real_i" stuff about?  This used to be two loops -
+     * from start_subvol to layout->cnt-1, then from 0 to start_subvol-1.
+     * That way is practically an open invitation to bugs when only one
+     * of the loops is updated.  Using real_i and modulo operators to make
+     * it one loop avoids this problem.  Remember, folks: it's everyone's
+     * responsibility to help stamp out copy/paste abuse.
+     */
+    bricks_used = 0;
+    for (real_i = 0; real_i < layout->cnt; real_i++) {
+        i = (real_i + start_subvol) % layout->cnt;
+        err = layout->list[i].err;
+        if ((err != -1) && (err != ENOENT)) {
+            continue;
+        }
+        if (weight_by_size) {
+            curr_size = dht_get_chunks_from_xl(this, layout->list[i].xlator);
+            if (!curr_size) {
+                continue;
+            }
+        } else {
+            curr_size = 1;
+        }
+        range_size = chunk * curr_size;
+        gf_msg_debug(this->name, 0, "assigning range size 0x%x to %s",
+                     range_size, layout->list[i].xlator->name);
+        DHT_SET_LAYOUT_RANGE(layout, i, start, range_size, loc->path);
+        if (++bricks_used >= bricks_to_use) {
+            layout->list[i].stop = 0xffffffff;
+            goto done;
+        }
+        start += range_size;
+    }
 
 done:
-        return;
+    return;
+}
+
+static int
+dht_selfheal_dir_getafix(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+{
+    dht_local_t *local = NULL;
+    uint32_t holes = 0;
+    int ret = -1;
+    int i = -1;
+    uint32_t overlaps = 0;
+
+    local = frame->local;
+
+    holes = local->selfheal.hole_cnt;
+    overlaps = local->selfheal.overlaps_cnt;
+
+    if (holes || overlaps) {
+        /* If the layout has anomalies which would change the hash
+         * ranges, then we need to reset the commit_hash for this
+         * directory, as the layout would change and things may not
+         * be in place as expected */
+        layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+        dht_selfheal_layout_new_directory(frame, loc, layout);
+        ret = 0;
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        /* directory not present */
+        if (layout->list[i].err == ENOENT) {
+            ret = 0;
+            break;
+        }
+    }
+
+    /* TODO: give a fix to these non-virgins */
+
+    return ret;
 }
 
 int
-dht_selfheal_dir_getafix (call_frame_t *frame, loc_t *loc,
-                          dht_layout_t *layout)
+dht_selfheal_new_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                           dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
-        xlator_t    *this = NULL;
-        dht_local_t *local = NULL;
-        int          missing = -1;
-        int          down = -1;
-        int          holes = -1;
-        int          ret = -1;
-        int          i = -1;
-        int          overlaps = -1;
+    dht_local_t *local = NULL;
+    int ret = 0;
+    inode_t *linked_inode = NULL, *inode = NULL;
+    loc_t *loc = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t op_errno = EIO;
 
-        this = frame->this;
-        conf = this->private;
-        local = frame->local;
+    local = frame->local;
 
-        missing = local->selfheal.missing;
-        down = local->selfheal.down;
-        holes = local->selfheal.hole_cnt;
-        overlaps = local->selfheal.overlaps_cnt;
+    loc = &local->loc;
 
-        if ((missing + down) == conf->subvolume_cnt) {
-                dht_selfheal_layout_new_directory (frame, loc, layout);
-                ret = 0;
-        }
+    gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
 
-        if (holes <= down) {
-                /* the down subvol might fill up the holes */
-                ret = 0;
-        }
+    linked_inode = inode_link(loc->inode, loc->parent, loc->name,
+                              &local->stbuf);
+    if (!linked_inode) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0, DHT_MSG_LINK_INODE_FAILED,
+                "pgfid=%s", pgfid, "name=%s", loc->name, "gfid=%s", gfid, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        if (holes || overlaps) {
-                dht_selfheal_layout_new_directory (frame, loc, layout);
-                ret = 0;
-        }
+    inode = loc->inode;
+    loc->inode = linked_inode;
+    inode_unref(inode);
 
-        for (i = 0; i < layout->cnt; i++) {
-                /* directory not present */
-                if (layout->list[i].err == ENOENT) {
-                        ret = 0;
-                        break;
-                }
-        }
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
+
+    dht_layout_sort_volname(layout);
+    dht_selfheal_layout_new_directory(frame, &local->loc, layout);
+
+    op_errno = ENOMEM;
+    ret = dht_selfheal_layout_lock(frame, layout, _gf_true,
+                                   dht_selfheal_dir_xattr,
+                                   dht_should_heal_layout);
 
-        /* TODO: give a fix to these non-virgins */
+out:
+    if (ret < 0) {
+        dir_cbk(frame, NULL, frame->this, -1, op_errno, NULL);
+    }
 
-        return ret;
+    return 0;
 }
 
 int
-dht_selfheal_new_directory (call_frame_t *frame,
-                            dht_selfheal_dir_cbk_t dir_cbk,
-                            dht_layout_t *layout)
+dht_fix_directory_layout(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                         dht_layout_t *layout)
 {
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *tmp_layout = NULL;
+    int ret = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
 
-        dht_layout_sort_volname (layout);
-        dht_selfheal_layout_new_directory (frame, &local->loc, layout);
-        dht_selfheal_dir_xattr (frame, &local->loc, layout);
-        return 0;
+    /* No layout sorting required here */
+    tmp_layout = dht_fix_layout_of_directory(frame, &local->loc, layout);
+    if (!tmp_layout) {
+        return -1;
+    }
+
+    ret = dht_selfheal_layout_lock(frame, tmp_layout, _gf_false,
+                                   dht_fix_dir_xattr, dht_should_fix_layout);
+
+    return ret;
+}
+
+int
+dht_selfheal_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                       loc_t *loc, dht_layout_t *layout)
+{
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    uint32_t down = 0;
+    uint32_t misc = 0;
+    int ret = 0;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    inode_t *linked_inode = NULL, *inode = NULL;
+
+    local = frame->local;
+    this = frame->this;
+
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(this, layout);
+
+    if (local->need_attrheal) {
+        if (__is_root_gfid(local->stbuf.ia_gfid)) {
+            local->stbuf.ia_gid = local->prebuf.ia_gid;
+            local->stbuf.ia_uid = local->prebuf.ia_uid;
+
+            local->stbuf.ia_ctime = local->prebuf.ia_ctime;
+            local->stbuf.ia_ctime_nsec = local->prebuf.ia_ctime_nsec;
+            local->stbuf.ia_prot = local->prebuf.ia_prot;
+
+        } else if (!IA_ISINVAL(local->mds_stbuf.ia_type)) {
+            local->stbuf = local->mds_stbuf;
+        }
+    }
+
+    if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+        gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+        gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+        linked_inode = inode_link(loc->inode, loc->parent, loc->name,
+                                  &local->stbuf);
+        if (!linked_inode) {
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LINK_INODE_FAILED,
+                    "pgfid=%s", pgfid, "name=%s", loc->name, "gfid=%s", gfid,
+                    NULL);
+            ret = 0;
+            goto sorry_no_fix;
+        }
+
+        inode = loc->inode;
+        loc->inode = linked_inode;
+        inode_unref(inode);
+    }
+
+    if (local->need_xattr_heal && (local->mds_xattr)) {
+        dht_dir_set_heal_xattr(this, local, local->xattr, local->mds_xattr,
+                               NULL, NULL);
+        dict_unref(local->mds_xattr);
+        local->mds_xattr = NULL;
+    }
+
+    dht_layout_anomalies(this, loc, layout, &local->selfheal.hole_cnt,
+                         &local->selfheal.overlaps_cnt,
+                         &local->selfheal.missing_cnt, &local->selfheal.down,
+                         &local->selfheal.misc, NULL);
+
+    down = local->selfheal.down;
+    misc = local->selfheal.misc;
+
+    if (down) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SELFHEAL_FAILED,
+                "path=%s", loc->path, "subvol-down=%d", down, "Not-fixing",
+                "gfid=%s", gfid, NULL);
+        ret = 0;
+        goto sorry_no_fix;
+    }
+
+    if (misc) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SELFHEAL_FAILED,
+                "path=%s", loc->path, "misc=%d", misc, "unrecoverable-errors",
+                "gfid=%s", gfid, NULL);
+
+        ret = 0;
+        goto sorry_no_fix;
+    }
+
+    dht_layout_sort_volname(layout);
+    local->heal_layout = _gf_true;
+
+    /* Ignore return value as it can be inferred from result of
+     * dht_layout_anomalies
+     */
+    dht_selfheal_dir_getafix(frame, loc, layout);
+
+    if (!(local->selfheal.hole_cnt || local->selfheal.overlaps_cnt ||
+          local->selfheal.missing_cnt)) {
+        local->heal_layout = _gf_false;
+    }
+
+    ret = dht_selfheal_dir_mkdir(frame, loc, layout, 0);
+    if (ret < 0) {
+        ret = 0;
+        goto sorry_no_fix;
+    }
+
+    return 0;
+
+sorry_no_fix:
+    /* TODO: need to put appropriate local->op_errno */
+    dht_selfheal_dir_finish(frame, this, ret, 1);
+
+    return 0;
 }
 
 int
-dht_fix_directory_layout (call_frame_t *frame,
-                          dht_selfheal_dir_cbk_t dir_cbk,
-                          dht_layout_t *layout)
+dht_selfheal_restore(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                     loc_t *loc, dht_layout_t *layout)
 {
-        dht_local_t  *local = NULL;
-        dht_layout_t *tmp_layout = NULL;
+    int ret = 0;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
 
-        /* No layout sorting required here */
-        tmp_layout = dht_fix_layout_of_directory (frame, &local->loc, layout);
-        dht_fix_dir_xattr (frame, &local->loc, tmp_layout);
+    ret = dht_selfheal_dir_mkdir(frame, loc, layout, 1);
 
-        return 0;
+    return ret;
 }
 
+int
+dht_dir_heal_xattrs(void *data)
+{
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dict_t *user_xattr = NULL;
+    dict_t *internal_xattr = NULL;
+    dict_t *mds_xattr = NULL;
+    dict_t *xdata = NULL;
+    int call_cnt = 0;
+    int ret = -1;
+    int uret = 0;
+    int uflag = 0;
+    int i = 0;
+    int xattr_hashed = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t allzero[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", data, out);
+
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
+    mds_subvol = local->mds_subvol;
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (!mds_subvol) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_MDS_SUBVOL, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    if ((local->loc.inode && gf_uuid_is_null(local->loc.inode->gfid)) ||
+        gf_uuid_is_null(local->loc.gfid)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_NOT_PRESENT,
+                "skip-heal path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    internal_xattr = dict_new();
+    if (!internal_xattr) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+    xdata = dict_new();
+    if (!xdata) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+
+    call_cnt = conf->subvolume_cnt;
+
+    user_xattr = dict_new();
+    if (!user_xattr) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+
+    ret = syncop_listxattr(local->mds_subvol, &local->loc, &mds_xattr, NULL,
+                           NULL);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LIST_XATTRS_FAILED,
+                "path=%s", local->loc.path, "name=%s", local->mds_subvol->name,
+                NULL);
+    }
+
+    if (!mds_xattr)
+        goto out;
+
+    dht_dir_set_heal_xattr(this, local, user_xattr, mds_xattr, &uret, &uflag);
+
+    /* To set quota related xattr need to set GLUSTERFS_INTERNAL_FOP_KEY
+     * key value to 1
+     */
+    if (dict_get(user_xattr, QUOTA_LIMIT_KEY) ||
+        dict_get(user_xattr, QUOTA_LIMIT_OBJECTS_KEY)) {
+        ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                    "key=%s", GLUSTERFS_INTERNAL_FOP_KEY, "path=%s",
+                    local->loc.path, NULL);
+            goto out;
+        }
+    }
+    if (uret <= 0 && !uflag)
+        goto out;
+
+    for (i = 0; i < call_cnt; i++) {
+        subvol = conf->subvolumes[i];
+        if (subvol == mds_subvol)
+            continue;
+        if (uret || uflag) {
+            /* Custom xattr heal is required - let posix handle it */
+            ret = dict_set_int8(xdata, "sync_backend_xattrs", _gf_true);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", local->loc.path, "key=%s",
+                        "sync_backend_xattrs", NULL);
+                goto out;
+            }
+
+            ret = syncop_setxattr(subvol, &local->loc, user_xattr, 0, xdata,
+                                  NULL);
+            if (ret) {
+                xattr_hashed = 1;
+                gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                        DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                        "set-user-xattr-failed path=%s", local->loc.path,
+                        "subvol=%s", subvol->name, "gfid=%s", gfid, NULL);
+            } else {
+                dict_del(xdata, "sync_backend_xattrs");
+            }
+        }
+    }
+    /* After heal all custom xattr reset internal MDS xattr to 0 */
+    if (!xattr_hashed) {
+        ret = dht_dict_set_array(internal_xattr, conf->mds_xattr_key, allzero,
+                                 1);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                    "key=%s", conf->mds_xattr_key, "path=%s", local->loc.path,
+                    NULL);
+            goto out;
+        }
+        ret = syncop_setxattr(mds_subvol, &local->loc, internal_xattr, 0, NULL,
+                              NULL);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", mds_subvol->name, "gfid=%s", gfid, NULL);
+        }
+    }
+
+out:
+    if (user_xattr)
+        dict_unref(user_xattr);
+    if (mds_xattr)
+        dict_unref(mds_xattr);
+    if (internal_xattr)
+        dict_unref(internal_xattr);
+    if (xdata)
+        dict_unref(xdata);
+    return 0;
+}
 
 int
-dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-                        loc_t *loc, dht_layout_t *layout)
+dht_dir_heal_xattrs_done(int ret, call_frame_t *sync_frame, void *data)
 {
-        dht_local_t *local    = NULL;
-        uint32_t     down     = 0;
-        uint32_t     misc     = 0;
-        int          ret      = 0;
-        xlator_t    *this     = NULL;
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
+}
 
-        local = frame->local;
-        this = frame->this;
+int
+dht_dir_attr_heal(void *data)
+{
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int ret = -1;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", data, out);
+
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO("dht", conf, out);
+
+    mds_subvol = local->mds_subvol;
+    call_cnt = conf->subvolume_cnt;
+
+    if (!__is_root_gfid(local->stbuf.ia_gfid) && (!mds_subvol)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_MDS_SUBVOL, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == mds_subvol) {
+                if (!conf->subvolume_status[i]) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            DHT_MSG_MDS_DOWN_UNABLE_TO_SET, "path=%s",
+                            local->loc.path, "gfid=%s", gfid, NULL);
+                    goto out;
+                }
+            }
+        }
+    }
+
+    for (i = 0; i < call_cnt; i++) {
+        subvol = conf->subvolumes[i];
+        if (!subvol || subvol == mds_subvol)
+            continue;
+        if (__is_root_gfid(local->stbuf.ia_gfid)) {
+            ret = syncop_setattr(
+                subvol, &local->loc, &local->stbuf,
+                (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL,
+                NULL, NULL, NULL);
+        } else {
+            ret = syncop_setattr(
+                subvol, &local->loc, &local->mds_stbuf,
+                (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL,
+                NULL, NULL, NULL);
+        }
+
+        if (ret) {
+            gf_uuid_unparse(local->loc.gfid, gfid);
 
-        dht_layout_anomalies (this, loc, layout,
-                              &local->selfheal.hole_cnt,
-                              &local->selfheal.overlaps_cnt,
-                              &local->selfheal.missing,
-                              &local->selfheal.down,
-                              &local->selfheal.misc);
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_DIR_ATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", subvol->name, "gfid=%s", gfid, NULL);
+        }
+    }
+out:
+    return 0;
+}
 
-        down     = local->selfheal.down;
-        misc     = local->selfheal.misc;
+int
+dht_dir_attr_heal_done(int ret, call_frame_t *sync_frame, void *data)
+{
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
+}
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (this, layout);
+/* EXIT: dht_update_commit_hash_for_layout */
+static int
+dht_update_commit_hash_for_layout_done(call_frame_t *frame, void *cookie,
+                                       xlator_t *this, int32_t op_ret,
+                                       int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
 
-        if (down) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%d subvolumes down -- not fixing", down);
-                ret = 0;
-                goto sorry_no_fix;
+    local = frame->local;
+
+    /* preserve oldest error */
+    if (op_ret && !local->op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    }
+
+    DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno, NULL);
+
+    return 0;
+}
+
+static int
+dht_update_commit_hash_for_layout_unlock(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+    int ret = 0;
+
+    local = frame->local;
+
+    ret = dht_unlock_inodelk(frame, local->lock[0].layout.my_layout.locks,
+                             local->lock[0].layout.my_layout.lk_count,
+                             dht_update_commit_hash_for_layout_done);
+    if (ret < 0) {
+        /* preserve oldest error, just ... */
+        if (!local->op_ret) {
+            local->op_errno = errno;
+            local->op_ret = -1;
         }
 
-        if (misc) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%d subvolumes have unrecoverable errors", misc);
-                ret = 0;
-                goto sorry_no_fix;
+        gf_smsg(this->name, GF_LOG_WARNING, errno, DHT_MSG_WIND_UNLOCK_FAILED,
+                "path=%s", local->loc.path, NULL);
+
+        dht_update_commit_hash_for_layout_done(frame, NULL, this, 0, 0, NULL);
+    }
+
+    return 0;
+}
+
+static int
+dht_update_commit_hash_for_layout_cbk(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int op_ret, int op_errno,
+                                      dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    /* store first failure, just because */
+    if (op_ret && !local->op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    }
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        dht_update_commit_hash_for_layout_unlock(frame, this);
+    }
+
+    return 0;
+}
+
+static int
+dht_update_commit_hash_for_layout_resume(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int32_t op_ret,
+                                         int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0, j = 0;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int32_t *disk_layout = NULL;
+    dict_t **xattr = NULL;
+
+    local = frame->local;
+    conf = frame->this->private;
+    count = conf->local_subvols_cnt;
+    layout = local->layout;
+
+    if (op_ret < 0) {
+        goto err_done;
+    }
+
+    /* We precreate the xattr list as we cannot change call count post the
+     * first wind as we may never continue from there. So we finish prep
+     * work before winding the setxattrs */
+    xattr = GF_CALLOC(count, sizeof(*xattr), gf_common_mt_char);
+    if (!xattr) {
+        local->op_errno = errno;
+
+        gf_smsg(this->name, GF_LOG_WARNING, errno, DHT_MSG_COMMIT_HASH_FAILED,
+                "allocation-failed path=%s", local->loc.path, NULL);
+
+        goto err;
+    }
+
+    for (i = 0; i < count; i++) {
+        /* find the layout index for the subvolume */
+        ret = dht_layout_index_for_subvol(layout, conf->local_subvols[i]);
+        if (ret < 0) {
+            local->op_errno = ENOENT;
+
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_COMMIT_HASH_FAILED,
+                    "path=%s", local->loc.path, "subvol=%s",
+                    conf->local_subvols[i]->name, "find-disk-layout-failed",
+                    NULL);
+
+            goto err;
         }
+        j = ret;
 
-        dht_layout_sort_volname (layout);
-        ret = dht_selfheal_dir_getafix (frame, loc, layout);
+        /* update the commit hash for the layout */
+        layout->list[j].commit_hash = layout->commit_hash;
 
+        /* extract the current layout */
+        ret = dht_disk_layout_extract(this, layout, j, &disk_layout);
         if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "not able to form layout for the directory");
-                goto sorry_no_fix;
+            local->op_errno = errno;
+
+            gf_smsg(this->name, GF_LOG_WARNING, errno,
+                    DHT_MSG_COMMIT_HASH_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", conf->local_subvols[i]->name,
+                    "extract-disk-layout-failed", NULL);
+
+            goto err;
         }
 
-        dht_selfheal_dir_mkdir (frame, loc, layout, 0);
+        xattr[i] = dict_new();
+        if (!xattr[i]) {
+            local->op_errno = errno;
 
-        return 0;
+            gf_smsg(this->name, GF_LOG_WARNING, errno,
+                    DHT_MSG_COMMIT_HASH_FAILED, "path=%s Allocation-failed",
+                    local->loc.path, NULL);
 
-sorry_no_fix:
-        /* TODO: need to put appropriate local->op_errno */
-        dht_selfheal_dir_finish (frame, this, ret);
+            goto err;
+        }
 
-        return 0;
-}
+        ret = dict_set_bin(xattr[i], conf->xattr_name, disk_layout, 4 * 4);
+        if (ret != 0) {
+            local->op_errno = ENOMEM;
 
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "path=%s",
+                    local->loc.path, "subvol=%s", conf->local_subvols[i]->name,
+                    "set-xattr-failed", NULL);
 
-int
-dht_selfheal_restore (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-                      loc_t *loc, dht_layout_t *layout)
-{
-        int          ret = 0;
-        dht_local_t *local    = NULL;
+            goto err;
+        }
+        disk_layout = NULL;
 
-        local = frame->local;
+        gf_msg_trace(this->name, 0,
+                     "setting commit hash %u on subvolume %s"
+                     " for %s",
+                     layout->list[j].commit_hash, conf->local_subvols[i]->name,
+                     local->loc.path);
+    }
+
+    /* wind the setting of the commit hash across the local subvols */
+    local->call_cnt = count;
+    local->op_ret = 0;
+    local->op_errno = 0;
+    for (i = 0; i < count; i++) {
+        STACK_WIND(frame, dht_update_commit_hash_for_layout_cbk,
+                   conf->local_subvols[i],
+                   conf->local_subvols[i]->fops->setxattr, &local->loc,
+                   xattr[i], 0, NULL);
+    }
+    for (i = 0; i < count; i++)
+        dict_unref(xattr[i]);
+    GF_FREE(xattr);
+
+    return 0;
+err:
+    if (xattr) {
+        for (i = 0; i < count; i++) {
+            if (xattr[i])
+                dict_unref(xattr[i]);
+        }
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+        GF_FREE(xattr);
+    }
 
-        ret = dht_selfheal_dir_mkdir (frame, loc, layout, 1);
+    GF_FREE(disk_layout);
+
+    local->op_ret = -1;
+
+    dht_update_commit_hash_for_layout_unlock(frame, this);
+
+    return 0;
+err_done:
+    local->op_ret = -1;
+
+    dht_update_commit_hash_for_layout_done(frame, NULL, this, 0, 0, NULL);
+
+    return 0;
+}
+
+/* ENTER: dht_update_commit_hash_for_layout (see EXIT above)
+ * This function is invoked from rebalance only.
+ * As a result, the check here is simple enough to see if defrag is present
+ * in the conf, as other data would be populated appropriately if so.
+ * If ever this was to be used in other code paths, checks would need to
+ * change.
+ *
+ * Functional details:
+ *  - Lock the inodes on the subvols that we want the commit hash updated
+ *  - Update each layout with the inode layout, modified to take in the new
+ *    commit hash.
+ *  - Unlock and return.
+ */
+int
+dht_update_commit_hash_for_layout(call_frame_t *frame)
+{
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0;
+    dht_lock_t **lk_array = NULL;
+    dht_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
+
+    local = frame->local;
+    conf = frame->this->private;
+
+    if (!conf->defrag)
+        goto err;
+
+    count = conf->local_subvols_cnt;
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+    if (lk_array == NULL)
+        goto err;
+
+    for (i = 0; i < count; i++) {
+        lk_array[i] = dht_lock_new(frame->this, conf->local_subvols[i],
+                                   &local->loc, F_WRLCK, DHT_LAYOUT_HEAL_DOMAIN,
+                                   NULL, FAIL_ON_ANY_ERROR);
+        if (lk_array[i] == NULL)
+            goto err;
+    }
+
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = count;
+
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_update_commit_hash_for_layout_resume);
+    if (ret < 0) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+        goto err;
+    }
+
+    return 0;
+err:
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        return ret;
+    return -1;
 }
diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
new file mode 100644
index 00000000000..bb72b0ffbb5
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -0,0 +1,1104 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* TODO: add NS locking */
+#include <glusterfs/statedump.h>
+#include "dht-common.h"
+#include "dht-messages.h"
+
+#ifndef MAX
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
+#endif
+
+/* TODO:
+   - use volumename in xattr instead of "dht"
+   - use NS locks
+   - handle all cases in self heal layout reconstruction
+   - complete linkfile selfheal
+*/
+
+static void
+dht_layout_dump(dht_layout_t *layout, const char *prefix)
+{
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+
+    if (!layout)
+        goto out;
+
+    gf_proc_dump_build_key(key, prefix, "cnt");
+    gf_proc_dump_write(key, "%d", layout->cnt);
+    gf_proc_dump_build_key(key, prefix, "preset");
+    gf_proc_dump_write(key, "%d", layout->preset);
+    gf_proc_dump_build_key(key, prefix, "gen");
+    gf_proc_dump_write(key, "%d", layout->gen);
+    if (layout->type != IA_INVAL) {
+        gf_proc_dump_build_key(key, prefix, "inode type");
+        gf_proc_dump_write(key, "%d", layout->type);
+    }
+
+    if (!IA_ISDIR(layout->type))
+        goto out;
+
+    for (i = 0; i < layout->cnt; i++) {
+        gf_proc_dump_build_key(key, prefix, "list[%d].err", i);
+        gf_proc_dump_write(key, "%d", layout->list[i].err);
+        gf_proc_dump_build_key(key, prefix, "list[%d].start", i);
+        gf_proc_dump_write(key, "0x%x", layout->list[i].start);
+        gf_proc_dump_build_key(key, prefix, "list[%d].stop", i);
+        gf_proc_dump_write(key, "0x%x", layout->list[i].stop);
+        if (layout->list[i].xlator) {
+            gf_proc_dump_build_key(key, prefix, "list[%d].xlator.type", i);
+            gf_proc_dump_write(key, "%s", layout->list[i].xlator->type);
+            gf_proc_dump_build_key(key, prefix, "list[%d].xlator.name", i);
+            gf_proc_dump_write(key, "%s", layout->list[i].xlator->name);
+        }
+    }
+
+out:
+    return;
+}
+
+int32_t
+dht_priv_dump(xlator_t *this)
+{
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+
+    if (!this)
+        goto out;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    ret = TRY_LOCK(&conf->subvolume_lock);
+    if (ret != 0) {
+        return ret;
+    }
+
+    gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
+    gf_proc_dump_build_key(key_prefix, "xlator.cluster.dht", "%s.priv",
+                           this->name);
+    gf_proc_dump_write("subvol_cnt", "%d", conf->subvolume_cnt);
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        snprintf(key, sizeof(key), "subvolumes[%d]", i);
+        gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
+                           conf->subvolumes[i]->name);
+        if (conf->file_layouts && conf->file_layouts[i]) {
+            snprintf(key, sizeof(key), "file_layouts[%d]", i);
+            dht_layout_dump(conf->file_layouts[i], key);
+        }
+        if (conf->dir_layouts && conf->dir_layouts[i]) {
+            snprintf(key, sizeof(key), "dir_layouts[%d]", i);
+            dht_layout_dump(conf->dir_layouts[i], key);
+        }
+        if (conf->subvolume_status) {
+            snprintf(key, sizeof(key), "subvolume_status[%d]", i);
+            gf_proc_dump_write(key, "%d", (int)conf->subvolume_status[i]);
+        }
+    }
+
+    gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
+    gf_proc_dump_write("gen", "%d", conf->gen);
+    gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
+    gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
+    gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
+    gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
+    gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
+    gf_proc_dump_write("use-readdirp", "%d", conf->use_readdirp);
+
+    if (conf->du_stats && conf->subvolume_status) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (!conf->subvolume_status[i])
+                continue;
+
+            snprintf(key, sizeof(key), "subvolumes[%d]", i);
+            gf_proc_dump_write(key, "%s", conf->subvolumes[i]->name);
+
+            snprintf(key, sizeof(key), "du_stats[%d].avail_percent", i);
+            gf_proc_dump_write(key, "%lf", conf->du_stats[i].avail_percent);
+
+            snprintf(key, sizeof(key), "du_stats[%d].avail_space", i);
+            gf_proc_dump_write(key, "%" PRIu64, conf->du_stats[i].avail_space);
+
+            snprintf(key, sizeof(key), "du_stats[%d].avail_inodes", i);
+            gf_proc_dump_write(key, "%lf", conf->du_stats[i].avail_inodes);
+
+            snprintf(key, sizeof(key), "du_stats[%d].log", i);
+            gf_proc_dump_write(key, "%" PRIu32, conf->du_stats[i].log);
+        }
+    }
+
+    if (conf->last_stat_fetch)
+        gf_proc_dump_write("last_stat_fetch", "%s",
+                           ctime(&conf->last_stat_fetch));
+
+    UNLOCK(&conf->subvolume_lock);
+
+out:
+    return ret;
+}
+
+int32_t
+dht_inodectx_dump(xlator_t *this, inode_t *inode)
+{
+    int ret = -1;
+    dht_layout_t *layout = NULL;
+
+    if (!this)
+        goto out;
+    if (!inode)
+        goto out;
+
+    ret = dht_inode_ctx_layout_get(inode, this, &layout);
+
+    if ((ret != 0) || !layout)
+        return ret;
+
+    gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
+    dht_layout_dump(layout, "layout");
+
+out:
+    return ret;
+}
+
+void
+dht_fini(xlator_t *this)
+{
+    int i = 0;
+    dht_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+
+    conf = this->private;
+    this->private = NULL;
+    if (conf) {
+        if (conf->file_layouts) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                GF_FREE(conf->file_layouts[i]);
+            }
+            GF_FREE(conf->file_layouts);
+        }
+
+        dict_unref(conf->leaf_to_subvol);
+
+        /* allocated in dht_init_subvolumes() */
+        GF_FREE(conf->subvolumes);
+        GF_FREE(conf->subvolume_status);
+        GF_FREE(conf->last_event);
+        GF_FREE(conf->subvol_up_time);
+        GF_FREE(conf->du_stats);
+        GF_FREE(conf->decommissioned_bricks);
+
+        /* allocated in dht_init() */
+        GF_FREE(conf->mds_xattr_key);
+        GF_FREE(conf->link_xattr_name);
+        GF_FREE(conf->commithash_xattr_name);
+        GF_FREE(conf->wild_xattr_name);
+
+        /* allocated in dht_init_regex() */
+        if (conf->rsync_regex_valid)
+            regfree(&conf->rsync_regex);
+        if (conf->extra_regex_valid)
+            regfree(&conf->extra_regex);
+
+        synclock_destroy(&conf->link_lock);
+
+        if (conf->lock_pool)
+            mem_pool_destroy(conf->lock_pool);
+
+        GF_FREE(conf);
+    }
+out:
+    return;
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+
+    ret = xlator_mem_acct_init(this, gf_dht_mt_end + 1);
+
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+               "Memory accounting init failed");
+        return ret;
+    }
+out:
+    return ret;
+}
+
+static int
+dht_parse_decommissioned_bricks(xlator_t *this, dht_conf_t *conf,
+                                const char *bricks)
+{
+    int i = 0;
+    int ret = -1;
+    char *tmpstr = NULL;
+    char *dup_brick = NULL;
+    char *node = NULL;
+
+    if (!conf || !bricks)
+        goto out;
+
+    dup_brick = gf_strdup(bricks);
+    if (dup_brick == NULL) {
+        goto out;
+    }
+
+    node = strtok_r(dup_brick, ",", &tmpstr);
+    while (node) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (!strcmp(conf->subvolumes[i]->name, node)) {
+                conf->decommissioned_bricks[i] = conf->subvolumes[i];
+                conf->decommission_subvols_cnt++;
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       DHT_MSG_SUBVOL_DECOMMISSION_INFO,
+                       "decommissioning subvolume %s",
+                       conf->subvolumes[i]->name);
+                break;
+            }
+        }
+        if (i == conf->subvolume_cnt) {
+            /* Wrong node given. */
+            goto out;
+        }
+        node = strtok_r(NULL, ",", &tmpstr);
+    }
+
+    ret = 0;
+    conf->decommission_in_progress = 1;
+out:
+    GF_FREE(dup_brick);
+
+    return ret;
+}
+
+static void
+dht_decommissioned_remove(xlator_t *this, dht_conf_t *conf)
+{
+    int i = 0;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->decommissioned_bricks[i]) {
+            conf->decommissioned_bricks[i] = NULL;
+            conf->decommission_subvols_cnt--;
+        }
+    }
+}
+
+static void
+dht_init_regex(xlator_t *this, dict_t *odict, char *name, regex_t *re,
+               gf_boolean_t *re_valid, dht_conf_t *conf)
+{
+    char *temp_str = NULL;
+
+    if (dict_get_str(odict, name, &temp_str) != 0) {
+        if (strcmp(name, "rsync-hash-regex")) {
+            return;
+        }
+        temp_str = "^\\.(.+)\\.[^.]+$";
+    }
+
+    LOCK(&conf->lock);
+    {
+        if (*re_valid) {
+            regfree(re);
+            *re_valid = _gf_false;
+        }
+
+        if (!strcmp(temp_str, "none")) {
+            goto unlock;
+        }
+
+        if (regcomp(re, temp_str, REG_EXTENDED) == 0) {
+            gf_msg_debug(this->name, 0, "using regex %s = %s", name, temp_str);
+            *re_valid = _gf_true;
+        } else {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_REGEX_INFO,
+                   "compiling regex %s failed", temp_str);
+        }
+    }
+unlock:
+    UNLOCK(&conf->lock);
+}
+
+int
+dht_set_subvol_range(xlator_t *this)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf)
+        goto out;
+
+    conf->leaf_to_subvol = dict_new();
+    if (!conf->leaf_to_subvol)
+        goto out;
+
+    ret = glusterfs_reachable_leaves(this, conf->leaf_to_subvol);
+
+out:
+    return ret;
+}
+
+static int
+dht_configure_throttle(xlator_t *this, dht_conf_t *conf, char *temp_str)
+{
+    int rebal_thread_count = 0;
+    int ret = 0;
+
+    pthread_mutex_lock(&conf->defrag->dfq_mutex);
+    {
+        if (!strcasecmp(temp_str, "lazy")) {
+            conf->defrag->recon_thread_count = 1;
+        } else if (!strcasecmp(temp_str, "normal")) {
+            conf->defrag->recon_thread_count = 2;
+        } else if (!strcasecmp(temp_str, "aggressive")) {
+            conf->defrag->recon_thread_count = MAX(MAX_REBAL_THREADS - 4, 4);
+        } else if ((gf_string2int(temp_str, &rebal_thread_count) == 0)) {
+            if ((rebal_thread_count > 0) &&
+                (rebal_thread_count <= MAX_REBAL_THREADS)) {
+                conf->defrag->recon_thread_count = rebal_thread_count;
+                pthread_mutex_unlock(&conf->defrag->dfq_mutex);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                       "rebal thread count configured to %d",
+                       rebal_thread_count);
+                goto out;
+            } else {
+                pthread_mutex_unlock(&conf->defrag->dfq_mutex);
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option: Reconfigure: "
+                       "rebal-throttle should be "
+                       "within range of 0 and maximum number of"
+                       " cores available");
+                ret = -1;
+                goto out;
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option: Reconfigure: "
+                   "rebal-throttle should be {lazy|normal|aggressive}"
+                   " or a number up to the number of cores available,"
+                   " not (%s), defaulting to (%d)",
+                   temp_str, conf->dthrottle);
+            ret = -1;
+        }
+    }
+    pthread_mutex_unlock(&conf->defrag->dfq_mutex);
+
+out:
+    return ret;
+}
+
+int
+dht_reconfigure(xlator_t *this, dict_t *options)
+{
+    dht_conf_t *conf = NULL;
+    char *temp_str = NULL;
+    gf_boolean_t search_unhashed;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", options, out);
+
+    conf = this->private;
+    if (!conf)
+        return 0;
+
+    if (dict_get_str(options, "lookup-unhashed", &temp_str) == 0) {
+        /* If option is not "auto", other options _should_ be boolean*/
+        if (strcasecmp(temp_str, "auto")) {
+            if (!gf_string2boolean(temp_str, &search_unhashed)) {
+                gf_msg_debug(this->name, 0,
+                             "Reconfigure: "
+                             "lookup-unhashed reconfigured(%s)",
+                             temp_str);
+                conf->search_unhashed = search_unhashed;
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option: Reconfigure: "
+                       "lookup-unhashed should be boolean,"
+                       " not (%s), defaulting to (%d)",
+                       temp_str, conf->search_unhashed);
+                ret = -1;
+                goto out;
+            }
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "Reconfigure:"
+                         " lookup-unhashed reconfigured auto ");
+            conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        }
+    }
+
+    GF_OPTION_RECONF("lookup-optimize", conf->lookup_optimize, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("min-free-disk", conf->min_free_disk, options,
+                     percent_or_size, out);
+    /* option can be any one of percent or bytes */
+    conf->disk_unit = 0;
+    if (conf->min_free_disk < 100.0)
+        conf->disk_unit = 'p';
+
+    GF_OPTION_RECONF("min-free-inodes", conf->min_free_inodes, options, percent,
+                     out);
+
+    GF_OPTION_RECONF("directory-layout-spread", conf->dir_spread_cnt, options,
+                     uint32, out);
+
+    GF_OPTION_RECONF("readdir-optimize", conf->readdir_optimize, options, bool,
+                     out);
+    GF_OPTION_RECONF("randomize-hash-range-by-gfid", conf->randomize_by_gfid,
+                     options, bool, out);
+
+    GF_OPTION_RECONF("lock-migration", conf->lock_migration_enabled, options,
+                     bool, out);
+
+    GF_OPTION_RECONF("force-migration", conf->force_migration, options, bool,
+                     out);
+
+    if (conf->defrag) {
+        if (dict_get_str(options, "rebal-throttle", &temp_str) == 0) {
+            ret = dht_configure_throttle(this, conf, temp_str);
+            if (ret == -1)
+                goto out;
+        }
+    }
+
+    if (conf->defrag) {
+        conf->defrag->lock_migration_enabled = conf->lock_migration_enabled;
+    }
+
+    if (conf->defrag) {
+        GF_OPTION_RECONF("rebalance-stats", conf->defrag->stats, options, bool,
+                         out);
+    }
+
+    if (dict_get_str(options, "decommissioned-bricks", &temp_str) == 0) {
+        ret = dht_parse_decommissioned_bricks(this, conf, temp_str);
+        if (ret == -1)
+            goto out;
+    } else {
+        dht_decommissioned_remove(this, conf);
+    }
+
+    dht_init_regex(this, options, "rsync-hash-regex", &conf->rsync_regex,
+                   &conf->rsync_regex_valid, conf);
+    dht_init_regex(this, options, "extra-hash-regex", &conf->extra_regex,
+                   &conf->extra_regex_valid, conf);
+
+    GF_OPTION_RECONF("weighted-rebalance", conf->do_weighting, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("use-readdirp", conf->use_readdirp, options, bool, out);
+    ret = 0;
+out:
+    return ret;
+}
+
+static int
+gf_defrag_pattern_list_fill(xlator_t *this, gf_defrag_info_t *defrag,
+                            char *data)
+{
+    int ret = -1;
+    char *tmp_str = NULL;
+    char *tmp_str1 = NULL;
+    char *dup_str = NULL;
+    char *num = NULL;
+    char *pattern_str = NULL;
+    char *pattern = NULL;
+    gf_defrag_pattern_list_t *temp_list = NULL;
+    gf_defrag_pattern_list_t *pattern_list = NULL;
+
+    if (!this || !defrag || !data)
+        goto out;
+
+    /* Get the pattern for pattern list. "pattern:<optional-size>"
+     * eg: *avi, *pdf:10MB, *:1TB
+     */
+    pattern_str = strtok_r(data, ",", &tmp_str);
+    while (pattern_str) {
+        dup_str = gf_strdup(pattern_str);
+        if (!dup_str)
+            goto out;
+        pattern_list = GF_CALLOC(1, sizeof(gf_defrag_pattern_list_t), 1);
+        if (!pattern_list) {
+            goto out;
+        }
+        pattern = strtok_r(dup_str, ":", &tmp_str1);
+        num = strtok_r(NULL, ":", &tmp_str1);
+        if (!pattern)
+            goto out;
+        if (!num) {
+            if (gf_string2bytesize_uint64(pattern, &pattern_list->size) == 0) {
+                pattern = "*";
+            }
+        } else if (gf_string2bytesize_uint64(num, &pattern_list->size) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option. Defrag pattern:"
+                   " Invalid number format \"%s\"",
+                   num);
+            goto out;
+        }
+        memcpy(pattern_list->path_pattern, pattern, strlen(dup_str));
+
+        if (!defrag->defrag_pattern)
+            temp_list = NULL;
+        else
+            temp_list = defrag->defrag_pattern;
+
+        pattern_list->next = temp_list;
+
+        defrag->defrag_pattern = pattern_list;
+        pattern_list = NULL;
+
+        GF_FREE(dup_str);
+        dup_str = NULL;
+
+        pattern_str = strtok_r(NULL, ",", &tmp_str);
+    }
+
+    ret = 0;
+out:
+    if (ret)
+        GF_FREE(pattern_list);
+    GF_FREE(dup_str);
+
+    return ret;
+}
+
+static int
+dht_init_methods(xlator_t *this)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+
+    conf = this->private;
+    methods = &(conf->methods);
+
+    methods->migration_get_dst_subvol = dht_migration_get_dst_subvol;
+    methods->migration_other = NULL;
+    methods->layout_search = dht_layout_search;
+
+    ret = 0;
+err:
+    return ret;
+}
+
+int
+dht_init(xlator_t *this)
+{
+    dht_conf_t *conf = NULL;
+    char *temp_str = NULL;
+    int ret = -1;
+    int i = 0;
+    gf_defrag_info_t *defrag = NULL;
+    int cmd = 0;
+    char *node_uuid = NULL;
+    uint32_t commit_hash = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+
+    if (!this->children) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_INVALID_CONFIGURATION,
+               "Distribute needs more than one subvolume");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_CONFIGURATION,
+               "dangling volume. check volfile");
+    }
+
+    conf = GF_CALLOC(1, sizeof(*conf), gf_dht_mt_dht_conf_t);
+    if (!conf) {
+        goto err;
+    }
+
+    LOCK_INIT(&conf->subvolume_lock);
+    LOCK_INIT(&conf->layout_lock);
+    LOCK_INIT(&conf->lock);
+    synclock_init(&conf->link_lock, SYNC_LOCK_DEFAULT);
+
+    /* We get the commit-hash to set only for rebalance process */
+    if (dict_get_uint32(this->options, "commit-hash", &commit_hash) == 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_COMMIT_HASH_INFO,
+               "%s using commit hash %u", __func__, commit_hash);
+        conf->vol_commit_hash = commit_hash;
+        conf->vch_forced = _gf_true;
+    }
+
+    ret = dict_get_int32(this->options, "rebalance-cmd", &cmd);
+
+    if (cmd) {
+        defrag = GF_CALLOC(1, sizeof(gf_defrag_info_t), gf_defrag_info_mt);
+
+        GF_VALIDATE_OR_GOTO(this->name, defrag, err);
+
+        LOCK_INIT(&defrag->lock);
+
+        defrag->is_exiting = 0;
+
+        conf->defrag = defrag;
+        defrag->this = this;
+
+        ret = dict_get_str(this->options, "node-uuid", &node_uuid);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_CONFIGURATION,
+                   "Invalid volume configuration: "
+                   "node-uuid not specified");
+            goto err;
+        }
+
+        if (gf_uuid_parse(node_uuid, defrag->node_uuid)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option:"
+                   " Cannot parse glusterd node uuid");
+            goto err;
+        }
+
+        defrag->cmd = cmd;
+
+        defrag->stats = _gf_false;
+
+        defrag->queue = NULL;
+
+        defrag->crawl_done = 0;
+
+        defrag->global_error = 0;
+
+        defrag->q_entry_count = 0;
+
+        defrag->wakeup_crawler = 0;
+
+        pthread_mutex_init(&defrag->dfq_mutex, 0);
+        pthread_cond_init(&defrag->parallel_migration_cond, 0);
+        pthread_cond_init(&defrag->rebalance_crawler_alarm, 0);
+        pthread_cond_init(&defrag->df_wakeup_thread, 0);
+
+        pthread_mutex_init(&defrag->fc_mutex, 0);
+        pthread_cond_init(&defrag->fc_wakeup_cond, 0);
+
+        defrag->global_error = 0;
+    }
+
+    conf->use_fallocate = 1;
+
+    conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
+    if (dict_get_str(this->options, "lookup-unhashed", &temp_str) == 0) {
+        /* If option is not "auto", other options _should_ be boolean */
+        if (strcasecmp(temp_str, "auto")) {
+            gf_boolean_t search_unhashed_bool;
+            ret = gf_string2boolean(temp_str, &search_unhashed_bool);
+            if (ret == -1) {
+                goto err;
+            }
+            conf->search_unhashed = search_unhashed_bool
+                                        ? GF_DHT_LOOKUP_UNHASHED_ON
+                                        : GF_DHT_LOOKUP_UNHASHED_OFF;
+        } else {
+            conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        }
+    }
+
+    GF_OPTION_INIT("lookup-optimize", conf->lookup_optimize, bool, err);
+
+    GF_OPTION_INIT("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool, err);
+
+    GF_OPTION_INIT("use-readdirp", conf->use_readdirp, bool, err);
+
+    GF_OPTION_INIT("min-free-disk", conf->min_free_disk, percent_or_size, err);
+
+    GF_OPTION_INIT("min-free-inodes", conf->min_free_inodes, percent, err);
+
+    conf->dir_spread_cnt = conf->subvolume_cnt;
+    GF_OPTION_INIT("directory-layout-spread", conf->dir_spread_cnt, uint32,
+                   err);
+
+    GF_OPTION_INIT("assert-no-child-down", conf->assert_no_child_down, bool,
+                   err);
+
+    GF_OPTION_INIT("readdir-optimize", conf->readdir_optimize, bool, err);
+
+    GF_OPTION_INIT("lock-migration", conf->lock_migration_enabled, bool, err);
+
+    GF_OPTION_INIT("force-migration", conf->force_migration, bool, err);
+
+    if (defrag) {
+        defrag->lock_migration_enabled = conf->lock_migration_enabled;
+
+        GF_OPTION_INIT("rebalance-stats", defrag->stats, bool, err);
+        if (dict_get_str(this->options, "rebalance-filter", &temp_str) == 0) {
+            if (gf_defrag_pattern_list_fill(this, defrag, temp_str) == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option:"
+                       " Cannot parse rebalance-filter (%s)",
+                       temp_str);
+
+                goto err;
+            }
+        }
+    }
+
+    /* option can be any one of percent or bytes */
+    conf->disk_unit = 0;
+    if (conf->min_free_disk < 100)
+        conf->disk_unit = 'p';
+
+    ret = dht_init_subvolumes(this, conf);
+    if (ret == -1) {
+        goto err;
+    }
+
+    if (cmd) {
+        ret = dht_init_local_subvolumes(this, conf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_INIT_LOCAL_SUBVOL_FAILED,
+                   "dht_init_local_subvolumes failed");
+            goto err;
+        }
+    }
+
+    if (dict_get_str(this->options, "decommissioned-bricks", &temp_str) == 0) {
+        ret = dht_parse_decommissioned_bricks(this, conf, temp_str);
+        if (ret == -1)
+            goto err;
+    }
+
+    dht_init_regex(this, this->options, "rsync-hash-regex", &conf->rsync_regex,
+                   &conf->rsync_regex_valid, conf);
+    dht_init_regex(this, this->options, "extra-hash-regex", &conf->extra_regex,
+                   &conf->extra_regex_valid, conf);
+
+    ret = dht_layouts_init(this, conf);
+    if (ret == -1) {
+        goto err;
+    }
+
+    conf->gen = 1;
+
+    this->local_pool = mem_pool_new(dht_local_t, 512);
+    if (!this->local_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               " DHT initialisation failed. "
+               "failed to create local_t's memory pool");
+        goto err;
+    }
+
+    GF_OPTION_INIT("randomize-hash-range-by-gfid", conf->randomize_by_gfid,
+                   bool, err);
+
+    if (defrag) {
+        GF_OPTION_INIT("rebal-throttle", temp_str, str, err);
+        if (temp_str) {
+            ret = dht_configure_throttle(this, conf, temp_str);
+            if (ret == -1)
+                goto err;
+        }
+    }
+
+    GF_OPTION_INIT("xattr-name", conf->xattr_name, str, err);
+    gf_asprintf(&conf->mds_xattr_key, "%s." DHT_MDS_STR, conf->xattr_name);
+    gf_asprintf(&conf->link_xattr_name, "%s." DHT_LINKFILE_STR,
+                conf->xattr_name);
+    gf_asprintf(&conf->commithash_xattr_name, "%s." DHT_COMMITHASH_STR,
+                conf->xattr_name);
+    gf_asprintf(&conf->wild_xattr_name, "%s*", conf->xattr_name);
+    if (!conf->link_xattr_name || !conf->wild_xattr_name) {
+        goto err;
+    }
+
+    GF_OPTION_INIT("weighted-rebalance", conf->do_weighting, bool, err);
+
+    conf->lock_pool = mem_pool_new(dht_lock_t, 512);
+    if (!conf->lock_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INIT_FAILED,
+               "failed to create lock mem_pool, failing "
+               "initialization");
+        goto err;
+    }
+
+    this->private = conf;
+
+    if (dht_set_subvol_range(this))
+        goto err;
+
+    if (dht_init_methods(this))
+        goto err;
+
+    return 0;
+
+err:
+    if (conf) {
+        if (conf->file_layouts) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                GF_FREE(conf->file_layouts[i]);
+            }
+            GF_FREE(conf->file_layouts);
+        }
+
+        GF_FREE(conf->subvolumes);
+
+        GF_FREE(conf->subvolume_status);
+
+        GF_FREE(conf->du_stats);
+
+        GF_FREE(conf->defrag);
+
+        GF_FREE(conf->xattr_name);
+        GF_FREE(conf->link_xattr_name);
+        GF_FREE(conf->wild_xattr_name);
+        GF_FREE(conf->mds_xattr_key);
+
+        if (conf->lock_pool)
+            mem_pool_destroy(conf->lock_pool);
+
+        GF_FREE(conf);
+    }
+
+    return -1;
+}
+
+struct volume_options dht_options[] = {
+    {
+        .key = {"lookup-unhashed"},
+        .value = {"auto", "yes", "no", "enable", "disable", "1", "0", "on",
+                  "off"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "on",
+        .description =
+            "This option if set to ON, does a lookup through "
+            "all the sub-volumes, in case a lookup didn't return any result "
+            "from the hash subvolume. If set to OFF, it does not do a lookup "
+            "on the remaining subvolumes.",
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+        .level = OPT_STATUS_BASIC,
+    },
+    {.key = {"lookup-optimize"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description =
+         "This option if set to ON enables the optimization "
+         "of -ve lookups, by not doing a lookup on non-hashed subvolumes for "
+         "files, in case the hashed subvolume does not return any result. "
+         "This option disregards the lookup-unhashed setting, when enabled.",
+     .op_version = {GD_OP_VERSION_3_7_2},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"min-free-disk"},
+     .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
+     .default_value = "10%",
+     .description =
+         "Percentage/Size of disk space, after which the "
+         "process starts balancing out the cluster, and logs will appear "
+         "in log files",
+     .op_version = {1},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"min-free-inodes"},
+     .type = GF_OPTION_TYPE_PERCENT,
+     .default_value = "5%",
+     .description = "after system has only N% of inodes, warnings "
+                    "starts to appear in log files",
+     .op_version = {1},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {
+        .key = {"unhashed-sticky-bit"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+    },
+    {.key = {"use-readdirp"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "This option if set to ON, forces the use of "
+                    "readdirp, and hence also displays the stats of the files.",
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"assert-no-child-down"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option if set to ON, in the event of "
+                    "CHILD_DOWN, will call exit."},
+    {
+        .key = {"directory-layout-spread"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .validate = GF_OPT_VALIDATE_MIN,
+        .description = "Specifies the directory layout spread. Takes number "
+                       "of subvolumes as default value.",
+
+        .op_version = {2},
+    },
+    {
+        .key = {"decommissioned-bricks"},
+        .type = GF_OPTION_TYPE_ANY,
+        .description =
+            "This option if set to ON, decommissions "
+            "the brick, so that no new data is allowed to be created "
+            "on that brick.",
+        .level = OPT_STATUS_ADVANCED,
+    },
+    {
+        .key = {"rebalance-cmd"},
+        .type = GF_OPTION_TYPE_INT,
+    },
+    {
+        .key = {"commit-hash"},
+        .type = GF_OPTION_TYPE_INT,
+    },
+    {
+        .key = {"node-uuid"},
+        .type = GF_OPTION_TYPE_STR,
+    },
+    {
+        .key = {"rebalance-stats"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description =
+            "This option if set to ON displays and logs the "
+            " time taken for migration of each file, during the rebalance "
+            "process. If set to OFF, the rebalance logs will only display the "
+            "time spent in each directory.",
+        .op_version = {2},
+        .level = OPT_STATUS_BASIC,
+    },
+    {.key = {"readdir-optimize"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description =
+         "This option if set to ON enables the optimization "
+         "that allows DHT to requests non-first subvolumes to filter out "
+         "directory entries.",
+     .op_version = {1},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"rsync-hash-regex"},
+     .type = GF_OPTION_TYPE_STR,
+     /* Setting a default here doesn't work.  See dht_init_regex. */
+     .description =
+         "Regular expression for stripping temporary-file "
+         "suffix and prefix used by rsync, to prevent relocation when the "
+         "file is renamed.",
+     .op_version = {3},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"extra-hash-regex"},
+     .type = GF_OPTION_TYPE_STR,
+     /* Setting a default here doesn't work.  See dht_init_regex. */
+     .description =
+         "Regular expression for stripping temporary-file "
+         "suffix and prefix used by an application, to prevent relocation when "
+         "the file is renamed.",
+     .op_version = {3},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {
+        .key = {"rebalance-filter"},
+        .type = GF_OPTION_TYPE_STR,
+    },
+
+    {
+        .key = {"xattr-name"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "trusted.glusterfs.dht",
+        .description =
+            "Base for extended attributes used by this "
+            "translator instance, to avoid conflicts with others above or "
+            "below it.",
+        .op_version = {3},
+    },
+
+    {.key = {"weighted-rebalance"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description =
+         "When enabled, files will be allocated to bricks "
+         "with a probability proportional to their size.  Otherwise, all "
+         "bricks will have the same probability (legacy behavior).",
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    /* NUFA option */
+    {.key = {"local-volume-name"}, .type = GF_OPTION_TYPE_XLATOR},
+
+    /* switch option */
+    {.key = {"pattern.switch.case"}, .type = GF_OPTION_TYPE_ANY},
+
+    {
+        .key = {"randomize-hash-range-by-gfid"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description =
+            "Use gfid of directory to determine the subvolume "
+            "from which hash ranges are allocated starting with 0. "
+            "Note that we still use a directory/file's name to determine the "
+            "subvolume to which it hashes",
+        .op_version = {GD_OP_VERSION_3_6_0},
+    },
+
+    {.key = {"rebal-throttle"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "normal",
+     .description = " Sets the maximum number of parallel file migrations "
+                    "allowed on a node during the rebalance operation. The"
+                    " default value is normal and allows a max of "
+                    "[($(processing units) - 4) / 2), 2]  files to be "
+                    "migrated at a time. Lazy will allow only one file to "
+                    "be migrated at a time and aggressive will allow "
+                    "max of [($(processing units) - 4) / 2), 4]",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
+
+    },
+
+    {.key = {"lock-migration"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = " If enabled this feature will migrate the posix locks"
+                    " associated with a file during rebalance",
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    {.key = {"force-migration"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "If disabled, rebalance will not migrate files that "
+                    "are being written to by an application",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    {.key = {NULL}},
+};
+
+#define NUM_DHT_OPTIONS (sizeof(dht_options) / sizeof(dht_options[0]))
+
+extern struct volume_options options[NUM_DHT_OPTIONS]
+    __attribute__((alias("dht_options")));
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index 6b156c1d354..53de8292704 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -8,601 +8,116 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-/* TODO: add NS locking */
-
-#include "statedump.h"
 #include "dht-common.h"
 
-/* TODO:
-   - use volumename in xattr instead of "dht"
-   - use NS locks
-   - handle all cases in self heal layout reconstruction
-   - complete linkfile selfheal
-*/
-struct volume_options options[];
-
-void
-dht_layout_dump (dht_layout_t  *layout, const char *prefix)
-{
-
-        char    key[GF_DUMP_MAX_BUF_LEN];
-        int     i = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", layout, out);
-        GF_VALIDATE_OR_GOTO ("dht", prefix, out);
-
-        gf_proc_dump_build_key(key, prefix, "cnt");
-        gf_proc_dump_write(key, "%d", layout->cnt);
-        gf_proc_dump_build_key(key, prefix, "preset");
-        gf_proc_dump_write(key, "%d", layout->preset);
-        gf_proc_dump_build_key(key, prefix, "gen");
-        gf_proc_dump_write(key, "%d", layout->gen);
-        gf_proc_dump_build_key(key, prefix, "type");
-        gf_proc_dump_write(key, "%d", layout->type);
-
-        for (i = 0; i < layout->cnt; i++) {
-                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
-                gf_proc_dump_write(key, "%d", layout->list[i].err);
-                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].start);
-                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].stop);
-                if (layout->list[i].xlator) {
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.type", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->type);
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.name", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->name);
-                }
-        }
-
-out:
-        return;
-}
-
-
-int32_t
-dht_priv_dump (xlator_t *this)
-{
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char            key[GF_DUMP_MAX_BUF_LEN];
-        int             i = 0;
-        dht_conf_t      *conf = NULL;
-        int             ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        conf = this->private;
-
-        if (!conf)
-                return -1;
-
-        ret = TRY_LOCK(&conf->subvolume_lock);
-
-        if (ret != 0) {
-                gf_log("", GF_LOG_WARNING, "Unable to lock dht subvolume %s",
-                       this->name);
-                return ret;
-        }
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
-        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
-                               this->name);
-        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                sprintf (key, "subvolumes[%d]", i);
-                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
-                                   conf->subvolumes[i]->name);
-                if (conf->file_layouts && conf->file_layouts[i]){
-                        sprintf (key, "file_layouts[%d]", i);
-                        dht_layout_dump(conf->file_layouts[i], key);
-                }
-                if (conf->dir_layouts && conf->dir_layouts[i]) {
-                        sprintf (key, "dir_layouts[%d]", i);
-                        dht_layout_dump(conf->dir_layouts[i], key);
-                }
-                if (conf->subvolume_status) {
-
-                        sprintf (key, "subvolume_status[%d]", i);
-                        gf_proc_dump_write(key, "%d",
-                                           (int)conf->subvolume_status[i]);
-                }
-
-        }
-
-        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
-        gf_proc_dump_write("gen", "%d", conf->gen);
-        gf_proc_dump_write("min_free_disk", "%lu", conf->min_free_disk);
-	gf_proc_dump_write("min_free_inodes", "%lu", conf->min_free_inodes);
-        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
-        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
-        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
-        if (conf ->du_stats) {
-                gf_proc_dump_write("du_stats.avail_percent", "%lf",
-                                   conf->du_stats->avail_percent);
-                gf_proc_dump_write("du_stats.avail_space", "%lu",
-                                   conf->du_stats->avail_space);
-		gf_proc_dump_write("du_stats.avail_inodes", "%lf",
-                                   conf->du_stats->avail_inodes);
-                gf_proc_dump_write("du_stats.log", "%lu", conf->du_stats->log);
-        }
-
-        if (conf->last_stat_fetch.tv_sec)
-                gf_proc_dump_write("last_stat_fetch", "%s",
-                                    ctime(&conf->last_stat_fetch.tv_sec));
-
-        UNLOCK(&conf->subvolume_lock);
-
-out:
-        return ret;
-}
-
-int32_t
-dht_inodectx_dump (xlator_t *this, inode_t *inode)
-{
-        int             ret = -1;
-        dht_layout_t    *layout = NULL;
-        uint64_t        tmp_layout = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", inode, out);
-
-        ret = inode_ctx_get (inode, this, &tmp_layout);
-
-        if (ret != 0)
-                return ret;
-
-        layout = (dht_layout_t *)(long)tmp_layout;
-
-        if (!layout)
-                return -1;
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
-        dht_layout_dump(layout, "layout");
-
-out:
-        return ret;
-}
-
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        int              ret = -1;
-        va_list          ap;
-        dict_t          *output = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-
-        if (!data)
-                goto out;
-
-        va_start (ap, data);
-        output = va_arg (ap, dict_t*);
-
-        ret = dht_notify (this, event, data, output);
-
-out:
-        return ret;
-}
-
-void
-fini (xlator_t *this)
-{
-        int         i = 0;
-        dht_conf_t *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        conf = this->private;
-        this->private = NULL;
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                GF_FREE (conf->subvolumes);
-
-                GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf);
-        }
-out:
-        return;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-
-int
-dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
-                                 const char *bricks)
-{
-        int         i  = 0;
-        int         ret  = -1;
-        char       *tmpstr = NULL;
-        char       *dup_brick = NULL;
-        char       *node = NULL;
-
-        if (!conf || !bricks)
-                goto out;
-
-        dup_brick = gf_strdup (bricks);
-        node = strtok_r (dup_brick, ",", &tmpstr);
-        while (node) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (!strcmp (conf->subvolumes[i]->name, node)) {
-                                conf->decommissioned_bricks[i] =
-                                        conf->subvolumes[i];
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "decommissioning subvolume %s",
-                                        conf->subvolumes[i]->name);
-                                break;
-                        }
-                }
-                if (i == conf->subvolume_cnt) {
-                        /* Wrong node given. */
-                        goto out;
-                }
-                node = strtok_r (NULL, ",", &tmpstr);
-        }
-
-        ret = 0;
-        conf->decommission_in_progress = 1;
-out:
-        GF_FREE (dup_brick);
-
-        return ret;
-}
-
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        dht_conf_t      *conf = NULL;
-        char            *temp_str = NULL;
-        gf_boolean_t     search_unhashed;
-        int              ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", options, out);
-
-        conf = this->private;
-        if (!conf)
-                return 0;
-
-        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean*/
-                if (strcasecmp (temp_str, "auto")) {
-                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
-                                gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
-                                       " lookup-unhashed reconfigured (%s)",
-                                       temp_str);
-                                conf->search_unhashed = search_unhashed;
-                        } else {
-                                gf_log(this->name, GF_LOG_ERROR, "Reconfigure:"
-                                       " lookup-unhashed should be boolean,"
-                                       " not (%s), defaulting to (%d)",
-                                       temp_str, conf->search_unhashed);
-                                //return -1;
-                                ret = -1;
-                                goto out;
-                        }
-                } else {
-                        gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
-                               " lookup-unhashed reconfigured auto ");
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-                }
-        }
-
-	GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
-                          percent_or_size, out);
-	GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
-                          percent, out);
-        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
-                          options, uint32, out);
-
-        if (conf->defrag) {
-                GF_OPTION_RECONF ("rebalance-stats", conf->defrag->stats,
-                                  options, bool, out);
-        }
-
-        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto out;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-int
-init (xlator_t *this)
-{
-        dht_conf_t                      *conf           = NULL;
-        char                            *temp_str       = NULL;
-        int                              ret            = -1;
-        int                              i              = 0;
-        gf_defrag_info_t                *defrag         = NULL;
-        int                              cmd            = 0;
-        char                            *node_uuid      = NULL;
-
-
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Distribute needs more than one subvolume");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        ret = dict_get_int32 (this->options, "rebalance-cmd", &cmd);
-
-        if (cmd) {
-                defrag = GF_CALLOC (1, sizeof (gf_defrag_info_t),
-                                    gf_defrag_info_mt);
-
-                GF_VALIDATE_OR_GOTO (this->name, defrag, err);
-
-                LOCK_INIT (&defrag->lock);
-
-                defrag->is_exiting = 0;
-
-                conf->defrag = defrag;
-
-                ret = dict_get_str (this->options, "node-uuid", &node_uuid);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "node-uuid not "
-                                "specified");
-                        goto err;
-                }
-
-                if (uuid_parse (node_uuid, defrag->node_uuid)) {
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot parse "
-                                "glusterd node uuid");
-                        goto err;
-                }
-
-                defrag->cmd = cmd;
-
-                defrag->stats = _gf_false;
-        }
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-        }
-
-        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
-                        err);
-
-        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
-
-	GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
-			err);
-
-        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
-                        err);
-
-        conf->dir_spread_cnt = conf->subvolume_cnt;
-        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
-                        uint32, err);
-
-        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
-                        bool, err);
-
-        if (defrag) {
-                GF_OPTION_INIT ("rebalance-stats", defrag->stats, bool, err);
-        }
-
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto err;
-        }
-
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
-
-        conf->gen = 1;
-
-        this->local_pool = mem_pool_new (dht_local_t, 512);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto err;
-        }
-
-        this->private = conf;
-
-        return 0;
-
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                GF_FREE (conf->subvolumes);
-
-                GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf->du_stats);
-
-                GF_FREE (conf->defrag);
-
-                GF_FREE (conf);
-        }
-
-        return -1;
-}
-
+struct xlator_fops dht_pt_fops = {
+    /* we need to keep mkdir to make sure we
+       have layout on new directory */
+    .mkdir = dht_pt_mkdir,
+    .getxattr = dht_pt_getxattr,
+    .fgetxattr = dht_pt_fgetxattr,
+
+    /* required to trace fop properly in changelog */
+    .rename = dht_pt_rename,
+
+    /* FIXME: commenting the '.lookup()' below made some of
+       the failing tests to pass. I would remove the below
+       line, but keeping it here as a reminder for people
+       to check for issues if they find concerns with DHT
+       pass-through logic  */
+    /*
+      .lookup = dht_lookup,
+      .readdir = dht_readdir,
+      .readdirp = dht_readdirp,
+    */
+    /* Keeping above as commented, mainly to support the
+       usecase of a gluster volume getting to 1x(anytype),
+       due to remove-brick (shrinking) exercise. In that case,
+       we would need above fops to be available, so we can
+       handle the case of dangling linkto files (if any) */
+};
 
 struct xlator_fops fops = {
-        .lookup      = dht_lookup,
-        .mknod       = dht_mknod,
-        .create      = dht_create,
-
-        .open        = dht_open,
-        .statfs      = dht_statfs,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
-
-        /* Inode read operations */
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .getxattr    = dht_getxattr,
-        .fgetxattr    = dht_fgetxattr,
-        .readv       = dht_readv,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .lk          = dht_lk,
-
-        /* Inode write operations */
-        .fremovexattr = dht_fremovexattr,
-        .removexattr = dht_removexattr,
-        .setxattr    = dht_setxattr,
-        .fsetxattr   = dht_fsetxattr,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .writev      = dht_writev,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
-        .fsetattr    = dht_fsetattr,
+    .ipc = dht_ipc,
+    .lookup = dht_lookup,
+    .mknod = dht_mknod,
+    .create = dht_create,
+
+    .open = dht_open,
+    .statfs = dht_statfs,
+    .opendir = dht_opendir,
+    .readdir = dht_readdir,
+    .readdirp = dht_readdirp,
+    .fsyncdir = dht_fsyncdir,
+    .symlink = dht_symlink,
+    .unlink = dht_unlink,
+    .link = dht_link,
+    .mkdir = dht_mkdir,
+    .rmdir = dht_rmdir,
+    .rename = dht_rename,
+    .entrylk = dht_entrylk,
+    .fentrylk = dht_fentrylk,
+
+    /* Inode read operations */
+    .stat = dht_stat,
+    .fstat = dht_fstat,
+    .access = dht_access,
+    .readlink = dht_readlink,
+    .getxattr = dht_getxattr,
+    .fgetxattr = dht_fgetxattr,
+    .readv = dht_readv,
+    .flush = dht_flush,
+    .fsync = dht_fsync,
+    .inodelk = dht_inodelk,
+    .finodelk = dht_finodelk,
+    .lk = dht_lk,
+    .lease = dht_lease,
+
+    /* Inode write operations */
+    .fremovexattr = dht_fremovexattr,
+    .removexattr = dht_removexattr,
+    .setxattr = dht_setxattr,
+    .fsetxattr = dht_fsetxattr,
+    .truncate = dht_truncate,
+    .ftruncate = dht_ftruncate,
+    .writev = dht_writev,
+    .xattrop = dht_xattrop,
+    .fxattrop = dht_fxattrop,
+    .setattr = dht_setattr,
+    .fsetattr = dht_fsetattr,
+    .fallocate = dht_fallocate,
+    .discard = dht_discard,
+    .zerofill = dht_zerofill,
 };
 
 struct xlator_dumpops dumpops = {
-        .priv = dht_priv_dump,
-        .inodectx = dht_inodectx_dump,
+    .priv = dht_priv_dump,
+    .inodectx = dht_inodectx_dump,
 };
 
-
 struct xlator_cbks cbks = {
-//      .release    = dht_release,
-//      .releasedir = dht_releasedir,
-        .forget     = dht_forget
+    .release = dht_release,
+    //      .releasedir = dht_releasedir,
+    .forget = dht_forget,
 };
 
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "on",
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-          .default_value = "10%",
-          .description = "Percentage/Size of disk space that must be "
-                         "kept free."
-        },
-	{ .key  = {"min-free-inodes"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "5%",
-          .description = "Percentage inodes that must be "
-                         "kept free."
-        },
-        { .key = {"unhashed-sticky-bit"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"use-readdirp"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key = {"assert-no-child-down"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key  = {"directory-layout-spread"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key  = {"decommissioned-bricks"},
-          .type = GF_OPTION_TYPE_ANY,
-        },
-        { .key  = {"rebalance-cmd"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key = {"node-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-        { .key = {"rebalance-stats"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-
-        { .key  = {NULL} },
+extern int32_t
+mem_acct_init(xlator_t *this);
+
+extern struct volume_options dht_options[];
+
+xlator_api_t xlator_api = {
+    .init = dht_init,
+    .fini = dht_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = dht_options,
+    .identifier = "distribute",
+    .pass_through_fops = &dht_pt_fops,
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c
index 53c66aa45aa..3648a564840 100644
--- a/xlators/cluster/dht/src/nufa.c
+++ b/xlators/cluster/dht/src/nufa.c
@@ -8,695 +8,650 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "dht-common.h"
 
 /* TODO: all 'TODO's in dht.c holds good */
 
+extern struct volume_options dht_options[];
+
 int
-nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                       struct iatt *postparent)
+nufa_local_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, dict_t *xattr,
+                      struct iatt *postparent)
 {
-        xlator_t     *subvol      = NULL;
-        char          is_linkfile = 0;
-        char          is_dir      = 0;
-        dht_conf_t   *conf        = NULL;
-        dht_local_t  *local       = NULL;
-        loc_t        *loc         = NULL;
-        int           i           = 0;
-        call_frame_t *prev        = NULL;
-        int           call_cnt    = 0;
-        int           ret         = 0;
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
-
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-                if (conf->search_unhashed) {
-                        local->op_errno = ENOENT;
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+    xlator_t *subvol = NULL;
+    char is_linkfile = 0;
+    char is_dir = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    int i = 0;
+    xlator_t *prev = NULL;
+    int call_cnt = 0;
+    int ret = 0;
+
+    conf = this->private;
+
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
+
+    if (ENTRY_MISSING(op_ret, op_errno)) {
+        if (conf->search_unhashed) {
+            local->op_errno = ENOENT;
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
+        }
+    }
+
+    if (op_ret == -1)
+        goto out;
+
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    is_dir = check_is_dir(inode, stbuf, xattr);
+
+    if (!is_dir && !is_linkfile) {
+        /* non-directory and not a linkfile */
+        ret = dht_layout_preset(this, prev, inode);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "could not set pre-set layout for subvol"
+                         " %s",
+                         prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (op_ret == -1)
-                goto out;
-
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
-        is_dir      = check_is_dir (inode, stbuf, xattr);
-
-        if (!is_dir && !is_linkfile) {
-                /* non-directory and not a linkfile */
-                ret = dht_layout_preset (this, prev->this, inode);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "could not set pre-set layout for subvol %s",
-                                prev->this->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                goto out;
-        }
+        goto out;
+    }
 
-        if (is_dir) {
-                call_cnt        = conf->subvolume_cnt;
-                local->call_cnt = call_cnt;
-
-                local->inode = inode_ref (inode);
-                local->xattr = dict_ref (xattr);
-
-                local->op_ret = 0;
-                local->op_errno = 0;
-
-                local->layout = dht_layout_new (this, conf->subvolume_cnt);
-                if (!local->layout) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_lookup_dir_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->lookup,
-                                    &local->loc, local->xattr_req);
-                }
-        }
+    if (is_dir) {
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-        if (is_linkfile) {
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
 
-                if (!subvol) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "linkfile not having link subvolume. path=%s",
-                                loc->path);
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+        local->op_ret = 0;
+        local->op_errno = 0;
 
-                STACK_WIND (frame, dht_lookup_linkfile_cbk,
-                            subvol, subvol->fops->lookup,
-                            &local->loc, local->xattr_req);
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
+        if (!local->layout) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto err;
         }
 
-        return 0;
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        }
+    }
 
-out:
-        if (!local->hashed_subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume in layout for path=%s",
-                        local->loc.path);
-                local->op_errno = ENOENT;
-                dht_lookup_everywhere (frame, this, loc);
-                return 0;
+    if (is_linkfile) {
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+
+        if (!subvol) {
+            gf_msg_debug(this->name, 0,
+                         "linkfile has no link subvolume. path=%s", loc->path);
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
 
-        STACK_WIND (frame, dht_lookup_cbk,
-                    local->hashed_subvol, local->hashed_subvol->fops->lookup,
-                    &local->loc, local->xattr_req);
+        STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
+
+    return 0;
 
+out:
+    if (!local->hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     local->loc.path);
+        local->op_errno = ENOENT;
+        dht_lookup_everywhere(frame, this, loc);
         return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                      &local->loc, local->xattr_req);
+
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
-                          inode, stbuf, xattr, postparent);
-        return 0;
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
+    return 0;
 }
 
 int
-nufa_lookup (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, dict_t *xattr_req)
+nufa_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+
+    local->hashed_subvol = hashed_subvol;
+
+    if (is_revalidate(loc)) {
+        layout = local->layout;
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "revalidate lookup without cache. "
+                         "path=%s",
+                         loc->path);
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        if (layout->gen && (layout->gen < conf->gen)) {
+            gf_msg_debug(this->name, 0, "incomplete layout failure for path=%s",
+                         loc->path);
+            dht_layout_unref(this, local->layout);
+            goto do_fresh_lookup;
+        }
+
+        local->inode = inode_ref(loc->inode);
+
+        local->call_cnt = layout->cnt;
+        call_cnt = local->call_cnt;
+
+        /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
+         *       revalidates directly go to the cached-subvolume.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
         }
 
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
-        } else {
-                local->xattr_req = dict_new ();
+        for (i = 0; i < layout->cnt; i++) {
+            subvol = layout->list[i].xlator;
+
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                              subvol->fops->lookup, loc, local->xattr_req);
+
+            if (!--call_cnt)
+                break;
+        }
+    } else {
+    do_fresh_lookup:
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
         }
 
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-
-        local->hashed_subvol = hashed_subvol;
-
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "revalidate without cache. path=%s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "incomplete layout failure for path=%s",
-                                loc->path);
-                        dht_layout_unref (this, local->layout);
-                        goto do_fresh_lookup;
-                }
-
-                local->inode = inode_ref (loc->inode);
-
-                local->call_cnt = layout->cnt;
-                call_cnt = local->call_cnt;
-
-                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
-                 *       revalidates directly go to the cached-subvolume.
-                 */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                for (i = 0; i < layout->cnt; i++) {
-                        subvol = layout->list[i].xlator;
-
-                        STACK_WIND (frame, dht_revalidate_cbk,
-                                    subvol, subvol->fops->lookup,
-                                    loc, local->xattr_req);
-
-                        if (!--call_cnt)
-                                break;
-                }
-        } else {
-        do_fresh_lookup:
-                ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht.linkto", 256);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                /* Send it to only local volume */
-                STACK_WIND (frame, nufa_local_lookup_cbk,
-                            (xlator_t *)conf->private,
-                            ((xlator_t *)conf->private)->fops->lookup,
-                            loc, local->xattr_req);
+        ret = dict_set_uint32(local->xattr_req, conf->link_xattr_name, 256);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
         }
 
-        return 0;
+        /* Send it to only local volume */
+        STACK_WIND_COOKIE(
+            frame, nufa_local_lookup_cbk, ((xlator_t *)conf->private),
+            ((xlator_t *)conf->private),
+            ((xlator_t *)conf->private)->fops->lookup, loc, local->xattr_req);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-nufa_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int op_ret, int op_errno,
-                                 inode_t *inode, struct iatt *stbuf,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent, dict_t *xdata)
+nufa_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int op_ret, int op_errno,
+                                inode_t *inode, struct iatt *stbuf,
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1)
-                goto err;
+    if (op_ret == -1)
+        goto err;
 
-        STACK_WIND (frame, dht_create_cbk,
-                    local->cached_subvol, local->cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode, local->umask,
-                    local->fd, local->params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, local->cached_subvol,
+                      local->cached_subvol, local->cached_subvol->fops->create,
+                      &local->loc, local->flags, local->mode, local->umask,
+                      local->fd, local->params);
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 int
-nufa_create (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int32_t flags, mode_t mode,
-             mode_t umask, fd_t *fd, dict_t *params)
+nufa_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    avail_subvol = conf->private;
+    if (dht_is_subvol_filled(this, (xlator_t *)conf->private)) {
+        avail_subvol = dht_free_disk_available_subvol(
+            this, (xlator_t *)conf->private, local);
+    }
+
+    if (subvol != avail_subvol) {
+        /* create a link file instead of actual file */
+        local->params = dict_ref(params);
+        local->mode = mode;
+        local->flags = flags;
+        local->umask = umask;
+        local->cached_subvol = avail_subvol;
+        dht_linkfile_create(frame, nufa_create_linkfile_create_cbk, this,
+                            avail_subvol, subvol, loc);
+        return 0;
+    }
 
-        conf  = this->private;
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        dht_get_du_info (frame, this, loc);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                      subvol->fops->create, loc, flags, mode, umask, fd,
+                      params);
 
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    return 0;
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume in layout for path=%s",
-                        loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        avail_subvol = conf->private;
-        if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private);
-        }
+    return 0;
+}
 
-        if (subvol != avail_subvol) {
-                /* create a link file instead of actual file */
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->flags = flags;
-                local->umask = umask;
-                local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame,
-                                     nufa_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
-                return 0;
-        }
+int
+nufa_mknod_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "creating %s on %s", loc->path, subvol->name);
+    local = frame->local;
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        op_ret = -1;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_create_cbk,
-                    subvol, subvol->fops->create,
-                    loc, flags, mode, umask, fd, params);
+    if (op_ret >= 0) {
+        STACK_WIND_COOKIE(
+            frame, dht_newfile_cbk, (void *)local->cached_subvol,
+            local->cached_subvol, local->cached_subvol->fops->mknod,
+            &local->loc, local->mode, local->rdev, local->umask, local->params);
 
         return 0;
-
+    }
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
+    WIPE(postparent);
+    WIPE(preparent);
 
-        return 0;
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
 int
-nufa_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, inode_t *inode,
-                         struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
+nufa_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           dev_t rdev, mode_t umask, dict_t *params)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    /* Consider the disksize in consideration */
+    avail_subvol = conf->private;
+    if (dht_is_subvol_filled(this, (xlator_t *)conf->private)) {
+        avail_subvol = dht_free_disk_available_subvol(
+            this, (xlator_t *)conf->private, local);
+    }
+
+    if (avail_subvol != subvol) {
+        /* Create linkfile first */
+
+        local->params = dict_ref(params);
+        local->mode = mode;
+        local->umask = umask;
+        local->rdev = rdev;
+        local->cached_subvol = avail_subvol;
+
+        dht_linkfile_create(frame, nufa_mknod_linkfile_cbk, this, avail_subvol,
+                            subvol, loc);
+        return 0;
+    }
 
-        local = frame->local;
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        if (op_ret >= 0) {
-                STACK_WIND (frame, dht_newfile_cbk,
-                            local->cached_subvol,
-                            local->cached_subvol->fops->mknod,
-                            &local->loc, local->mode, local->rdev,
-                            local->umask, local->params);
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                      subvol->fops->mknod, loc, mode, rdev, umask, params);
 
-                return 0;
-        }
+    return 0;
 
-        WIPE (postparent);
-        WIPE (preparent);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent, xdata);
-        return 0;
+    return 0;
 }
 
-
-int
-nufa_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
+gf_boolean_t
+same_first_part(char *str1, char term1, char *str2, char term2)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    gf_boolean_t ended1;
+    gf_boolean_t ended2;
 
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume in layout for path=%s",
-                        loc->path);
-                op_errno = ENOENT;
-                goto err;
+    for (;;) {
+        ended1 = ((*str1 == '\0') || (*str1 == term1));
+        ended2 = ((*str2 == '\0') || (*str2 == term2));
+        if (ended1 && ended2) {
+            return _gf_true;
         }
-
-        /* Consider the disksize in consideration */
-        avail_subvol = conf->private;
-        if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private);
-        }
-
-        if (avail_subvol != subvol) {
-                /* Create linkfile first */
-
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->umask = umask;
-                local->rdev = rdev;
-                local->cached_subvol = avail_subvol;
-
-                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk,
-                                     avail_subvol, subvol, loc);
-                return 0;
+        if (ended1 || ended2 || (*str1 != *str2)) {
+            return _gf_false;
         }
+        ++str1;
+        ++str2;
+    }
+}
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "creating %s on %s", loc->path, subvol->name);
+typedef struct nufa_args {
+    xlator_t *this;
+    char *volname;
+    gf_boolean_t addr_match;
+} nufa_args_t;
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->mknod,
-                    loc, mode, rdev, umask, params);
+static void
+nufa_find_local_brick(xlator_t *xl, void *data)
+{
+    nufa_args_t *args = data;
+    xlator_t *this = args->this;
+    char *local_volname = args->volname;
+    gf_boolean_t addr_match = args->addr_match;
+    char *brick_host = NULL;
+    dht_conf_t *conf = this->private;
+    int ret = -1;
+
+    /*This means a local subvol was already found. We pick the first brick
+     * that is local*/
+    if (conf->private)
+        return;
 
-        return 0;
+    if (strcmp(xl->name, local_volname) == 0) {
+        conf->private = xl;
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Using specified subvol %s", local_volname);
+        return;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    if (!addr_match)
+        return;
 
-        return 0;
+    ret = dict_get_str(xl->options, "remote-host", &brick_host);
+    if ((ret == 0) && (gf_is_same_address(local_volname, brick_host) ||
+                       gf_is_local_addr(brick_host))) {
+        conf->private = xl;
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Using the first local "
+               "subvol %s",
+               xl->name);
+        return;
+    }
 }
 
-
-int
-notify (xlator_t *this, int event, void *data, ...)
+static void
+nufa_to_dht(xlator_t *this)
 {
-        int ret = -1;
+    GF_ASSERT(this);
+    GF_ASSERT(this->fops);
 
-        ret = dht_notify (this, event, data);
-
-        return ret;
+    this->fops->lookup = dht_lookup;
+    this->fops->create = dht_create;
+    this->fops->mknod = dht_mknod;
 }
 
-void
-fini (xlator_t *this)
+int
+nufa_find_local_subvol(xlator_t *this, void (*fn)(xlator_t *each, void *data),
+                       void *data)
 {
-        int         i = 0;
-        dht_conf_t *conf = NULL;
-
-        conf = this->private;
-
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                GF_FREE (conf->subvolumes);
-
-                GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf);
-        }
-
-        return;
+    int ret = -1;
+    dht_conf_t *conf = this->private;
+    xlator_list_t *trav = NULL;
+    xlator_t *parent = NULL;
+    xlator_t *candidate = NULL;
+
+    xlator_foreach_depth_first(this, fn, data);
+    if (!conf->private) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_BRICK_ERROR,
+               "Couldn't find a local "
+               "brick");
+        return -1;
+    }
+
+    candidate = conf->private;
+    trav = candidate->parents;
+    while (trav) {
+        parent = trav->xlator;
+        if (strcmp(parent->type, "cluster/nufa") == 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "Found local subvol, "
+                   "%s",
+                   candidate->name);
+            ret = 0;
+            conf->private = candidate;
+            break;
+        }
+
+        candidate = parent;
+        trav = parent->parents;
+    }
+
+    return ret;
 }
 
 int
-init (xlator_t *this)
+nufa_init(xlator_t *this)
 {
-        dht_conf_t    *conf = NULL;
-        xlator_list_t *trav = NULL;
-        data_t        *data = NULL;
-        char          *local_volname = NULL;
-        char          *temp_str = NULL;
-        int            ret = -1;
-        int            i = 0;
-        char           my_hostname[256];
-        uint32_t       temp_free_disk = 0;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "NUFA needs more than one subvolume");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf),
-                          gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-        }
-
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
+    data_t *data = NULL;
+    char *local_volname = NULL;
+    int ret = -1;
+    char my_hostname[256];
+    gf_boolean_t addr_match = _gf_false;
+    nufa_args_t args = {
+        0,
+    };
+
+    ret = dht_init(this);
+    if (ret) {
+        return ret;
+    }
 
-        conf->gen = 1;
+    if ((data = dict_get(this->options, "local-volume-name"))) {
+        local_volname = data->data;
 
+    } else {
+        addr_match = _gf_true;
         local_volname = "localhost";
-        ret = gethostname (my_hostname, 256);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "could not find hostname (%s)",
-                        strerror (errno));
-        }
-
+        ret = gethostname(my_hostname, 256);
         if (ret == 0)
-                local_volname = my_hostname;
-
-        data = dict_get (this->options, "local-volume-name");
-        if (data) {
-                local_volname = data->data;
-        }
-
-        trav = this->children;
-        while (trav) {
-                if (strcmp (trav->xlator->name, local_volname) == 0)
-                        break;
-                trav = trav->next;
-        }
-
-        if (!trav) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not find subvolume named '%s'. "
-                        "Please define volume with the name as the hostname "
-                        "or override it with 'option local-volume-name'",
-                        local_volname);
-                goto err;
-        }
-        /* The volume specified exists */
-        conf->private = trav->xlator;
-
-        conf->min_free_disk = 10;
-        conf->disk_unit = 'p';
-
-        if (dict_get_str (this->options, "min-free-disk",
-                          &temp_str) == 0) {
-                if (gf_string2percent (temp_str,
-                                       &temp_free_disk) == 0) {
-                        if (temp_free_disk > 100) {
-                                gf_string2bytesize (temp_str,
-                                                    &conf->min_free_disk);
-                                conf->disk_unit = 'b';
-                        } else {
-                                conf->min_free_disk = (uint64_t)temp_free_disk;
-                                conf->disk_unit = 'p';
-                        }
-                } else {
-                        gf_string2bytesize (temp_str,
-                                            &conf->min_free_disk);
-                        conf->disk_unit = 'b';
-                }
-        }
-
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_dht_mt_dht_du_t);
-        if (!conf->du_stats) {
-                goto err;
-        }
-
-        this->local_pool = mem_pool_new (dht_local_t, 128);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto err;
-        }
-
-        this->private = conf;
-
-        return 0;
-
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                GF_FREE (conf->subvolumes);
-
-                GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
-
-        return -1;
+            local_volname = my_hostname;
+
+        else
+            gf_msg(this->name, GF_LOG_WARNING, errno,
+                   DHT_MSG_GET_HOSTNAME_FAILED, "could not find hostname");
+    }
+
+    args.this = this;
+    args.volname = local_volname;
+    args.addr_match = addr_match;
+    ret = nufa_find_local_subvol(this, nufa_find_local_brick, &args);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Unable to find local subvolume, switching "
+               "to dht mode");
+        nufa_to_dht(this);
+    }
+    return 0;
 }
 
-
-struct xlator_fops fops = {
-        .lookup      = nufa_lookup,
-        .create      = nufa_create,
-        .mknod       = nufa_mknod,
-
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .setxattr    = dht_setxattr,
-        .getxattr    = dht_getxattr,
-        .removexattr = dht_removexattr,
-        .open        = dht_open,
-        .readv       = dht_readv,
-        .writev      = dht_writev,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .statfs      = dht_statfs,
-        .lk          = dht_lk,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
+dht_methods_t dht_methods = {
+    .migration_get_dst_subvol = dht_migration_get_dst_subvol,
+    .layout_search = dht_layout_search,
 };
 
-
-struct xlator_cbks cbks = {
-        .forget     = dht_forget
+struct xlator_fops fops = {
+    .lookup = nufa_lookup,
+    .create = nufa_create,
+    .mknod = nufa_mknod,
+
+    .stat = dht_stat,
+    .fstat = dht_fstat,
+    .truncate = dht_truncate,
+    .ftruncate = dht_ftruncate,
+    .access = dht_access,
+    .readlink = dht_readlink,
+    .setxattr = dht_setxattr,
+    .getxattr = dht_getxattr,
+    .removexattr = dht_removexattr,
+    .open = dht_open,
+    .readv = dht_readv,
+    .writev = dht_writev,
+    .flush = dht_flush,
+    .fsync = dht_fsync,
+    .statfs = dht_statfs,
+    .lk = dht_lk,
+    .opendir = dht_opendir,
+    .readdir = dht_readdir,
+    .readdirp = dht_readdirp,
+    .fsyncdir = dht_fsyncdir,
+    .symlink = dht_symlink,
+    .unlink = dht_unlink,
+    .link = dht_link,
+    .mkdir = dht_mkdir,
+    .rmdir = dht_rmdir,
+    .rename = dht_rename,
+    .inodelk = dht_inodelk,
+    .finodelk = dht_finodelk,
+    .entrylk = dht_entrylk,
+    .fentrylk = dht_fentrylk,
+    .xattrop = dht_xattrop,
+    .fxattrop = dht_fxattrop,
+    .setattr = dht_setattr,
 };
 
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR
-        },
-        { .key  = {"local-volume-name"},
-          .type = GF_OPTION_TYPE_XLATOR
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-        },
-        { .key  = {NULL} },
+struct xlator_cbks cbks = {.forget = dht_forget};
+extern int32_t
+mem_acct_init(xlator_t *this);
+
+xlator_api_t xlator_api = {
+    .init = nufa_init,
+    .fini = dht_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = dht_options,
+    .identifier = "nufa",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c
index 0542d7f9afc..207d109a025 100644
--- a/xlators/cluster/dht/src/switch.c
+++ b/xlators/cluster/dht/src/switch.c
@@ -8,12 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "dht-common.h"
 #include "dht-mem-types.h"
 
@@ -22,1001 +16,876 @@
 #include <fnmatch.h>
 #include <string.h>
 
+extern struct volume_options dht_options[];
+
 struct switch_sched_array {
-        xlator_t *xl;
-        int32_t   eligible;
-        int32_t   considered;
+    xlator_t *xl;
+    int32_t eligible;
+    int32_t considered;
 };
 
 /* Select one of this struct based on the path's pattern match */
 struct switch_struct {
-        struct switch_struct      *next;
-        struct switch_sched_array *array;
-        int32_t                    node_index; /* Index of the node in
-                                                  this pattern. */
-        int32_t                    num_child;  /* Total num of child nodes
-                                                  with this pattern. */
-        char                       path_pattern[256];
+    struct switch_struct *next;
+    struct switch_sched_array *array;
+    int32_t node_index; /* Index of the node in
+                           this pattern. */
+    int32_t num_child;  /* Total num of child nodes
+                           with this pattern. */
+    char path_pattern[256];
 };
 
 /* TODO: all 'TODO's in dht.c holds good */
 /* This function should return child node as '*:subvolumes' is inserterd */
 
 static int32_t
-gf_switch_valid_child (xlator_t *this, const char *child)
+gf_switch_valid_child(xlator_t *this, const char *child)
 {
-        xlator_list_t *children = NULL;
-        int32_t        ret = 0;
+    xlator_list_t *children = NULL;
+    int32_t ret = 0;
 
-        children = this->children;
-        while (children) {
-                if (!strcmp (child, children->xlator->name)) {
-                        ret = 1;
-                        break;
-                }
-                children = children->next;
+    children = this->children;
+    while (children) {
+        if (!strcmp(child, children->xlator->name)) {
+            ret = 1;
+            break;
         }
+        children = children->next;
+    }
 
-        return ret;
+    return ret;
 }
 
 static xlator_t *
-get_switch_matching_subvol (const char *path, dht_conf_t *conf,
-                            xlator_t *hashed_subvol)
+get_switch_matching_subvol(const char *path, dht_conf_t *conf,
+                           xlator_t *hashed_subvol)
 {
-        struct switch_struct *cond      = NULL;
-        struct switch_struct *trav      = NULL;
-        char                 *pathname  = NULL;
-        int                   idx       = 0;
-        xlator_t             *subvol    = NULL;
-
-        cond = conf->private;
-        subvol = hashed_subvol;
-        if (!cond)
-                goto out;
-
-        pathname = gf_strdup (path);
-        if (!pathname)
-                goto out;
-
-        trav = cond;
-        while (trav) {
-                if (fnmatch (trav->path_pattern,
-                             pathname, FNM_NOESCAPE) == 0) {
-                        for (idx = 0; idx < trav->num_child; idx++) {
-                                if (trav->array[idx].xl == hashed_subvol)
-                                        goto out;
-                        }
-                        idx = trav->node_index++;
-                        trav->node_index %= trav->num_child;
-                        subvol = trav->array[idx].xl;
-                        goto out;
-                }
-                trav = trav->next;
-        }
+    struct switch_struct *cond = NULL;
+    struct switch_struct *trav = NULL;
+    char *pathname = NULL;
+    int idx = 0;
+    xlator_t *subvol = NULL;
+
+    cond = conf->private;
+    subvol = hashed_subvol;
+    if (!cond)
+        goto out;
+
+    pathname = gf_strdup(path);
+    if (!pathname)
+        goto out;
+
+    trav = cond;
+    while (trav) {
+        if (fnmatch(trav->path_pattern, pathname, FNM_NOESCAPE) == 0) {
+            for (idx = 0; idx < trav->num_child; idx++) {
+                if (trav->array[idx].xl == hashed_subvol)
+                    goto out;
+            }
+            idx = trav->node_index++;
+            trav->node_index %= trav->num_child;
+            subvol = trav->array[idx].xl;
+            goto out;
+        }
+        trav = trav->next;
+    }
 out:
-        GF_FREE (pathname);
+    GF_FREE(pathname);
 
-        return subvol;
+    return subvol;
 }
 
-
 int
-switch_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
+switch_local_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
 {
-        xlator_t     *subvol      = NULL;
-        char          is_linkfile = 0;
-        char          is_dir      = 0;
-        dht_conf_t   *conf        = NULL;
-        dht_local_t  *local       = NULL;
-        loc_t        *loc         = NULL;
-        int           i           = 0;
-        call_frame_t *prev        = NULL;
-        int           call_cnt    = 0;
-        int           ret         = 0;
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
-
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-                if (conf->search_unhashed) {
-                        local->op_errno = ENOENT;
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
-        }
+    xlator_t *subvol = NULL;
+    char is_linkfile = 0;
+    char is_dir = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    int i = 0;
+    xlator_t *prev = NULL;
+    int call_cnt = 0;
+    int ret = 0;
 
-        if (op_ret == -1)
-                goto out;
+    conf = this->private;
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
-        is_dir      = check_is_dir (inode, stbuf, xattr);
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
 
-        if (!is_dir && !is_linkfile) {
-                /* non-directory and not a linkfile */
+    if (ENTRY_MISSING(op_ret, op_errno)) {
+        if (conf->search_unhashed) {
+            local->op_errno = ENOENT;
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
+        }
+    }
 
-                ret = dht_layout_preset (this, prev->this, inode);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "could not set pre-set layout for subvol %s",
-                                prev->this->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    if (op_ret == -1)
+        goto out;
 
-                goto out;
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    is_dir = check_is_dir(inode, stbuf, xattr);
+
+    if (!is_dir && !is_linkfile) {
+        /* non-directory and not a linkfile */
+
+        ret = dht_layout_preset(this, prev, inode);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "could not set pre-set layout "
+                         "for subvol %s",
+                         prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (is_dir) {
-                call_cnt        = conf->subvolume_cnt;
-                local->call_cnt = call_cnt;
+        goto out;
+    }
 
-                local->inode = inode_ref (inode);
-                local->xattr = dict_ref (xattr);
+    if (is_dir) {
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-                local->op_ret = 0;
-                local->op_errno = 0;
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
 
-                local->layout = dht_layout_new (this, conf->subvolume_cnt);
-                if (!local->layout) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "memory allocation failed :(");
-                        goto err;
-                }
+        local->op_ret = 0;
+        local->op_errno = 0;
 
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_lookup_dir_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->lookup,
-                                    &local->loc, local->xattr_req);
-                }
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
+        if (!local->layout) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg_debug(this->name, 0, "memory allocation failed :(");
+            goto err;
         }
 
-        if (is_linkfile) {
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        }
+    }
 
-                if (!subvol) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "linkfile not having link subvolume. path=%s",
-                                loc->path);
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+    if (is_linkfile) {
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
 
-                STACK_WIND (frame, dht_lookup_linkfile_cbk,
-                            subvol, subvol->fops->lookup,
-                            &local->loc, local->xattr_req);
+        if (!subvol) {
+            gf_msg_debug(this->name, 0,
+                         "linkfile has no link subvolume.path=%s", loc->path);
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
 
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
+
+    return 0;
 
 out:
-        if (!local->hashed_subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume in layout for path=%s",
-                        local->loc.path);
-                local->op_errno = ENOENT;
-                dht_lookup_everywhere (frame, this, loc);
-                return 0;
-        }
+    if (!local->hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     local->loc.path);
+        local->op_errno = ENOENT;
+        dht_lookup_everywhere(frame, this, loc);
+        return 0;
+    }
 
-        STACK_WIND (frame, dht_lookup_cbk,
-                    local->hashed_subvol, local->hashed_subvol->fops->lookup,
-                    &local->loc, local->xattr_req);
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                      &local->loc, local->xattr_req);
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
-                          inode, stbuf, xattr, NULL);
-        return 0;
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     NULL);
+    return 0;
 }
 
 int
-switch_lookup (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *xattr_req)
+switch_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xattr_req)
 {
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
-        xlator_t     *subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+    cached_subvol = local->cached_subvol;
+
+    local->hashed_subvol = hashed_subvol;
+
+    if (is_revalidate(loc)) {
+        layout = local->layout;
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "revalidate lookup without cache. path=%s", loc->path);
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        if (layout->gen && (layout->gen < conf->gen)) {
+            gf_msg_debug(this->name, 0, "incomplete layout failure for path=%s",
+                         loc->path);
+            dht_layout_unref(this, local->layout);
+            goto do_fresh_lookup;
+        }
+
+        local->inode = inode_ref(loc->inode);
+
+        local->call_cnt = layout->cnt;
+        call_cnt = local->call_cnt;
+
+        /* NOTE: we don't require 'trusted.glusterfs.dht.linkto'
+         * attribute, revalidates directly go to the cached-subvolume.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->xattr_name);
+
+        for (i = 0; i < layout->cnt; i++) {
+            subvol = layout->list[i].xlator;
+
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                              subvol->fops->lookup, loc, local->xattr_req);
+
+            if (!--call_cnt)
+                break;
+        }
+    } else {
+    do_fresh_lookup:
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->xattr_name);
+
+        ret = dict_set_uint32(local->xattr_req, conf->link_xattr_name, 256);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->link_xattr_name);
+
+        if (!hashed_subvol) {
+            gf_msg_debug(this->name, 0,
+                         "no subvolume in layout for path=%s, "
+                         "checking on all the subvols to see if "
+                         "it is a directory",
+                         loc->path);
+            call_cnt = conf->subvolume_cnt;
+            local->call_cnt = call_cnt;
+
+            local->layout = dht_layout_new(this, conf->subvolume_cnt);
+            if (!local->layout) {
                 op_errno = ENOMEM;
                 goto err;
-        }
-
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
+            }
+
+            for (i = 0; i < call_cnt; i++) {
+                STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk,
+                                  conf->subvolumes[i], conf->subvolumes[i],
+                                  conf->subvolumes[i]->fops->lookup,
+                                  &local->loc, local->xattr_req);
+            }
+            return 0;
+        }
+
+        /*  */
+        cached_subvol = get_switch_matching_subvol(loc->path, conf,
+                                                   hashed_subvol);
+        if (cached_subvol == hashed_subvol) {
+            STACK_WIND_COOKIE(frame, dht_lookup_cbk, hashed_subvol,
+                              hashed_subvol, hashed_subvol->fops->lookup, loc,
+                              local->xattr_req);
         } else {
-                local->xattr_req = dict_new ();
-        }
-
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-        cached_subvol = local->cached_subvol;
-
-        local->hashed_subvol = hashed_subvol;
-
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "revalidate without cache. path=%s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "incomplete layout failure for path=%s",
-                                loc->path);
-                        dht_layout_unref (this, local->layout);
-                        goto do_fresh_lookup;
-                }
-
-                local->inode    = inode_ref (loc->inode);
-
-                local->call_cnt = layout->cnt;
-                call_cnt = local->call_cnt;
-
-                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto'
-                 * attribute, revalidates directly go to the cached-subvolume.
-                 */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht");
-
-                for (i = 0; i < layout->cnt; i++) {
-                        subvol = layout->list[i].xlator;
-
-                        STACK_WIND (frame, dht_revalidate_cbk,
-                                    subvol, subvol->fops->lookup,
-                                    loc, local->xattr_req);
-
-                        if (!--call_cnt)
-                                break;
-                }
-        } else {
-        do_fresh_lookup:
-                ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht");
-
-                ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht.linkto", 256);
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht.linkto");
-
-                if (!hashed_subvol) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no subvolume in layout for path=%s, "
-                                "checking on all the subvols to see if "
-                                "it is a directory", loc->path);
-                        call_cnt        = conf->subvolume_cnt;
-                        local->call_cnt = call_cnt;
-
-                        local->layout = dht_layout_new (this,
-                                                        conf->subvolume_cnt);
-                        if (!local->layout) {
-                                op_errno = ENOMEM;
-                                goto err;
-                        }
-
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND (frame, dht_lookup_dir_cbk,
-                                            conf->subvolumes[i],
-                                            conf->subvolumes[i]->fops->lookup,
-                                            &local->loc, local->xattr_req);
-                        }
-                        return 0;
-                }
-
-                /*  */
-                cached_subvol = get_switch_matching_subvol (loc->path, conf,
-                                                            hashed_subvol);
-                if (cached_subvol == hashed_subvol) {
-                        STACK_WIND (frame, dht_lookup_cbk,
-                                    hashed_subvol,
-                                    hashed_subvol->fops->lookup,
-                                    loc, local->xattr_req);
-                } else {
-                        STACK_WIND (frame, switch_local_lookup_cbk,
-                                    cached_subvol,
-                                    cached_subvol->fops->lookup,
-                                    loc, local->xattr_req);
-                }
+            STACK_WIND_COOKIE(frame, switch_local_lookup_cbk, cached_subvol,
+                              cached_subvol, cached_subvol->fops->lookup, loc,
+                              local->xattr_req);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-switch_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                   xlator_t *this, int op_ret, int op_errno,
-                                   inode_t *inode, struct iatt *stbuf,
-                                   struct iatt *preparent,
-                                   struct iatt *postparent, dict_t *xdata)
+switch_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int op_ret, int op_errno,
+                                  inode_t *inode, struct iatt *stbuf,
+                                  struct iatt *preparent,
+                                  struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1)
-                goto err;
+    if (op_ret == -1)
+        goto err;
 
-        STACK_WIND (frame, dht_create_cbk,
-                    local->cached_subvol, local->cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode, local->umask,
-                    local->fd, local->params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, local->cached_subvol,
+                      local->cached_subvol, local->cached_subvol->fops->create,
+                      &local->loc, local->flags, local->mode, local->umask,
+                      local->fd, local->params);
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 int
-switch_create (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, int32_t flags, mode_t mode,
-               mode_t umask, fd_t *fd, dict_t *params)
+switch_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume in layout for path=%s",
-                        loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
-        if (dht_is_subvol_filled (this, avail_subvol)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol);
-        }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    avail_subvol = get_switch_matching_subvol(loc->path, conf, subvol);
+    if (dht_is_subvol_filled(this, avail_subvol)) {
+        avail_subvol = dht_free_disk_available_subvol(this, avail_subvol,
+                                                      local);
+    }
+
+    if (subvol != avail_subvol) {
+        /* create a link file instead of actual file */
+        local->mode = mode;
+        local->flags = flags;
+        local->umask = umask;
+        local->cached_subvol = avail_subvol;
+        dht_linkfile_create(frame, switch_create_linkfile_create_cbk, this,
+                            avail_subvol, subvol, loc);
+        return 0;
+    }
 
-        if (subvol != avail_subvol) {
-                /* create a link file instead of actual file */
-                local->mode = mode;
-                local->flags = flags;
-                local->umask = umask;
-                local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame,
-                                     switch_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
-                return 0;
-        }
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "creating %s on %s", loc->path, subvol->name);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                      subvol->fops->create, loc, flags, mode, umask, fd,
+                      params);
 
-        STACK_WIND (frame, dht_create_cbk,
-                    subvol, subvol->fops->create,
-                    loc, flags, mode, umask, fd, params);
-
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-switch_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, inode_t *inode,
-                           struct iatt *stbuf, struct iatt *preparent,
-                           struct iatt *postparent, dict_t *xdata)
+switch_mknod_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *stbuf, struct iatt *preparent,
+                          struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-
-        local = frame->local;
+    dht_local_t *local = NULL;
 
-        if (op_ret >= 0) {
-                STACK_WIND (frame, dht_newfile_cbk,
-                            local->cached_subvol,
-                            local->cached_subvol->fops->mknod,
-                            &local->loc, local->mode, local->rdev,
-                            local->umask, local->params);
+    local = frame->local;
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        op_ret = -1;
+        goto err;
+    }
 
-                return 0;
-        }
+    if (op_ret >= 0) {
+        STACK_WIND_COOKIE(
+            frame, dht_newfile_cbk, (void *)local->cached_subvol,
+            local->cached_subvol, local->cached_subvol->fops->mknod,
+            &local->loc, local->mode, local->rdev, local->umask, local->params);
 
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent, xdata);
         return 0;
+    }
+err:
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-
 int
-switch_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, mode_t umask, dict_t *params)
+switch_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             dev_t rdev, mode_t umask, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume in layout for path=%s",
-                        loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        /* Consider the disksize in consideration */
-        avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
-        if (dht_is_subvol_filled (this, avail_subvol)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol);
-        }
-
-        if (avail_subvol != subvol) {
-                /* Create linkfile first */
-
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->umask = umask;
-                local->rdev = rdev;
-                local->cached_subvol = avail_subvol;
-
-                dht_linkfile_create (frame, switch_mknod_linkfile_cbk,
-                                     avail_subvol, subvol, loc);
-                return 0;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "creating %s on %s", loc->path, subvol->name);
-
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->mknod,
-                    loc, mode, rdev, umask, params);
-
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
author	Ravishankar N <ravishankar@redhat.com>	2016-12-29 12:10:00 +0000
committer	Pranith Kumar Karampuri <pkarampu@redhat.com>	2017-01-02 22:13:51 -0800
commit	76fff8cb2a164b596ca67e65c99623f5b68361fd (patch)
tree	667273fe2b0f96cd817697478ab4fb118e0a8078 /tests/basic/afr
parent	11b6a2c9fc5232b58774cab29873406c0fbfef19 (diff)